xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/TD-20802

This commit is contained in:
dapan1121 2022-12-02 10:11:39 +08:00 committed by GitHub
parent 7ad0226e32 90b857da3f
commit dff4c36092
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
143 changed files with 6984 additions and 4474 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG d5df76d GIT_TAG 823fae5
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

42
docs/zh/27-train-faq/01-faq.md
View File

@ -201,3 +201,45 @@ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端
OOM 是操作系统的保护机制，当操作系统内存(包括 SWAP )不足时，会杀掉某些进程，从而保证操作系统的稳定运行。通常内存不足主要是如下两个原因导致，一是剩余内存小于 vm.min_free_kbytes ；二是程序请求的内存大于剩余内存。还有一种情况是内存充足但程序占用了特殊的内存地址，也会触发 OOM 。 OOM 是操作系统的保护机制，当操作系统内存(包括 SWAP )不足时，会杀掉某些进程，从而保证操作系统的稳定运行。通常内存不足主要是如下两个原因导致，一是剩余内存小于 vm.min_free_kbytes ；二是程序请求的内存大于剩余内存。还有一种情况是内存充足但程序占用了特殊的内存地址，也会触发 OOM 。
TDengine 会预先为每个 VNode 分配好内存，每个 Database 的 VNode 个数受 建库时的vgroups参数影响，每个 VNode 占用的内存大小受 buffer参数 影响。要防止 OOM，需要在项目建设之初合理规划内存，并合理设置 SWAP ，除此之外查询过量的数据也有可能导致内存暴涨，这取决于具体的查询语句。TDengine 企业版对内存管理做了优化，采用了新的内存分配器，对稳定性有更高要求的用户可以考虑选择企业版。 TDengine 会预先为每个 VNode 分配好内存，每个 Database 的 VNode 个数受 建库时的vgroups参数影响，每个 VNode 占用的内存大小受 buffer参数 影响。要防止 OOM，需要在项目建设之初合理规划内存，并合理设置 SWAP ，除此之外查询过量的数据也有可能导致内存暴涨，这取决于具体的查询语句。TDengine 企业版对内存管理做了优化，采用了新的内存分配器，对稳定性有更高要求的用户可以考虑选择企业版。
### 19. 在macOS上遇到Too many open files怎么办？
taosd日志文件报错Too many open file，是由于taosd打开文件数超过系统设置的上限所致。
解决方案如下：
1. 新建文件 /Library/LaunchDaemons/limit.maxfiles.plist，写入以下内容(以下示例将limit和maxfiles改为10万，可按需修改)：
```
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>limit.maxfiles</string>
<key>ProgramArguments</key>
<array>
<string>launchctl</string>
<string>limit</string>
<string>maxfiles</string>
<string>100000</string>
<string>100000</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>ServiceIPC</key>
<false/>
</dict>
</plist>
```
2. 修改文件权限
```
sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
sudo chmod 644 /Library/LaunchDaemons/limit.maxfiles.plist
```
3. 加载 plist 文件 (或重启系统后生效。launchd在启动时会自动加载该目录的 plist)
```
sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist
```
4.确认更改后的限制
```
launchctl limit maxfiles
```

1
include/common/systable.h
View File

@ -47,6 +47,7 @@ extern "C" {
#define TSDB_INS_TABLE_TOPICS "ins_topics" #define TSDB_INS_TABLE_TOPICS "ins_topics"
#define TSDB_INS_TABLE_STREAMS "ins_streams" #define TSDB_INS_TABLE_STREAMS "ins_streams"
#define TSDB_INS_TABLE_STREAM_TASKS "ins_stream_tasks" #define TSDB_INS_TABLE_STREAM_TASKS "ins_stream_tasks"
#define TSDB_INS_TABLE_USER_PRIVILEGES "ins_user_privileges"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas" #define TSDB_PERFS_TABLE_SMAS "perf_smas"

1
include/common/tglobal.h
View File

@ -137,6 +137,7 @@ extern int64_t tsWalFsyncDataSizeLimit;
// internal // internal
extern int32_t tsTransPullupInterval; extern int32_t tsTransPullupInterval;
extern int32_t tsMqRebalanceInterval; extern int32_t tsMqRebalanceInterval;
extern int32_t tsStreamCheckpointTickInterval;
extern int32_t tsTtlUnit; extern int32_t tsTtlUnit;
extern int32_t tsTtlPushInterval; extern int32_t tsTtlPushInterval;
extern int32_t tsGrantHBInterval; extern int32_t tsGrantHBInterval;

41
include/common/tmsg.h
View File

@ -120,6 +120,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_VNODES, TSDB_MGMT_TABLE_VNODES,
TSDB_MGMT_TABLE_APPS, TSDB_MGMT_TABLE_APPS,
TSDB_MGMT_TABLE_STREAM_TASKS, TSDB_MGMT_TABLE_STREAM_TASKS,
TSDB_MGMT_TABLE_PRIVILEGES,
TSDB_MGMT_TABLE_MAX, TSDB_MGMT_TABLE_MAX,
} EShowType; } EShowType;
@ -141,16 +142,18 @@ typedef enum _mgmt_table {
#define TSDB_FILL_PREV 4 #define TSDB_FILL_PREV 4
#define TSDB_FILL_NEXT 5 #define TSDB_FILL_NEXT 5
#define TSDB_ALTER_USER_PASSWD 0x1 #define TSDB_ALTER_USER_PASSWD 0x1
#define TSDB_ALTER_USER_SUPERUSER 0x2 #define TSDB_ALTER_USER_SUPERUSER 0x2
#define TSDB_ALTER_USER_ADD_READ_DB 0x3 #define TSDB_ALTER_USER_ADD_READ_DB 0x3
#define TSDB_ALTER_USER_REMOVE_READ_DB 0x4 #define TSDB_ALTER_USER_REMOVE_READ_DB 0x4
#define TSDB_ALTER_USER_ADD_WRITE_DB 0x5 #define TSDB_ALTER_USER_ADD_WRITE_DB 0x5
#define TSDB_ALTER_USER_REMOVE_WRITE_DB 0x6 #define TSDB_ALTER_USER_REMOVE_WRITE_DB 0x6
#define TSDB_ALTER_USER_ADD_ALL_DB 0x7 #define TSDB_ALTER_USER_ADD_ALL_DB 0x7
#define TSDB_ALTER_USER_REMOVE_ALL_DB 0x8 #define TSDB_ALTER_USER_REMOVE_ALL_DB 0x8
#define TSDB_ALTER_USER_ENABLE 0x9 #define TSDB_ALTER_USER_ENABLE 0x9
#define TSDB_ALTER_USER_SYSINFO 0xA #define TSDB_ALTER_USER_SYSINFO 0xA
#define TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC 0xB
#define TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC 0xC
#define TSDB_ALTER_USER_PRIVILEGES 0x2 #define TSDB_ALTER_USER_PRIVILEGES 0x2
@ -620,7 +623,7 @@ typedef struct {
int8_t enable; int8_t enable;
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char pass[TSDB_USET_PASSWORD_LEN]; char pass[TSDB_USET_PASSWORD_LEN];
char dbname[TSDB_DB_FNAME_LEN]; char objname[TSDB_DB_FNAME_LEN]; // db or topic
} SAlterUserReq; } SAlterUserReq;
int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq);
@ -1059,6 +1062,7 @@ typedef struct {
int32_t vgId; int32_t vgId;
int8_t syncState; int8_t syncState;
int8_t syncRestore; int8_t syncRestore;
int8_t syncCanRead;
int64_t cacheUsage; int64_t cacheUsage;
int64_t numOfTables; int64_t numOfTables;
int64_t numOfTimeSeries; int64_t numOfTimeSeries;
@ -1144,6 +1148,13 @@ typedef struct {
int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
typedef struct {
int64_t tick;
} SMStreamTickReq;
int32_t tSerializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
typedef struct { typedef struct {
int32_t id; int32_t id;
uint16_t port; // node sync Port uint16_t port; // node sync Port
@ -1744,6 +1755,8 @@ typedef struct {
int64_t watermark; int64_t watermark;
int32_t numOfTags; int32_t numOfTags;
SArray* pTags; // array of SField SArray* pTags; // array of SField
// 3.0.20
int64_t checkpointFreq; // ms
} SCMCreateStreamReq; } SCMCreateStreamReq;
typedef struct { typedef struct {
@ -1943,6 +1956,12 @@ typedef struct {
SHashObj* rebSubHash; // SHashObj<key, SMqRebSubscribe> SHashObj* rebSubHash; // SHashObj<key, SMqRebSubscribe>
} SMqDoRebalanceMsg; } SMqDoRebalanceMsg;
typedef struct {
int64_t streamId;
int64_t checkpointId;
char streamName[TSDB_STREAM_FNAME_LEN];
} SMStreamDoCheckpointMsg;
typedef struct { typedef struct {
int64_t status; int64_t status;
} SMVSubscribeRsp; } SMVSubscribeRsp;

10
include/common/tmsgdef.h
View File

@ -172,6 +172,8 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_SERVER_VERSION, "server-version", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SERVER_VERSION, "server-version", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_VND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG)
@ -241,8 +243,11 @@ enum {
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "stream-recover-finish", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "stream-task-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT, "stream-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESTORE_CHECKPOINT, "stream-restore-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_MON_MSG) TD_NEW_MSG_SEG(TDMT_MON_MSG)
@ -282,6 +287,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, "vnode-stream-recover1", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, "vnode-stream-recover1", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, "vnode-stream-recover2", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, "vnode-stream-recover2", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_CHECK_POINT_SOURCE, "vnode-stream-checkpoint-source", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)

554
include/common/ttokendef.h
View File

@ -58,282 +58,284 @@
#define TK_TO 40 #define TK_TO 40
#define TK_REVOKE 41 #define TK_REVOKE 41
#define TK_FROM 42 #define TK_FROM 42
#define TK_NK_COMMA 43 #define TK_SUBSCRIBE 43
#define TK_READ 44 #define TK_NK_COMMA 44
#define TK_WRITE 45 #define TK_READ 45
#define TK_NK_DOT 46 #define TK_WRITE 46
#define TK_DNODE 47 #define TK_NK_DOT 47
#define TK_PORT 48 #define TK_DNODE 48
#define TK_DNODES 49 #define TK_PORT 49
#define TK_NK_IPTOKEN 50 #define TK_DNODES 50
#define TK_FORCE 51 #define TK_NK_IPTOKEN 51
#define TK_LOCAL 52 #define TK_FORCE 52
#define TK_QNODE 53 #define TK_LOCAL 53
#define TK_BNODE 54 #define TK_QNODE 54
#define TK_SNODE 55 #define TK_BNODE 55
#define TK_MNODE 56 #define TK_SNODE 56
#define TK_DATABASE 57 #define TK_MNODE 57
#define TK_USE 58 #define TK_DATABASE 58
#define TK_FLUSH 59 #define TK_USE 59
#define TK_TRIM 60 #define TK_FLUSH 60
#define TK_IF 61 #define TK_TRIM 61
#define TK_NOT 62 #define TK_IF 62
#define TK_EXISTS 63 #define TK_NOT 63
#define TK_BUFFER 64 #define TK_EXISTS 64
#define TK_CACHEMODEL 65 #define TK_BUFFER 65
#define TK_CACHESIZE 66 #define TK_CACHEMODEL 66
#define TK_COMP 67 #define TK_CACHESIZE 67
#define TK_DURATION 68 #define TK_COMP 68
#define TK_NK_VARIABLE 69 #define TK_DURATION 69
#define TK_MAXROWS 70 #define TK_NK_VARIABLE 70
#define TK_MINROWS 71 #define TK_MAXROWS 71
#define TK_KEEP 72 #define TK_MINROWS 72
#define TK_PAGES 73 #define TK_KEEP 73
#define TK_PAGESIZE 74 #define TK_PAGES 74
#define TK_TSDB_PAGESIZE 75 #define TK_PAGESIZE 75
#define TK_PRECISION 76 #define TK_TSDB_PAGESIZE 76
#define TK_REPLICA 77 #define TK_PRECISION 77
#define TK_STRICT 78 #define TK_REPLICA 78
#define TK_VGROUPS 79 #define TK_STRICT 79
#define TK_SINGLE_STABLE 80 #define TK_VGROUPS 80
#define TK_RETENTIONS 81 #define TK_SINGLE_STABLE 81
#define TK_SCHEMALESS 82 #define TK_RETENTIONS 82
#define TK_WAL_LEVEL 83 #define TK_SCHEMALESS 83
#define TK_WAL_FSYNC_PERIOD 84 #define TK_WAL_LEVEL 84
#define TK_WAL_RETENTION_PERIOD 85 #define TK_WAL_FSYNC_PERIOD 85
#define TK_WAL_RETENTION_SIZE 86 #define TK_WAL_RETENTION_PERIOD 86
#define TK_WAL_ROLL_PERIOD 87 #define TK_WAL_RETENTION_SIZE 87
#define TK_WAL_SEGMENT_SIZE 88 #define TK_WAL_ROLL_PERIOD 88
#define TK_STT_TRIGGER 89 #define TK_WAL_SEGMENT_SIZE 89
#define TK_TABLE_PREFIX 90 #define TK_STT_TRIGGER 90
#define TK_TABLE_SUFFIX 91 #define TK_TABLE_PREFIX 91
#define TK_NK_COLON 92 #define TK_TABLE_SUFFIX 92
#define TK_MAX_SPEED 93 #define TK_NK_COLON 93
#define TK_TABLE 94 #define TK_MAX_SPEED 94
#define TK_NK_LP 95 #define TK_TABLE 95
#define TK_NK_RP 96 #define TK_NK_LP 96
#define TK_STABLE 97 #define TK_NK_RP 97
#define TK_ADD 98 #define TK_STABLE 98
#define TK_COLUMN 99 #define TK_ADD 99
#define TK_MODIFY 100 #define TK_COLUMN 100
#define TK_RENAME 101 #define TK_MODIFY 101
#define TK_TAG 102 #define TK_RENAME 102
#define TK_SET 103 #define TK_TAG 103
#define TK_NK_EQ 104 #define TK_SET 104
#define TK_USING 105 #define TK_NK_EQ 105
#define TK_TAGS 106 #define TK_USING 106
#define TK_COMMENT 107 #define TK_TAGS 107
#define TK_BOOL 108 #define TK_COMMENT 108
#define TK_TINYINT 109 #define TK_BOOL 109
#define TK_SMALLINT 110 #define TK_TINYINT 110
#define TK_INT 111 #define TK_SMALLINT 111
#define TK_INTEGER 112 #define TK_INT 112
#define TK_BIGINT 113 #define TK_INTEGER 113
#define TK_FLOAT 114 #define TK_BIGINT 114
#define TK_DOUBLE 115 #define TK_FLOAT 115
#define TK_BINARY 116 #define TK_DOUBLE 116
#define TK_TIMESTAMP 117 #define TK_BINARY 117
#define TK_NCHAR 118 #define TK_TIMESTAMP 118
#define TK_UNSIGNED 119 #define TK_NCHAR 119
#define TK_JSON 120 #define TK_UNSIGNED 120
#define TK_VARCHAR 121 #define TK_JSON 121
#define TK_MEDIUMBLOB 122 #define TK_VARCHAR 122
#define TK_BLOB 123 #define TK_MEDIUMBLOB 123
#define TK_VARBINARY 124 #define TK_BLOB 124
#define TK_DECIMAL 125 #define TK_VARBINARY 125
#define TK_MAX_DELAY 126 #define TK_DECIMAL 126
#define TK_WATERMARK 127 #define TK_MAX_DELAY 127
#define TK_ROLLUP 128 #define TK_WATERMARK 128
#define TK_TTL 129 #define TK_ROLLUP 129
#define TK_SMA 130 #define TK_TTL 130
#define TK_FIRST 131 #define TK_SMA 131
#define TK_LAST 132 #define TK_FIRST 132
#define TK_SHOW 133 #define TK_LAST 133
#define TK_DATABASES 134 #define TK_SHOW 134
#define TK_TABLES 135 #define TK_PRIVILEGES 135
#define TK_STABLES 136 #define TK_DATABASES 136
#define TK_MNODES 137 #define TK_TABLES 137
#define TK_QNODES 138 #define TK_STABLES 138
#define TK_FUNCTIONS 139 #define TK_MNODES 139
#define TK_INDEXES 140 #define TK_QNODES 140
#define TK_ACCOUNTS 141 #define TK_FUNCTIONS 141
#define TK_APPS 142 #define TK_INDEXES 142
#define TK_CONNECTIONS 143 #define TK_ACCOUNTS 143
#define TK_LICENCES 144 #define TK_APPS 144
#define TK_GRANTS 145 #define TK_CONNECTIONS 145
#define TK_QUERIES 146 #define TK_LICENCES 146
#define TK_SCORES 147 #define TK_GRANTS 147
#define TK_TOPICS 148 #define TK_QUERIES 148
#define TK_VARIABLES 149 #define TK_SCORES 149
#define TK_CLUSTER 150 #define TK_TOPICS 150
#define TK_BNODES 151 #define TK_VARIABLES 151
#define TK_SNODES 152 #define TK_CLUSTER 152
#define TK_TRANSACTIONS 153 #define TK_BNODES 153
#define TK_DISTRIBUTED 154 #define TK_SNODES 154
#define TK_CONSUMERS 155 #define TK_TRANSACTIONS 155
#define TK_SUBSCRIPTIONS 156 #define TK_DISTRIBUTED 156
#define TK_VNODES 157 #define TK_CONSUMERS 157
#define TK_LIKE 158 #define TK_SUBSCRIPTIONS 158
#define TK_TBNAME 159 #define TK_VNODES 159
#define TK_QTAGS 160 #define TK_LIKE 160
#define TK_AS 161 #define TK_TBNAME 161
#define TK_INDEX 162 #define TK_QTAGS 162
#define TK_FUNCTION 163 #define TK_AS 163
#define TK_INTERVAL 164 #define TK_INDEX 164
#define TK_TOPIC 165 #define TK_FUNCTION 165
#define TK_WITH 166 #define TK_INTERVAL 166
#define TK_META 167 #define TK_TOPIC 167
#define TK_CONSUMER 168 #define TK_WITH 168
#define TK_GROUP 169 #define TK_META 169
#define TK_DESC 170 #define TK_CONSUMER 170
#define TK_DESCRIBE 171 #define TK_GROUP 171
#define TK_RESET 172 #define TK_DESC 172
#define TK_QUERY 173 #define TK_DESCRIBE 173
#define TK_CACHE 174 #define TK_RESET 174
#define TK_EXPLAIN 175 #define TK_QUERY 175
#define TK_ANALYZE 176 #define TK_CACHE 176
#define TK_VERBOSE 177 #define TK_EXPLAIN 177
#define TK_NK_BOOL 178 #define TK_ANALYZE 178
#define TK_RATIO 179 #define TK_VERBOSE 179
#define TK_NK_FLOAT 180 #define TK_NK_BOOL 180
#define TK_OUTPUTTYPE 181 #define TK_RATIO 181
#define TK_AGGREGATE 182 #define TK_NK_FLOAT 182
#define TK_BUFSIZE 183 #define TK_OUTPUTTYPE 183
#define TK_STREAM 184 #define TK_AGGREGATE 184
#define TK_INTO 185 #define TK_BUFSIZE 185
#define TK_TRIGGER 186 #define TK_STREAM 186
#define TK_AT_ONCE 187 #define TK_INTO 187
#define TK_WINDOW_CLOSE 188 #define TK_TRIGGER 188
#define TK_IGNORE 189 #define TK_AT_ONCE 189
#define TK_EXPIRED 190 #define TK_WINDOW_CLOSE 190
#define TK_FILL_HISTORY 191 #define TK_IGNORE 191
#define TK_SUBTABLE 192 #define TK_EXPIRED 192
#define TK_KILL 193 #define TK_FILL_HISTORY 193
#define TK_CONNECTION 194 #define TK_SUBTABLE 194
#define TK_TRANSACTION 195 #define TK_KILL 195
#define TK_BALANCE 196 #define TK_CONNECTION 196
#define TK_VGROUP 197 #define TK_TRANSACTION 197
#define TK_MERGE 198 #define TK_BALANCE 198
#define TK_REDISTRIBUTE 199 #define TK_VGROUP 199
#define TK_SPLIT 200 #define TK_MERGE 200
#define TK_DELETE 201 #define TK_REDISTRIBUTE 201
#define TK_INSERT 202 #define TK_SPLIT 202
#define TK_NULL 203 #define TK_DELETE 203
#define TK_NK_QUESTION 204 #define TK_INSERT 204
#define TK_NK_ARROW 205 #define TK_NULL 205
#define TK_ROWTS 206 #define TK_NK_QUESTION 206
#define TK_QSTART 207 #define TK_NK_ARROW 207
#define TK_QEND 208 #define TK_ROWTS 208
#define TK_QDURATION 209 #define TK_QSTART 209
#define TK_WSTART 210 #define TK_QEND 210
#define TK_WEND 211 #define TK_QDURATION 211
#define TK_WDURATION 212 #define TK_WSTART 212
#define TK_IROWTS 213 #define TK_WEND 213
#define TK_CAST 214 #define TK_WDURATION 214
#define TK_NOW 215 #define TK_IROWTS 215
#define TK_TODAY 216 #define TK_CAST 216
#define TK_TIMEZONE 217 #define TK_NOW 217
#define TK_CLIENT_VERSION 218 #define TK_TODAY 218
#define TK_SERVER_VERSION 219 #define TK_TIMEZONE 219
#define TK_SERVER_STATUS 220 #define TK_CLIENT_VERSION 220
#define TK_CURRENT_USER 221 #define TK_SERVER_VERSION 221
#define TK_COUNT 222 #define TK_SERVER_STATUS 222
#define TK_LAST_ROW 223 #define TK_CURRENT_USER 223
#define TK_CASE 224 #define TK_COUNT 224
#define TK_END 225 #define TK_LAST_ROW 225
#define TK_WHEN 226 #define TK_CASE 226
#define TK_THEN 227 #define TK_END 227
#define TK_ELSE 228 #define TK_WHEN 228
#define TK_BETWEEN 229 #define TK_THEN 229
#define TK_IS 230 #define TK_ELSE 230
#define TK_NK_LT 231 #define TK_BETWEEN 231
#define TK_NK_GT 232 #define TK_IS 232
#define TK_NK_LE 233 #define TK_NK_LT 233
#define TK_NK_GE 234 #define TK_NK_GT 234
#define TK_NK_NE 235 #define TK_NK_LE 235
#define TK_MATCH 236 #define TK_NK_GE 236
#define TK_NMATCH 237 #define TK_NK_NE 237
#define TK_CONTAINS 238 #define TK_MATCH 238
#define TK_IN 239 #define TK_NMATCH 239
#define TK_JOIN 240 #define TK_CONTAINS 240
#define TK_INNER 241 #define TK_IN 241
#define TK_SELECT 242 #define TK_JOIN 242
#define TK_DISTINCT 243 #define TK_INNER 243
#define TK_WHERE 244 #define TK_SELECT 244
#define TK_PARTITION 245 #define TK_DISTINCT 245
#define TK_BY 246 #define TK_WHERE 246
#define TK_SESSION 247 #define TK_PARTITION 247
#define TK_STATE_WINDOW 248 #define TK_BY 248
#define TK_SLIDING 249 #define TK_SESSION 249
#define TK_FILL 250 #define TK_STATE_WINDOW 250
#define TK_VALUE 251 #define TK_SLIDING 251
#define TK_NONE 252 #define TK_FILL 252
#define TK_PREV 253 #define TK_VALUE 253
#define TK_LINEAR 254 #define TK_NONE 254
#define TK_NEXT 255 #define TK_PREV 255
#define TK_HAVING 256 #define TK_LINEAR 256
#define TK_RANGE 257 #define TK_NEXT 257
#define TK_EVERY 258 #define TK_HAVING 258
#define TK_ORDER 259 #define TK_RANGE 259
#define TK_SLIMIT 260 #define TK_EVERY 260
#define TK_SOFFSET 261 #define TK_ORDER 261
#define TK_LIMIT 262 #define TK_SLIMIT 262
#define TK_OFFSET 263 #define TK_SOFFSET 263
#define TK_ASC 264 #define TK_LIMIT 264
#define TK_NULLS 265 #define TK_OFFSET 265
#define TK_ABORT 266 #define TK_ASC 266
#define TK_AFTER 267 #define TK_NULLS 267
#define TK_ATTACH 268 #define TK_ABORT 268
#define TK_BEFORE 269 #define TK_AFTER 269
#define TK_BEGIN 270 #define TK_ATTACH 270
#define TK_BITAND 271 #define TK_BEFORE 271
#define TK_BITNOT 272 #define TK_BEGIN 272
#define TK_BITOR 273 #define TK_BITAND 273
#define TK_BLOCKS 274 #define TK_BITNOT 274
#define TK_CHANGE 275 #define TK_BITOR 275
#define TK_COMMA 276 #define TK_BLOCKS 276
#define TK_COMPACT 277 #define TK_CHANGE 277
#define TK_CONCAT 278 #define TK_COMMA 278
#define TK_CONFLICT 279 #define TK_COMPACT 279
#define TK_COPY 280 #define TK_CONCAT 280
#define TK_DEFERRED 281 #define TK_CONFLICT 281
#define TK_DELIMITERS 282 #define TK_COPY 282
#define TK_DETACH 283 #define TK_DEFERRED 283
#define TK_DIVIDE 284 #define TK_DELIMITERS 284
#define TK_DOT 285 #define TK_DETACH 285
#define TK_EACH 286 #define TK_DIVIDE 286
#define TK_FAIL 287 #define TK_DOT 287
#define TK_FILE 288 #define TK_EACH 288
#define TK_FOR 289 #define TK_FAIL 289
#define TK_GLOB 290 #define TK_FILE 290
#define TK_ID 291 #define TK_FOR 291
#define TK_IMMEDIATE 292 #define TK_GLOB 292
#define TK_IMPORT 293 #define TK_ID 293
#define TK_INITIALLY 294 #define TK_IMMEDIATE 294
#define TK_INSTEAD 295 #define TK_IMPORT 295
#define TK_ISNULL 296 #define TK_INITIALLY 296
#define TK_KEY 297 #define TK_INSTEAD 297
#define TK_MODULES 298 #define TK_ISNULL 298
#define TK_NK_BITNOT 299 #define TK_KEY 299
#define TK_NK_SEMI 300 #define TK_MODULES 300
#define TK_NOTNULL 301 #define TK_NK_BITNOT 301
#define TK_OF 302 #define TK_NK_SEMI 302
#define TK_PLUS 303 #define TK_NOTNULL 303
#define TK_PRIVILEGE 304 #define TK_OF 304
#define TK_RAISE 305 #define TK_PLUS 305
#define TK_REPLACE 306 #define TK_PRIVILEGE 306
#define TK_RESTRICT 307 #define TK_RAISE 307
#define TK_ROW 308 #define TK_REPLACE 308
#define TK_SEMI 309 #define TK_RESTRICT 309
#define TK_STAR 310 #define TK_ROW 310
#define TK_STATEMENT 311 #define TK_SEMI 311
#define TK_STRING 312 #define TK_STAR 312
#define TK_TIMES 313 #define TK_STATEMENT 313
#define TK_UPDATE 314 #define TK_STRING 314
#define TK_VALUES 315 #define TK_TIMES 315
#define TK_VARIABLE 316 #define TK_UPDATE 316
#define TK_VIEW 317 #define TK_VALUES 317
#define TK_WAL 318 #define TK_VARIABLE 318
#define TK_VIEW 319
#define TK_WAL 320
#define TK_NK_SPACE 600 #define TK_NK_SPACE 600
#define TK_NK_COMMENT 601 #define TK_NK_COMMENT 601

9
include/libs/nodes/cmdnodes.h
View File

@ -42,9 +42,10 @@ extern "C" {
#define PRIVILEGE_TYPE_MASK(n) (1 << n) #define PRIVILEGE_TYPE_MASK(n) (1 << n)
#define PRIVILEGE_TYPE_ALL PRIVILEGE_TYPE_MASK(0) #define PRIVILEGE_TYPE_ALL PRIVILEGE_TYPE_MASK(0)
#define PRIVILEGE_TYPE_READ PRIVILEGE_TYPE_MASK(1) #define PRIVILEGE_TYPE_READ PRIVILEGE_TYPE_MASK(1)
#define PRIVILEGE_TYPE_WRITE PRIVILEGE_TYPE_MASK(2) #define PRIVILEGE_TYPE_WRITE PRIVILEGE_TYPE_MASK(2)
#define PRIVILEGE_TYPE_SUBSCRIBE PRIVILEGE_TYPE_MASK(3)
#define PRIVILEGE_TYPE_TEST_MASK(val, mask) (((val) & (mask)) != 0) #define PRIVILEGE_TYPE_TEST_MASK(val, mask) (((val) & (mask)) != 0)
@ -423,7 +424,7 @@ typedef struct SDropFunctionStmt {
typedef struct SGrantStmt { typedef struct SGrantStmt {
ENodeType type; ENodeType type;
char userName[TSDB_USER_LEN]; char userName[TSDB_USER_LEN];
char dbName[TSDB_DB_NAME_LEN]; char objName[TSDB_DB_NAME_LEN]; // db or topic
int64_t privileges; int64_t privileges;
} SGrantStmt; } SGrantStmt;

1
include/libs/nodes/nodes.h
View File

@ -187,6 +187,7 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_TRANSACTIONS_STMT, QUERY_NODE_SHOW_TRANSACTIONS_STMT,
QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT,
QUERY_NODE_SHOW_VNODES_STMT, QUERY_NODE_SHOW_VNODES_STMT,
QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
QUERY_NODE_SHOW_CREATE_DATABASE_STMT, QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
QUERY_NODE_SHOW_CREATE_TABLE_STMT, QUERY_NODE_SHOW_CREATE_TABLE_STMT,
QUERY_NODE_SHOW_CREATE_STABLE_STMT, QUERY_NODE_SHOW_CREATE_STABLE_STMT,

91
include/libs/stream/tstream.h
View File

@ -275,31 +275,6 @@ typedef struct {
SEpSet epSet; SEpSet epSet;
} SStreamChildEpInfo; } SStreamChildEpInfo;
typedef struct {
int32_t srcNodeId;
int32_t srcChildId;
int64_t stateSaveVer;
int64_t stateProcessedVer;
} SStreamCheckpointInfo;
typedef struct {
int64_t streamId;
int64_t checkTs;
int32_t checkpointId; // incremental
int32_t taskId;
SArray* checkpointVer; // SArray<SStreamCheckpointInfo>
} SStreamMultiVgCheckpointInfo;
typedef struct {
int32_t taskId;
int32_t checkpointId; // incremental
} SStreamCheckpointKey;
typedef struct {
int32_t taskId;
SArray* checkpointVer;
} SStreamRecoveringState;
typedef struct SStreamTask { typedef struct SStreamTask {
int64_t streamId; int64_t streamId;
int32_t taskId; int32_t taskId;
@ -364,6 +339,10 @@ typedef struct SStreamTask {
int64_t checkReqId; int64_t checkReqId;
SArray* checkReqIds; // shuffle SArray* checkReqIds; // shuffle
int32_t refCnt; int32_t refCnt;
int64_t checkpointingId;
int32_t checkpointAlignCnt;
} SStreamTask; } SStreamTask;
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
@ -509,6 +488,60 @@ typedef struct {
int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq); int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq);
int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq); int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq);
typedef struct {
int64_t streamId;
int64_t checkpointId;
int32_t taskId;
int32_t nodeId;
int64_t expireTime;
} SStreamCheckpointSourceReq;
typedef struct {
int64_t streamId;
int64_t checkpointId;
int32_t taskId;
int32_t nodeId;
int64_t expireTime;
} SStreamCheckpointSourceRsp;
int32_t tEncodeSStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq);
int32_t tDecodeSStreamCheckpointSourceReq(SDecoder* pDecoder, SStreamCheckpointSourceReq* pReq);
int32_t tEncodeSStreamCheckpointSourceRsp(SEncoder* pEncoder, const SStreamCheckpointSourceRsp* pRsp);
int32_t tDecodeSStreamCheckpointSourceRsp(SDecoder* pDecoder, SStreamCheckpointSourceRsp* pRsp);
typedef struct {
SMsgHead msgHead;
int64_t streamId;
int64_t checkpointId;
int32_t downstreamTaskId;
int32_t downstreamNodeId;
int32_t upstreamTaskId;
int32_t upstreamNodeId;
int32_t childId;
int64_t expireTime;
int8_t taskLevel;
} SStreamCheckpointReq;
typedef struct {
SMsgHead msgHead;
int64_t streamId;
int64_t checkpointId;
int32_t downstreamTaskId;
int32_t downstreamNodeId;
int32_t upstreamTaskId;
int32_t upstreamNodeId;
int32_t childId;
int64_t expireTime;
int8_t taskLevel;
} SStreamCheckpointRsp;
int32_t tEncodeSStreamCheckpointReq(SEncoder* pEncoder, const SStreamCheckpointReq* pReq);
int32_t tDecodeSStreamCheckpointReq(SDecoder* pDecoder, SStreamCheckpointReq* pReq);
int32_t tEncodeSStreamCheckpointRsp(SEncoder* pEncoder, const SStreamCheckpointRsp* pRsp);
int32_t tDecodeSStreamCheckpointRsp(SDecoder* pDecoder, SStreamCheckpointRsp* pRsp);
typedef struct { typedef struct {
int64_t streamId; int64_t streamId;
int32_t downstreamTaskId; int32_t downstreamTaskId;
@ -598,18 +631,22 @@ void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask);
int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen); int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen);
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId); SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId);
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
void streamMetaRemoveTask1(SStreamMeta* pMeta, int32_t taskId); void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
int32_t streamMetaBegin(SStreamMeta* pMeta); int32_t streamMetaBegin(SStreamMeta* pMeta);
int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta);
int32_t streamMetaRollBack(SStreamMeta* pMeta); int32_t streamMetaRollBack(SStreamMeta* pMeta);
int32_t streamLoadTasks(SStreamMeta* pMeta); int32_t streamLoadTasks(SStreamMeta* pMeta);
// checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
int32_t streamProcessCheckpointReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointReq* pReq);
int32_t streamProcessCheckpointRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointRsp* pRsp);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

23
include/libs/sync/sync.h
View File

@ -40,6 +40,8 @@ extern "C" {
#define SNAPSHOT_MAX_CLOCK_SKEW_MS 1000 * 10 #define SNAPSHOT_MAX_CLOCK_SKEW_MS 1000 * 10
#define SNAPSHOT_WAIT_MS 1000 * 30 #define SNAPSHOT_WAIT_MS 1000 * 30
#define SYNC_MAX_RETRY_BACKOFF 5
#define SYNC_LOG_REPL_RETRY_WAIT_MS 100
#define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000
#define SYNC_HEART_TIMEOUT_MS 1000 * 8 #define SYNC_HEART_TIMEOUT_MS 1000 * 8
@ -49,7 +51,7 @@ extern "C" {
#define SYNC_MAX_BATCH_SIZE 1 #define SYNC_MAX_BATCH_SIZE 1
#define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_BEGIN 0
#define SYNC_INDEX_INVALID -1 #define SYNC_INDEX_INVALID -1
#define SYNC_TERM_INVALID 0xFFFFFFFFFFFFFFFF #define SYNC_TERM_INVALID -1 // 0xFFFFFFFFFFFFFFFF
typedef enum { typedef enum {
SYNC_STRATEGY_NO_SNAPSHOT = 0, SYNC_STRATEGY_NO_SNAPSHOT = 0,
@ -60,7 +62,7 @@ typedef enum {
typedef uint64_t SyncNodeId; typedef uint64_t SyncNodeId;
typedef int32_t SyncGroupId; typedef int32_t SyncGroupId;
typedef int64_t SyncIndex; typedef int64_t SyncIndex;
typedef uint64_t SyncTerm; typedef int64_t SyncTerm;
typedef struct SSyncNode SSyncNode; typedef struct SSyncNode SSyncNode;
typedef struct SWal SWal; typedef struct SWal SWal;
@ -136,13 +138,13 @@ typedef struct SSnapshotMeta {
typedef struct SSyncFSM { typedef struct SSyncFSM {
void* data; void* data;
void (*FpCommitCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
void (*FpPreCommitCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
void (*FpRollBackCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm); void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm);
void (*FpReConfigCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SReConfigCbMeta* pMeta); void (*FpReConfigCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SReConfigCbMeta* pMeta);
void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm); bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm);
int32_t (*FpApplyQueueItems)(const struct SSyncFSM* pFsm); int32_t (*FpApplyQueueItems)(const struct SSyncFSM* pFsm);
@ -211,15 +213,20 @@ typedef struct SSyncInfo {
int32_t (*syncEqCtrlMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg); int32_t (*syncEqCtrlMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg);
} SSyncInfo; } SSyncInfo;
// if state == leader
// if restored, display "leader"
// if !restored && canRead, display "leader*"
// if !restored && !canRead, display "leader**"
typedef struct SSyncState { typedef struct SSyncState {
ESyncState state; ESyncState state;
bool restored; bool restored;
bool canRead;
} SSyncState; } SSyncState;
int32_t syncInit(); int32_t syncInit();
void syncCleanUp(); void syncCleanUp();
int64_t syncOpen(SSyncInfo* pSyncInfo); int64_t syncOpen(SSyncInfo* pSyncInfo);
void syncStart(int64_t rid); int32_t syncStart(int64_t rid);
void syncStop(int64_t rid); void syncStop(int64_t rid);
void syncPreStop(int64_t rid); void syncPreStop(int64_t rid);
int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak); int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak);

2
include/libs/wal/wal.h
View File

@ -170,7 +170,7 @@ int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo
// Assign version automatically and return to caller, // Assign version automatically and return to caller,
// -1 will be returned for failed writes // -1 will be returned for failed writes
int64_t walAppendLog(SWal *, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen); int64_t walAppendLog(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
void walFsync(SWal *, bool force); void walFsync(SWal *, bool force);

8
include/os/osTime.h
View File

@ -35,6 +35,7 @@ extern "C" {
#ifdef WINDOWS #ifdef WINDOWS
#define CLOCK_REALTIME 0 #define CLOCK_REALTIME 0
#define CLOCK_MONOTONIC 0
#define MILLISECOND_PER_SECOND (1000i64) #define MILLISECOND_PER_SECOND (1000i64)
#else #else
@ -82,6 +83,13 @@ static FORCE_INLINE int64_t taosGetTimestampNs() {
return (int64_t)systemTime.tv_sec * 1000000000LL + (int64_t)systemTime.tv_nsec; return (int64_t)systemTime.tv_sec * 1000000000LL + (int64_t)systemTime.tv_nsec;
} }
//@return timestamp of monotonic clock in millisecond
static FORCE_INLINE int64_t taosGetMonoTimestampMs() {
struct timespec systemTime = {0};
taosClockGetTime(CLOCK_MONOTONIC, &systemTime);
return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_nsec / 1000000;
}
char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm); char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm);
struct tm *taosLocalTime(const time_t *timep, struct tm *result); struct tm *taosLocalTime(const time_t *timep, struct tm *result);
struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst); struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst);

1
include/util/tdef.h
View File

@ -281,6 +281,7 @@ typedef enum ELogicConditionType {
#define TSDB_DNODE_ROLE_VNODE 2 #define TSDB_DNODE_ROLE_VNODE 2
#define TSDB_MAX_REPLICA 5 #define TSDB_MAX_REPLICA 5
#define TSDB_SYNC_LOG_BUFFER_SIZE 4096
#define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_TBNAME_COLUMN_INDEX (-1)
#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta

25
source/client/src/clientSml.c
View File

@ -139,6 +139,8 @@ typedef struct {
int32_t numOfSTables; int32_t numOfSTables;
int32_t numOfCTables; int32_t numOfCTables;
int32_t numOfCreateSTables; int32_t numOfCreateSTables;
int32_t numOfAlterColSTables;
int32_t numOfAlterTagSTables;
int64_t parseTime; int64_t parseTime;
int64_t schemaTime; int64_t schemaTime;
@ -512,6 +514,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end; goto end;
} }
info->cost.numOfAlterTagSTables++;
taosMemoryFreeClear(pTableMeta); taosMemoryFreeClear(pTableMeta);
code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -559,6 +562,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end; goto end;
} }
info->cost.numOfAlterColSTables++;
taosMemoryFreeClear(pTableMeta);
code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
@ -820,11 +825,6 @@ static int8_t smlGetTsTypeByPrecision(int8_t precision) {
} }
static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) {
void *tmp = taosMemoryCalloc(1, len + 1);
memcpy(tmp, data, len);
uDebug("SML:0x%" PRIx64 " smlParseInfluxTime tslen:%d, ts:%s", info->id, len, (char*)tmp);
taosMemoryFree(tmp);
if (len == 0 || (len == 1 && data[0] == '0')) { if (len == 0 || (len == 1 && data[0] == '0')) {
return taosGetTimestampNs(); return taosGetTimestampNs();
} }
@ -877,7 +877,10 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra
} }
uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts); uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts);
if (ts == -1) return TSDB_CODE_INVALID_TIMESTAMP; if (ts <= 0) {
uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts);
return TSDB_CODE_INVALID_TIMESTAMP;
}
// add ts to // add ts to
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
@ -2076,10 +2079,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *
static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) {
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
void *tmp = taosMemoryCalloc(1, len + 1); uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? "rawdata" : sql));
memcpy(tmp, sql, len);
uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? (char*)tmp : sql));
taosMemoryFree(tmp);
int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
@ -2372,11 +2372,12 @@ static int32_t smlInsertData(SSmlHandle *info) {
static void smlPrintStatisticInfo(SSmlHandle *info) { static void smlPrintStatisticInfo(SSmlHandle *info) {
uError("SML:0x%" PRIx64 uError("SML:0x%" PRIx64
" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d \ " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
"", "",
info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables,
info->cost.numOfCreateSTables, info->cost.schemaTime - info->cost.parseTime, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables,
info->cost.schemaTime - info->cost.parseTime,
info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
} }

7
source/common/src/systable.c
View File

@ -273,6 +273,12 @@ static const SSysDbTableSchema vnodesSchema[] = {
{.name = "dnode_ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "dnode_ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
}; };
static const SSysDbTableSchema userUserPrivilegesSchema[] = {
{.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "object_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysTableMeta infosMeta[] = { static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true}, {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true}, {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
@ -297,6 +303,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_STREAMS, streamSchema, tListLen(streamSchema), false}, {TSDB_INS_TABLE_STREAMS, streamSchema, tListLen(streamSchema), false},
{TSDB_INS_TABLE_STREAM_TASKS, streamTaskSchema, tListLen(streamTaskSchema), false}, {TSDB_INS_TABLE_STREAM_TASKS, streamTaskSchema, tListLen(streamTaskSchema), false},
{TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true}, {TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
{TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), false},
}; };
static const SSysDbTableSchema connectionsSchema[] = { static const SSysDbTableSchema connectionsSchema[] = {

1
source/common/src/tglobal.c
View File

@ -167,6 +167,7 @@ int64_t tsWalFsyncDataSizeLimit = (100 * 1024 * 1024L);
// internal // internal
int32_t tsTransPullupInterval = 2; int32_t tsTransPullupInterval = 2;
int32_t tsMqRebalanceInterval = 2; int32_t tsMqRebalanceInterval = 2;
int32_t tsStreamCheckpointTickInterval = 1;
int32_t tsTtlUnit = 86400; int32_t tsTtlUnit = 86400;
int32_t tsTtlPushInterval = 86400; int32_t tsTtlPushInterval = 86400;
int32_t tsGrantHBInterval = 60; int32_t tsGrantHBInterval = 60;

39
source/common/src/tmsg.c
View File

@ -992,15 +992,20 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tEncodeI32(&encoder, vlen) < 0) return -1; if (tEncodeI32(&encoder, vlen) < 0) return -1;
for (int32_t i = 0; i < vlen; ++i) { for (int32_t i = 0; i < vlen; ++i) {
SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i); SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i);
int64_t reserved = 0;
if (tEncodeI32(&encoder, pload->vgId) < 0) return -1; if (tEncodeI32(&encoder, pload->vgId) < 0) return -1;
if (tEncodeI8(&encoder, pload->syncState) < 0) return -1; if (tEncodeI8(&encoder, pload->syncState) < 0) return -1;
if (tEncodeI8(&encoder, pload->syncRestore) < 0) return -1; if (tEncodeI8(&encoder, pload->syncRestore) < 0) return -1;
if (tEncodeI8(&encoder, pload->syncCanRead) < 0) return -1;
if (tEncodeI64(&encoder, pload->cacheUsage) < 0) return -1; if (tEncodeI64(&encoder, pload->cacheUsage) < 0) return -1;
if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1; if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1;
if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1; if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1;
if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1; if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1;
if (tEncodeI64(&encoder, pload->compStorage) < 0) return -1; if (tEncodeI64(&encoder, pload->compStorage) < 0) return -1;
if (tEncodeI64(&encoder, pload->pointsWritten) < 0) return -1; if (tEncodeI64(&encoder, pload->pointsWritten) < 0) return -1;
if (tEncodeI64(&encoder, reserved) < 0) return -1;
if (tEncodeI64(&encoder, reserved) < 0) return -1;
if (tEncodeI64(&encoder, reserved) < 0) return -1;
} }
// mnode loads // mnode loads
@ -1065,15 +1070,20 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
for (int32_t i = 0; i < vlen; ++i) { for (int32_t i = 0; i < vlen; ++i) {
SVnodeLoad vload = {0}; SVnodeLoad vload = {0};
int64_t reserved = 0;
if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1; if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1;
if (tDecodeI8(&decoder, &vload.syncState) < 0) return -1; if (tDecodeI8(&decoder, &vload.syncState) < 0) return -1;
if (tDecodeI8(&decoder, &vload.syncRestore) < 0) return -1; if (tDecodeI8(&decoder, &vload.syncRestore) < 0) return -1;
if (tDecodeI8(&decoder, &vload.syncCanRead) < 0) return -1;
if (tDecodeI64(&decoder, &vload.cacheUsage) < 0) return -1; if (tDecodeI64(&decoder, &vload.cacheUsage) < 0) return -1;
if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1; if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1;
if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1; if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1;
if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1; if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1;
if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1; if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1;
if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1; if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1;
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
if (taosArrayPush(pReq->pVloads, &vload) == NULL) { if (taosArrayPush(pReq->pVloads, &vload) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
@ -1288,7 +1298,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
if (tEncodeI8(&encoder, pReq->enable) < 0) return -1; if (tEncodeI8(&encoder, pReq->enable) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->dbname) < 0) return -1; if (tEncodeCStr(&encoder, pReq->objname) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -1307,7 +1317,7 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
if (tDecodeI8(&decoder, &pReq->enable) < 0) return -1; if (tDecodeI8(&decoder, &pReq->enable) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->dbname) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->objname) < 0) return -1;
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -3738,6 +3748,31 @@ int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
return 0; return 0;
} }
int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI64(&encoder, pReq->tick) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) { int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) {
if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1; if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1;
if (tEncodeU16(pEncoder, pReplica->port) < 0) return -1; if (tEncodeU16(pEncoder, pReplica->port) < 0) return -1;

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -196,7 +196,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
code = 0; code = 0;

2
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -162,11 +162,13 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
if (pMsg == NULL) return -1; if (pMsg == NULL) return -1;
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL;
dTrace("msg:%p, is created and will put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, is created and will put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg); int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg);
if (code != 0) { if (code != 0) {
dTrace("msg:%p, is freed", pMsg); dTrace("msg:%p, is freed", pMsg);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
return code; return code;

2
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -61,6 +61,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
if (pMsg == NULL) return -1; if (pMsg == NULL) return -1;
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL;
switch (qtype) { switch (qtype) {
case QUERY_QUEUE: case QUERY_QUEUE:
@ -74,6 +75,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
return 0; return 0;
default: default:
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
return -1; return -1;
} }

1
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -151,6 +151,7 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
pHead->contLen = htonl(pHead->contLen); pHead->contLen = htonl(pHead->contLen);
pHead->vgId = SNODE_HANDLE; pHead->vgId = SNODE_HANDLE;
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL;
switch (qtype) { switch (qtype) {
case STREAM_QUEUE: case STREAM_QUEUE:

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -464,7 +464,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
code = 0; code = 0;

2
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -246,12 +246,12 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
pHead->contLen = htonl(pHead->contLen); pHead->contLen = htonl(pHead->contLen);
pHead->vgId = htonl(pHead->vgId); pHead->vgId = htonl(pHead->vgId);
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL;
int32_t code = vmPutMsgToQueue(pMgmt, pMsg, qtype); int32_t code = vmPutMsgToQueue(pMgmt, pMsg, qtype);
if (code != 0) { if (code != 0) {
dTrace("msg:%p, is freed", pMsg); dTrace("msg:%p, is freed", pMsg);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pRpc->pCont = NULL;
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }

20
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -71,6 +71,9 @@ typedef enum {
MND_OPER_READ_DB, MND_OPER_READ_DB,
MND_OPER_READ_OR_WRITE_DB, MND_OPER_READ_OR_WRITE_DB,
MND_OPER_SHOW_VARIBALES, MND_OPER_SHOW_VARIBALES,
MND_OPER_SUBSCRIBE,
MND_OPER_CREATE_TOPIC,
MND_OPER_DROP_TOPIC,
} EOperType; } EOperType;
typedef enum { typedef enum {
@ -273,6 +276,7 @@ typedef struct {
int32_t authVersion; int32_t authVersion;
SHashObj* readDbs; SHashObj* readDbs;
SHashObj* writeDbs; SHashObj* writeDbs;
SHashObj* topics;
SRWLatch lock; SRWLatch lock;
} SUserObj; } SUserObj;
@ -328,6 +332,7 @@ typedef struct {
int32_t dnodeId; int32_t dnodeId;
ESyncState syncState; ESyncState syncState;
bool syncRestore; bool syncRestore;
bool syncCanRead;
} SVnodeGid; } SVnodeGid;
typedef struct { typedef struct {
@ -635,10 +640,14 @@ typedef struct {
SArray* tasks; // SArray<SArray<SStreamTask>> SArray* tasks; // SArray<SArray<SStreamTask>>
SSchemaWrapper outputSchema; SSchemaWrapper outputSchema;
SSchemaWrapper tagSchema; SSchemaWrapper tagSchema;
// 3.0.20
int64_t checkpointFreq; // ms
int64_t currentTick; // do not serialize
} SStreamObj; } SStreamObj;
int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj);
int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj); int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj, int32_t sver);
void tFreeStreamObj(SStreamObj* pObj); void tFreeStreamObj(SStreamObj* pObj);
typedef struct { typedef struct {
@ -648,15 +657,6 @@ typedef struct {
SArray* childInfo; // SArray<SStreamChildEpInfo> SArray* childInfo; // SArray<SStreamChildEpInfo>
} SStreamCheckpointObj; } SStreamCheckpointObj;
#if 0
typedef struct {
int64_t uid;
int64_t streamId;
int8_t status;
int8_t stage;
} SStreamRecoverObj;
#endif
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
source/dnode/mnode/impl/inc/mndPrivilege.h
View File

@ -28,6 +28,8 @@ void mndCleanupPrivilege(SMnode *pMnode);
int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType); int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType);
int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb);
int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname); int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname);
int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic);
int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName);
int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname); int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname);
int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter);
int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp); int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp);

2
source/dnode/mnode/impl/inc/mndTopic.h
View File

@ -25,7 +25,7 @@ extern "C" {
int32_t mndInitTopic(SMnode *pMnode); int32_t mndInitTopic(SMnode *pMnode);
void mndCleanupTopic(SMnode *pMnode); void mndCleanupTopic(SMnode *pMnode);
SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName); SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName);
void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic); void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic);
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic); SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic);

8
source/dnode/mnode/impl/src/mndAcct.c
View File

@ -147,6 +147,8 @@ _OVER:
static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SAcctObj *pAcct = NULL;
SSdbRow *pRow = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -156,10 +158,10 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SAcctObj)); pRow = sdbAllocRow(sizeof(SAcctObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SAcctObj *pAcct = sdbGetRowObj(pRow); pAcct = sdbGetRowObj(pRow);
if (pAcct == NULL) goto _OVER; if (pAcct == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -186,7 +188,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("acct:%s, failed to decode from raw:%p since %s", pAcct->acct, pRaw, terrstr()); mError("acct:%s, failed to decode from raw:%p since %s", pAcct == NULL ? "null" : pAcct->acct, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }

9
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -157,6 +157,8 @@ _OVER:
static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SClusterObj *pCluster = NULL;
SSdbRow *pRow = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -166,10 +168,10 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SClusterObj)); pRow = sdbAllocRow(sizeof(SClusterObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SClusterObj *pCluster = sdbGetRowObj(pRow); pCluster = sdbGetRowObj(pRow);
if (pCluster == NULL) goto _OVER; if (pCluster == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -184,7 +186,8 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster->id, pRaw, terrstr()); mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster == NULL ? 0 : pCluster->id, pRaw,
terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }

11
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -712,7 +712,9 @@ CM_ENCODE_OVER:
SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
void *buf = NULL; SSdbRow *pRow = NULL;
SMqConsumerObj *pConsumer = NULL;
void *buf = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER;
@ -722,10 +724,10 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
goto CM_DECODE_OVER; goto CM_DECODE_OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SMqConsumerObj)); pRow = sdbAllocRow(sizeof(SMqConsumerObj));
if (pRow == NULL) goto CM_DECODE_OVER; if (pRow == NULL) goto CM_DECODE_OVER;
SMqConsumerObj *pConsumer = sdbGetRowObj(pRow); pConsumer = sdbGetRowObj(pRow);
if (pConsumer == NULL) goto CM_DECODE_OVER; if (pConsumer == NULL) goto CM_DECODE_OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -745,7 +747,8 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
CM_DECODE_OVER: CM_DECODE_OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer == NULL ? 0 : pConsumer->consumerId,
pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }

8
source/dnode/mnode/impl/src/mndDb.c
View File

@ -147,6 +147,8 @@ _OVER:
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SDbObj *pDb = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -156,10 +158,10 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj)); pRow = sdbAllocRow(sizeof(SDbObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SDbObj *pDb = sdbGetRowObj(pRow); pDb = sdbGetRowObj(pRow);
if (pDb == NULL) goto _OVER; if (pDb == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -232,7 +234,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("db:%s, failed to decode from raw:%p since %s", pDb->name, pRaw, terrstr()); mError("db:%s, failed to decode from raw:%p since %s", pDb == NULL ? "null" : pDb->name, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }

9
source/dnode/mnode/impl/src/mndDef.c
View File

@ -76,11 +76,14 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1;
// 3.0.20
if (tEncodeI64(pEncoder, pObj->checkpointFreq) < 0) return -1;
tEndEncode(pEncoder); tEndEncode(pEncoder);
return pEncoder->pos; return pEncoder->pos;
} }
int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
if (tStartDecode(pDecoder) < 0) return -1; if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1; if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
@ -139,6 +142,10 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1;
// 3.0.20
if (sver >= 2) {
if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1;
}
tEndDecode(pDecoder); tEndDecode(pDecoder);
return 0; return 0;
} }

28
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -154,8 +154,9 @@ _OVER:
} }
static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
SSdbRow *pRow = NULL;
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SDnodeObj *pDnode = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -166,7 +167,8 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
pRow = sdbAllocRow(sizeof(SDnodeObj)); pRow = sdbAllocRow(sizeof(SDnodeObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SDnodeObj *pDnode = sdbGetRowObj(pRow);
pDnode = sdbGetRowObj(pRow);
if (pDnode == NULL) goto _OVER; if (pDnode == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -181,7 +183,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("dnode:%d, failed to decode from raw:%p since %s", pDnode->id, pRaw, terrstr()); mError("dnode:%d, failed to decode from raw:%p since %s", pDnode == NULL ? 0 : pDnode->id, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }
@ -375,14 +377,18 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
} }
bool roleChanged = false; bool roleChanged = false;
for (int32_t vg = 0; vg < pVgroup->replica; ++vg) { for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) { SVnodeGid *pGid = &pVgroup->vnodeGid[vg];
if (pVgroup->vnodeGid[vg].syncState != pVload->syncState || if (pGid->dnodeId == statusReq.dnodeId) {
pVgroup->vnodeGid[vg].syncRestore != pVload->syncRestore) { if (pGid->syncState != pVload->syncState || pGid->syncRestore != pVload->syncRestore ||
mInfo("vgId:%d, state changed by status msg, old state:%s restored:%d new state:%s restored:%d", pGid->syncCanRead != pVload->syncCanRead) {
pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore, mInfo(
syncStr(pVload->syncState), pVload->syncRestore); "vgId:%d, state changed by status msg, old state:%s restored:%d canRead:%d new state:%s restored:%d "
pVgroup->vnodeGid[vg].syncState = pVload->syncState; "canRead:%d",
pVgroup->vnodeGid[vg].syncRestore = pVload->syncRestore; pVgroup->vgId, syncStr(pGid->syncState), pGid->syncRestore, pGid->syncCanRead,
syncStr(pVload->syncState), pVload->syncRestore, pVload->syncCanRead);
pGid->syncState = pVload->syncState;
pGid->syncRestore = pVload->syncRestore;
pGid->syncCanRead = pVload->syncCanRead;
roleChanged = true; roleChanged = true;
} }
break; break;

8
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -101,6 +101,8 @@ _OVER:
static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SFuncObj *pFunc = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -110,10 +112,10 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SFuncObj)); pRow = sdbAllocRow(sizeof(SFuncObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SFuncObj *pFunc = sdbGetRowObj(pRow); pFunc = sdbGetRowObj(pRow);
if (pFunc == NULL) goto _OVER; if (pFunc == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -148,7 +150,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("func:%s, failed to decode from raw:%p since %s", pFunc->name, pRaw, terrstr()); mError("func:%s, failed to decode from raw:%p since %s", pFunc == NULL ? "null" : pFunc->name, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }

59
source/dnode/mnode/impl/src/mndMain.c
View File

@ -85,6 +85,21 @@ static void *mndBuildTimerMsg(int32_t *pContLen) {
return pReq; return pReq;
} }
static void *mndBuildCheckpointTickMsg(int32_t *pContLen, int64_t sec) {
SMStreamTickReq timerReq = {
.tick = sec,
};
int32_t contLen = tSerializeSMStreamTickMsg(NULL, 0, &timerReq);
if (contLen <= 0) return NULL;
void *pReq = rpcMallocCont(contLen);
if (pReq == NULL) return NULL;
tSerializeSMStreamTickMsg(pReq, contLen, &timerReq);
*pContLen = contLen;
return pReq;
}
static void mndPullupTrans(SMnode *pMnode) { static void mndPullupTrans(SMnode *pMnode) {
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
@ -105,7 +120,24 @@ static void mndCalMqRebalance(SMnode *pMnode) {
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) { if (pReq != NULL) {
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen}; SRpcMsg rpcMsg = {
.msgType = TDMT_MND_TMQ_TIMER,
.pCont = pReq,
.contLen = contLen,
};
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
}
}
static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) {
int32_t contLen = 0;
void *pReq = mndBuildCheckpointTickMsg(&contLen, sec);
if (pReq != NULL) {
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_STREAM_CHECKPOINT_TIMER,
.pCont = pReq,
.contLen = contLen,
};
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
} }
} }
@ -150,12 +182,16 @@ static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs)
bool roleChanged = false; bool roleChanged = false;
for (int32_t vg = 0; vg < pVgroup->replica; ++vg) { for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
if (pVgroup->vnodeGid[vg].dnodeId == dnodeId) { SVnodeGid *pGid = &pVgroup->vnodeGid[vg];
if (pVgroup->vnodeGid[vg].syncState != TAOS_SYNC_STATE_OFFLINE) { if (pGid->dnodeId == dnodeId) {
mInfo("vgId:%d, state changed by offline check, old state:%s restored:%d new state:error restored:0", if (pGid->syncState != TAOS_SYNC_STATE_OFFLINE) {
pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore); mInfo(
pVgroup->vnodeGid[vg].syncState = TAOS_SYNC_STATE_OFFLINE; "vgId:%d, state changed by offline check, old state:%s restored:%d canRead:%d new state:error restored:0 "
pVgroup->vnodeGid[vg].syncRestore = 0; "canRead:0",
pVgroup->vgId, syncStr(pGid->syncState), pGid->syncRestore, pGid->syncCanRead);
pGid->syncState = TAOS_SYNC_STATE_OFFLINE;
pGid->syncRestore = 0;
pGid->syncCanRead = 0;
roleChanged = true; roleChanged = true;
} }
break; break;
@ -220,6 +256,12 @@ static void *mndThreadFp(void *param) {
mndCalMqRebalance(pMnode); mndCalMqRebalance(pMnode);
} }
#if 0
if (sec % tsStreamCheckpointTickInterval == 0) {
mndStreamCheckpointTick(pMnode, sec);
}
#endif
if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) { if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) {
mndPullupTelem(pMnode); mndPullupTelem(pMnode);
} }
@ -491,7 +533,7 @@ void mndPreClose(SMnode *pMnode) {
if (pMnode != NULL) { if (pMnode != NULL) {
syncLeaderTransfer(pMnode->syncMgmt.sync); syncLeaderTransfer(pMnode->syncMgmt.sync);
syncPreStop(pMnode->syncMgmt.sync); syncPreStop(pMnode->syncMgmt.sync);
#if 0
while (syncSnapshotRecving(pMnode->syncMgmt.sync)) { while (syncSnapshotRecving(pMnode->syncMgmt.sync)) {
mInfo("vgId:1, snapshot is recving"); mInfo("vgId:1, snapshot is recving");
taosMsleep(300); taosMsleep(300);
@ -500,6 +542,7 @@ void mndPreClose(SMnode *pMnode) {
mInfo("vgId:1, snapshot is sending"); mInfo("vgId:1, snapshot is sending");
taosMsleep(300); taosMsleep(300);
} }
#endif
} }
} }

8
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -142,6 +142,8 @@ _OVER:
static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SMnodeObj *pObj = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
@ -151,10 +153,10 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SMnodeObj)); pRow = sdbAllocRow(sizeof(SMnodeObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SMnodeObj *pObj = sdbGetRowObj(pRow); pObj = sdbGetRowObj(pRow);
if (pObj == NULL) goto _OVER; if (pObj == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -167,7 +169,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("mnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); mError("mnode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }

4
source/dnode/mnode/impl/src/mndPrivilege.c
View File

@ -28,6 +28,10 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType
int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname) { int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname) {
return 0; return 0;
} }
int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic) { return 0; }
int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName) {
return 0;
}
int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) {
memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); memcpy(pRsp->user, pUser->user, TSDB_USER_LEN);
pRsp->superAuth = 1; pRsp->superAuth = 1;

8
source/dnode/mnode/impl/src/mndQnode.c
View File

@ -100,6 +100,8 @@ _OVER:
static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SQnodeObj *pObj = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -109,10 +111,10 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) {
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SQnodeObj)); pRow = sdbAllocRow(sizeof(SQnodeObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SQnodeObj *pObj = sdbGetRowObj(pRow); pObj = sdbGetRowObj(pRow);
if (pObj == NULL) goto _OVER; if (pObj == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -125,7 +127,7 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("qnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); mError("qnode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }

8
source/dnode/mnode/impl/src/mndSma.c
View File

@ -132,6 +132,8 @@ _OVER:
static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SSmaObj *pSma = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -141,10 +143,10 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) {
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SSmaObj)); pRow = sdbAllocRow(sizeof(SSmaObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SSmaObj *pSma = sdbGetRowObj(pRow); pSma = sdbGetRowObj(pRow);
if (pSma == NULL) goto _OVER; if (pSma == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -200,7 +202,7 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr()); mError("sma:%s, failed to decode from raw:%p since %s", pSma == NULL ? "null" : pSma->name, pRaw, terrstr());
taosMemoryFreeClear(pSma->expr); taosMemoryFreeClear(pSma->expr);
taosMemoryFreeClear(pSma->tagsFilter); taosMemoryFreeClear(pSma->tagsFilter);
taosMemoryFreeClear(pSma->sql); taosMemoryFreeClear(pSma->sql);

8
source/dnode/mnode/impl/src/mndSnode.c
View File

@ -105,6 +105,8 @@ _OVER:
static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SSnodeObj *pObj = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -114,10 +116,10 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) {
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SSnodeObj)); pRow = sdbAllocRow(sizeof(SSnodeObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SSnodeObj *pObj = sdbGetRowObj(pRow); pObj = sdbGetRowObj(pRow);
if (pObj == NULL) goto _OVER; if (pObj == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -130,7 +132,7 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("snode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); mError("snode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }

16
source/dnode/mnode/impl/src/mndStb.c
View File

@ -162,6 +162,8 @@ _OVER:
static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SStbObj *pStb = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -171,10 +173,10 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SStbObj)); pRow = sdbAllocRow(sizeof(SStbObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SStbObj *pStb = sdbGetRowObj(pRow); pStb = sdbGetRowObj(pRow);
if (pStb == NULL) goto _OVER; if (pStb == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -254,10 +256,12 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("stb:%s, failed to decode from raw:%p since %s", pStb->name, pRaw, terrstr()); mError("stb:%s, failed to decode from raw:%p since %s", pStb == NULL ? "null" : pStb->name, pRaw, terrstr());
taosMemoryFreeClear(pStb->pColumns); if (pStb != NULL) {
taosMemoryFreeClear(pStb->pTags); taosMemoryFreeClear(pStb->pColumns);
taosMemoryFreeClear(pStb->comment); taosMemoryFreeClear(pStb->pTags);
taosMemoryFreeClear(pStb->comment);
}
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }

253
source/dnode/mnode/impl/src/mndStream.c
View File

@ -28,7 +28,7 @@
#include "parser.h" #include "parser.h"
#include "tname.h" #include "tname.h"
#define MND_STREAM_VER_NUMBER 1 #define MND_STREAM_VER_NUMBER 2
#define MND_STREAM_RESERVE_SIZE 64 #define MND_STREAM_RESERVE_SIZE 64
static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
@ -36,6 +36,8 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream);
static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq);
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/ /*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/
static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq);
static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
@ -62,6 +64,10 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
@ -120,21 +126,22 @@ STREAM_ENCODE_OVER:
SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
void *buf = NULL; SSdbRow *pRow = NULL;
SStreamObj *pStream = NULL;
void *buf = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER;
if (sver != MND_STREAM_VER_NUMBER) { if (sver != 1 && sver != 2) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto STREAM_DECODE_OVER; goto STREAM_DECODE_OVER;
} }
int32_t size = sizeof(SStreamObj); pRow = sdbAllocRow(sizeof(SStreamObj));
SSdbRow *pRow = sdbAllocRow(size);
if (pRow == NULL) goto STREAM_DECODE_OVER; if (pRow == NULL) goto STREAM_DECODE_OVER;
SStreamObj *pStream = sdbGetRowObj(pRow); pStream = sdbGetRowObj(pRow);
if (pStream == NULL) goto STREAM_DECODE_OVER; if (pStream == NULL) goto STREAM_DECODE_OVER;
int32_t tlen; int32_t tlen;
@ -146,7 +153,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, buf, tlen + 1); tDecoderInit(&decoder, buf, tlen + 1);
if (tDecodeSStreamObj(&decoder, pStream) < 0) { if (tDecodeSStreamObj(&decoder, pStream, sver) < 0) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
goto STREAM_DECODE_OVER; goto STREAM_DECODE_OVER;
} }
@ -157,7 +164,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
STREAM_DECODE_OVER: STREAM_DECODE_OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
mError("stream:%s, failed to decode from raw:%p since %s", pStream->name, pRaw, terrstr()); mError("stream:%s, failed to decode from raw:%p since %s", pStream == NULL ? "null" : pStream->name, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }
@ -679,6 +686,164 @@ _OVER:
tFreeStreamObj(&streamObj); tFreeStreamObj(&streamObj);
return code; return code;
} }
static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SStreamObj *pStream = NULL;
// iterate all stream obj
while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) break;
// incr tick
int64_t currentTick = atomic_add_fetch_64(&pStream->currentTick, 1);
// if >= checkpointFreq, build msg TDMT_MND_STREAM_BEGIN_CHECKPOINT, put into write q
if (currentTick >= pStream->checkpointFreq) {
atomic_store_64(&pStream->currentTick, 0);
SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
pMsg->streamId = pStream->uid;
pMsg->checkpointId = tGenIdPI64();
memcpy(pMsg->streamName, pStream->name, TSDB_STREAM_FNAME_LEN);
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT,
.pCont = pMsg,
.contLen = sizeof(SMStreamDoCheckpointMsg),
};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
}
return 0;
}
static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, const SStreamTask *pTask,
SMStreamDoCheckpointMsg *pMsg) {
SStreamCheckpointSourceReq req = {0};
req.checkpointId = pMsg->checkpointId;
req.nodeId = pTask->nodeId;
req.expireTime = -1;
req.streamId = pTask->streamId;
req.taskId = pTask->taskId;
int32_t code;
int32_t blen;
tEncodeSize(tEncodeSStreamCheckpointSourceReq, &req, blen, code);
if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
int32_t tlen = sizeof(SMsgHead) + blen;
void *buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen);
tEncodeSStreamCheckpointSourceReq(&encoder, &req);
SMsgHead *pMsgHead = (SMsgHead *)buf;
pMsgHead->contLen = htonl(tlen);
pMsgHead->vgId = htonl(pTask->nodeId);
tEncoderClear(&encoder);
*pBuf = buf;
*pLen = tlen;
return 0;
}
static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont;
SStreamObj *pStream = mndAcquireStream(pMnode, pMsg->streamName);
if (pStream == NULL || pStream->uid != pMsg->streamId) {
mError("start checkpointing failed since stream %s not found", pMsg->streamName);
return -1;
}
// build new transaction:
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "stream-checkpoint");
if (pTrans == NULL) return -1;
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb);
taosRLockLatch(&pStream->lock);
// 1. redo action: broadcast checkpoint source msg for all source vg
int32_t totLevel = taosArrayGetSize(pStream->tasks);
for (int32_t i = 0; i < totLevel; i++) {
SArray *pLevel = taosArrayGetP(pStream->tasks, i);
SStreamTask *pTask = taosArrayGetP(pLevel, 0);
if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
int32_t sz = taosArrayGetSize(pLevel);
for (int32_t j = 0; j < sz; j++) {
SStreamTask *pTask = taosArrayGetP(pLevel, j);
ASSERT(pTask->nodeId > 0);
SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->nodeId);
if (pVgObj == NULL) {
ASSERT(0);
taosRUnLockLatch(&pStream->lock);
mndReleaseStream(pMnode, pStream);
mndTransDrop(pTrans);
return -1;
}
void *buf;
int32_t tlen;
if (mndBuildStreamCheckpointSourceReq(&buf, &tlen, pTask, pMsg) < 0) {
taosRUnLockLatch(&pStream->lock);
mndReleaseStream(pMnode, pStream);
mndTransDrop(pTrans);
return -1;
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
action.pCont = buf;
action.contLen = tlen;
action.msgType = TDMT_VND_STREAM_CHECK_POINT_SOURCE;
mndReleaseVgroup(pMnode, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(buf);
taosRUnLockLatch(&pStream->lock);
mndReleaseStream(pMnode, pStream);
mndTransDrop(pTrans);
return -1;
}
}
}
}
// 2. reset tick
atomic_store_64(&pStream->currentTick, 0);
// 3. commit log: stream checkpoint info
taosRUnLockLatch(&pStream->lock);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("failed to prepare trans rebalance since %s", terrstr());
mndTransDrop(pTrans);
mndReleaseStream(pMnode, pStream);
return -1;
}
mndReleaseStream(pMnode, pStream);
mndTransDrop(pTrans);
return 0;
}
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
@ -747,71 +912,6 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return TSDB_CODE_ACTION_IN_PROGRESS; return TSDB_CODE_ACTION_IN_PROGRESS;
} }
#if 0
static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL;
/*SDbObj *pDb = NULL;*/
/*SUserObj *pUser = NULL;*/
SMRecoverStreamReq recoverReq = {0};
if (tDeserializeSMRecoverStreamReq(pReq->pCont, pReq->contLen, &recoverReq) < 0) {
ASSERT(0);
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
pStream = mndAcquireStream(pMnode, recoverReq.name);
if (pStream == NULL) {
if (recoverReq.igNotExists) {
mInfo("stream:%s, not exist, ignore not exist is set", recoverReq.name);
sdbRelease(pMnode->pSdb, pStream);
return 0;
} else {
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
return -1;
}
}
if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) {
return -1;
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq);
if (pTrans == NULL) {
mError("stream:%s, failed to recover since %s", recoverReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
return -1;
}
mInfo("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name);
// broadcast to recover all tasks
if (mndRecoverStreamTasks(pMnode, pTrans, pStream) < 0) {
mError("stream:%s, failed to recover task since %s", recoverReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
return -1;
}
// update stream status
if (mndSetStreamRecover(pMnode, pTrans, pStream) < 0) {
sdbRelease(pMnode->pSdb, pStream);
return -1;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare recover stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
sdbRelease(pMnode->pSdb, pStream);
return TSDB_CODE_ACTION_IN_PROGRESS;
}
#endif
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
@ -846,13 +946,6 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
} }
} }
#if 0
if (mndSetDropOffsetStreamLogs(pMnode, pTrans, pStream) < 0) {
sdbRelease(pSdb, pStream);
goto END;
}
#endif
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
} }

37
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -440,9 +440,9 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
} }
static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "persist-reb"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb");
mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL);
if (pTrans == NULL) return -1; if (pTrans == NULL) return -1;
mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL);
// make txn: // make txn:
// 1. redo action: action to all vg // 1. redo action: action to all vg
@ -523,28 +523,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
tDeleteSMqConsumerObj(pConsumerNew); tDeleteSMqConsumerObj(pConsumerNew);
taosMemoryFree(pConsumerNew); taosMemoryFree(pConsumerNew);
} }
#if 0
if (consumerNum) {
char topic[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CGROUP_LEN];
mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
if (pTopic) {
// TODO make topic complete
SMqTopicObj topicObj = {0};
memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
topicObj.refConsumerCnt = pTopic->refConsumerCnt - consumerNum;
// TODO is that correct?
pTopic->refConsumerCnt = topicObj.refConsumerCnt;
mInfo("subscribe topic %s unref %d consumer cgroup %s, refcnt %d", pTopic->name, consumerNum, cgroup,
topicObj.refConsumerCnt);
if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) {
ASSERT(0);
goto REB_FAIL;
}
}
}
#endif
// 4. TODO commit log: modification log // 4. TODO commit log: modification log
@ -743,7 +721,9 @@ SUB_ENCODE_OVER:
static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
void *buf = NULL; SSdbRow *pRow = NULL;
SMqSubscribeObj *pSub = NULL;
void *buf = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER;
@ -753,11 +733,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
goto SUB_DECODE_OVER; goto SUB_DECODE_OVER;
} }
int32_t size = sizeof(SMqSubscribeObj); pRow = sdbAllocRow(sizeof(SMqSubscribeObj));
SSdbRow *pRow = sdbAllocRow(size);
if (pRow == NULL) goto SUB_DECODE_OVER; if (pRow == NULL) goto SUB_DECODE_OVER;
SMqSubscribeObj *pSub = sdbGetRowObj(pRow); pSub = sdbGetRowObj(pRow);
if (pSub == NULL) goto SUB_DECODE_OVER; if (pSub == NULL) goto SUB_DECODE_OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -777,7 +756,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
SUB_DECODE_OVER: SUB_DECODE_OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); mError("subscribe:%s, failed to decode from raw:%p since %s", pSub == NULL ? "null" : pSub->key, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }

17
source/dnode/mnode/impl/src/mndSync.c
View File

@ -72,15 +72,11 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
return code; return code;
} }
void mndSyncCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { void mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
SMnode *pMnode = pFsm->data; SMnode *pMnode = pFsm->data;
SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSyncMgmt *pMgmt = &pMnode->syncMgmt;
SSdbRaw *pRaw = pMsg->pCont; SSdbRaw *pRaw = pMsg->pCont;
// delete msg handle
SRpcMsg rpcMsg = {0};
rpcMsg.info = pMsg->info;
int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw);
pMgmt->errCode = pMeta->code; pMgmt->errCode = pMeta->code;
mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64 mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64
@ -120,6 +116,12 @@ void mndSyncCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMet
} }
} }
void mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
mndProcessWriteMsg(pFsm, pMsg, pMeta);
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) { int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) {
mInfo("start to read snapshot from sdb in atomic way"); mInfo("start to read snapshot from sdb in atomic way");
SMnode *pMnode = pFsm->data; SMnode *pMnode = pFsm->data;
@ -361,7 +363,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
void mndSyncStart(SMnode *pMnode) { void mndSyncStart(SMnode *pMnode) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSyncMgmt *pMgmt = &pMnode->syncMgmt;
syncStart(pMgmt->sync); if (syncStart(pMgmt->sync) < 0) {
mError("vgId:1, failed to start sync, id:%" PRId64, pMgmt->sync);
return;
}
mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync); mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync);
} }

20
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -152,8 +152,10 @@ TOPIC_ENCODE_OVER:
SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SMqTopicObj *pTopic = NULL;
void *buf = NULL;
void *buf = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER;
@ -162,11 +164,10 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
goto TOPIC_DECODE_OVER; goto TOPIC_DECODE_OVER;
} }
int32_t size = sizeof(SMqTopicObj); pRow = sdbAllocRow(sizeof(SMqTopicObj));
SSdbRow *pRow = sdbAllocRow(size);
if (pRow == NULL) goto TOPIC_DECODE_OVER; if (pRow == NULL) goto TOPIC_DECODE_OVER;
SMqTopicObj *pTopic = sdbGetRowObj(pRow); pTopic = sdbGetRowObj(pRow);
if (pTopic == NULL) goto TOPIC_DECODE_OVER; if (pTopic == NULL) goto TOPIC_DECODE_OVER;
int32_t len; int32_t len;
@ -251,7 +252,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
TOPIC_DECODE_OVER: TOPIC_DECODE_OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
mError("topic:%s, failed to decode from raw:%p since %s", pTopic->name, pRaw, terrstr()); mError("topic:%s, failed to decode from raw:%p since %s", pTopic == NULL ? "null" : pTopic->name, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }
@ -288,7 +289,7 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopic
return 0; return 0;
} }
SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName) { SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SMqTopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName); SMqTopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName);
if (pTopic == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { if (pTopic == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
@ -573,7 +574,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) { if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC) != 0) {
goto _OVER; goto _OVER;
} }
@ -633,6 +634,11 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
} }
} }
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC) != 0) {
mndReleaseTopic(pMnode, pTopic);
return -1;
}
void *pIter = NULL; void *pIter = NULL;
SMqConsumerObj *pConsumer; SMqConsumerObj *pConsumer;
while (1) { while (1) {

213
source/dnode/mnode/impl/src/mndUser.c
View File

@ -18,10 +18,11 @@
#include "mndDb.h" #include "mndDb.h"
#include "mndPrivilege.h" #include "mndPrivilege.h"
#include "mndShow.h" #include "mndShow.h"
#include "mndTopic.h"
#include "mndTrans.h" #include "mndTrans.h"
#include "tbase64.h" #include "tbase64.h"
#define USER_VER_NUMBER 1 #define USER_VER_NUMBER 2
#define USER_RESERVE_SIZE 64 #define USER_RESERVE_SIZE 64
static int32_t mndCreateDefaultUsers(SMnode *pMnode); static int32_t mndCreateDefaultUsers(SMnode *pMnode);
@ -36,6 +37,8 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq);
static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq); static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq);
static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter);
static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter);
int32_t mndInitUser(SMnode *pMnode) { int32_t mndInitUser(SMnode *pMnode) {
SSdbTable table = { SSdbTable table = {
@ -56,6 +59,8 @@ int32_t mndInitUser(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_USER, mndRetrieveUsers); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_USER, mndRetrieveUsers);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_USER, mndCancelGetNextUser); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_USER, mndCancelGetNextUser);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_PRIVILEGES, mndRetrievePrivileges);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_PRIVILEGES, mndCancelGetNextPrivileges);
return sdbSetTable(pMnode->pSdb, table); return sdbSetTable(pMnode->pSdb, table);
} }
@ -119,7 +124,9 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN; int32_t numOfTopics = taosHashGetSize(pUser->topics);
int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN +
numOfTopics * TSDB_TOPIC_FNAME_LEN;
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
if (pRaw == NULL) goto _OVER; if (pRaw == NULL) goto _OVER;
@ -137,6 +144,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfTopics, _OVER)
char *db = taosHashIterate(pUser->readDbs, NULL); char *db = taosHashIterate(pUser->readDbs, NULL);
while (db != NULL) { while (db != NULL) {
@ -150,6 +158,12 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
db = taosHashIterate(pUser->writeDbs, db); db = taosHashIterate(pUser->writeDbs, db);
} }
char *topic = taosHashIterate(pUser->topics, NULL);
while (topic != NULL) {
SDB_SET_BINARY(pRaw, dataPos, topic, TSDB_TOPIC_FNAME_LEN, _OVER);
db = taosHashIterate(pUser->topics, topic);
}
SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -168,19 +182,21 @@ _OVER:
static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SUserObj *pUser = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
if (sver != USER_VER_NUMBER) { if (sver != 1 && sver != 2) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SUserObj)); pRow = sdbAllocRow(sizeof(SUserObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SUserObj *pUser = sdbGetRowObj(pRow); pUser = sdbGetRowObj(pRow);
if (pUser == NULL) goto _OVER; if (pUser == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -197,12 +213,18 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
int32_t numOfReadDbs = 0; int32_t numOfReadDbs = 0;
int32_t numOfWriteDbs = 0; int32_t numOfWriteDbs = 0;
int32_t numOfTopics = 0;
SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, _OVER) SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, _OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, _OVER) SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, _OVER)
if (sver >= 2) {
SDB_GET_INT32(pRaw, dataPos, &numOfTopics, _OVER)
}
pUser->readDbs = taosHashInit(numOfReadDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pUser->readDbs = taosHashInit(numOfReadDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->writeDbs = pUser->writeDbs =
taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto _OVER; pUser->topics = taosHashInit(numOfTopics, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pUser->readDbs == NULL || pUser->writeDbs == NULL || pUser->topics == NULL) goto _OVER;
for (int32_t i = 0; i < numOfReadDbs; ++i) { for (int32_t i = 0; i < numOfReadDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
@ -218,6 +240,15 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN); taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN);
} }
if (sver >= 2) {
for (int32_t i = 0; i < numOfTopics; ++i) {
char topic[TSDB_TOPIC_FNAME_LEN] = {0};
SDB_GET_BINARY(pRaw, dataPos, topic, TSDB_TOPIC_FNAME_LEN, _OVER)
int32_t len = strlen(topic) + 1;
taosHashPut(pUser->topics, topic, len, topic, TSDB_TOPIC_FNAME_LEN);
}
}
SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pUser->lock); taosInitRWLatch(&pUser->lock);
@ -225,9 +256,12 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr()); mError("user:%s, failed to decode from raw:%p since %s", pUser == NULL ? "null" : pUser->user, pRaw, terrstr());
taosHashCleanup(pUser->readDbs); if (pUser != NULL) {
taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->readDbs);
taosHashCleanup(pUser->writeDbs);
taosHashCleanup(pUser->topics);
}
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }
@ -255,8 +289,10 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser); mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser);
taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->readDbs);
taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->writeDbs);
taosHashCleanup(pUser->topics);
pUser->readDbs = NULL; pUser->readDbs = NULL;
pUser->writeDbs = NULL; pUser->writeDbs = NULL;
pUser->topics = NULL;
return 0; return 0;
} }
@ -270,6 +306,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN);
TSWAP(pOld->readDbs, pNew->readDbs); TSWAP(pOld->readDbs, pNew->readDbs);
TSWAP(pOld->writeDbs, pNew->writeDbs); TSWAP(pOld->writeDbs, pNew->writeDbs);
TSWAP(pOld->topics, pNew->topics);
taosWUnLockLatch(&pOld->lock); taosWUnLockLatch(&pOld->lock);
return 0; return 0;
@ -482,9 +519,10 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
taosRLockLatch(&pUser->lock); taosRLockLatch(&pUser->lock);
newUser.readDbs = mndDupDbHash(pUser->readDbs); newUser.readDbs = mndDupDbHash(pUser->readDbs);
newUser.writeDbs = mndDupDbHash(pUser->writeDbs); newUser.writeDbs = mndDupDbHash(pUser->writeDbs);
newUser.topics = mndDupDbHash(pUser->topics);
taosRUnLockLatch(&pUser->lock); taosRUnLockLatch(&pUser->lock);
if (newUser.readDbs == NULL || newUser.writeDbs == NULL) { if (newUser.readDbs == NULL || newUser.writeDbs == NULL || newUser.topics == NULL) {
goto _OVER; goto _OVER;
} }
@ -507,14 +545,14 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
} }
if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) { if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
if (strcmp(alterReq.dbname, "1.*") != 0) { if (strcmp(alterReq.objname, "1.*") != 0) {
int32_t len = strlen(alterReq.dbname) + 1; int32_t len = strlen(alterReq.objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname); SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
if (pDb == NULL) { if (pDb == NULL) {
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
goto _OVER; goto _OVER;
} }
if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) { if (taosHashPut(newUser.readDbs, alterReq.objname, len, alterReq.objname, TSDB_DB_FNAME_LEN) != 0) {
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
goto _OVER; goto _OVER;
} }
@ -531,14 +569,14 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
} }
if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) { if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
if (strcmp(alterReq.dbname, "1.*") != 0) { if (strcmp(alterReq.objname, "1.*") != 0) {
int32_t len = strlen(alterReq.dbname) + 1; int32_t len = strlen(alterReq.objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname); SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
if (pDb == NULL) { if (pDb == NULL) {
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
goto _OVER; goto _OVER;
} }
if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) { if (taosHashPut(newUser.writeDbs, alterReq.objname, len, alterReq.objname, TSDB_DB_FNAME_LEN) != 0) {
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
goto _OVER; goto _OVER;
} }
@ -555,33 +593,53 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
} }
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) { if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
if (strcmp(alterReq.dbname, "1.*") != 0) { if (strcmp(alterReq.objname, "1.*") != 0) {
int32_t len = strlen(alterReq.dbname) + 1; int32_t len = strlen(alterReq.objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname); SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
if (pDb == NULL) { if (pDb == NULL) {
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
goto _OVER; goto _OVER;
} }
taosHashRemove(newUser.readDbs, alterReq.dbname, len); taosHashRemove(newUser.readDbs, alterReq.objname, len);
} else { } else {
taosHashClear(newUser.readDbs); taosHashClear(newUser.readDbs);
} }
} }
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) { if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
if (strcmp(alterReq.dbname, "1.*") != 0) { if (strcmp(alterReq.objname, "1.*") != 0) {
int32_t len = strlen(alterReq.dbname) + 1; int32_t len = strlen(alterReq.objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname); SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
if (pDb == NULL) { if (pDb == NULL) {
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
goto _OVER; goto _OVER;
} }
taosHashRemove(newUser.writeDbs, alterReq.dbname, len); taosHashRemove(newUser.writeDbs, alterReq.objname, len);
} else { } else {
taosHashClear(newUser.writeDbs); taosHashClear(newUser.writeDbs);
} }
} }
if (alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) {
int32_t len = strlen(alterReq.objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
if (pTopic == NULL) {
mndReleaseTopic(pMnode, pTopic);
goto _OVER;
}
taosHashPut(newUser.topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN);
}
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC) {
int32_t len = strlen(alterReq.objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
if (pTopic == NULL) {
mndReleaseTopic(pMnode, pTopic);
goto _OVER;
}
taosHashRemove(newUser.topics, alterReq.objname, len);
}
code = mndAlterUser(pMnode, pUser, &newUser, pReq); code = mndAlterUser(pMnode, pUser, &newUser, pReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
@ -594,6 +652,7 @@ _OVER:
mndReleaseUser(pMnode, pUser); mndReleaseUser(pMnode, pUser);
taosHashCleanup(newUser.writeDbs); taosHashCleanup(newUser.writeDbs);
taosHashCleanup(newUser.readDbs); taosHashCleanup(newUser.readDbs);
taosHashCleanup(newUser.topics);
return code; return code;
} }
@ -756,6 +815,108 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
} }
static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SUserObj *pUser = NULL;
int32_t cols = 0;
char *pWrite;
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser);
if (pShow->pIter == NULL) break;
int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
int32_t numOfTopics = taosHashGetSize(pUser->topics);
if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics >= rows) break;
char *db = taosHashIterate(pUser->readDbs, NULL);
while (db != NULL) {
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
char privilege[20] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(privilege, "read", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
SName name = {0};
char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&name, db, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&name, varDataVal(objName));
varDataSetLen(objName, strlen(varDataVal(objName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)objName, false);
numOfRows++;
db = taosHashIterate(pUser->readDbs, db);
}
db = taosHashIterate(pUser->writeDbs, NULL);
while (db != NULL) {
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
char privilege[20] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(privilege, "write", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
SName name = {0};
char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&name, db, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&name, varDataVal(objName));
varDataSetLen(objName, strlen(varDataVal(objName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)objName, false);
numOfRows++;
db = taosHashIterate(pUser->writeDbs, db);
}
char *topic = taosHashIterate(pUser->topics, NULL);
while (topic != NULL) {
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
char privilege[20] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(privilege, "subscribe", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0};
tstrncpy(varDataVal(topicName), mndGetDbStr(topic), TSDB_TOPIC_NAME_LEN - 2);
varDataSetLen(topicName, strlen(varDataVal(topicName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
numOfRows++;
db = taosHashIterate(pUser->topics, topic);
}
sdbRelease(pSdb, pUser);
}
pShow->numOfRows += numOfRows;
return numOfRows;
}
static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
}
int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp,
int32_t *pRspLen) { int32_t *pRspLen) {
SUserAuthBatchRsp batchRsp = {0}; SUserAuthBatchRsp batchRsp = {0};

21
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -113,6 +113,8 @@ _OVER:
SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SVgObj *pVgroup = NULL;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -122,10 +124,10 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
goto _OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj)); pRow = sdbAllocRow(sizeof(SVgObj));
if (pRow == NULL) goto _OVER; if (pRow == NULL) goto _OVER;
SVgObj *pVgroup = sdbGetRowObj(pRow); pVgroup = sdbGetRowObj(pRow);
if (pVgroup == NULL) goto _OVER; if (pVgroup == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -152,7 +154,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup == NULL ? 0 : pVgroup->vgId, pRaw, terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }
@ -186,6 +188,7 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) {
if (pNewGid->dnodeId == pOldGid->dnodeId) { if (pNewGid->dnodeId == pOldGid->dnodeId) {
pNewGid->syncState = pOldGid->syncState; pNewGid->syncState = pOldGid->syncState;
pNewGid->syncRestore = pOldGid->syncRestore; pNewGid->syncRestore = pOldGid->syncRestore;
pNewGid->syncCanRead = pOldGid->syncCanRead;
} }
} }
} }
@ -696,8 +699,16 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
if (!exist) { if (!exist) {
strcpy(role, "dropping"); strcpy(role, "dropping");
} else if (online) { } else if (online) {
bool show = (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER && !pVgroup->vnodeGid[i].syncRestore); char *star = "";
snprintf(role, sizeof(role), "%s%s", syncStr(pVgroup->vnodeGid[i].syncState), show ? "*" : ""); if (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER) {
if (!pVgroup->vnodeGid[i].syncRestore && !pVgroup->vnodeGid[i].syncCanRead) {
star = "**";
} else if (!pVgroup->vnodeGid[i].syncRestore && pVgroup->vnodeGid[i].syncCanRead) {
star = "*";
} else {
}
}
snprintf(role, sizeof(role), "%s%s", syncStr(pVgroup->vnodeGid[i].syncState), star);
} else { } else {
} }
STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes);

2
source/dnode/snode/src/snode.c
View File

@ -168,7 +168,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) { int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) {
SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg; SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg;
streamMetaRemoveTask1(pSnode->pMeta, pReq->taskId); streamMetaRemoveTask(pSnode->pMeta, pReq->taskId);
return 0; return 0;
} }

2
source/dnode/vnode/src/inc/vnd.h
View File

@ -97,7 +97,7 @@ bool vnodeShouldRollback(SVnode* pVnode);
// vnodeSync.c // vnodeSync.c
int32_t vnodeSyncOpen(SVnode* pVnode, char* path); int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
void vnodeSyncStart(SVnode* pVnode); int32_t vnodeSyncStart(SVnode* pVnode);
void vnodeSyncPreClose(SVnode* pVnode); void vnodeSyncPreClose(SVnode* pVnode);
void vnodeSyncClose(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode);
void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg); void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg);

2
source/dnode/vnode/src/tq/tq.c
View File

@ -1425,7 +1425,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
streamMetaRemoveTask1(pTq->pStreamMeta, pReq->taskId); streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId);
return 0; return 0;
} }

6
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -811,7 +811,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
int32_t lino = 0; int32_t lino = 0;
memset(pCommitter, 0, sizeof(*pCommitter)); memset(pCommitter, 0, sizeof(*pCommitter));
ASSERT(pTsdb->mem && pTsdb->imem == NULL); ASSERT(pTsdb->mem && pTsdb->imem == NULL && "last tsdb commit incomplete");
taosThreadRwlockWrlock(&pTsdb->rwLock); taosThreadRwlockWrlock(&pTsdb->rwLock);
pTsdb->imem = pTsdb->mem; pTsdb->imem = pTsdb->mem;
@ -1261,7 +1261,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
} }
} }
} else { } else {
ASSERT(0); ASSERT(0 && "dup rows not allowed");
} }
if (pBDataW->nRow >= pCommitter->maxRow) { if (pBDataW->nRow >= pCommitter->maxRow) {
@ -1682,4 +1682,4 @@ _exit:
tsdbInfo("vgId:%d, tsdb rollback commit", TD_VID(pTsdb->pVnode)); tsdbInfo("vgId:%d, tsdb rollback commit", TD_VID(pTsdb->pVnode));
} }
return code; return code;
} }

3
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -728,7 +728,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
taosArraySet(pMerger->pArray, iCol, pColVal); taosArraySet(pMerger->pArray, iCol, pColVal);
} }
} else { } else {
ASSERT(0); ASSERT(0 && "dup versions not allowed");
} }
} }
@ -902,7 +902,6 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _clear; goto _clear;
} }
midx = (sidx + eidx) / 2; midx = (sidx + eidx) / 2;
code = tsdbBuildDeleteSkyline(aDelData, sidx, midx, aSkyline1); code = tsdbBuildDeleteSkyline(aDelData, sidx, midx, aSkyline1);

3
source/dnode/vnode/src/vnd/vnodeBufPool.c
View File

@ -176,6 +176,9 @@ void vnodeBufPoolRef(SVBufPool *pPool) {
} }
void vnodeBufPoolUnRef(SVBufPool *pPool) { void vnodeBufPoolUnRef(SVBufPool *pPool) {
if (pPool == NULL) {
return;
}
int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1); int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1);
if (nRef == 0) { if (nRef == 0) {
SVnode *pVnode = pPool->pVnode; SVnode *pVnode = pPool->pVnode;

4
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -209,8 +209,8 @@ int vnodeCommit(SVnode *pVnode) {
SVnodeInfo info = {0}; SVnodeInfo info = {0};
char dir[TSDB_FILENAME_LEN]; char dir[TSDB_FILENAME_LEN];
vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64, TD_VID(pVnode), pVnode->state.commitID, vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode),
pVnode->state.applied); pVnode->state.commitID, pVnode->state.applied, pVnode->state.applyTerm);
// persist wal before starting // persist wal before starting
if (walPersist(pVnode->pWal) < 0) { if (walPersist(pVnode->pWal) < 0) {

9
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -144,9 +144,9 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
pVnode->config = info.config; pVnode->config = info.config;
pVnode->state.committed = info.state.committed; pVnode->state.committed = info.state.committed;
pVnode->state.commitTerm = info.state.commitTerm; pVnode->state.commitTerm = info.state.commitTerm;
pVnode->state.applied = info.state.committed;
pVnode->state.commitID = info.state.commitID; pVnode->state.commitID = info.state.commitID;
pVnode->state.commitTerm = info.state.commitTerm; pVnode->state.applied = info.state.committed;
pVnode->state.applyTerm = info.state.commitTerm;
pVnode->pTfs = pTfs; pVnode->pTfs = pTfs;
pVnode->msgCb = msgCb; pVnode->msgCb = msgCb;
taosThreadMutexInit(&pVnode->lock, NULL); taosThreadMutexInit(&pVnode->lock, NULL);
@ -269,10 +269,7 @@ void vnodeClose(SVnode *pVnode) {
} }
// start the sync timer after the queue is ready // start the sync timer after the queue is ready
int32_t vnodeStart(SVnode *pVnode) { int32_t vnodeStart(SVnode *pVnode) { return vnodeSyncStart(pVnode); }
vnodeSyncStart(pVnode);
return 0;
}
void vnodeStop(SVnode *pVnode) {} void vnodeStop(SVnode *pVnode) {}

1
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -380,6 +380,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
pLoad->vgId = TD_VID(pVnode); pLoad->vgId = TD_VID(pVnode);
pLoad->syncState = state.state; pLoad->syncState = state.state;
pLoad->syncRestore = state.restored; pLoad->syncRestore = state.restored;
pLoad->syncCanRead = state.canRead;
pLoad->cacheUsage = tsdbCacheGetUsage(pVnode); pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta); pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);

8
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -178,8 +178,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
return -1; return -1;
} }
if (version <= pVnode->state.applied) {
vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version,
pVnode->state.applied);
pRsp->info.handle = NULL;
return -1;
}
vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
version); version);
ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm);
pVnode->state.applied = version; pVnode->state.applied = version;
pVnode->state.applyTerm = pMsg->info.conn.applyTerm; pVnode->state.applyTerm = pMsg->info.conn.applyTerm;

39
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -295,36 +295,31 @@ static int32_t vnodeSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot)
return 0; return 0;
} }
static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
pMsg->info.conn.applyIndex = pMeta->index;
SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; pMsg->info.conn.applyTerm = pMeta->term;
rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
rpcMsg.info = pMsg->info;
rpcMsg.info.conn.applyIndex = pMeta->index;
rpcMsg.info.conn.applyTerm = pMeta->term;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64
", weak:%d, code:%d, state:%d %s, type:%s", ", weak:%d, code:%d, state:%d %s, type:%s",
pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, pMeta->code, pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, pMsg->info.conn.applyIndex, pMeta->isWeak, pMeta->code,
pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType));
tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg);
} }
static void vnodeSyncCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { static void vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
vnodeSyncApplyMsg(pFsm, pMsg, pMeta); vnodeSyncApplyMsg(pFsm, pMsg, pMeta);
} }
static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
if (pMeta->isWeak == 1) { if (pMeta->isWeak == 1) {
vnodeSyncApplyMsg(pFsm, pMsg, pMeta); vnodeSyncApplyMsg(pFsm, pMsg, pMeta);
} }
} }
static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s", vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s",
pVnode->config.vgId, pFsm, pMeta->index, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state), pVnode->config.vgId, pFsm, pMeta->index, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state),
@ -404,7 +399,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm) {
walApplyVer(pVnode->pWal, pVnode->state.applied); walApplyVer(pVnode->pWal, pVnode->state.applied);
pVnode->restored = true; pVnode->restored = true;
vDebug("vgId:%d, sync restore finished", pVnode->config.vgId); vInfo("vgId:%d, sync restore finished", pVnode->config.vgId);
} }
static void vnodeBecomeFollower(const SSyncFSM *pFsm) { static void vnodeBecomeFollower(const SSyncFSM *pFsm) {
@ -506,16 +501,20 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
return 0; return 0;
} }
void vnodeSyncStart(SVnode *pVnode) { int32_t vnodeSyncStart(SVnode *pVnode) {
vInfo("vgId:%d, start sync", pVnode->config.vgId); vInfo("vgId:%d, start sync", pVnode->config.vgId);
syncStart(pVnode->sync); if (syncStart(pVnode->sync) < 0) {
vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, terrstr());
return -1;
}
return 0;
} }
void vnodeSyncPreClose(SVnode *pVnode) { void vnodeSyncPreClose(SVnode *pVnode) {
vInfo("vgId:%d, pre close sync", pVnode->config.vgId); vInfo("vgId:%d, pre close sync", pVnode->config.vgId);
syncLeaderTransfer(pVnode->sync); syncLeaderTransfer(pVnode->sync);
syncPreStop(pVnode->sync); syncPreStop(pVnode->sync);
#if 0
while (syncSnapshotRecving(pVnode->sync)) { while (syncSnapshotRecving(pVnode->sync)) {
vInfo("vgId:%d, snapshot is recving", pVnode->config.vgId); vInfo("vgId:%d, snapshot is recving", pVnode->config.vgId);
taosMsleep(300); taosMsleep(300);
@ -524,7 +523,7 @@ void vnodeSyncPreClose(SVnode *pVnode) {
vInfo("vgId:%d, snapshot is sending", pVnode->config.vgId); vInfo("vgId:%d, snapshot is sending", pVnode->config.vgId);
taosMsleep(300); taosMsleep(300);
} }
#endif
taosThreadMutexLock(&pVnode->lock); taosThreadMutexLock(&pVnode->lock);
if (pVnode->blocked) { if (pVnode->blocked) {
vInfo("vgId:%d, post block after close sync", pVnode->config.vgId); vInfo("vgId:%d, post block after close sync", pVnode->config.vgId);
@ -553,12 +552,12 @@ bool vnodeIsLeader(SVnode *pVnode) {
} else { } else {
terrno = TSDB_CODE_APP_NOT_READY; terrno = TSDB_CODE_APP_NOT_READY;
} }
vDebug("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncStr(state.state), state.restored); vInfo("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncStr(state.state), state.restored);
return false; return false;
} }
if (!pVnode->restored) { if (!pVnode->restored) {
vDebug("vgId:%d, vnode not restored", pVnode->config.vgId); vInfo("vgId:%d, vnode not restored", pVnode->config.vgId);
terrno = TSDB_CODE_APP_NOT_READY; terrno = TSDB_CODE_APP_NOT_READY;
return false; return false;
} }

1
source/libs/executor/inc/executorimpl.h
View File

@ -536,6 +536,7 @@ typedef struct SStreamIntervalOperatorInfo {
SArray* pChildren; SArray* pChildren;
SStreamState* pState; SStreamState* pState;
SWinKey delKey; SWinKey delKey;
uint64_t numOfDatapack;
} SStreamIntervalOperatorInfo; } SStreamIntervalOperatorInfo;
typedef struct SDataGroupInfo { typedef struct SDataGroupInfo {

4
source/libs/executor/src/filloperator.c
View File

@ -1146,7 +1146,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
if (delTs > nextKey.ts) { if (delTs > nextKey.ts) {
break; break;
} }
endTs = delTs;
SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; SWinKey delKey = {.groupId = delGroupId, .ts = delTs};
if (delTs == nextKey.ts) { if (delTs == nextKey.ts) {
code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur);
@ -1159,7 +1159,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
streamStateFreeCur(pCur); streamStateFreeCur(pCur);
pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey);
} }
endTs = TMAX(ts, nextKey.ts - 1); endTs = TMAX(delTs, nextKey.ts - 1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
break; break;
} }

2
source/libs/executor/src/scanoperator.c
View File

@ -1113,7 +1113,7 @@ static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t
if (hasGroup) { if (hasGroup) {
(*pRowIndex) += 1; (*pRowIndex) += 1;
} else { } else {
while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] < endWin.ekey)) { while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] <= endWin.ekey)) {
(*pRowIndex) += 1; (*pRowIndex) += 1;
if ((*pRowIndex) == pDataBlockInfo->rows) { if ((*pRowIndex) == pDataBlockInfo->rows) {
break; break;

9
source/libs/executor/src/timewindowoperator.c
View File

@ -2516,9 +2516,11 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) { if (pBlock == NULL) {
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); qDebug("===stream===return data:%s. recv datablock num:%" PRIu64 , IS_FINAL_OP(pInfo) ? "interval final" : "interval semi", pInfo->numOfDatapack);
pInfo->numOfDatapack = 0;
break; break;
} }
pInfo->numOfDatapack++;
printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv"); printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
ASSERT(pBlock->info.type != STREAM_INVERT); ASSERT(pBlock->info.type != STREAM_INVERT);
@ -2734,6 +2736,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey)); pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
pInfo->delKey.ts = INT64_MAX; pInfo->delKey.ts = INT64_MAX;
pInfo->delKey.groupId = 0; pInfo->delKey.groupId = 0;
pInfo->numOfDatapack = 0;
pOperator->operatorType = pPhyNode->type; pOperator->operatorType = pPhyNode->type;
pOperator->blocking = true; pOperator->blocking = true;
@ -4700,8 +4703,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
while (1) { while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) { if (pBlock == NULL) {
qDebug("===stream===return data:single interval. recv datablock num:%" PRIu64, pInfo->numOfDatapack);
pInfo->numOfDatapack = 0;
break; break;
} }
pInfo->numOfDatapack++;
printDataBlock(pBlock, "single interval recv"); printDataBlock(pBlock, "single interval recv");
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
@ -4851,6 +4857,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->pChildren = NULL; pInfo->pChildren = NULL;
pInfo->delKey.ts = INT64_MAX; pInfo->delKey.ts = INT64_MAX;
pInfo->delKey.groupId = 0; pInfo->delKey.groupId = 0;
pInfo->numOfDatapack = 0;
setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,
pInfo, pTaskInfo); pInfo, pTaskInfo);

4
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -424,6 +424,7 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_SHOW_TRANSACTIONS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
case QUERY_NODE_SHOW_TAGS_STMT: case QUERY_NODE_SHOW_TAGS_STMT:
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
return makeNode(type, sizeof(SShowStmt)); return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_SHOW_TABLE_TAGS_STMT: case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
return makeNode(type, sizeof(SShowTableTagsStmt)); return makeNode(type, sizeof(SShowTableTagsStmt));
@ -943,7 +944,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT:
case QUERY_NODE_SHOW_TRANSACTIONS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
case QUERY_NODE_SHOW_TAGS_STMT: { case QUERY_NODE_SHOW_TAGS_STMT:
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: {
SShowStmt* pStmt = (SShowStmt*)pNode; SShowStmt* pStmt = (SShowStmt*)pNode;
nodesDestroyNode(pStmt->pDbName); nodesDestroyNode(pStmt->pDbName);
nodesDestroyNode(pStmt->pTbName); nodesDestroyNode(pStmt->pTbName);

3
source/libs/parser/inc/sql.y
View File

@ -101,6 +101,7 @@ cmd ::= REVOKE privileges(A) ON priv_level(B) FROM user_name(C).
%destructor privileges { } %destructor privileges { }
privileges(A) ::= ALL. { A = PRIVILEGE_TYPE_ALL; } privileges(A) ::= ALL. { A = PRIVILEGE_TYPE_ALL; }
privileges(A) ::= priv_type_list(B). { A = B; } privileges(A) ::= priv_type_list(B). { A = B; }
privileges(A) ::= SUBSCRIBE. { A = PRIVILEGE_TYPE_SUBSCRIBE; }
%type priv_type_list { int64_t } %type priv_type_list { int64_t }
%destructor priv_type_list { } %destructor priv_type_list { }
@ -116,6 +117,7 @@ priv_type(A) ::= WRITE.
%destructor priv_level { } %destructor priv_level { }
priv_level(A) ::= NK_STAR(B) NK_DOT NK_STAR. { A = B; } priv_level(A) ::= NK_STAR(B) NK_DOT NK_STAR. { A = B; }
priv_level(A) ::= db_name(B) NK_DOT NK_STAR. { A = B; } priv_level(A) ::= db_name(B) NK_DOT NK_STAR. { A = B; }
priv_level(A) ::= topic_name(B). { A = B; }
/************************************************ create/drop/alter dnode *********************************************/ /************************************************ create/drop/alter dnode *********************************************/
cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); } cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); }
@ -393,6 +395,7 @@ col_name(A) ::= column_name(B).
/************************************************ show ****************************************************************/ /************************************************ show ****************************************************************/
cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); }
cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); }
cmd ::= SHOW USER PRIVILEGES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); }
cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); }
cmd ::= SHOW db_name_cond_opt(A) TABLES like_pattern_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, A, B, OP_TYPE_LIKE); } cmd ::= SHOW db_name_cond_opt(A) TABLES like_pattern_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, A, B, OP_TYPE_LIKE); }
cmd ::= SHOW db_name_cond_opt(A) STABLES like_pattern_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, A, B, OP_TYPE_LIKE); } cmd ::= SHOW db_name_cond_opt(A) STABLES like_pattern_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, A, B, OP_TYPE_LIKE); }

4
source/libs/parser/src/parAstCreater.c
View File

@ -1815,7 +1815,7 @@ SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbN
SGrantStmt* pStmt = (SGrantStmt*)nodesMakeNode(QUERY_NODE_GRANT_STMT); SGrantStmt* pStmt = (SGrantStmt*)nodesMakeNode(QUERY_NODE_GRANT_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
pStmt->privileges = privileges; pStmt->privileges = privileges;
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); COPY_STRING_FORM_ID_TOKEN(pStmt->objName, pDbName);
COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName);
return (SNode*)pStmt; return (SNode*)pStmt;
} }
@ -1828,7 +1828,7 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb
SRevokeStmt* pStmt = (SRevokeStmt*)nodesMakeNode(QUERY_NODE_REVOKE_STMT); SRevokeStmt* pStmt = (SRevokeStmt*)nodesMakeNode(QUERY_NODE_REVOKE_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
pStmt->privileges = privileges; pStmt->privileges = privileges;
COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); COPY_STRING_FORM_ID_TOKEN(pStmt->objName, pDbName);
COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName);
return (SNode*)pStmt; return (SNode*)pStmt;
} }

7
source/libs/parser/src/parAstParser.c
View File

@ -504,6 +504,11 @@ static int32_t collectMetaKeyFromShowVnodes(SCollectMetaKeyCxt* pCxt, SShowVnode
pCxt->pMetaCache); pCxt->pMetaCache);
} }
static int32_t collectMetaKeyFromShowUserPrivileges(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_PRIVILEGES,
pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) { static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) {
return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
} }
@ -648,6 +653,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromShowDnodeVariables(pCxt, (SShowDnodeVariablesStmt*)pStmt); return collectMetaKeyFromShowDnodeVariables(pCxt, (SShowDnodeVariablesStmt*)pStmt);
case QUERY_NODE_SHOW_VNODES_STMT: case QUERY_NODE_SHOW_VNODES_STMT:
return collectMetaKeyFromShowVnodes(pCxt, (SShowVnodesStmt*)pStmt); return collectMetaKeyFromShowVnodes(pCxt, (SShowVnodesStmt*)pStmt);
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
return collectMetaKeyFromShowUserPrivileges(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt); return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT:

2
source/libs/parser/src/parTokenizer.c
View File

@ -161,6 +161,7 @@ static SKeyword keywordTable[] = {
{"PPS", TK_PPS}, {"PPS", TK_PPS},
{"PRECISION", TK_PRECISION}, {"PRECISION", TK_PRECISION},
{"PREV", TK_PREV}, {"PREV", TK_PREV},
{"PRIVILEGES", TK_PRIVILEGES},
{"QNODE", TK_QNODE}, {"QNODE", TK_QNODE},
{"QNODES", TK_QNODES}, {"QNODES", TK_QNODES},
{"QTIME", TK_QTIME}, {"QTIME", TK_QTIME},
@ -202,6 +203,7 @@ static SKeyword keywordTable[] = {
{"STREAM", TK_STREAM}, {"STREAM", TK_STREAM},
{"STREAMS", TK_STREAMS}, {"STREAMS", TK_STREAMS},
{"STRICT", TK_STRICT}, {"STRICT", TK_STRICT},
{"SUBSCRIBE", TK_SUBSCRIBE},
{"SUBSCRIPTIONS", TK_SUBSCRIPTIONS}, {"SUBSCRIPTIONS", TK_SUBSCRIPTIONS},
{"SUBTABLE", TK_SUBTABLE}, {"SUBTABLE", TK_SUBTABLE},
{"SYSINFO", TK_SYSINFO}, {"SYSINFO", TK_SYSINFO},

17
source/libs/parser/src/parTranslater.c
View File

@ -251,6 +251,12 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
.numOfShowCols = 1, .numOfShowCols = 1,
.pShowCols = {"*"} .pShowCols = {"*"}
}, },
{ .showType = QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_TABLE_USER_PRIVILEGES,
.numOfShowCols = 1,
.pShowCols = {"*"}
},
}; };
// clang-format on // clang-format on
@ -5031,7 +5037,7 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt
alterReq.sysInfo = pStmt->sysinfo; alterReq.sysInfo = pStmt->sysinfo;
snprintf(alterReq.pass, sizeof(alterReq.pass), "%s", pStmt->password); snprintf(alterReq.pass, sizeof(alterReq.pass), "%s", pStmt->password);
if (NULL != pCxt->pParseCxt->db) { if (NULL != pCxt->pParseCxt->db) {
snprintf(alterReq.dbname, sizeof(alterReq.dbname), "%s", pCxt->pParseCxt->db); snprintf(alterReq.objname, sizeof(alterReq.objname), "%s", pCxt->pParseCxt->db);
} }
return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &alterReq); return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &alterReq);
@ -5710,9 +5716,11 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
req.alterType = TSDB_ALTER_USER_ADD_READ_DB; req.alterType = TSDB_ALTER_USER_ADD_READ_DB;
} else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) { } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) {
req.alterType = TSDB_ALTER_USER_ADD_WRITE_DB; req.alterType = TSDB_ALTER_USER_ADD_WRITE_DB;
} else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) {
req.alterType = TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC;
} }
strcpy(req.user, pStmt->userName); strcpy(req.user, pStmt->userName);
sprintf(req.dbname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->dbName); sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName);
return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
} }
@ -5726,9 +5734,11 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) {
req.alterType = TSDB_ALTER_USER_REMOVE_READ_DB; req.alterType = TSDB_ALTER_USER_REMOVE_READ_DB;
} else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) { } else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) {
req.alterType = TSDB_ALTER_USER_REMOVE_WRITE_DB; req.alterType = TSDB_ALTER_USER_REMOVE_WRITE_DB;
} else if (PRIVILEGE_TYPE_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) {
req.alterType = TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC;
} }
strcpy(req.user, pStmt->userName); strcpy(req.user, pStmt->userName);
sprintf(req.dbname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->dbName); sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName);
return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
} }
@ -7504,6 +7514,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_SHOW_CONSUMERS_STMT: case QUERY_NODE_SHOW_CONSUMERS_STMT:
case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
case QUERY_NODE_SHOW_TAGS_STMT: case QUERY_NODE_SHOW_TAGS_STMT:
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
code = rewriteShow(pCxt, pQuery); code = rewriteShow(pCxt, pQuery);
break; break;
case QUERY_NODE_SHOW_VGROUPS_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT:

6863
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

8
source/libs/parser/test/mockCatalog.cpp
View File

@ -101,6 +101,10 @@ void generateInformationSchema(MockCatalogService* mcs) {
.addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN)
.addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN)
.done(); .done();
mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_PRIVILEGES, TSDB_SYSTEM_TABLE, 2)
.addColumn("user_name", TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN)
.addColumn("privilege", TSDB_DATA_TYPE_BINARY, 10)
.done();
} }
void generatePerformanceSchema(MockCatalogService* mcs) { void generatePerformanceSchema(MockCatalogService* mcs) {
@ -248,8 +252,8 @@ int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, con
return g_mockCatalogService->catalogGetTableDistVgInfo(pTableName, pVgList); return g_mockCatalogService->catalogGetTableDistVgInfo(pTableName, pVgList);
} }
int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum,
int32_t* tableNum, int64_t* stateTs) { int64_t* stateTs) {
return 0; return 0;
} }

4
source/libs/parser/test/parAlterToBalanceTest.cpp
View File

@ -613,7 +613,7 @@ TEST_F(ParserInitialATest, alterUser) {
if (nullptr != pPass) { if (nullptr != pPass) {
strcpy(expect.pass, pPass); strcpy(expect.pass, pPass);
} }
strcpy(expect.dbname, "test"); strcpy(expect.objname, "test");
}; };
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
@ -627,7 +627,7 @@ TEST_F(ParserInitialATest, alterUser) {
ASSERT_EQ(req.enable, expect.enable); ASSERT_EQ(req.enable, expect.enable);
ASSERT_EQ(std::string(req.user), std::string(expect.user)); ASSERT_EQ(std::string(req.user), std::string(expect.user));
ASSERT_EQ(std::string(req.pass), std::string(expect.pass)); ASSERT_EQ(std::string(req.pass), std::string(expect.pass));
ASSERT_EQ(std::string(req.dbname), std::string(expect.dbname)); ASSERT_EQ(std::string(req.objname), std::string(expect.objname));
}); });
setAlterUserReq("wxy", TSDB_ALTER_USER_PASSWD, "123456"); setAlterUserReq("wxy", TSDB_ALTER_USER_PASSWD, "123456");

28
source/libs/parser/test/parExplainToSyncdbTest.cpp
View File

@ -34,10 +34,38 @@ TEST_F(ParserExplainToSyncdbTest, explain) {
TEST_F(ParserExplainToSyncdbTest, grant) { TEST_F(ParserExplainToSyncdbTest, grant) {
useDb("root", "test"); useDb("root", "test");
SAlterUserReq expect = {0};
auto setAlterUserReq = [&](int8_t alterType, const string& user, const string& obj) {
expect.alterType = alterType;
snprintf(expect.user, sizeof(expect.user), "%s", user.c_str());
snprintf(expect.objname, sizeof(expect.objname), "%s", obj.c_str());
};
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_GRANT_STMT);
ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_USER);
SAlterUserReq req = {0};
ASSERT_EQ(tDeserializeSAlterUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS);
ASSERT_EQ(req.alterType, expect.alterType);
ASSERT_EQ(string(req.user), string(expect.user));
ASSERT_EQ(string(req.objname), string(expect.objname));
});
setAlterUserReq(TSDB_ALTER_USER_ADD_ALL_DB, "wxy", "0.test");
run("GRANT ALL ON test.* TO wxy"); run("GRANT ALL ON test.* TO wxy");
setAlterUserReq(TSDB_ALTER_USER_ADD_READ_DB, "wxy", "0.test");
run("GRANT READ ON test.* TO wxy"); run("GRANT READ ON test.* TO wxy");
setAlterUserReq(TSDB_ALTER_USER_ADD_WRITE_DB, "wxy", "0.test");
run("GRANT WRITE ON test.* TO wxy"); run("GRANT WRITE ON test.* TO wxy");
setAlterUserReq(TSDB_ALTER_USER_ADD_ALL_DB, "wxy", "0.test");
run("GRANT READ, WRITE ON test.* TO wxy"); run("GRANT READ, WRITE ON test.* TO wxy");
setAlterUserReq(TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC, "wxy", "0.tp1");
run("GRANT SUBSCRIBE ON tp1 TO wxy");
} }
TEST_F(ParserExplainToSyncdbTest, insert) { TEST_F(ParserExplainToSyncdbTest, insert) {

8
source/libs/parser/test/parShowToUse.cpp
View File

@ -213,7 +213,13 @@ TEST_F(ParserShowToUseTest, showTags) {
TEST_F(ParserShowToUseTest, showUsers) { TEST_F(ParserShowToUseTest, showUsers) {
useDb("root", "test"); useDb("root", "test");
run("SHOW users"); run("SHOW USERS");
}
TEST_F(ParserShowToUseTest, showUserPrivileges) {
useDb("root", "test");
run("SHOW USER PRIVILEGES");
} }
TEST_F(ParserShowToUseTest, showVariables) { TEST_F(ParserShowToUseTest, showVariables) {

6
source/libs/stream/inc/streamInc.h
View File

@ -17,7 +17,6 @@
#define _STREAM_INC_H_ #define _STREAM_INC_H_
#include "executor.h" #include "executor.h"
#include "tref.h"
#include "tstream.h" #include "tstream.h"
#ifdef __cplusplus #ifdef __cplusplus
@ -25,9 +24,8 @@ extern "C" {
#endif #endif
typedef struct { typedef struct {
int8_t inited; int8_t inited;
int32_t refPool; void* timer;
void* timer;
} SStreamGlobalEnv; } SStreamGlobalEnv;
static SStreamGlobalEnv streamEnv; static SStreamGlobalEnv streamEnv;

194
source/libs/stream/src/streamCheckpoint.c Normal file
View File

@ -0,0 +1,194 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "streamInc.h"
int32_t tEncodeSStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) {
if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->checkpointId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->nodeId) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->expireTime) < 0) return -1;
tEndEncode(pEncoder);
return pEncoder->pos;
}
int32_t tDecodeSStreamCheckpointSourceReq(SDecoder* pDecoder, SStreamCheckpointSourceReq* pReq) {
if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
if (tDecodeI64(pDecoder, &pReq->checkpointId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->nodeId) < 0) return -1;
if (tDecodeI64(pDecoder, &pReq->expireTime) < 0) return -1;
tEndDecode(pDecoder);
return 0;
}
int32_t tEncodeSStreamCheckpointSourceRsp(SEncoder* pEncoder, const SStreamCheckpointSourceRsp* pRsp) {
if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->checkpointId) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->nodeId) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->expireTime) < 0) return -1;
tEndEncode(pEncoder);
return pEncoder->pos;
}
int32_t tDecodeSStreamCheckpointSourceRsp(SDecoder* pDecoder, SStreamCheckpointSourceRsp* pRsp) {
if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
if (tDecodeI64(pDecoder, &pRsp->checkpointId) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->taskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->nodeId) < 0) return -1;
if (tDecodeI64(pDecoder, &pRsp->expireTime) < 0) return -1;
tEndDecode(pDecoder);
return 0;
}
int32_t tEncodeSStreamCheckpointReq(SEncoder* pEncoder, const SStreamCheckpointReq* pReq) {
if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->checkpointId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->downstreamNodeId) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->upstreamTaskId) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->upstreamNodeId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->childId) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->expireTime) < 0) return -1;
if (tEncodeI8(pEncoder, pReq->taskLevel) < 0) return -1;
tEndEncode(pEncoder);
return pEncoder->pos;
}
int32_t tDecodeSStreamCheckpointReq(SDecoder* pDecoder, SStreamCheckpointReq* pReq) {
if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
if (tDecodeI64(pDecoder, &pReq->checkpointId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->downstreamNodeId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->childId) < 0) return -1;
if (tDecodeI64(pDecoder, &pReq->expireTime) < 0) return -1;
if (tDecodeI8(pDecoder, &pReq->taskLevel) < 0) return -1;
tEndDecode(pDecoder);
return 0;
}
int32_t tEncodeSStreamCheckpointRsp(SEncoder* pEncoder, const SStreamCheckpointRsp* pRsp) {
if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->checkpointId) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->downstreamNodeId) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->upstreamTaskId) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->upstreamNodeId) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->childId) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->expireTime) < 0) return -1;
if (tEncodeI8(pEncoder, pRsp->taskLevel) < 0) return -1;
tEndEncode(pEncoder);
return pEncoder->pos;
}
int32_t tDecodeSStreamCheckpointRsp(SDecoder* pDecoder, SStreamCheckpointRsp* pRsp) {
if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
if (tDecodeI64(pDecoder, &pRsp->checkpointId) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->downstreamNodeId) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->upstreamTaskId) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->upstreamNodeId) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->childId) < 0) return -1;
if (tDecodeI64(pDecoder, &pRsp->expireTime) < 0) return -1;
if (tDecodeI8(pDecoder, &pRsp->taskLevel) < 0) return -1;
tEndDecode(pDecoder);
return 0;
}
static int32_t streamAlignCheckpoint(SStreamTask* pTask, int64_t checkpointId, int32_t childId) {
if (pTask->checkpointingId == 0) {
pTask->checkpointingId = checkpointId;
pTask->checkpointAlignCnt = taosArrayGetSize(pTask->childEpInfo);
}
ASSERT(pTask->checkpointingId == checkpointId);
return atomic_sub_fetch_32(&pTask->checkpointAlignCnt, 1);
}
static int32_t streamDoCheckpoint(SStreamMeta* pMeta, SStreamTask* pTask, int64_t checkpointId) {
// commit tdb state
streamStateCommit(pTask->pState);
// commit non-tdb state
// copy and save new state
// report to mnode
// send checkpoint req to downstream
return 0;
}
static int32_t streamDoSourceCheckpoint(SStreamMeta* pMeta, SStreamTask* pTask, int64_t checkpointId) {
// ref wal
// set status checkpointing
// do checkpoint
return 0;
}
int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq) {
int32_t code;
int64_t checkpointId = pReq->checkpointId;
code = streamDoSourceCheckpoint(pMeta, pTask, checkpointId);
if (code < 0) {
// rsp error
return -1;
}
return 0;
}
int32_t streamProcessCheckpointReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointReq* pReq) {
int32_t code;
int64_t checkpointId = pReq->checkpointId;
int32_t childId = pReq->childId;
if (taosArrayGetSize(pTask->childEpInfo) > 0) {
code = streamAlignCheckpoint(pTask, checkpointId, childId);
if (code > 0) {
return 0;
}
if (code < 0) {
ASSERT(0);
return -1;
}
}
code = streamDoCheckpoint(pMeta, pTask, checkpointId);
if (code < 0) {
// rsp error
return -1;
}
// send rsp to all children
return 0;
}
int32_t streamProcessCheckpointRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointRsp* pRsp) {
// recover step2, scan from wal
// unref wal
// set status normal
return 0;
}

31
source/libs/stream/src/streamMeta.c
View File

@ -202,7 +202,7 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) {
} }
} }
void streamMetaRemoveTask1(SStreamMeta* pMeta, int32_t taskId) { void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
if (ppTask) { if (ppTask) {
SStreamTask* pTask = *ppTask; SStreamTask* pTask = *ppTask;
@ -219,35 +219,6 @@ void streamMetaRemoveTask1(SStreamMeta* pMeta, int32_t taskId) {
} }
} }
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
if (ppTask) {
SStreamTask* pTask = *ppTask;
taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t));
atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING);
if (tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), &pMeta->txn) < 0) {
/*return -1;*/
}
if (pTask->triggerParam != 0) {
taosTmrStop(pTask->timer);
}
while (1) {
int8_t schedStatus =
atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__DROPPING);
if (schedStatus != TASK_SCHED_STATUS__ACTIVE) {
tFreeSStreamTask(pTask);
break;
}
taosMsleep(10);
}
}
return 0;
}
int32_t streamMetaBegin(SStreamMeta* pMeta) { int32_t streamMetaBegin(SStreamMeta* pMeta) {
if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) { 0) {

43
source/libs/stream/src/streamRecover.c
View File

@ -325,46 +325,3 @@ int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishR
tEndDecode(pDecoder); tEndDecode(pDecoder);
return 0; return 0;
} }
int32_t tEncodeSStreamCheckpointInfo(SEncoder* pEncoder, const SStreamCheckpointInfo* pCheckpoint) {
if (tEncodeI32(pEncoder, pCheckpoint->srcNodeId) < 0) return -1;
if (tEncodeI32(pEncoder, pCheckpoint->srcChildId) < 0) return -1;
if (tEncodeI64(pEncoder, pCheckpoint->stateProcessedVer) < 0) return -1;
return 0;
}
int32_t tDecodeSStreamCheckpointInfo(SDecoder* pDecoder, SStreamCheckpointInfo* pCheckpoint) {
if (tDecodeI32(pDecoder, &pCheckpoint->srcNodeId) < 0) return -1;
if (tDecodeI32(pDecoder, &pCheckpoint->srcChildId) < 0) return -1;
if (tDecodeI64(pDecoder, &pCheckpoint->stateProcessedVer) < 0) return -1;
return 0;
}
int32_t tEncodeSStreamMultiVgCheckpointInfo(SEncoder* pEncoder, const SStreamMultiVgCheckpointInfo* pCheckpoint) {
if (tEncodeI64(pEncoder, pCheckpoint->streamId) < 0) return -1;
if (tEncodeI64(pEncoder, pCheckpoint->checkTs) < 0) return -1;
if (tEncodeI32(pEncoder, pCheckpoint->checkpointId) < 0) return -1;
if (tEncodeI32(pEncoder, pCheckpoint->taskId) < 0) return -1;
int32_t sz = taosArrayGetSize(pCheckpoint->checkpointVer);
if (tEncodeI32(pEncoder, sz) < 0) return -1;
for (int32_t i = 0; i < sz; i++) {
SStreamCheckpointInfo* pOneVgCkpoint = taosArrayGet(pCheckpoint->checkpointVer, i);
if (tEncodeSStreamCheckpointInfo(pEncoder, pOneVgCkpoint) < 0) return -1;
}
return 0;
}
int32_t tDecodeSStreamMultiVgCheckpointInfo(SDecoder* pDecoder, SStreamMultiVgCheckpointInfo* pCheckpoint) {
if (tDecodeI64(pDecoder, &pCheckpoint->streamId) < 0) return -1;
if (tDecodeI64(pDecoder, &pCheckpoint->checkTs) < 0) return -1;
if (tDecodeI32(pDecoder, &pCheckpoint->checkpointId) < 0) return -1;
if (tDecodeI32(pDecoder, &pCheckpoint->taskId) < 0) return -1;
int32_t sz;
if (tDecodeI32(pDecoder, &sz) < 0) return -1;
for (int32_t i = 0; i < sz; i++) {
SStreamCheckpointInfo oneVgCheckpoint;
if (tDecodeSStreamCheckpointInfo(pDecoder, &oneVgCheckpoint) < 0) return -1;
taosArrayPush(pCheckpoint->checkpointVer, &oneVgCheckpoint);
}
return 0;
}

5
source/libs/sync/inc/syncCommit.h
View File

@ -47,8 +47,13 @@ extern "C" {
// //
void syncOneReplicaAdvance(SSyncNode* pSyncNode); void syncOneReplicaAdvance(SSyncNode* pSyncNode);
void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode); void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode);
bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index); bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index);
bool syncAgree(SSyncNode* pSyncNode, SyncIndex index); bool syncAgree(SSyncNode* pSyncNode, SyncIndex index);
bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index);
int64_t syncNodeUpdateCommitIndex(SSyncNode* ths, SyncIndex commitIndex);
int64_t syncNodeCheckCommitIndex(SSyncNode* ths, SyncIndex indexLikely);
#ifdef __cplusplus #ifdef __cplusplus
} }

15
source/libs/sync/inc/syncInt.h
View File

@ -50,6 +50,8 @@ typedef struct SyncPreSnapshotReply SyncPreSnapshotReply;
typedef struct SyncHeartbeatReply SyncHeartbeatReply; typedef struct SyncHeartbeatReply SyncHeartbeatReply;
typedef struct SyncHeartbeat SyncHeartbeat; typedef struct SyncHeartbeat SyncHeartbeat;
typedef struct SyncPreSnapshot SyncPreSnapshot; typedef struct SyncPreSnapshot SyncPreSnapshot;
typedef struct SSyncLogBuffer SSyncLogBuffer;
typedef struct SSyncLogReplMgr SSyncLogReplMgr;
typedef struct SRaftId { typedef struct SRaftId {
SyncNodeId addr; SyncNodeId addr;
@ -97,6 +99,7 @@ typedef struct SSyncNode {
char configPath[TSDB_FILENAME_LEN * 2]; char configPath[TSDB_FILENAME_LEN * 2];
// sync io // sync io
SSyncLogBuffer* pLogBuf;
SWal* pWal; SWal* pWal;
const SMsgCb* msgcb; const SMsgCb* msgcb;
int32_t (*syncSendMSg)(const SEpSet* pEpSet, SRpcMsg* pMsg); int32_t (*syncSendMSg)(const SEpSet* pEpSet, SRpcMsg* pMsg);
@ -179,6 +182,9 @@ typedef struct SSyncNode {
SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; SSyncSnapshotSender* senders[TSDB_MAX_REPLICA];
SSyncSnapshotReceiver* pNewNodeReceiver; SSyncSnapshotReceiver* pNewNodeReceiver;
// log replication mgr
SSyncLogReplMgr* logReplMgrs[TSDB_MAX_REPLICA];
SPeerState peerStates[TSDB_MAX_REPLICA]; SPeerState peerStates[TSDB_MAX_REPLICA];
// is config changing // is config changing
@ -205,11 +211,12 @@ typedef struct SSyncNode {
// open/close -------------- // open/close --------------
SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo); SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo);
void syncNodeStart(SSyncNode* pSyncNode); int32_t syncNodeStart(SSyncNode* pSyncNode);
void syncNodeStartStandBy(SSyncNode* pSyncNode); int32_t syncNodeStartStandBy(SSyncNode* pSyncNode);
void syncNodeClose(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode);
void syncNodePreClose(SSyncNode* pSyncNode); void syncNodePreClose(SSyncNode* pSyncNode);
int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak); int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak);
int32_t syncNodeRestore(SSyncNode* pSyncNode);
void syncHbTimerDataFree(SSyncHbTimerData* pData); void syncHbTimerDataFree(SSyncHbTimerData* pData);
// on message --------------------- // on message ---------------------
@ -244,6 +251,7 @@ int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHa
bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode); bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode);
bool syncNodeSnapshotSending(SSyncNode* pSyncNode); bool syncNodeSnapshotSending(SSyncNode* pSyncNode);
bool syncNodeSnapshotRecving(SSyncNode* pSyncNode); bool syncNodeSnapshotRecving(SSyncNode* pSyncNode);
bool syncNodeIsReadyForRead(SSyncNode* pSyncNode);
// raft state change -------------- // raft state change --------------
void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term);
@ -260,6 +268,9 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode);
void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId);
void syncNodeVoteForSelf(SSyncNode* pSyncNode); void syncNodeVoteForSelf(SSyncNode* pSyncNode);
// log replication
SSyncLogReplMgr* syncNodeGetLogReplMgr(SSyncNode* pNode, SRaftId* pDestId);
// snapshot -------------- // snapshot --------------
bool syncNodeHasSnapshot(SSyncNode* pSyncNode); bool syncNodeHasSnapshot(SSyncNode* pSyncNode);
void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode); void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode);

4
source/libs/sync/inc/syncMessage.h
View File

@ -106,7 +106,7 @@ typedef struct SyncAppendEntriesReply {
SRaftId destId; SRaftId destId;
// private data // private data
SyncTerm term; SyncTerm term;
SyncTerm privateTerm; SyncTerm lastMatchTerm;
bool success; bool success;
SyncIndex matchIndex; SyncIndex matchIndex;
SyncIndex lastSendIndex; SyncIndex lastSendIndex;
@ -246,6 +246,8 @@ int32_t syncBuildRequestVote(SRpcMsg* pMsg, int32_t vgId);
int32_t syncBuildRequestVoteReply(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildRequestVoteReply(SRpcMsg* pMsg, int32_t vgId);
int32_t syncBuildAppendEntries(SRpcMsg* pMsg, int32_t dataLen, int32_t vgId); int32_t syncBuildAppendEntries(SRpcMsg* pMsg, int32_t dataLen, int32_t vgId);
int32_t syncBuildAppendEntriesReply(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildAppendEntriesReply(SRpcMsg* pMsg, int32_t vgId);
int32_t syncBuildAppendEntriesFromRaftLog(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm,
SRpcMsg* pRpcMsg);
int32_t syncBuildHeartbeat(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildHeartbeat(SRpcMsg* pMsg, int32_t vgId);
int32_t syncBuildHeartbeatReply(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildHeartbeatReply(SRpcMsg* pMsg, int32_t vgId);
int32_t syncBuildPreSnapshot(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildPreSnapshot(SRpcMsg* pMsg, int32_t vgId);

116
source/libs/sync/inc/syncPipeline.h Normal file
View File

@ -0,0 +1,116 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_PIPELINE_H
#define _TD_LIBS_SYNC_PIPELINE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "syncInt.h"
typedef struct SSyncReplInfo {
bool barrier;
bool acked;
int64_t timeMs;
int64_t term;
} SSyncReplInfo;
typedef struct SSyncLogReplMgr {
SSyncReplInfo states[TSDB_SYNC_LOG_BUFFER_SIZE];
int64_t startIndex;
int64_t matchIndex;
int64_t endIndex;
int64_t size;
bool restored;
int64_t peerStartTime;
int32_t retryBackoff;
int32_t peerId;
} SSyncLogReplMgr;
typedef struct SSyncLogBufEntry {
SSyncRaftEntry* pItem;
SyncIndex prevLogIndex;
SyncTerm prevLogTerm;
} SSyncLogBufEntry;
typedef struct SSyncLogBuffer {
SSyncLogBufEntry entries[TSDB_SYNC_LOG_BUFFER_SIZE];
int64_t startIndex;
int64_t commitIndex;
int64_t matchIndex;
int64_t endIndex;
int64_t size;
TdThreadMutex mutex;
TdThreadMutexAttr attr;
} SSyncLogBuffer;
// SSyncLogRepMgr
SSyncLogReplMgr* syncLogReplMgrCreate();
void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr);
int32_t syncLogReplMgrReset(SSyncLogReplMgr* pMgr);
int32_t syncNodeLogReplMgrInit(SSyncNode* pNode);
void syncNodeLogReplMgrDestroy(SSyncNode* pNode);
// access
static FORCE_INLINE int64_t syncLogGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) {
return (1 << pMgr->retryBackoff) * SYNC_LOG_REPL_RETRY_WAIT_MS;
}
static FORCE_INLINE int32_t syncLogGetNextRetryBackoff(SSyncLogReplMgr* pMgr) {
return TMIN(pMgr->retryBackoff + 1, SYNC_MAX_RETRY_BACKOFF);
}
SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index);
int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
int32_t syncLogBufferReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm,
SRaftId* pDestId, bool* pBarrier);
int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index);
int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg);
int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg);
int32_t syncLogReplMgrProcessReplyInNormalMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg);
int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg);
int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode);
// SSyncLogBuffer
SSyncLogBuffer* syncLogBufferCreate();
void syncLogBufferDestroy(SSyncLogBuffer* pBuf);
int32_t syncLogBufferInit(SSyncLogBuffer* pBuf, SSyncNode* pNode);
int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode);
// access
int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf);
int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry);
int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm);
int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* pMatchTerm);
int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t commitIndex);
int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode);
// private
SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex index, bool* pInBuf);
int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf);
int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SyncIndex toIndex);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_PIPELINE_H*/

6
source/libs/sync/inc/syncRaftEntry.h
View File

@ -42,9 +42,13 @@ SSyncRaftEntry* syncEntryBuildFromClientRequest(const SyncClientRequest* pMsg, S
SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, SyncIndex index); SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, SyncIndex index);
SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg); SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg);
SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId); SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId);
void syncEntryDestory(SSyncRaftEntry* pEntry); void syncEntryDestroy(SSyncRaftEntry* pEntry);
void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7 void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7
static FORCE_INLINE bool syncLogIsReplicationBarrier(SSyncRaftEntry* pEntry) {
return pEntry->originalRpcType == TDMT_SYNC_NOOP;
}
typedef struct SRaftEntryHashCache { typedef struct SRaftEntryHashCache {
SHashObj* pEntryHash; SHashObj* pEntryHash;
int32_t maxCount; int32_t maxCount;

4
source/libs/sync/inc/syncReplication.h
View File

@ -52,6 +52,10 @@ int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* pDestId, SRpc
int32_t syncNodeReplicate(SSyncNode* pSyncNode); int32_t syncNodeReplicate(SSyncNode* pSyncNode);
int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapshot); int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapshot);
int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode);
int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

2
source/libs/sync/inc/syncSnapshot.h
View File

@ -87,6 +87,8 @@ void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceive
int32_t syncNodeOnSnapshot(SSyncNode *ths, const SRpcMsg *pMsg); int32_t syncNodeOnSnapshot(SSyncNode *ths, const SRpcMsg *pMsg);
int32_t syncNodeOnSnapshotReply(SSyncNode *ths, const SRpcMsg *pMsg); int32_t syncNodeOnSnapshotReply(SSyncNode *ths, const SRpcMsg *pMsg);
SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode *pSyncNode, SyncIndex snapshotLastApplyIndex);
// start // start
#ifdef __cplusplus #ifdef __cplusplus

7
source/libs/sync/inc/syncUtil.h
View File

@ -94,11 +94,12 @@ void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const c
void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s); void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s);
void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s); void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s);
void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, bool printX, int64_t timerElapsed, int64_t execTime); void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, bool printX, int64_t timerElapsed,
void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, int64_t timeDiff); int64_t execTime);
void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, int64_t timeDiff, const char* s);
void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s); void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s);
void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, int64_t timeDiff); void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, int64_t timeDiff, const char* s);
void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s); void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s);
void syncLogRecvSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s); void syncLogRecvSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s);

135
source/libs/sync/src/syncAppendEntries.c
View File

@ -15,10 +15,13 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "syncAppendEntries.h" #include "syncAppendEntries.h"
#include "syncPipeline.h"
#include "syncMessage.h" #include "syncMessage.h"
#include "syncRaftLog.h" #include "syncRaftLog.h"
#include "syncRaftStore.h" #include "syncRaftStore.h"
#include "syncReplication.h"
#include "syncUtil.h" #include "syncUtil.h"
#include "syncCommit.h"
// TLA+ Spec // TLA+ Spec
// HandleAppendEntriesRequest(i, j, m) == // HandleAppendEntriesRequest(i, j, m) ==
@ -124,9 +127,113 @@ int32_t syncNodeFollowerCommit(SSyncNode* ths, SyncIndex newCommitIndex) {
return 0; return 0;
} }
SSyncRaftEntry* syncLogAppendEntriesToRaftEntry(const SyncAppendEntries* pMsg) {
SSyncRaftEntry* pEntry = taosMemoryMalloc(pMsg->dataLen);
if (pEntry == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
(void)memcpy(pEntry, pMsg->data, pMsg->dataLen);
ASSERT(pEntry->bytes == pMsg->dataLen);
return pEntry;
}
int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
SyncAppendEntries* pMsg = pRpcMsg->pCont; SyncAppendEntries* pMsg = pRpcMsg->pCont;
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
bool accepted = false;
// if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
syncLogRecvAppendEntries(ths, pMsg, "not in my config");
goto _IGNORE;
}
int32_t code = syncBuildAppendEntriesReply(&rpcRsp, ths->vgId);
if (code != 0) {
syncLogRecvAppendEntries(ths, pMsg, "build rsp error");
goto _IGNORE;
}
SyncAppendEntriesReply* pReply = rpcRsp.pCont;
// prepare response msg
pReply->srcId = ths->myRaftId;
pReply->destId = pMsg->srcId;
pReply->term = ths->pRaftStore->currentTerm;
pReply->success = false;
pReply->matchIndex = SYNC_INDEX_INVALID;
pReply->lastSendIndex = pMsg->prevLogIndex + 1;
pReply->startTime = ths->startTime;
if (pMsg->term < ths->pRaftStore->currentTerm) {
goto _SEND_RESPONSE;
}
if (pMsg->term > ths->pRaftStore->currentTerm) {
pReply->term = pMsg->term;
}
syncNodeStepDown(ths, pMsg->term);
syncNodeResetElectTimer(ths);
if (pMsg->dataLen < (int32_t)sizeof(SSyncRaftEntry)) {
sError("vgId:%d, incomplete append entries received. prev index:%" PRId64 ", term:%" PRId64 ", datalen:%d",
ths->vgId, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
goto _IGNORE;
}
SSyncRaftEntry* pEntry = syncLogAppendEntriesToRaftEntry(pMsg);
if (pEntry == NULL) {
sError("vgId:%d, failed to get raft entry from append entries since %s", ths->vgId, terrstr());
goto _IGNORE;
}
if (pMsg->prevLogIndex + 1 != pEntry->index || pEntry->term < 0) {
sError("vgId:%d, invalid previous log index in msg. index:%" PRId64 ", term:%" PRId64 ", prevLogIndex:%" PRId64
", prevLogTerm:%" PRId64,
ths->vgId, pEntry->index, pEntry->term, pMsg->prevLogIndex, pMsg->prevLogTerm);
goto _IGNORE;
}
sTrace("vgId:%d, recv append entries msg. index:%" PRId64 ", term:%" PRId64 ", preLogIndex:%" PRId64
", prevLogTerm:%" PRId64 " commitIndex:%" PRId64 "",
pMsg->vgId, pMsg->prevLogIndex + 1, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex);
// accept
if (syncLogBufferAccept(ths->pLogBuf, ths, pEntry, pMsg->prevLogTerm) < 0) {
goto _SEND_RESPONSE;
}
accepted = true;
_SEND_RESPONSE:
pReply->matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, &pReply->lastMatchTerm);
bool matched = (pReply->matchIndex >= pReply->lastSendIndex);
if (accepted && matched) {
pReply->success = true;
// update commit index only after matching
(void)syncNodeUpdateCommitIndex(ths, pMsg->commitIndex);
}
// ack, i.e. send response
(void)syncNodeSendMsgById(&pReply->destId, ths, &rpcRsp);
// commit index, i.e. leader notice me
if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) {
sError("vgId:%d, failed to commit raft fsm log since %s.", ths->vgId, terrstr());
goto _out;
}
_out:
return 0;
_IGNORE:
rpcFreeCont(rpcRsp.pCont);
return 0;
}
int32_t syncNodeOnAppendEntriesOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
SyncAppendEntries* pMsg = pRpcMsg->pCont;
SRpcMsg rpcRsp = {0};
// if already drop replica, do not process // if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
@ -233,18 +340,20 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
if (hLocal) { if (hLocal) {
taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false); taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false);
} else { } else {
syncEntryDestory(pLocalEntry); syncEntryDestroy(pLocalEntry);
} }
if (hAppend) { if (hAppend) {
taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false); taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false);
} else { } else {
syncEntryDestory(pAppendEntry); syncEntryDestroy(pAppendEntry);
} }
goto _IGNORE; goto _IGNORE;
} }
ASSERT(pAppendEntry->index == appendIndex);
// append // append
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
if (code != 0) { if (code != 0) {
@ -255,13 +364,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
if (hLocal) { if (hLocal) {
taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false); taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false);
} else { } else {
syncEntryDestory(pLocalEntry); syncEntryDestroy(pLocalEntry);
} }
if (hAppend) { if (hAppend) {
taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false); taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false);
} else { } else {
syncEntryDestory(pAppendEntry); syncEntryDestroy(pAppendEntry);
} }
goto _IGNORE; goto _IGNORE;
@ -281,8 +390,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
snprintf(logBuf, sizeof(logBuf), "ignore, log not exist, truncate error, append-index:%" PRId64, appendIndex); snprintf(logBuf, sizeof(logBuf), "ignore, log not exist, truncate error, append-index:%" PRId64, appendIndex);
syncLogRecvAppendEntries(ths, pMsg, logBuf); syncLogRecvAppendEntries(ths, pMsg, logBuf);
syncEntryDestory(pLocalEntry); syncEntryDestroy(pLocalEntry);
syncEntryDestory(pAppendEntry); syncEntryDestroy(pAppendEntry);
goto _IGNORE; goto _IGNORE;
} }
@ -296,13 +405,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
if (hLocal) { if (hLocal) {
taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false); taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false);
} else { } else {
syncEntryDestory(pLocalEntry); syncEntryDestroy(pLocalEntry);
} }
if (hAppend) { if (hAppend) {
taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false); taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false);
} else { } else {
syncEntryDestory(pAppendEntry); syncEntryDestroy(pAppendEntry);
} }
goto _IGNORE; goto _IGNORE;
@ -320,13 +429,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
if (hLocal) { if (hLocal) {
taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false); taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false);
} else { } else {
syncEntryDestory(pLocalEntry); syncEntryDestroy(pLocalEntry);
} }
if (hAppend) { if (hAppend) {
taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false); taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false);
} else { } else {
syncEntryDestory(pAppendEntry); syncEntryDestroy(pAppendEntry);
} }
goto _IGNORE; goto _IGNORE;
@ -339,13 +448,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
if (hLocal) { if (hLocal) {
taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false); taosLRUCacheRelease(ths->pLogStore->pCache, hLocal, false);
} else { } else {
syncEntryDestory(pLocalEntry); syncEntryDestroy(pLocalEntry);
} }
if (hAppend) { if (hAppend) {
taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false); taosLRUCacheRelease(ths->pLogStore->pCache, hAppend, false);
} else { } else {
syncEntryDestory(pAppendEntry); syncEntryDestroy(pAppendEntry);
} }
} else { } else {
@ -373,4 +482,4 @@ _SEND_RESPONSE:
// send response // send response
syncNodeSendMsgById(&pReply->destId, ths, &rpcRsp); syncNodeSendMsgById(&pReply->destId, ths, &rpcRsp);
return 0; return 0;
} }

54
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -17,7 +17,9 @@
#include "syncAppendEntriesReply.h" #include "syncAppendEntriesReply.h"
#include "syncCommit.h" #include "syncCommit.h"
#include "syncIndexMgr.h" #include "syncIndexMgr.h"
#include "syncPipeline.h"
#include "syncMessage.h" #include "syncMessage.h"
#include "syncRaftEntry.h"
#include "syncRaftStore.h" #include "syncRaftStore.h"
#include "syncReplication.h" #include "syncReplication.h"
#include "syncSnapshot.h" #include "syncSnapshot.h"
@ -38,8 +40,58 @@
// //
int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
int32_t ret = 0;
SyncAppendEntriesReply* pMsg = pRpcMsg->pCont; SyncAppendEntriesReply* pMsg = pRpcMsg->pCont;
int32_t ret = 0;
// if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
syncLogRecvAppendEntriesReply(ths, pMsg, "not in my config");
return 0;
}
// drop stale response
if (pMsg->term < ths->pRaftStore->currentTerm) {
syncLogRecvAppendEntriesReply(ths, pMsg, "drop stale response");
return 0;
}
if (ths->state == TAOS_SYNC_STATE_LEADER) {
if (pMsg->term > ths->pRaftStore->currentTerm) {
syncLogRecvAppendEntriesReply(ths, pMsg, "error term");
syncNodeStepDown(ths, pMsg->term);
return -1;
}
ASSERT(pMsg->term == ths->pRaftStore->currentTerm);
sTrace("vgId:%d received append entries reply. srcId:0x%016" PRIx64 ", term:%" PRId64 ", matchIndex:%" PRId64 "",
pMsg->vgId, pMsg->srcId.addr, pMsg->term, pMsg->matchIndex);
if (pMsg->success) {
SyncIndex oldMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId));
if (pMsg->matchIndex > oldMatchIndex) {
syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex);
}
// commit if needed
SyncIndex indexLikely = TMIN(pMsg->matchIndex, ths->pLogBuf->matchIndex);
SyncIndex commitIndex = syncNodeCheckCommitIndex(ths, indexLikely);
(void)syncLogBufferCommit(ths->pLogBuf, ths, commitIndex);
}
// replicate log
SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(ths, &pMsg->srcId);
if (pMgr == NULL) {
sError("vgId:%d, failed to get log repl mgr for src addr: 0x%016" PRIx64, ths->vgId, pMsg->srcId.addr);
return -1;
}
(void)syncLogReplMgrProcessReply(pMgr, ths, pMsg);
}
return 0;
}
int32_t syncNodeOnAppendEntriesReplyOld(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {
int32_t ret = 0;
// if already drop replica, do not process // if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {

46
source/libs/sync/src/syncCommit.c
View File

@ -84,6 +84,7 @@ void syncOneReplicaAdvance(SSyncNode* pSyncNode) {
} }
void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
ASSERT(false && "deprecated");
if (pSyncNode == NULL) { if (pSyncNode == NULL) {
sError("pSyncNode is NULL"); sError("pSyncNode is NULL");
return; return;
@ -138,7 +139,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
if (h) { if (h) {
taosLRUCacheRelease(pCache, h, false); taosLRUCacheRelease(pCache, h, false);
} else { } else {
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
break; break;
@ -150,7 +151,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
if (h) { if (h) {
taosLRUCacheRelease(pCache, h, false); taosLRUCacheRelease(pCache, h, false);
} else { } else {
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
} }
} }
@ -286,15 +287,48 @@ bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) {
} }
*/ */
bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) { bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index) {
int count = 0;
SSyncIndexMgr* pMatches = pNode->pMatchIndex;
ASSERT(pNode->replicaNum == pMatches->replicaNum);
for (int i = 0; i < pNode->replicaNum; i++) {
SyncIndex matchIndex = pMatches->index[i];
if (matchIndex >= index) {
count++;
}
}
return count >= pNode->quorum;
}
bool syncAgree(SSyncNode* pNode, SyncIndex index) {
int agreeCount = 0; int agreeCount = 0;
for (int i = 0; i < pSyncNode->replicaNum; ++i) { for (int i = 0; i < pNode->replicaNum; ++i) {
if (syncAgreeIndex(pSyncNode, &(pSyncNode->replicasId[i]), index)) { if (syncAgreeIndex(pNode, &(pNode->replicasId[i]), index)) {
++agreeCount; ++agreeCount;
} }
if (agreeCount >= pSyncNode->quorum) { if (agreeCount >= pNode->quorum) {
return true; return true;
} }
} }
return false; return false;
} }
int64_t syncNodeUpdateCommitIndex(SSyncNode* ths, SyncIndex commitIndex) {
SyncIndex lastVer = ths->pLogStore->syncLogLastIndex(ths->pLogStore);
commitIndex = TMAX(commitIndex, ths->commitIndex);
ths->commitIndex = TMIN(commitIndex, lastVer);
ths->pLogStore->syncLogUpdateCommitIndex(ths->pLogStore, ths->commitIndex);
return ths->commitIndex;
}
int64_t syncNodeCheckCommitIndex(SSyncNode* ths, SyncIndex indexLikely) {
if (indexLikely > ths->commitIndex && syncNodeAgreedUpon(ths, indexLikely)) {
SyncIndex commitIndex = indexLikely;
syncNodeUpdateCommitIndex(ths, commitIndex);
sTrace("vgId:%d, agreed upon. role:%d, term:%" PRId64 ", index: %" PRId64 "", ths->vgId, ths->state,
ths->pRaftStore->currentTerm, commitIndex);
}
return ths->commitIndex;
}

4
source/libs/sync/src/syncEnv.c
View File

@ -100,7 +100,9 @@ SSyncNode *syncNodeAcquire(int64_t rid) {
return pNode; return pNode;
} }
void syncNodeRelease(SSyncNode *pNode) { taosReleaseRef(gNodeRefId, pNode->rid); } void syncNodeRelease(SSyncNode *pNode) {
if (pNode) taosReleaseRef(gNodeRefId, pNode->rid);
}
int64_t syncHbTimerDataAdd(SSyncHbTimerData *pData) { int64_t syncHbTimerDataAdd(SSyncHbTimerData *pData) {
pData->rid = taosAddRef(gHbDataRefId, pData); pData->rid = taosAddRef(gHbDataRefId, pData);

9
source/libs/sync/src/syncIndexMgr.c
View File

@ -81,6 +81,15 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId,
index); index);
} }
SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pDestId) {
for (int i = 0; i < pNode->replicaNum; i++) {
if (syncUtilSameId(&(pNode->replicasId[i]), pDestId)) {
return pNode->logReplMgrs[i];
}
}
return NULL;
}
SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
if (pSyncIndexMgr == NULL) { if (pSyncIndexMgr == NULL) {
return SYNC_INDEX_INVALID; return SYNC_INDEX_INVALID;

320
source/libs/sync/src/syncMain.c
View File

@ -22,6 +22,7 @@
#include "syncEnv.h" #include "syncEnv.h"
#include "syncIndexMgr.h" #include "syncIndexMgr.h"
#include "syncInt.h" #include "syncInt.h"
#include "syncPipeline.h"
#include "syncMessage.h" #include "syncMessage.h"
#include "syncRaftCfg.h" #include "syncRaftCfg.h"
#include "syncRaftLog.h" #include "syncRaftLog.h"
@ -34,6 +35,7 @@
#include "syncTimeout.h" #include "syncTimeout.h"
#include "syncUtil.h" #include "syncUtil.h"
#include "syncVoteMgr.h" #include "syncVoteMgr.h"
#include "tref.h"
static void syncNodeEqPingTimer(void* param, void* tmrId); static void syncNodeEqPingTimer(void* param, void* tmrId);
static void syncNodeEqElectTimer(void* param, void* tmrId); static void syncNodeEqElectTimer(void* param, void* tmrId);
@ -56,7 +58,6 @@ static int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeade
static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry); static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry);
static ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode); static ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode);
static SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapshotLastApplyIndex);
int64_t syncOpen(SSyncInfo* pSyncInfo) { int64_t syncOpen(SSyncInfo* pSyncInfo) {
SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
@ -80,12 +81,29 @@ int64_t syncOpen(SSyncInfo* pSyncInfo) {
return pSyncNode->rid; return pSyncNode->rid;
} }
void syncStart(int64_t rid) { int32_t syncStart(int64_t rid) {
SSyncNode* pSyncNode = syncNodeAcquire(rid); SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode != NULL) { if (pSyncNode == NULL) {
syncNodeStart(pSyncNode); sError("failed to acquire rid: %" PRId64 " of tsNodeReftId for pSyncNode", rid);
syncNodeRelease(pSyncNode); return -1;
} }
if (syncNodeRestore(pSyncNode) < 0) {
sError("vgId:%d, failed to restore sync log buffer since %s", pSyncNode->vgId, terrstr());
goto _err;
}
if (syncNodeStart(pSyncNode) < 0) {
sError("vgId:%d, failed to start sync node since %s", pSyncNode->vgId, terrstr());
goto _err;
}
syncNodeRelease(pSyncNode);
return 0;
_err:
syncNodeRelease(pSyncNode);
return -1;
} }
void syncStop(int64_t rid) { void syncStop(int64_t rid) {
@ -381,15 +399,13 @@ int32_t syncStepDown(int64_t rid, SyncTerm newTerm) {
return 0; return 0;
} }
bool syncIsReadyForRead(int64_t rid) { bool syncNodeIsReadyForRead(SSyncNode* pSyncNode) {
SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode == NULL) { if (pSyncNode == NULL) {
sError("sync ready for read error"); sError("sync ready for read error");
return false; return false;
} }
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && pSyncNode->restoreFinish) { if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && pSyncNode->restoreFinish) {
syncNodeRelease(pSyncNode);
return true; return true;
} }
@ -428,7 +444,7 @@ bool syncIsReadyForRead(int64_t rid) {
if (h) { if (h) {
taosLRUCacheRelease(pCache, h, false); taosLRUCacheRelease(pCache, h, false);
} else { } else {
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
} }
} }
@ -443,6 +459,18 @@ bool syncIsReadyForRead(int64_t rid) {
} }
} }
return ready;
}
bool syncIsReadyForRead(int64_t rid) {
SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode == NULL) {
sError("sync ready for read error");
return false;
}
bool ready = syncNodeIsReadyForRead(pSyncNode);
syncNodeRelease(pSyncNode); syncNodeRelease(pSyncNode);
return ready; return ready;
} }
@ -521,6 +549,7 @@ SSyncState syncGetState(int64_t rid) {
if (pSyncNode != NULL) { if (pSyncNode != NULL) {
state.state = pSyncNode->state; state.state = pSyncNode->state;
state.restored = pSyncNode->restoreFinish; state.restored = pSyncNode->restoreFinish;
state.canRead = syncNodeIsReadyForRead(pSyncNode);
syncNodeRelease(pSyncNode); syncNodeRelease(pSyncNode);
} }
@ -543,7 +572,7 @@ int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapsho
int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index, &pEntry); int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index, &pEntry);
if (code != 0) { if (code != 0) {
if (pEntry != NULL) { if (pEntry != NULL) {
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
syncNodeRelease(pSyncNode); syncNodeRelease(pSyncNode);
return -1; return -1;
@ -555,7 +584,7 @@ int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapsho
pSnapshot->lastApplyTerm = pEntry->term; pSnapshot->lastApplyTerm = pEntry->term;
pSnapshot->lastConfigIndex = syncNodeGetSnapshotConfigIndex(pSyncNode, index); pSnapshot->lastConfigIndex = syncNodeGetSnapshotConfigIndex(pSyncNode, index);
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
syncNodeRelease(pSyncNode); syncNodeRelease(pSyncNode);
return 0; return 0;
} }
@ -626,7 +655,7 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) {
tstrncpy(pEp->fqdn, pSyncNode->pRaftCfg->cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN); tstrncpy(pEp->fqdn, pSyncNode->pRaftCfg->cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN);
pEp->port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; pEp->port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort;
pEpSet->numOfEps++; pEpSet->numOfEps++;
sInfo("vgId:%d, sync get retry epset, index:%d %s:%d", pSyncNode->vgId, i, pEp->fqdn, pEp->port); sDebug("vgId:%d, sync get retry epset, index:%d %s:%d", pSyncNode->vgId, i, pEp->fqdn, pEp->port);
} }
if (pEpSet->numOfEps > 0) { if (pEpSet->numOfEps > 0) {
pEpSet->inUse = (pSyncNode->pRaftCfg->cfg.myIndex + 1) % pEpSet->numOfEps; pEpSet->inUse = (pSyncNode->pRaftCfg->cfg.myIndex + 1) % pEpSet->numOfEps;
@ -756,6 +785,29 @@ static int32_t syncHbTimerStop(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) {
return ret; return ret;
} }
int32_t syncNodeLogStoreRestoreOnNeed(SSyncNode* pNode) {
ASSERT(pNode->pLogStore != NULL && "log store not created");
ASSERT(pNode->pFsm != NULL && "pFsm not registered");
ASSERT(pNode->pFsm->FpGetSnapshotInfo != NULL && "FpGetSnapshotInfo not registered");
SSnapshot snapshot;
if (pNode->pFsm->FpGetSnapshotInfo(pNode->pFsm, &snapshot) < 0) {
sError("vgId:%d, failed to get snapshot info since %s", pNode->vgId, terrstr());
return -1;
}
SyncIndex commitIndex = snapshot.lastApplyIndex;
SyncIndex firstVer = pNode->pLogStore->syncLogBeginIndex(pNode->pLogStore);
SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore);
if (lastVer < commitIndex || firstVer > commitIndex + 1) {
if (pNode->pLogStore->syncLogRestoreFromSnapshot(pNode->pLogStore, commitIndex)) {
sError("vgId:%d, failed to restore log store from snapshot since %s. lastVer: %" PRId64 ", snapshotVer: %" PRId64,
pNode->vgId, terrstr(), lastVer, commitIndex);
return -1;
}
}
return 0;
}
// open/close --------------
SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
SSyncNode* pSyncNode = taosMemoryCalloc(1, sizeof(SSyncNode)); SSyncNode* pSyncNode = taosMemoryCalloc(1, sizeof(SSyncNode));
if (pSyncNode == NULL) { if (pSyncNode == NULL) {
@ -831,6 +883,13 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
pSyncNode->syncEqMsg = pSyncInfo->syncEqMsg; pSyncNode->syncEqMsg = pSyncInfo->syncEqMsg;
pSyncNode->syncEqCtrlMsg = pSyncInfo->syncEqCtrlMsg; pSyncNode->syncEqCtrlMsg = pSyncInfo->syncEqCtrlMsg;
// create raft log ring buffer
pSyncNode->pLogBuf = syncLogBufferCreate();
if (pSyncNode->pLogBuf == NULL) {
sError("failed to init sync log buffer since %s. vgId:%d", terrstr(), pSyncNode->vgId);
goto _error;
}
// init raft config // init raft config
pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath); pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath);
if (pSyncNode->pRaftCfg == NULL) { if (pSyncNode->pRaftCfg == NULL) {
@ -956,6 +1015,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
} }
pSyncNode->commitIndex = commitIndex; pSyncNode->commitIndex = commitIndex;
if (syncNodeLogStoreRestoreOnNeed(pSyncNode) < 0) {
goto _error;
}
// timer ms init // timer ms init
pSyncNode->pingBaseLine = PING_TIMER_MS; pSyncNode->pingBaseLine = PING_TIMER_MS;
pSyncNode->electBaseLine = ELECT_TIMER_MS_MIN; pSyncNode->electBaseLine = ELECT_TIMER_MS_MIN;
@ -1013,9 +1075,13 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
// is config changing // is config changing
pSyncNode->changing = false; pSyncNode->changing = false;
// replication mgr
syncNodeLogReplMgrInit(pSyncNode);
// peer state // peer state
syncNodePeerStateInit(pSyncNode); syncNodePeerStateInit(pSyncNode);
//
// min match index // min match index
pSyncNode->minMatchIndex = SYNC_INDEX_INVALID; pSyncNode->minMatchIndex = SYNC_INDEX_INVALID;
@ -1031,6 +1097,12 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
// snapshotting // snapshotting
atomic_store_64(&pSyncNode->snapshottingIndex, SYNC_INDEX_INVALID); atomic_store_64(&pSyncNode->snapshottingIndex, SYNC_INDEX_INVALID);
// init log buffer
if (syncLogBufferInit(pSyncNode->pLogBuf, pSyncNode) < 0) {
sError("vgId:%d, failed to init sync log buffer since %s", pSyncNode->vgId, terrstr());
goto _error;
}
pSyncNode->isStart = true; pSyncNode->isStart = true;
pSyncNode->electNum = 0; pSyncNode->electNum = 0;
pSyncNode->becomeLeaderNum = 0; pSyncNode->becomeLeaderNum = 0;
@ -1064,7 +1136,49 @@ void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) {
} }
} }
void syncNodeStart(SSyncNode* pSyncNode) { int32_t syncNodeRestore(SSyncNode* pSyncNode) {
ASSERT(pSyncNode->pLogStore != NULL && "log store not created");
ASSERT(pSyncNode->pLogBuf != NULL && "ring log buffer not created");
SyncIndex lastVer = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
SyncIndex commitIndex = pSyncNode->pLogStore->syncLogCommitIndex(pSyncNode->pLogStore);
SyncIndex endIndex = pSyncNode->pLogBuf->endIndex;
if (lastVer != -1 && endIndex != lastVer + 1) {
terrno = TSDB_CODE_WAL_LOG_INCOMPLETE;
sError("vgId:%d, failed to restore sync node since %s. expected lastLogIndex: %" PRId64 ", lastVer: %" PRId64 "",
pSyncNode->vgId, terrstr(), endIndex - 1, lastVer);
return -1;
}
ASSERT(endIndex == lastVer + 1);
commitIndex = TMAX(pSyncNode->commitIndex, commitIndex);
if (syncLogBufferCommit(pSyncNode->pLogBuf, pSyncNode, commitIndex) < 0) {
return -1;
}
return 0;
}
int32_t syncNodeStart(SSyncNode* pSyncNode) {
// start raft
if (pSyncNode->replicaNum == 1) {
raftStoreNextTerm(pSyncNode->pRaftStore);
syncNodeBecomeLeader(pSyncNode, "one replica start");
// Raft 3.6.2 Committing entries from previous terms
syncNodeAppendNoop(pSyncNode);
} else {
syncNodeBecomeFollower(pSyncNode, "first start");
}
int32_t ret = 0;
ret = syncNodeStartPingTimer(pSyncNode);
ASSERT(ret == 0);
return ret;
}
void syncNodeStartOld(SSyncNode* pSyncNode) {
// start raft // start raft
if (pSyncNode->replicaNum == 1) { if (pSyncNode->replicaNum == 1) {
raftStoreNextTerm(pSyncNode->pRaftStore); raftStoreNextTerm(pSyncNode->pRaftStore);
@ -1083,7 +1197,7 @@ void syncNodeStart(SSyncNode* pSyncNode) {
ASSERT(ret == 0); ASSERT(ret == 0);
} }
void syncNodeStartStandBy(SSyncNode* pSyncNode) { int32_t syncNodeStartStandBy(SSyncNode* pSyncNode) {
// state change // state change
pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
syncNodeStopHeartbeatTimer(pSyncNode); syncNodeStopHeartbeatTimer(pSyncNode);
@ -1096,6 +1210,7 @@ void syncNodeStartStandBy(SSyncNode* pSyncNode) {
ret = 0; ret = 0;
ret = syncNodeStartPingTimer(pSyncNode); ret = syncNodeStartPingTimer(pSyncNode);
ASSERT(ret == 0); ASSERT(ret == 0);
return ret;
} }
void syncNodePreClose(SSyncNode* pSyncNode) { void syncNodePreClose(SSyncNode* pSyncNode) {
@ -1116,6 +1231,7 @@ void syncNodeClose(SSyncNode* pSyncNode) {
ASSERT(ret == 0); ASSERT(ret == 0);
pSyncNode->pRaftStore = NULL; pSyncNode->pRaftStore = NULL;
syncNodeLogReplMgrDestroy(pSyncNode);
syncRespMgrDestroy(pSyncNode->pSyncRespMgr); syncRespMgrDestroy(pSyncNode->pSyncRespMgr);
pSyncNode->pSyncRespMgr = NULL; pSyncNode->pSyncRespMgr = NULL;
voteGrantedDestroy(pSyncNode->pVotesGranted); voteGrantedDestroy(pSyncNode->pVotesGranted);
@ -1128,6 +1244,8 @@ void syncNodeClose(SSyncNode* pSyncNode) {
pSyncNode->pMatchIndex = NULL; pSyncNode->pMatchIndex = NULL;
logStoreDestory(pSyncNode->pLogStore); logStoreDestory(pSyncNode->pLogStore);
pSyncNode->pLogStore = NULL; pSyncNode->pLogStore = NULL;
syncLogBufferDestroy(pSyncNode->pLogBuf);
pSyncNode->pLogBuf = NULL;
raftCfgClose(pSyncNode->pRaftCfg); raftCfgClose(pSyncNode->pRaftCfg);
pSyncNode->pRaftCfg = NULL; pSyncNode->pRaftCfg = NULL;
@ -1529,7 +1647,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
// Raft 3.6.2 Committing entries from previous terms // Raft 3.6.2 Committing entries from previous terms
syncNodeAppendNoop(pSyncNode); syncNodeAppendNoop(pSyncNode);
syncMaybeAdvanceCommitIndex(pSyncNode); // syncMaybeAdvanceCommitIndex(pSyncNode);
} else { } else {
syncNodeBecomeFollower(pSyncNode, tmpbuf); syncNodeBecomeFollower(pSyncNode, tmpbuf);
@ -1617,6 +1735,9 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) {
// min match index // min match index
pSyncNode->minMatchIndex = SYNC_INDEX_INVALID; pSyncNode->minMatchIndex = SYNC_INDEX_INVALID;
// reset log buffer
syncLogBufferReset(pSyncNode->pLogBuf, pSyncNode);
// trace log // trace log
sNTrace(pSyncNode, "become follower %s", debugStr); sNTrace(pSyncNode, "become follower %s", debugStr);
} }
@ -1712,6 +1833,9 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) {
// min match index // min match index
pSyncNode->minMatchIndex = SYNC_INDEX_INVALID; pSyncNode->minMatchIndex = SYNC_INDEX_INVALID;
// reset log buffer
syncLogBufferReset(pSyncNode->pLogBuf, pSyncNode);
// trace log // trace log
sNInfo(pSyncNode, "become leader %s", debugStr); sNInfo(pSyncNode, "become leader %s", debugStr);
} }
@ -1723,6 +1847,22 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {
sNTrace(pSyncNode, "state change syncNodeCandidate2Leader"); sNTrace(pSyncNode, "state change syncNodeCandidate2Leader");
int32_t ret = syncNodeAppendNoop(pSyncNode);
if (ret < 0) {
sError("vgId:%d, failed to append noop entry since %s", pSyncNode->vgId, terrstr());
}
SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
ASSERT(lastIndex >= 0);
sInfo("vgId:%d, become leader. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64 "",
pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex);
}
void syncNodeCandidate2LeaderOld(SSyncNode* pSyncNode) {
ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
ASSERT(voteGrantedMajority(pSyncNode->pVotesGranted));
syncNodeBecomeLeader(pSyncNode, "candidate to leader");
// Raft 3.6.2 Committing entries from previous terms // Raft 3.6.2 Committing entries from previous terms
syncNodeAppendNoop(pSyncNode); syncNodeAppendNoop(pSyncNode);
syncMaybeAdvanceCommitIndex(pSyncNode); syncMaybeAdvanceCommitIndex(pSyncNode);
@ -1746,18 +1886,30 @@ int32_t syncNodePeerStateInit(SSyncNode* pSyncNode) {
void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { void syncNodeFollower2Candidate(SSyncNode* pSyncNode) {
ASSERT(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); ASSERT(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER);
pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE;
SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
sInfo("vgId:%d, become candidate from follower. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64,
pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex);
sNTrace(pSyncNode, "follower to candidate"); sNTrace(pSyncNode, "follower to candidate");
} }
void syncNodeLeader2Follower(SSyncNode* pSyncNode) { void syncNodeLeader2Follower(SSyncNode* pSyncNode) {
ASSERT(pSyncNode->state == TAOS_SYNC_STATE_LEADER); ASSERT(pSyncNode->state == TAOS_SYNC_STATE_LEADER);
syncNodeBecomeFollower(pSyncNode, "leader to follower"); syncNodeBecomeFollower(pSyncNode, "leader to follower");
SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
sInfo("vgId:%d, become follower from leader. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64,
pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex);
sNTrace(pSyncNode, "leader to follower"); sNTrace(pSyncNode, "leader to follower");
} }
void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { void syncNodeCandidate2Follower(SSyncNode* pSyncNode) {
ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
syncNodeBecomeFollower(pSyncNode, "candidate to follower"); syncNodeBecomeFollower(pSyncNode, "candidate to follower");
SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
sInfo("vgId:%d, become follower from candidate. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64,
pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex);
sNTrace(pSyncNode, "candidate to follower"); sNTrace(pSyncNode, "candidate to follower");
} }
@ -1911,7 +2063,7 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
if (h) { if (h) {
taosLRUCacheRelease(pCache, h, false); taosLRUCacheRelease(pCache, h, false);
} else { } else {
syncEntryDestory(pPreEntry); syncEntryDestroy(pPreEntry);
} }
return preTerm; return preTerm;
@ -2153,7 +2305,7 @@ static int32_t syncNodeEqNoop(SSyncNode* pNode) {
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
int32_t code = syncBuildClientRequestFromNoopEntry(&rpcMsg, pEntry, pNode->vgId); int32_t code = syncBuildClientRequestFromNoopEntry(&rpcMsg, pEntry, pNode->vgId);
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
sNTrace(pNode, "propose msg, type:noop"); sNTrace(pNode, "propose msg, type:noop");
code = (*pNode->syncEqMsg)(pNode->msgcb, &rpcMsg); code = (*pNode->syncEqMsg)(pNode->msgcb, &rpcMsg);
@ -2181,6 +2333,37 @@ int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHand
return code; return code;
} }
int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) {
// append to log buffer
if (syncLogBufferAppend(ths->pLogBuf, ths, pEntry) < 0) {
sError("vgId:%d, failed to enqueue sync log buffer. index:%" PRId64 "", ths->vgId, pEntry->index);
return -1;
}
// proceed match index, with replicating on needed
SyncIndex matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, NULL);
sTrace("vgId:%d, append raft entry. index: %" PRId64 ", term: %" PRId64 " pBuf: [%" PRId64 " %" PRId64 " %" PRId64
", %" PRId64 ")",
ths->vgId, pEntry->index, pEntry->term, ths->pLogBuf->startIndex, ths->pLogBuf->commitIndex,
ths->pLogBuf->matchIndex, ths->pLogBuf->endIndex);
// multi replica
if (ths->replicaNum > 1) {
return 0;
}
// single replica
(void)syncNodeUpdateCommitIndex(ths, matchIndex);
if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) {
sError("vgId:%d, failed to commit until commitIndex:%" PRId64 "", ths->vgId, ths->commitIndex);
return -1;
}
return 0;
}
bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) {
if (pSyncNode->replicaNum == 1) { if (pSyncNode->replicaNum == 1) {
return false; return false;
@ -2224,6 +2407,20 @@ bool syncNodeSnapshotRecving(SSyncNode* pSyncNode) {
} }
static int32_t syncNodeAppendNoop(SSyncNode* ths) { static int32_t syncNodeAppendNoop(SSyncNode* ths) {
SyncIndex index = syncLogBufferGetEndIndex(ths->pLogBuf);
SyncTerm term = ths->pRaftStore->currentTerm;
SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId);
if (pEntry == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
int32_t ret = syncNodeAppend(ths, pEntry);
return 0;
}
static int32_t syncNodeAppendNoopOld(SSyncNode* ths) {
int32_t ret = 0; int32_t ret = 0;
SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore);
@ -2246,7 +2443,7 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
if (h) { if (h) {
taosLRUCacheRelease(ths->pLogStore->pCache, h, false); taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
} else { } else {
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
return ret; return ret;
@ -2255,9 +2452,13 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
SyncHeartbeat* pMsg = pRpcMsg->pCont; SyncHeartbeat* pMsg = pRpcMsg->pCont;
const STraceId* trace = &pRpcMsg->info.traceId;
char tbuf[40] = {0};
TRACE_TO_STR(trace, tbuf);
int64_t tsMs = taosGetTimestampMs(); int64_t tsMs = taosGetTimestampMs();
int64_t timeDiff = tsMs - pMsg->timeStamp; int64_t timeDiff = tsMs - pMsg->timeStamp;
syncLogRecvHeartbeat(ths, pMsg, timeDiff); syncLogRecvHeartbeat(ths, pMsg, timeDiff, tbuf);
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
(void)syncBuildHeartbeatReply(&rpcMsg, ths->vgId); (void)syncBuildHeartbeatReply(&rpcMsg, ths->vgId);
@ -2267,6 +2468,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
pMsgReply->srcId = ths->myRaftId; pMsgReply->srcId = ths->myRaftId;
pMsgReply->term = ths->pRaftStore->currentTerm; pMsgReply->term = ths->pRaftStore->currentTerm;
pMsgReply->privateTerm = 8864; // magic number pMsgReply->privateTerm = 8864; // magic number
pMsgReply->startTime = ths->startTime;
pMsgReply->timeStamp = tsMs; pMsgReply->timeStamp = tsMs;
if (pMsg->term == ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { if (pMsg->term == ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) {
@ -2330,11 +2532,35 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
} }
int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
const STraceId* trace = &pRpcMsg->info.traceId;
char tbuf[40] = {0};
TRACE_TO_STR(trace, tbuf);
SyncHeartbeatReply* pMsg = pRpcMsg->pCont; SyncHeartbeatReply* pMsg = pRpcMsg->pCont;
SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(ths, &pMsg->srcId);
if (pMgr == NULL) {
sError("vgId:%d, failed to get log repl mgr for the peer at addr 0x016%" PRIx64 "", ths->vgId, pMsg->srcId.addr);
return -1;
}
int64_t tsMs = taosGetTimestampMs();
syncLogRecvHeartbeatReply(ths, pMsg, tsMs - pMsg->timeStamp, tbuf);
syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs);
return syncLogReplMgrProcessHeartbeatReply(pMgr, ths, pMsg);
}
int32_t syncNodeOnHeartbeatReplyOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
SyncHeartbeatReply* pMsg = pRpcMsg->pCont;
const STraceId* trace = &pRpcMsg->info.traceId;
char tbuf[40] = {0};
TRACE_TO_STR(trace, tbuf);
int64_t tsMs = taosGetTimestampMs(); int64_t tsMs = taosGetTimestampMs();
int64_t timeDiff = tsMs - pMsg->timeStamp; int64_t timeDiff = tsMs - pMsg->timeStamp;
syncLogRecvHeartbeatReply(ths, pMsg, timeDiff); syncLogRecvHeartbeatReply(ths, pMsg, timeDiff, tbuf);
// update last reply time, make decision whether the other node is alive or not // update last reply time, make decision whether the other node is alive or not
syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs); syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs);
@ -2345,6 +2571,26 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
SyncLocalCmd* pMsg = pRpcMsg->pCont; SyncLocalCmd* pMsg = pRpcMsg->pCont;
syncLogRecvLocalCmd(ths, pMsg, ""); syncLogRecvLocalCmd(ths, pMsg, "");
if (pMsg->cmd == SYNC_LOCAL_CMD_STEP_DOWN) {
syncNodeStepDown(ths, pMsg->sdNewTerm);
} else if (pMsg->cmd == SYNC_LOCAL_CMD_FOLLOWER_CMT) {
(void)syncNodeUpdateCommitIndex(ths, pMsg->fcIndex);
if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) {
sError("vgId:%d, failed to commit raft log since %s. commit index: %" PRId64 "", ths->vgId, terrstr(),
ths->commitIndex);
}
} else {
sError("error local cmd");
}
return 0;
}
int32_t syncNodeOnLocalCmdOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
SyncLocalCmd* pMsg = pRpcMsg->pCont;
syncLogRecvLocalCmd(ths, pMsg, "");
if (pMsg->cmd == SYNC_LOCAL_CMD_STEP_DOWN) { if (pMsg->cmd == SYNC_LOCAL_CMD_STEP_DOWN) {
syncNodeStepDown(ths, pMsg->sdNewTerm); syncNodeStepDown(ths, pMsg->sdNewTerm);
@ -2372,6 +2618,31 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex) { int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex) {
sNTrace(ths, "on client request"); sNTrace(ths, "on client request");
int32_t code = 0;
SyncIndex index = syncLogBufferGetEndIndex(ths->pLogBuf);
SyncTerm term = ths->pRaftStore->currentTerm;
SSyncRaftEntry* pEntry = NULL;
if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
pEntry = syncEntryBuildFromClientRequest(pMsg->pCont, term, index);
} else {
pEntry = syncEntryBuildFromRpcMsg(pMsg, term, index);
}
if (ths->state == TAOS_SYNC_STATE_LEADER) {
if (pRetIndex) {
(*pRetIndex) = index;
}
return syncNodeAppend(ths, pEntry);
}
return -1;
}
int32_t syncNodeOnClientRequestOld(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex) {
sNTrace(ths, "on client request");
int32_t ret = 0; int32_t ret = 0;
int32_t code = 0; int32_t code = 0;
@ -2395,7 +2666,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn
if (h) { if (h) {
taosLRUCacheRelease(ths->pLogStore->pCache, h, false); taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
} else { } else {
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
return -1; return -1;
@ -2418,7 +2689,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn
if (h) { if (h) {
taosLRUCacheRelease(ths->pLogStore->pCache, h, false); taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
} else { } else {
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
return -1; return -1;
@ -2453,7 +2724,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn
if (h) { if (h) {
taosLRUCacheRelease(ths->pLogStore->pCache, h, false); taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
} else { } else {
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
return ret; return ret;
@ -2563,6 +2834,7 @@ bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) {
} }
int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) {
ASSERT(false);
if (beginIndex > endIndex) { if (beginIndex > endIndex) {
return 0; return 0;
} }
@ -2676,7 +2948,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde
if (h) { if (h) {
taosLRUCacheRelease(pCache, h, false); taosLRUCacheRelease(pCache, h, false);
} else { } else {
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
} }
} }

29
source/libs/sync/src/syncMessage.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "syncMessage.h" #include "syncMessage.h"
#include "syncRaftEntry.h" #include "syncRaftEntry.h"
#include "syncRaftStore.h"
int32_t syncBuildTimeout(SRpcMsg* pMsg, ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t syncBuildTimeout(SRpcMsg* pMsg, ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS,
SSyncNode* pNode) { SSyncNode* pNode) {
@ -153,6 +154,34 @@ int32_t syncBuildAppendEntriesReply(SRpcMsg* pMsg, int32_t vgId) {
return 0; return 0;
} }
int32_t syncBuildAppendEntriesFromRaftLog(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm,
SRpcMsg* pRpcMsg) {
uint32_t dataLen = pEntry->bytes;
uint32_t bytes = sizeof(SyncAppendEntries) + dataLen;
pRpcMsg->contLen = bytes;
pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
if (pRpcMsg->pCont == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
SyncAppendEntries* pMsg = pRpcMsg->pCont;
pMsg->bytes = pRpcMsg->contLen;
pMsg->msgType = pRpcMsg->msgType = TDMT_SYNC_APPEND_ENTRIES;
pMsg->dataLen = dataLen;
(void)memcpy(pMsg->data, pEntry, dataLen);
pMsg->prevLogIndex = pEntry->index - 1;
pMsg->prevLogTerm = prevLogTerm;
pMsg->vgId = pNode->vgId;
pMsg->srcId = pNode->myRaftId;
pMsg->term = pNode->pRaftStore->currentTerm;
pMsg->commitIndex = pNode->commitIndex;
pMsg->privateTerm = 0;
return 0;
}
int32_t syncBuildHeartbeat(SRpcMsg* pMsg, int32_t vgId) { int32_t syncBuildHeartbeat(SRpcMsg* pMsg, int32_t vgId) {
int32_t bytes = sizeof(SyncHeartbeat); int32_t bytes = sizeof(SyncHeartbeat);
pMsg->pCont = rpcMallocCont(bytes); pMsg->pCont = rpcMallocCont(bytes);

1089
source/libs/sync/src/syncPipeline.c Normal file
View File

File diff suppressed because it is too large Load Diff

1
source/libs/sync/src/syncRaftCfg.c
View File

@ -395,4 +395,3 @@ int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg) {
cJSON_Delete(pRoot); cJSON_Delete(pRoot);
return 0; return 0;
} }

10
source/libs/sync/src/syncRaftEntry.c
View File

@ -20,7 +20,7 @@
SSyncRaftEntry* syncEntryBuild(int32_t dataLen) { SSyncRaftEntry* syncEntryBuild(int32_t dataLen) {
int32_t bytes = sizeof(SSyncRaftEntry) + dataLen; int32_t bytes = sizeof(SSyncRaftEntry) + dataLen;
SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); SSyncRaftEntry* pEntry = taosMemoryCalloc(1, bytes);
if (pEntry == NULL) { if (pEntry == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
@ -89,7 +89,7 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId)
return pEntry; return pEntry;
} }
void syncEntryDestory(SSyncRaftEntry* pEntry) { void syncEntryDestroy(SSyncRaftEntry* pEntry) {
if (pEntry != NULL) { if (pEntry != NULL) {
sTrace("free entry: %p", pEntry); sTrace("free entry: %p", pEntry);
taosMemoryFree(pEntry); taosMemoryFree(pEntry);
@ -265,7 +265,7 @@ static int cmpFn(const void* p1, const void* p2) { return memcmp(p1, p2, sizeof(
static void freeRaftEntry(void* param) { static void freeRaftEntry(void* param) {
SSyncRaftEntry* pEntry = (SSyncRaftEntry*)param; SSyncRaftEntry* pEntry = (SSyncRaftEntry*)param;
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
SRaftEntryCache* raftEntryCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) { SRaftEntryCache* raftEntryCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) {
@ -395,7 +395,7 @@ int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count) {
SSkipListNode* pNode = tSkipListIterGet(pIter); SSkipListNode* pNode = tSkipListIterGet(pIter);
ASSERT(pNode != NULL); ASSERT(pNode != NULL);
SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode); SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode);
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
++returnCnt; ++returnCnt;
} }
tSkipListDestroyIter(pIter); tSkipListDestroyIter(pIter);
@ -424,7 +424,7 @@ int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count) {
++returnCnt; ++returnCnt;
SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode); SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode);
// syncEntryDestory(pEntry); // syncEntryDestroy(pEntry);
taosRemoveRef(pCache->refMgr, pEntry->rid); taosRemoveRef(pCache->refMgr, pEntry->rid);
} }
tSkipListDestroyIter(pIter); tSkipListDestroyIter(pIter);

10
source/libs/sync/src/syncRaftLog.c
View File

@ -201,9 +201,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
syncMeta.isWeek = pEntry->isWeak; syncMeta.isWeek = pEntry->isWeak;
syncMeta.seqNum = pEntry->seqNum; syncMeta.seqNum = pEntry->seqNum;
syncMeta.term = pEntry->term; syncMeta.term = pEntry->term;
int64_t tsWriteBegin = taosGetTimestampNs(); int64_t tsWriteBegin = taosGetTimestampNs();
index = walAppendLog(pWal, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); index = walAppendLog(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
int64_t tsWriteEnd = taosGetTimestampNs(); int64_t tsWriteEnd = taosGetTimestampNs();
int64_t tsElapsed = tsWriteEnd - tsWriteBegin; int64_t tsElapsed = tsWriteEnd - tsWriteBegin;
@ -217,7 +216,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
pEntry->index, err, err, errStr, sysErr, sysErrStr); pEntry->index, err, err, errStr, sysErr, sysErrStr);
return -1; return -1;
} }
pEntry->index = index;
ASSERT(pEntry->index == index);
sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index, sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index,
TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType), tsElapsed); TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType), tsElapsed);
@ -344,8 +344,6 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn
const char* sysErrStr = strerror(errno); const char* sysErrStr = strerror(errno);
sError("vgId:%d, wal truncate error, from-index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", sError("vgId:%d, wal truncate error, from-index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
pData->pSyncNode->vgId, fromIndex, err, err, errStr, sysErr, sysErrStr); pData->pSyncNode->vgId, fromIndex, err, err, errStr, sysErr, sysErrStr);
// ASSERT(0);
} }
// event log // event log
@ -397,8 +395,6 @@ int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) {
const char* sysErrStr = strerror(errno); const char* sysErrStr = strerror(errno);
sError("vgId:%d, wal update commit index error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", sError("vgId:%d, wal update commit index error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
pData->pSyncNode->vgId, index, err, err, errStr, sysErr, sysErrStr); pData->pSyncNode->vgId, index, err, err, errStr, sysErr, sysErrStr);
ASSERT(0);
return -1; return -1;
} }
return 0; return 0;

43
source/libs/sync/src/syncReplication.c
View File

@ -16,13 +16,11 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "syncReplication.h" #include "syncReplication.h"
#include "syncIndexMgr.h" #include "syncIndexMgr.h"
#include "syncPipeline.h"
#include "syncRaftEntry.h" #include "syncRaftEntry.h"
#include "syncRaftStore.h" #include "syncRaftStore.h"
#include "syncUtil.h" #include "syncUtil.h"
static int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
static int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
// TLA+ Spec // TLA+ Spec
// AppendEntries(i, j) == // AppendEntries(i, j) ==
// /\ i /= j // /\ i /= j
@ -48,7 +46,10 @@ static int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pNode, const SRaftId* d
// mdest |-> j]) // mdest |-> j])
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>> // /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapshot) { int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapshot) {
ASSERT(false && "deprecated");
// next index // next index
SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId);
@ -117,7 +118,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh
if (h) { if (h) {
taosLRUCacheRelease(pCache, h, false); taosLRUCacheRelease(pCache, h, false);
} else { } else {
syncEntryDestory(pEntry); syncEntryDestroy(pEntry);
} }
// prepare msg // prepare msg
@ -136,7 +137,29 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh
return 0; return 0;
} }
int32_t syncNodeReplicate(SSyncNode* pSyncNode) { int32_t syncNodeReplicate(SSyncNode* pNode) {
SSyncLogBuffer* pBuf = pNode->pLogBuf;
taosThreadMutexLock(&pBuf->mutex);
int32_t ret = syncNodeReplicateWithoutLock(pNode);
taosThreadMutexUnlock(&pBuf->mutex);
return ret;
}
int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) {
if (pNode->state != TAOS_SYNC_STATE_LEADER || pNode->replicaNum == 1) {
return -1;
}
for (int32_t i = 0; i < pNode->replicaNum; i++) {
if (syncUtilSameId(&pNode->replicasId[i], &pNode->myRaftId)) {
continue;
}
SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i];
(void)syncLogReplMgrReplicateOnce(pMgr, pNode);
}
return 0;
}
int32_t syncNodeReplicateOld(SSyncNode* pSyncNode) {
if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) {
return -1; return -1;
} }
@ -159,6 +182,14 @@ int32_t syncNodeReplicate(SSyncNode* pSyncNode) {
} }
int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) { int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) {
SyncAppendEntries* pMsg = pRpcMsg->pCont;
int32_t ret = 0;
pMsg->destId = *destRaftId;
syncNodeSendMsgById(destRaftId, pSyncNode, pRpcMsg);
return 0;
}
int32_t syncNodeSendAppendEntriesOld(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) {
int32_t ret = 0; int32_t ret = 0;
SyncAppendEntries* pMsg = pRpcMsg->pCont; SyncAppendEntries* pMsg = pRpcMsg->pCont;
if (pMsg == NULL) { if (pMsg == NULL) {
@ -237,4 +268,4 @@ int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) {
} }
return 0; return 0;
} }

2
source/libs/sync/src/syncRequestVote.c
View File

@ -124,7 +124,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
// send msg // send msg
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
ret = syncBuildRequestVoteReply(&rpcMsg, ths->vgId); ret = syncBuildRequestVoteReply(&rpcMsg, ths->vgId);
ASSERT(ret == 0 ); ASSERT(ret == 0);
SyncRequestVoteReply* pReply = rpcMsg.pCont; SyncRequestVoteReply* pReply = rpcMsg.pCont;
pReply->srcId = ths->myRaftId; pReply->srcId = ths->myRaftId;

21
source/libs/sync/src/syncSnapshot.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "syncSnapshot.h" #include "syncSnapshot.h"
#include "syncIndexMgr.h" #include "syncIndexMgr.h"
#include "syncPipeline.h"
#include "syncRaftCfg.h" #include "syncRaftCfg.h"
#include "syncRaftLog.h" #include "syncRaftLog.h"
#include "syncRaftStore.h" #include "syncRaftStore.h"
@ -273,6 +274,11 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) {
return 1; return 1;
} }
char host[64];
uint16_t port;
syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port);
sInfo("vgId:%d, start snapshot for peer: %s:%d", pSyncNode->vgId, host, port);
code = snapshotSenderStart(pSender); code = snapshotSenderStart(pSender);
if (code != 0) { if (code != 0) {
sNError(pSyncNode, "snapshot sender start error"); sNError(pSyncNode, "snapshot sender start error");
@ -372,7 +378,10 @@ int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapsh
} }
int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pPreMsg) { int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pPreMsg) {
ASSERT(!snapshotReceiverIsStart(pReceiver)); if (snapshotReceiverIsStart(pReceiver)) {
sWarn("vgId:%d, snapshot receiver has started.", pReceiver->pSyncNode->vgId);
return 0;
}
pReceiver->start = true; pReceiver->start = true;
pReceiver->ack = SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT; pReceiver->ack = SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT;
@ -742,6 +751,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
} else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) {
syncNodeOnSnapshotEnd(pSyncNode, pMsg); syncNodeOnSnapshotEnd(pSyncNode, pMsg);
(void)syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode);
} else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) {
// force close, no response // force close, no response
@ -873,11 +883,10 @@ int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
// receive ack is finish, close sender // receive ack is finish, close sender
if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) {
snapshotSenderStop(pSender, true); snapshotSenderStop(pSender, true);
SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId);
// update next-index if (pMgr) {
syncIndexMgrSetIndex(pSyncNode->pNextIndex, &(pMsg->srcId), pMsg->lastIndex + 1); syncLogReplMgrReset(pMgr);
syncNodeReplicateOne(pSyncNode, &(pMsg->srcId), false); }
return 0; return 0;
} }

2
source/libs/sync/src/syncTimeout.c
View File

@ -131,4 +131,4 @@ int32_t syncNodeOnTimeout(SSyncNode* ths, const SRpcMsg* pRpc) {
} }
return ret; return ret;
} }

Some files were not shown because too many files have changed in this diff Show More