xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #25256 from taosdata/feat/TD-24834 

Feat/td 24834
This commit is contained in:
Hongze Cheng 2024-04-19 13:34:37 +08:00 committed by GitHub
parent 62260bff79 a044308150
commit fc5c84422e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
93 changed files with 8353 additions and 7291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/common/systable.h
View File

@ -56,6 +56,7 @@ extern "C" {
#define TSDB_INS_TABLE_GRANTS_FULL "ins_grants_full"
#define TSDB_INS_TABLE_GRANTS_LOGS "ins_grants_logs"
#define TSDB_INS_TABLE_MACHINES "ins_machines"
#define TSDB_INS_TABLE_ENCRYPTIONS "ins_encryptions"
#define TSDB_INS_TABLE_TSMAS "ins_tsmas"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"

24
include/common/tglobal.h
View File

@ -29,6 +29,17 @@ extern "C" {
#define SLOW_LOG_TYPE_OTHERS 0x4
#define SLOW_LOG_TYPE_ALL 0xFFFFFFFF
typedef enum {
DND_CA_SM4 = 1,
} EEncryptAlgor;
typedef enum {
DND_CS_TSDB = 1,
DND_CS_VNODE_WAL = 2,
DND_CS_SDB = 4,
DND_CS_MNODE_WAL = 8,
} EEncryptScope;
// cluster
extern char tsFirst[];
extern char tsSecond[];
@ -39,6 +50,12 @@ extern uint16_t tsServerPort;
extern int32_t tsVersion;
extern int32_t tsStatusInterval;
extern int32_t tsNumOfSupportVnodes;
extern char tsEncryptAlgorithm[];
extern char tsEncryptScope[];
extern EEncryptAlgor tsiEncryptAlgorithm;
extern EEncryptScope tsiEncryptScope;
//extern char tsAuthCode[];
extern char tsEncryptKey[];
// common
extern int32_t tsMaxShellConns;
@ -94,8 +111,6 @@ extern char tsSnodeAddress[]; // 127.0.0.1:873
// mnode
extern int64_t tsMndSdbWriteDelta;
extern int64_t tsMndLogRetention;
extern int8_t tsGrant;
extern int32_t tsMndGrantMode;
extern bool tsMndSkipGrant;
extern bool tsEnableWhiteList;
@ -104,6 +119,11 @@ extern int64_t tsDndStart;
extern int64_t tsDndStartOsUptime;
extern int64_t tsDndUpTime;
// dnode misc
extern uint32_t tsEncryptionKeyChksum;
extern int8_t tsEncryptionKeyStat;
extern int8_t tsGrant;
// monitor
extern bool tsEnableMonitor;
extern int32_t tsMonitorInterval;

4
include/common/tgrant.h
View File

@ -58,9 +58,11 @@ typedef enum {
TSDB_GRANT_BACKUP_RESTORE,
} EGrantType;
int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);
int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode);
int32_t grantCheck(EGrantType grant);
int32_t grantCheckExpire(EGrantType grant);
char* tGetMachineId();
char *tGetMachineId();
// #ifndef GRANTS_CFG
#ifdef TD_ENTERPRISE

21
include/common/tmsg.h
View File

@ -154,6 +154,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_GRANTS_LOGS,
TSDB_MGMT_TABLE_MACHINES,
TSDB_MGMT_TABLE_ARBGROUP,
TSDB_MGMT_TABLE_ENCRYPTIONS,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -374,6 +375,7 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_GRANTS_FULL_STMT,
QUERY_NODE_SHOW_GRANTS_LOGS_STMT,
QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT,
QUERY_NODE_SHOW_ENCRYPTIONS_STMT,
QUERY_NODE_SHOW_TSMAS_STMT,
QUERY_NODE_CREATE_TSMA_STMT,
QUERY_NODE_SHOW_CREATE_TSMA_STMT,
@ -1247,6 +1249,7 @@ typedef struct {
int32_t sqlLen;
char* sql;
int8_t withArbitrator;
int8_t encryptAlgorithm;
} SCreateDbReq;
int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq);
@ -1405,6 +1408,7 @@ typedef struct {
int8_t replications;
int8_t strict;
int8_t cacheLast;
int8_t encryptAlgorithm;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
@ -1605,13 +1609,15 @@ void tFreeSFuncInfo(SFuncInfo* pInfo);
void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp);
typedef struct {
int32_t statusInterval;
int64_t checkTime; // 1970-01-01 00:00:00.000
char timezone[TD_TIMEZONE_LEN]; // tsTimezone
char locale[TD_LOCALE_LEN]; // tsLocale
char charset[TD_LOCALE_LEN]; // tsCharset
int8_t ttlChangeOnWrite;
int8_t enableWhiteList;
int32_t statusInterval;
int64_t checkTime; // 1970-01-01 00:00:00.000
char timezone[TD_TIMEZONE_LEN]; // tsTimezone
char locale[TD_LOCALE_LEN]; // tsLocale
char charset[TD_LOCALE_LEN]; // tsCharset
int8_t ttlChangeOnWrite;
int8_t enableWhiteList;
int8_t encryptionKeyStat;
uint32_t encryptionKeyChksum;
} SClusterCfg;
typedef struct {
@ -1823,6 +1829,7 @@ typedef struct {
int8_t learnerSelfIndex;
SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA];
int32_t changeVersion;
int8_t encryptAlgorithm;
} SCreateVnodeReq;
int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);

2
include/common/tmsgdef.h
View File

@ -106,6 +106,7 @@
TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE_TYPE, "dnode-alter-mnode-type", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CREATE_ENCRYPT_KEY, "create-encrypt-key", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_DND_MSG)
@ -220,6 +221,7 @@
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ENCRYPT_KEY, "create-encrypt-key", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)

747
include/common/ttokendef.h
View File

@ -16,378 +16,381 @@
#ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_NK_COMMA 33
#define TK_HOST 34
#define TK_USER 35
#define TK_ENABLE 36
#define TK_NK_INTEGER 37
#define TK_SYSINFO 38
#define TK_ADD 39
#define TK_DROP 40
#define TK_GRANT 41
#define TK_ON 42
#define TK_TO 43
#define TK_REVOKE 44
#define TK_FROM 45
#define TK_SUBSCRIBE 46
#define TK_READ 47
#define TK_WRITE 48
#define TK_NK_DOT 49
#define TK_WITH 50
#define TK_DNODE 51
#define TK_PORT 52
#define TK_DNODES 53
#define TK_RESTORE 54
#define TK_NK_IPTOKEN 55
#define TK_FORCE 56
#define TK_UNSAFE 57
#define TK_CLUSTER 58
#define TK_LOCAL 59
#define TK_QNODE 60
#define TK_BNODE 61
#define TK_SNODE 62
#define TK_MNODE 63
#define TK_VNODE 64
#define TK_DATABASE 65
#define TK_USE 66
#define TK_FLUSH 67
#define TK_TRIM 68
#define TK_S3MIGRATE 69
#define TK_COMPACT 70
#define TK_IF 71
#define TK_NOT 72
#define TK_EXISTS 73
#define TK_BUFFER 74
#define TK_CACHEMODEL 75
#define TK_CACHESIZE 76
#define TK_COMP 77
#define TK_DURATION 78
#define TK_NK_VARIABLE 79
#define TK_MAXROWS 80
#define TK_MINROWS 81
#define TK_KEEP 82
#define TK_PAGES 83
#define TK_PAGESIZE 84
#define TK_TSDB_PAGESIZE 85
#define TK_PRECISION 86
#define TK_REPLICA 87
#define TK_VGROUPS 88
#define TK_SINGLE_STABLE 89
#define TK_RETENTIONS 90
#define TK_SCHEMALESS 91
#define TK_WAL_LEVEL 92
#define TK_WAL_FSYNC_PERIOD 93
#define TK_WAL_RETENTION_PERIOD 94
#define TK_WAL_RETENTION_SIZE 95
#define TK_WAL_ROLL_PERIOD 96
#define TK_WAL_SEGMENT_SIZE 97
#define TK_STT_TRIGGER 98
#define TK_TABLE_PREFIX 99
#define TK_TABLE_SUFFIX 100
#define TK_S3_CHUNKSIZE 101
#define TK_S3_KEEPLOCAL 102
#define TK_S3_COMPACT 103
#define TK_KEEP_TIME_OFFSET 104
#define TK_NK_COLON 105
#define TK_BWLIMIT 106
#define TK_START 107
#define TK_TIMESTAMP 108
#define TK_END 109
#define TK_TABLE 110
#define TK_NK_LP 111
#define TK_NK_RP 112
#define TK_STABLE 113
#define TK_COLUMN 114
#define TK_MODIFY 115
#define TK_RENAME 116
#define TK_TAG 117
#define TK_SET 118
#define TK_NK_EQ 119
#define TK_USING 120
#define TK_TAGS 121
#define TK_BOOL 122
#define TK_TINYINT 123
#define TK_SMALLINT 124
#define TK_INT 125
#define TK_INTEGER 126
#define TK_BIGINT 127
#define TK_FLOAT 128
#define TK_DOUBLE 129
#define TK_BINARY 130
#define TK_NCHAR 131
#define TK_UNSIGNED 132
#define TK_JSON 133
#define TK_VARCHAR 134
#define TK_MEDIUMBLOB 135
#define TK_BLOB 136
#define TK_VARBINARY 137
#define TK_GEOMETRY 138
#define TK_DECIMAL 139
#define TK_COMMENT 140
#define TK_MAX_DELAY 141
#define TK_WATERMARK 142
#define TK_ROLLUP 143
#define TK_TTL 144
#define TK_SMA 145
#define TK_DELETE_MARK 146
#define TK_FIRST 147
#define TK_LAST 148
#define TK_SHOW 149
#define TK_PRIVILEGES 150
#define TK_DATABASES 151
#define TK_TABLES 152
#define TK_STABLES 153
#define TK_MNODES 154
#define TK_QNODES 155
#define TK_ARBGROUPS 156
#define TK_FUNCTIONS 157
#define TK_INDEXES 158
#define TK_ACCOUNTS 159
#define TK_APPS 160
#define TK_CONNECTIONS 161
#define TK_LICENCES 162
#define TK_GRANTS 163
#define TK_FULL 164
#define TK_LOGS 165
#define TK_MACHINES 166
#define TK_QUERIES 167
#define TK_SCORES 168
#define TK_TOPICS 169
#define TK_VARIABLES 170
#define TK_BNODES 171
#define TK_SNODES 172
#define TK_TRANSACTIONS 173
#define TK_DISTRIBUTED 174
#define TK_CONSUMERS 175
#define TK_SUBSCRIPTIONS 176
#define TK_VNODES 177
#define TK_ALIVE 178
#define TK_VIEWS 179
#define TK_VIEW 180
#define TK_COMPACTS 181
#define TK_NORMAL 182
#define TK_CHILD 183
#define TK_LIKE 184
#define TK_TBNAME 185
#define TK_QTAGS 186
#define TK_AS 187
#define TK_SYSTEM 188
#define TK_TSMA 189
#define TK_INTERVAL 190
#define TK_RECURSIVE 191
#define TK_TSMAS 192
#define TK_FUNCTION 193
#define TK_INDEX 194
#define TK_COUNT 195
#define TK_LAST_ROW 196
#define TK_META 197
#define TK_ONLY 198
#define TK_TOPIC 199
#define TK_CONSUMER 200
#define TK_GROUP 201
#define TK_DESC 202
#define TK_DESCRIBE 203
#define TK_RESET 204
#define TK_QUERY 205
#define TK_CACHE 206
#define TK_EXPLAIN 207
#define TK_ANALYZE 208
#define TK_VERBOSE 209
#define TK_NK_BOOL 210
#define TK_RATIO 211
#define TK_NK_FLOAT 212
#define TK_OUTPUTTYPE 213
#define TK_AGGREGATE 214
#define TK_BUFSIZE 215
#define TK_LANGUAGE 216
#define TK_REPLACE 217
#define TK_STREAM 218
#define TK_INTO 219
#define TK_PAUSE 220
#define TK_RESUME 221
#define TK_PRIMARY 222
#define TK_KEY 223
#define TK_TRIGGER 224
#define TK_AT_ONCE 225
#define TK_WINDOW_CLOSE 226
#define TK_IGNORE 227
#define TK_EXPIRED 228
#define TK_FILL_HISTORY 229
#define TK_UPDATE 230
#define TK_SUBTABLE 231
#define TK_UNTREATED 232
#define TK_KILL 233
#define TK_CONNECTION 234
#define TK_TRANSACTION 235
#define TK_BALANCE 236
#define TK_VGROUP 237
#define TK_LEADER 238
#define TK_MERGE 239
#define TK_REDISTRIBUTE 240
#define TK_SPLIT 241
#define TK_DELETE 242
#define TK_INSERT 243
#define TK_NK_BIN 244
#define TK_NK_HEX 245
#define TK_NULL 246
#define TK_NK_QUESTION 247
#define TK_NK_ALIAS 248
#define TK_NK_ARROW 249
#define TK_ROWTS 250
#define TK_QSTART 251
#define TK_QEND 252
#define TK_QDURATION 253
#define TK_WSTART 254
#define TK_WEND 255
#define TK_WDURATION 256
#define TK_IROWTS 257
#define TK_ISFILLED 258
#define TK_CAST 259
#define TK_NOW 260
#define TK_TODAY 261
#define TK_TIMEZONE 262
#define TK_CLIENT_VERSION 263
#define TK_SERVER_VERSION 264
#define TK_SERVER_STATUS 265
#define TK_CURRENT_USER 266
#define TK_CASE 267
#define TK_WHEN 268
#define TK_THEN 269
#define TK_ELSE 270
#define TK_BETWEEN 271
#define TK_IS 272
#define TK_NK_LT 273
#define TK_NK_GT 274
#define TK_NK_LE 275
#define TK_NK_GE 276
#define TK_NK_NE 277
#define TK_MATCH 278
#define TK_NMATCH 279
#define TK_CONTAINS 280
#define TK_IN 281
#define TK_JOIN 282
#define TK_INNER 283
#define TK_LEFT 284
#define TK_RIGHT 285
#define TK_OUTER 286
#define TK_SEMI 287
#define TK_ANTI 288
#define TK_ASOF 289
#define TK_WINDOW 290
#define TK_WINDOW_OFFSET 291
#define TK_JLIMIT 292
#define TK_SELECT 293
#define TK_NK_HINT 294
#define TK_DISTINCT 295
#define TK_WHERE 296
#define TK_PARTITION 297
#define TK_BY 298
#define TK_SESSION 299
#define TK_STATE_WINDOW 300
#define TK_EVENT_WINDOW 301
#define TK_COUNT_WINDOW 302
#define TK_SLIDING 303
#define TK_FILL 304
#define TK_VALUE 305
#define TK_VALUE_F 306
#define TK_NONE 307
#define TK_PREV 308
#define TK_NULL_F 309
#define TK_LINEAR 310
#define TK_NEXT 311
#define TK_HAVING 312
#define TK_RANGE 313
#define TK_EVERY 314
#define TK_ORDER 315
#define TK_SLIMIT 316
#define TK_SOFFSET 317
#define TK_LIMIT 318
#define TK_OFFSET 319
#define TK_ASC 320
#define TK_NULLS 321
#define TK_ABORT 322
#define TK_AFTER 323
#define TK_ATTACH 324
#define TK_BEFORE 325
#define TK_BEGIN 326
#define TK_BITAND 327
#define TK_BITNOT 328
#define TK_BITOR 329
#define TK_BLOCKS 330
#define TK_CHANGE 331
#define TK_COMMA 332
#define TK_CONCAT 333
#define TK_CONFLICT 334
#define TK_COPY 335
#define TK_DEFERRED 336
#define TK_DELIMITERS 337
#define TK_DETACH 338
#define TK_DIVIDE 339
#define TK_DOT 340
#define TK_EACH 341
#define TK_FAIL 342
#define TK_FILE 343
#define TK_FOR 344
#define TK_GLOB 345
#define TK_ID 346
#define TK_IMMEDIATE 347
#define TK_IMPORT 348
#define TK_INITIALLY 349
#define TK_INSTEAD 350
#define TK_ISNULL 351
#define TK_MODULES 352
#define TK_NK_BITNOT 353
#define TK_NK_SEMI 354
#define TK_NOTNULL 355
#define TK_OF 356
#define TK_PLUS 357
#define TK_PRIVILEGE 358
#define TK_RAISE 359
#define TK_RESTRICT 360
#define TK_ROW 361
#define TK_STAR 362
#define TK_STATEMENT 363
#define TK_STRICT 364
#define TK_STRING 365
#define TK_TIMES 366
#define TK_VALUES 367
#define TK_VARIABLE 368
#define TK_WAL 369
#define TK_ENCODE 370
#define TK_COMPRESS 371
#define TK_LEVEL 372
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_NK_COMMA 33
#define TK_HOST 34
#define TK_USER 35
#define TK_ENABLE 36
#define TK_NK_INTEGER 37
#define TK_SYSINFO 38
#define TK_ADD 39
#define TK_DROP 40
#define TK_GRANT 41
#define TK_ON 42
#define TK_TO 43
#define TK_REVOKE 44
#define TK_FROM 45
#define TK_SUBSCRIBE 46
#define TK_READ 47
#define TK_WRITE 48
#define TK_NK_DOT 49
#define TK_WITH 50
#define TK_ENCRYPT_KEY 51
#define TK_DNODE 52
#define TK_PORT 53
#define TK_DNODES 54
#define TK_RESTORE 55
#define TK_NK_IPTOKEN 56
#define TK_FORCE 57
#define TK_UNSAFE 58
#define TK_CLUSTER 59
#define TK_LOCAL 60
#define TK_QNODE 61
#define TK_BNODE 62
#define TK_SNODE 63
#define TK_MNODE 64
#define TK_VNODE 65
#define TK_DATABASE 66
#define TK_USE 67
#define TK_FLUSH 68
#define TK_TRIM 69
#define TK_S3MIGRATE 70
#define TK_COMPACT 71
#define TK_IF 72
#define TK_NOT 73
#define TK_EXISTS 74
#define TK_BUFFER 75
#define TK_CACHEMODEL 76
#define TK_CACHESIZE 77
#define TK_COMP 78
#define TK_DURATION 79
#define TK_NK_VARIABLE 80
#define TK_MAXROWS 81
#define TK_MINROWS 82
#define TK_KEEP 83
#define TK_PAGES 84
#define TK_PAGESIZE 85
#define TK_TSDB_PAGESIZE 86
#define TK_PRECISION 87
#define TK_REPLICA 88
#define TK_VGROUPS 89
#define TK_SINGLE_STABLE 90
#define TK_RETENTIONS 91
#define TK_SCHEMALESS 92
#define TK_WAL_LEVEL 93
#define TK_WAL_FSYNC_PERIOD 94
#define TK_WAL_RETENTION_PERIOD 95
#define TK_WAL_RETENTION_SIZE 96
#define TK_WAL_ROLL_PERIOD 97
#define TK_WAL_SEGMENT_SIZE 98
#define TK_STT_TRIGGER 99
#define TK_TABLE_PREFIX 100
#define TK_TABLE_SUFFIX 101
#define TK_S3_CHUNKSIZE 102
#define TK_S3_KEEPLOCAL 103
#define TK_S3_COMPACT 104
#define TK_KEEP_TIME_OFFSET 105
#define TK_ENCRYPT_ALGORITHM 106
#define TK_NK_COLON 107
#define TK_BWLIMIT 108
#define TK_START 109
#define TK_TIMESTAMP 110
#define TK_END 111
#define TK_TABLE 112
#define TK_NK_LP 113
#define TK_NK_RP 114
#define TK_STABLE 115
#define TK_COLUMN 116
#define TK_MODIFY 117
#define TK_RENAME 118
#define TK_TAG 119
#define TK_SET 120
#define TK_NK_EQ 121
#define TK_USING 122
#define TK_TAGS 123
#define TK_BOOL 124
#define TK_TINYINT 125
#define TK_SMALLINT 126
#define TK_INT 127
#define TK_INTEGER 128
#define TK_BIGINT 129
#define TK_FLOAT 130
#define TK_DOUBLE 131
#define TK_BINARY 132
#define TK_NCHAR 133
#define TK_UNSIGNED 134
#define TK_JSON 135
#define TK_VARCHAR 136
#define TK_MEDIUMBLOB 137
#define TK_BLOB 138
#define TK_VARBINARY 139
#define TK_GEOMETRY 140
#define TK_DECIMAL 141
#define TK_COMMENT 142
#define TK_MAX_DELAY 143
#define TK_WATERMARK 144
#define TK_ROLLUP 145
#define TK_TTL 146
#define TK_SMA 147
#define TK_DELETE_MARK 148
#define TK_FIRST 149
#define TK_LAST 150
#define TK_SHOW 151
#define TK_PRIVILEGES 152
#define TK_DATABASES 153
#define TK_TABLES 154
#define TK_STABLES 155
#define TK_MNODES 156
#define TK_QNODES 157
#define TK_ARBGROUPS 158
#define TK_FUNCTIONS 159
#define TK_INDEXES 160
#define TK_ACCOUNTS 161
#define TK_APPS 162
#define TK_CONNECTIONS 163
#define TK_LICENCES 164
#define TK_GRANTS 165
#define TK_FULL 166
#define TK_LOGS 167
#define TK_MACHINES 168
#define TK_ENCRYPTIONS 169
#define TK_QUERIES 170
#define TK_SCORES 171
#define TK_TOPICS 172
#define TK_VARIABLES 173
#define TK_BNODES 174
#define TK_SNODES 175
#define TK_TRANSACTIONS 176
#define TK_DISTRIBUTED 177
#define TK_CONSUMERS 178
#define TK_SUBSCRIPTIONS 179
#define TK_VNODES 180
#define TK_ALIVE 181
#define TK_VIEWS 182
#define TK_VIEW 183
#define TK_COMPACTS 184
#define TK_NORMAL 185
#define TK_CHILD 186
#define TK_LIKE 187
#define TK_TBNAME 188
#define TK_QTAGS 189
#define TK_AS 190
#define TK_SYSTEM 191
#define TK_TSMA 192
#define TK_INTERVAL 193
#define TK_RECURSIVE 194
#define TK_TSMAS 195
#define TK_FUNCTION 196
#define TK_INDEX 197
#define TK_COUNT 198
#define TK_LAST_ROW 199
#define TK_META 200
#define TK_ONLY 201
#define TK_TOPIC 202
#define TK_CONSUMER 203
#define TK_GROUP 204
#define TK_DESC 205
#define TK_DESCRIBE 206
#define TK_RESET 207
#define TK_QUERY 208
#define TK_CACHE 209
#define TK_EXPLAIN 210
#define TK_ANALYZE 211
#define TK_VERBOSE 212
#define TK_NK_BOOL 213
#define TK_RATIO 214
#define TK_NK_FLOAT 215
#define TK_OUTPUTTYPE 216
#define TK_AGGREGATE 217
#define TK_BUFSIZE 218
#define TK_LANGUAGE 219
#define TK_REPLACE 220
#define TK_STREAM 221
#define TK_INTO 222
#define TK_PAUSE 223
#define TK_RESUME 224
#define TK_PRIMARY 225
#define TK_KEY 226
#define TK_TRIGGER 227
#define TK_AT_ONCE 228
#define TK_WINDOW_CLOSE 229
#define TK_IGNORE 230
#define TK_EXPIRED 231
#define TK_FILL_HISTORY 232
#define TK_UPDATE 233
#define TK_SUBTABLE 234
#define TK_UNTREATED 235
#define TK_KILL 236
#define TK_CONNECTION 237
#define TK_TRANSACTION 238
#define TK_BALANCE 239
#define TK_VGROUP 240
#define TK_LEADER 241
#define TK_MERGE 242
#define TK_REDISTRIBUTE 243
#define TK_SPLIT 244
#define TK_DELETE 245
#define TK_INSERT 246
#define TK_NK_BIN 247
#define TK_NK_HEX 248
#define TK_NULL 249
#define TK_NK_QUESTION 250
#define TK_NK_ALIAS 251
#define TK_NK_ARROW 252
#define TK_ROWTS 253
#define TK_QSTART 254
#define TK_QEND 255
#define TK_QDURATION 256
#define TK_WSTART 257
#define TK_WEND 258
#define TK_WDURATION 259
#define TK_IROWTS 260
#define TK_ISFILLED 261
#define TK_CAST 262
#define TK_NOW 263
#define TK_TODAY 264
#define TK_TIMEZONE 265
#define TK_CLIENT_VERSION 266
#define TK_SERVER_VERSION 267
#define TK_SERVER_STATUS 268
#define TK_CURRENT_USER 269
#define TK_CASE 270
#define TK_WHEN 271
#define TK_THEN 272
#define TK_ELSE 273
#define TK_BETWEEN 274
#define TK_IS 275
#define TK_NK_LT 276
#define TK_NK_GT 277
#define TK_NK_LE 278
#define TK_NK_GE 279
#define TK_NK_NE 280
#define TK_MATCH 281
#define TK_NMATCH 282
#define TK_CONTAINS 283
#define TK_IN 284
#define TK_JOIN 285
#define TK_INNER 286
#define TK_LEFT 287
#define TK_RIGHT 288
#define TK_OUTER 289
#define TK_SEMI 290
#define TK_ANTI 291
#define TK_ASOF 292
#define TK_WINDOW 293
#define TK_WINDOW_OFFSET 294
#define TK_JLIMIT 295
#define TK_SELECT 296
#define TK_NK_HINT 297
#define TK_DISTINCT 298
#define TK_WHERE 299
#define TK_PARTITION 300
#define TK_BY 301
#define TK_SESSION 302
#define TK_STATE_WINDOW 303
#define TK_EVENT_WINDOW 304
#define TK_COUNT_WINDOW 305
#define TK_SLIDING 306
#define TK_FILL 307
#define TK_VALUE 308
#define TK_VALUE_F 309
#define TK_NONE 310
#define TK_PREV 311
#define TK_NULL_F 312
#define TK_LINEAR 313
#define TK_NEXT 314
#define TK_HAVING 315
#define TK_RANGE 316
#define TK_EVERY 317
#define TK_ORDER 318
#define TK_SLIMIT 319
#define TK_SOFFSET 320
#define TK_LIMIT 321
#define TK_OFFSET 322
#define TK_ASC 323
#define TK_NULLS 324
#define TK_ABORT 325
#define TK_AFTER 326
#define TK_ATTACH 327
#define TK_BEFORE 328
#define TK_BEGIN 329
#define TK_BITAND 330
#define TK_BITNOT 331
#define TK_BITOR 332
#define TK_BLOCKS 333
#define TK_CHANGE 334
#define TK_COMMA 335
#define TK_CONCAT 336
#define TK_CONFLICT 337
#define TK_COPY 338
#define TK_DEFERRED 339
#define TK_DELIMITERS 340
#define TK_DETACH 341
#define TK_DIVIDE 342
#define TK_DOT 343
#define TK_EACH 344
#define TK_FAIL 345
#define TK_FILE 346
#define TK_FOR 347
#define TK_GLOB 348
#define TK_ID 349
#define TK_IMMEDIATE 350
#define TK_IMPORT 351
#define TK_INITIALLY 352
#define TK_INSTEAD 353
#define TK_ISNULL 354
#define TK_MODULES 355
#define TK_NK_BITNOT 356
#define TK_NK_SEMI 357
#define TK_NOTNULL 358
#define TK_OF 359
#define TK_PLUS 360
#define TK_PRIVILEGE 361
#define TK_RAISE 362
#define TK_RESTRICT 363
#define TK_ROW 364
#define TK_STAR 365
#define TK_STATEMENT 366
#define TK_STRICT 367
#define TK_STRING 368
#define TK_TIMES 369
#define TK_VALUES 370
#define TK_VARIABLE 371
#define TK_WAL 372
#define TK_ENCODE 373
#define TK_COMPRESS 374
#define TK_LEVEL 375
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

39
include/libs/crypt/crypt.h Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _CRYPT_H_
#define _CRYPT_H_
#include "tdef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SCryptOpts{
int32_t len;
char* source;
char* result;
int32_t unitLen;
unsigned char key[17];
}SCryptOpts;
int32_t CBC_Decrypt(SCryptOpts *opts);
int32_t CBC_Encrypt(SCryptOpts *opts);
#ifdef __cplusplus
}
#endif
#endif // _CRYPT_H_

2
include/libs/nodes/cmdnodes.h
View File

@ -70,7 +70,9 @@ typedef struct SDatabaseOptions {
int8_t cacheModel;
int32_t cacheLastSize;
int8_t compressionLevel;
int8_t encryptAlgorithm;
int32_t daysPerFile;
char encryptAlgorithmStr[TSDB_ENCRYPT_ALGO_STR_LEN];
SValueNode* pDaysPerFile;
int32_t fsyncPeriod;
int32_t maxRowsPerBlock;

63
include/libs/sm4/sm4.h Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <memory.h>
#include <time.h>
#ifndef _SM4_H_
#define _SM4_H_
#ifdef __cplusplus
extern "C" {
#endif
int SM4_ECB_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
int SM4_ECB_Decrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
int SM4_CBC_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
int SM4_CBC_Decrypt(unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
#ifdef __cplusplus
}
#endif
#endif // _SM4_H_

3
include/libs/wal/wal.h
View File

@ -57,6 +57,8 @@ typedef struct {
int64_t retentionSize;
int64_t segSize;
EWalType level; // wal level
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
} SWalCfg;
typedef struct {
@ -196,6 +198,7 @@ SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id);
void walCloseReader(SWalReader *pRead);
void walReadReset(SWalReader *pReader);
int32_t walReadVer(SWalReader *pRead, int64_t ver);
void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func);
int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver);
int32_t walNextValidMsg(SWalReader *pRead);
int64_t walReaderGetCurrentVer(const SWalReader *pReader);

14
include/util/taoserror.h
View File

@ -306,6 +306,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) //
#define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B)
#define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C)
#define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D)
#define TSDB_CODE_MND_INVALID_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038E)
// #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x
// #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x
// #define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x0392) // 2.x
@ -314,6 +316,7 @@ int32_t* taosGetErrno();
// #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) // 2.x
#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) //
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
// mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)
@ -421,6 +424,17 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal
#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416)
#define TSDB_CODE_DNODE_NO_MACHINE_CODE TAOS_DEF_ERROR_CODE(0, 0x0417)
#define TSDB_CODE_DNODE_NO_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x0418)
#define TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0419)
#define TSDB_CODE_DNODE_INVALID_ENCRYPTKEY TAOS_DEF_ERROR_CODE(0, 0x0420)
#define TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED TAOS_DEF_ERROR_CODE(0, 0x0421)
#define TSDB_CODE_DNODE_INVALID_ENCRYPT_KLEN TAOS_DEF_ERROR_CODE(0, 0x0422)
#define TSDB_CODE_DNODE_INVALID_STATUS_INTERVAL TAOS_DEF_ERROR_CODE(0, 0x0423)
#define TSDB_CODE_DNODE_INVALID_TIMEZONE TAOS_DEF_ERROR_CODE(0, 0x0424)
#define TSDB_CODE_DNODE_INVALID_CHARSET TAOS_DEF_ERROR_CODE(0, 0x0425)
#define TSDB_CODE_DNODE_INVALID_LOCALE TAOS_DEF_ERROR_CODE(0, 0x0426)
#define TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR TAOS_DEF_ERROR_CODE(0, 0x0427)
#define TSDB_CODE_DNODE_INVALID_EN_WHITELIST TAOS_DEF_ERROR_CODE(0, 0x0428)
// mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)

13
include/util/tdef.h
View File

@ -188,6 +188,10 @@ typedef enum ELogicConditionType {
LOGIC_COND_TYPE_NOT,
} ELogicConditionType;
#define ENCRYPTED_LEN(len) (len/16) * 16 + (len%16?1:0) * 16
#define ENCRYPT_KEY_LEN 16
#define ENCRYPT_KEY_LEN_MIN 8
#define TSDB_INT32_ID_LEN 11
#define TSDB_NAME_DELIMITER_LEN 1
@ -388,6 +392,14 @@ typedef enum ELogicConditionType {
#define TSDB_CACHE_MODEL_LAST_ROW 1
#define TSDB_CACHE_MODEL_LAST_VALUE 2
#define TSDB_CACHE_MODEL_BOTH 3
#define TSDB_ENCRYPT_ALGO_STR_LEN 16
#define TSDB_ENCRYPT_ALGO_NONE_STR "none"
#define TSDB_ENCRYPT_ALGO_SM4_STR "sm4"
#define TSDB_ENCRYPT_ALGO_NONE 0
#define TSDB_ENCRYPT_ALGO_SM4 1
#define TSDB_DEFAULT_ENCRYPT_ALGO TSDB_ENCRYPT_ALGO_NONE
#define TSDB_MIN_ENCRYPT_ALGO TSDB_ENCRYPT_ALGO_NONE
#define TSDB_MAX_ENCRYPT_ALGO TSDB_ENCRYPT_ALGO_SM4
#define TSDB_DEFAULT_CACHE_MODEL TSDB_CACHE_MODEL_NONE
#define TSDB_MIN_DB_CACHE_SIZE 1 // MB
#define TSDB_MAX_DB_CACHE_SIZE 65536
@ -523,6 +535,7 @@ typedef enum ELogicConditionType {
enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED };
enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK };
enum { ENCRYPT_KEY_STAT_UNKNOWN = 0, ENCRYPT_KEY_STAT_UNSET, ENCRYPT_KEY_STAT_SET, ENCRYPT_KEY_STAT_LOADED};
typedef struct {
char dir[TSDB_FILENAME_LEN];

9
source/common/src/systable.c
View File

@ -121,6 +121,7 @@ static const SSysDbTableSchema userDBSchema[] = {
{.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "encrypt_algorithm", .bytes = TSDB_ENCRYPT_ALGO_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema userFuncSchema[] = {
@ -400,12 +401,15 @@ static const SSysDbTableSchema userGrantsLogsSchema[] = {
static const SSysDbTableSchema userMachinesSchema[] = {
{.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
#ifndef TD_UNIQ_GRANT
{.name = "dnode_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
#endif
{.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema encryptionsSchema[] = {
{.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "key_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
@ -439,6 +443,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_GRANTS_LOGS, userGrantsLogsSchema, tListLen(userGrantsLogsSchema), true},
{TSDB_INS_TABLE_MACHINES, userMachinesSchema, tListLen(userMachinesSchema), true},
{TSDB_INS_TABLE_ARBGROUPS, arbGroupsSchema, tListLen(arbGroupsSchema), true},
{TSDB_INS_TABLE_ENCRYPTIONS, encryptionsSchema, tListLen(encryptionsSchema), true},
{TSDB_INS_TABLE_TSMAS, tsmaSchema, tListLen(tsmaSchema), false},
};

23
source/common/src/tglobal.c
View File

@ -41,6 +41,13 @@ uint16_t tsServerPort = 6030;
int32_t tsVersion = 30000000;
int32_t tsStatusInterval = 1; // second
int32_t tsNumOfSupportVnodes = 256;
char tsEncryptAlgorithm[16] = {0};
char tsEncryptScope[100] = {0};
EEncryptAlgor tsiEncryptAlgorithm = 0;
EEncryptScope tsiEncryptScope = 0;
//char tsAuthCode[500] = {0};
//char tsEncryptKey[17] = {0};
char tsEncryptKey[17] = {0};
// common
int32_t tsMaxShellConns = 50000;
@ -77,8 +84,6 @@ int32_t tsSnapReplMaxWaitN = 128;
// mnode
int64_t tsMndSdbWriteDelta = 200;
int64_t tsMndLogRetention = 2000;
int8_t tsGrant = 1;
int32_t tsMndGrantMode = 0;
bool tsMndSkipGrant = false;
bool tsEnableWhiteList = false; // ip white list cfg
@ -92,6 +97,11 @@ int64_t tsDndStart = 0;
int64_t tsDndStartOsUptime = 0;
int64_t tsDndUpTime = 0;
// dnode misc
uint32_t tsEncryptionKeyChksum = 0;
int8_t tsEncryptionKeyStat = ENCRYPT_KEY_STAT_UNSET;
int8_t tsGrant = 1;
// monitor
bool tsEnableMonitor = true;
int32_t tsMonitorInterval = 30;
@ -609,6 +619,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
//if (cfgAddString(pCfg, "authCode", tsAuthCode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
return -1;
@ -705,7 +719,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
return -1;
if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -1163,6 +1176,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32;
tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32;
tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32;
tstrncpy(tsEncryptAlgorithm, cfgGetItem(pCfg, "encryptAlgorithm")->str, 16);
tstrncpy(tsEncryptScope, cfgGetItem(pCfg, "encryptScope")->str, 100);
//tstrncpy(tsAuthCode, cfgGetItem(pCfg, "authCode")->str, 100);
tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32;
tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32;
@ -1238,7 +1254,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64;
tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64;
tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval;
tsMndGrantMode = cfgGetItem(pCfg, "grantMode")->i32;
tsEnableWhiteList = cfgGetItem(pCfg, "enableWhiteList")->bval;
tsStartUdfd = cfgGetItem(pCfg, "udf")->bval;

36
source/common/src/tmsg.c
View File

@ -1216,6 +1216,8 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tEncodeCStr(&encoder, pReq->clusterCfg.locale) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->clusterCfg.charset) < 0) return -1;
if (tEncodeI8(&encoder, pReq->clusterCfg.enableWhiteList) < 0) return -1;
if (tEncodeI8(&encoder, pReq->clusterCfg.encryptionKeyStat) < 0) return -1;
if (tEncodeU32(&encoder, pReq->clusterCfg.encryptionKeyChksum) < 0) return -1;
// vnode loads
int32_t vlen = (int32_t)taosArrayGetSize(pReq->pVloads);
@ -1308,6 +1310,8 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tDecodeCStrTo(&decoder, pReq->clusterCfg.locale) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->clusterCfg.charset) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->clusterCfg.enableWhiteList) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->clusterCfg.encryptionKeyStat) < 0) return -1;
if (tDecodeU32(&decoder, &pReq->clusterCfg.encryptionKeyChksum) < 0) return -1;
// vnode loads
int32_t vlen = 0;
@ -3080,13 +3084,14 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
}
if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
ENCODESQL();
if (tEncodeI8(&encoder, pReq->withArbitrator) < 0) return -1;
if (tEncodeI8(&encoder, pReq->encryptAlgorithm) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
tEndEncode(&encoder);
@ -3155,22 +3160,21 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
DECODESQL();
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->withArbitrator) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->encryptAlgorithm) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
DECODESQL();
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->withArbitrator) < 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -4137,6 +4141,7 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) {
if (tEncodeI16(encoder, pRsp->sstTrigger) < 0) return -1;
if (tEncodeI32(encoder, pRsp->keepTimeOffset) < 0) return -1;
if (tEncodeI8(encoder, pRsp->withArbitrator) < 0) return -1;
if (tEncodeI8(encoder, pRsp->encryptAlgorithm) < 0) return -1;
if (tEncodeI32(encoder, pRsp->s3ChunkSize) < 0) return -1;
if (tEncodeI32(encoder, pRsp->s3KeepLocal) < 0) return -1;
if (tEncodeI8(encoder, pRsp->s3Compact) < 0) return -1;
@ -4213,14 +4218,13 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) {
if (tDecodeI32(decoder, &pRsp->keepTimeOffset) < 0) return -1;
}
pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (!tDecodeIsEnd(decoder)) {
if (tDecodeI8(decoder, &pRsp->withArbitrator) < 0) return -1;
}
pRsp->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(decoder)) {
if (tDecodeI8(decoder, &pRsp->withArbitrator) < 0) return -1;
if (tDecodeI8(decoder, &pRsp->encryptAlgorithm) < 0) return -1;
if (tDecodeI32(decoder, &pRsp->s3ChunkSize) < 0) return -1;
if (tDecodeI32(decoder, &pRsp->s3KeepLocal) < 0) return -1;
if (tDecodeI8(decoder, &pRsp->s3Compact) < 0) return -1;
@ -5282,6 +5286,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
}
if (tEncodeI32(&encoder, pReq->changeVersion) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI8(&encoder, pReq->encryptAlgorithm) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
@ -5378,11 +5383,12 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->encryptAlgorithm) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;

4
source/dnode/mgmt/CMakeLists.txt
View File

@ -31,8 +31,8 @@ ENDIF ()
IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEPENDENCIES(taosd jemalloc)
target_link_libraries(taosd dnode ${LINK_JEMALLOC})
target_link_libraries(taosd dnode crypt ${LINK_JEMALLOC})
ELSE ()
target_link_libraries(taosd dnode)
target_link_libraries(taosd dnode crypt)
ENDIF ()

41
source/dnode/mgmt/exe/dmMain.c
View File

@ -22,6 +22,7 @@
#ifdef TD_JEMALLOC_ENABLED
#include "jemalloc/jemalloc.h"
#endif
#include "dmUtil.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@ -49,6 +50,8 @@
#define DM_VERSION "Print program version."
#define DM_EMAIL "<support@taosdata.com>"
#define DM_MEM_DBG "Enable memory debug"
#define DM_SET_ENCRYPTKEY "Set encrypt key. such as: -y 1234567890abcdef, the length should be less or equal to 16."
// clang-format on
static struct {
#ifdef WINDOWS
@ -67,6 +70,8 @@ static struct {
const char **envCmd;
SArray *pArgs; // SConfigPair
int64_t startTime;
bool generateCode;
char encryptKey[ENCRYPT_KEY_LEN + 1];
} global = {0};
static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetGlobalDebugFlag(143); }
@ -195,6 +200,23 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
}
} else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true;
} else if (strcmp(argv[i], "-y") == 0) {
global.generateCode = true;
if(i < argc - 1) {
int32_t len = strlen(argv[++i]);
if (len < ENCRYPT_KEY_LEN_MIN) {
printf("encrypt key is too short, it should be great or equal to %d\n", ENCRYPT_KEY_LEN_MIN);
return -1;
}
if (len > ENCRYPT_KEY_LEN) {
printf("encrypt key overflow, it should be less or equal to %d\n", ENCRYPT_KEY_LEN);
return -1;
}
tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
} else {
printf("'-y' requires a parameter\n");
return -1;
}
} else if (strcmp(argv[i], "-C") == 0) {
global.dumpConfig = true;
} else if (strcmp(argv[i], "-V") == 0) {
@ -259,6 +281,7 @@ static void dmPrintHelp() {
printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE);
printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY);
printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG);
printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION);
@ -364,6 +387,24 @@ int mainWindows(int argc, char **argv) {
return -1;
}
if(global.generateCode) {
if(dmCheckRunning(tsDataDir) == NULL) {
dError("failed to generate encrypt code since taosd is running, please stop it first");
return -1;
}
int ret = dmUpdateEncryptKey(global.encryptKey);
taosCloseLog();
taosCleanupArgs();
return ret;
}
if(dmGetEncryptKey() != 0){
dError("failed to start since failed to get encrypt key");
taosCloseLog();
taosCleanupArgs();
return -1;
};
if (taosConvInit() != 0) {
dError("failed to init conv");
taosCloseLog();

1
source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
View File

@ -57,6 +57,7 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg);
int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg);
int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
// dmWorker.c
int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);

30
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "dmInt.h"
#include "systable.h"
#include "tchecksum.h"
extern SConfig *tsCfg;
@ -113,6 +114,8 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.clusterCfg.checkTime = 0;
req.clusterCfg.ttlChangeOnWrite = tsTtlChangeOnWrite;
req.clusterCfg.enableWhiteList = tsEnableWhiteList ? 1 : 0;
req.clusterCfg.encryptionKeyStat = tsEncryptionKeyStat;
req.clusterCfg.encryptionKeyChksum = tsEncryptionKeyChksum;
char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN);
@ -221,6 +224,32 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return 0;
}
int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
#ifdef TD_ENTERPRISE
int32_t code = 0;
SDCfgDnodeReq cfgReq = {0};
if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
code = dmUpdateEncryptKey(cfgReq.value);
if (code == 0) {
tsEncryptionKeyChksum = taosCalcChecksum(0, cfgReq.value, strlen(cfgReq.value));
tsEncryptionKeyStat = ENCRYPT_KEY_STAT_LOADED;
tstrncpy(tsEncryptKey, cfgReq.value, ENCRYPT_KEY_LEN + 1);
}
_exit:
pMsg->code = code;
pMsg->info.rsp = NULL;
pMsg->info.rspLen = 0;
return code;
#else
return 0;
#endif
}
static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK;
pStatus->details[0] = 0;
@ -421,6 +450,7 @@ SArray *dmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_ENCRYPT_KEY, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
// Requests handled by MNODE
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;

3
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -359,6 +359,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
case TDMT_MND_GRANT_NOTIFY:
code = dmProcessGrantNotify(NULL, pMsg);
break;
case TDMT_DND_CREATE_ENCRYPT_KEY:
code = dmProcessCreateEncryptKeyReq(pMgmt, pMsg);
break;
default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed in mgmt queue", pMsg);

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -113,6 +113,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CONFIG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_ENCRYPT_KEY_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -204,6 +205,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ENCRYPT_KEY, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;

38
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -142,6 +142,14 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
if ((pRetention->freq >= 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
}
}
#if defined(TD_ENTERPRISE)
pCfg->tsdbCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
pCfg->tsdbCfg.encryptAlgorithm = 0;
#endif
pCfg->walCfg.vgId = pCreate->vgId;
pCfg->walCfg.fsyncPeriod = pCreate->walFsyncPeriod;
@ -150,6 +158,23 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->walCfg.retentionSize = pCreate->walRetentionSize;
pCfg->walCfg.segSize = pCreate->walSegmentSize;
pCfg->walCfg.level = pCreate->walLevel;
#if defined(TD_ENTERPRISE)
pCfg->walCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
pCfg->walCfg.encryptAlgorithm = 0;
#endif
#if defined(TD_ENTERPRISE)
pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
pCfg->tdbEncryptAlgorithm = 0;
#endif
pCfg->sttTrigger = pCreate->sstTrigger;
pCfg->hashBegin = pCreate->hashBegin;
@ -246,14 +271,15 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
"precision:%d compression:%d minRows:%d maxRows:%d"
", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64
", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d "
"learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d",
"learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d encryptAlgorithm:%d",
req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024,
(uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize,
req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression,
req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion);
req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion,
req.encryptAlgorithm);
for (int32_t i = 0; i < req.replica; ++i) {
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port,
@ -278,6 +304,14 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
if(req.encryptAlgorithm == DND_CA_SM4){
if(strlen(tsEncryptKey) == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId);
return -1;
}
}
vmGenerateVnodeCfg(&req, &vnodeCfg);
if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) {

40
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -161,6 +161,46 @@ int32_t dmInitVars(SDnode *pDnode) {
return -1;
}
#if defined(TD_ENTERPRISE)
tsiEncryptAlgorithm = pData->encryptAlgorigthm;
tsiEncryptScope = pData->encryptScope;
/*
if(tsiEncryptAlgorithm != 0) {
if(pData->machineId != NULL && strlen(pData->machineId) > 0){
dInfo("get crypt key at startup, machineId:%s", pData->machineId);
int32_t code = 0;
//code = taosGetCryptKey(tsAuthCode, pData->machineId, tsCryptKey);
code = 0;
strncpy(tsEncryptKey, tsAuthCode, 16);
if(code != 0) {
if(code == -1){
terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
dError("machine code changed, can't get crypt key");
}
if(code == -2){
terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
dError("failed to get crypt key");
}
return -1;
}
if(strlen(tsEncryptKey) == 0){
terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
dError("failed to get crypt key at startup since key is null, machineId:%s", pData->machineId);
return -1;
}
}
else{
terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
dError("failed to get crypt key at startup, machineId:%s", pData->machineId);
return -1;
}
}
*/
#endif
if (pData->dropped) {
dError("dnode will not start since its already dropped");
return -1;

5
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -110,6 +110,8 @@ typedef struct {
bool validMnodeEps;
int64_t ipWhiteVer;
char machineId[TSDB_MACHINE_ID_LEN + 1];
EEncryptAlgor encryptAlgorigthm;
EEncryptScope encryptScope;
} SDnodeData;
typedef struct {
@ -191,7 +193,8 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
bool dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port);
void dmRemoveDnodePairs(SDnodeData *pData);
void dmGetDnodeEp(void *pData, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort);
int32_t dmUpdateEncryptKey(char *key);
int32_t dmGetEncryptKey();
#ifdef __cplusplus
}
#endif

99
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -64,7 +64,12 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) {
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "dropped", pData->dropped, code);
if (code < 0) return -1;
#ifdef TD_ENTERPRISE
tjsonGetInt32ValueFromDouble(pJson, "encryptAlgor", pData->encryptAlgorigthm, code);
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "encryptScope", pData->encryptScope, code);
if (code < 0) return -1;
#endif
SJson *dnodes = tjsonGetObjectItem(pJson, "dnodes");
if (dnodes == NULL) return 0;
int32_t numOfDnodes = tjsonGetArraySize(dnodes);
@ -89,6 +94,25 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) {
return 0;
}
int dmOccurrences(char *str, char *toSearch) {
int count = 0;
char *ptr = str;
while ((ptr = strstr(ptr, toSearch)) != NULL) {
count++;
ptr++;
}
return count;
}
void dmSplitStr(char** arr, char* str, const char* del) {
char *lasts;
char* s = strsep(&str, del);
while (s != NULL) {
*arr++ = s;
s = strsep(&str, del);
}
}
int32_t dmReadEps(SDnodeData *pData) {
int32_t code = -1;
TdFilePtr pFile = NULL;
@ -106,6 +130,74 @@ int32_t dmReadEps(SDnodeData *pData) {
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("dnode file:%s not exist", file);
#ifdef TD_ENTERPRISE
if(strlen(tsEncryptAlgorithm) > 0){
if(strcmp(tsEncryptAlgorithm, "sm4") == 0) {
pData->encryptAlgorigthm = DND_CA_SM4;
}
else{
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptAlgorithm:%s", tsEncryptAlgorithm);
goto _OVER;
}
dInfo("start to parse encryptScope:%s", tsEncryptScope);
int32_t scopeLen = strlen(tsEncryptScope);
if(scopeLen == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptScope:%s", tsEncryptScope);
goto _OVER;
}
char* tmp = taosMemoryMalloc(scopeLen + 1);
memset(tmp, 0, scopeLen + 1);
memcpy(tmp, tsEncryptScope, scopeLen);
int32_t count = dmOccurrences(tmp, ",");
char** array = taosMemoryMalloc(sizeof(char*) * (count + 1));
memset(array, 0, sizeof(char*) * (count + 1));
dmSplitStr(array, tmp, ",");
for(int32_t i = 0; i < count + 1; i++){
char* str = *(array + i);
bool success = false;
if(strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_TSDB;
success = true;
}
if(strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_VNODE_WAL;
success = true;
}
if(strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_SDB;
success = true;
}
if(strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_MNODE_WAL;
success = true;
}
if(!success){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
taosMemoryFree(tmp);
taosMemoryFree(array);
dError("invalid tsEncryptScope:%s", tsEncryptScope);
goto _OVER;
}
}
taosMemoryFree(tmp);
taosMemoryFree(array);
dInfo("set tsCryptAlgorithm:%s, tsCryptScope:%s from cfg", tsEncryptAlgorithm, tsEncryptScope);
}
#endif
code = 0;
goto _OVER;
}
@ -191,7 +283,10 @@ static int32_t dmEncodeEps(SJson *pJson, SDnodeData *pData) {
if (tjsonAddIntegerToObject(pJson, "engineVer", pData->engineVer) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "clusterId", pData->clusterId) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "dropped", pData->dropped) < 0) return -1;
#ifdef TD_ENTERPRISE
if (tjsonAddDoubleToObject(pJson, "encryptAlgor", pData->encryptAlgorigthm) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "encryptScope", pData->encryptScope) < 0) return -1;
#endif
SJson *dnodes = tjsonCreateArray();
if (dnodes == NULL) return -1;
if (tjsonAddItemToObject(pJson, "dnodes", dnodes) < 0) return -1;

299
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -16,8 +16,14 @@
#define _DEFAULT_SOURCE
#include "dmUtil.h"
#include "tjson.h"
#include "tgrant.h"
#include "crypt.h"
#include "tchecksum.h"
#define MAXLEN 1024
#define DM_KEY_INDICATOR "this indicator!"
#define DM_ENCRYPT_CODE_FILE "encryptCode.cfg"
#define DM_CHECK_CODE_FILE "checkCode.bin"
static int32_t dmDecodeFile(SJson *pJson, bool *deployed) {
int32_t code = 0;
@ -177,3 +183,296 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
dDebug("lock file:%s to prevent repeated starts", filepath);
return pFile;
}
extern int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode);
static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key){
TdFilePtr pFile = NULL;
char *result = NULL;
int32_t code = -1;
int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR));
result = taosMemoryMalloc(len);
SCryptOpts opts;
strncpy(opts.key, key, ENCRYPT_KEY_LEN);
opts.len = len;
opts.source = DM_KEY_INDICATOR;
opts.result = result;
opts.unitLen = 16;
CBC_Encrypt(&opts);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (taosWriteFile(pFile, opts.result, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
dInfo("succeed to write checkCode file:%s", realfile);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
if(result != NULL) taosMemoryFree(result);
return code;
}
static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptCode){
TdFilePtr pFile = NULL;
int32_t code = -1;
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
int32_t len = strlen(encryptCode);
if (taosWriteFile(pFile, encryptCode, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
dInfo("succeed to write encryptCode file:%s", realfile);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
return code;
}
static int32_t dmCompareEncryptKey(char* file, char* key){
char *content = NULL;
int64_t size = 0;
TdFilePtr pFile = NULL;
char *result = NULL;
int32_t code = -1;
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content = taosMemoryMalloc(size);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER;
}
dInfo("succeed to read checkCode file:%s", file);
int len = ENCRYPTED_LEN(size);
result = taosMemoryMalloc(len);
SCryptOpts opts = {0};
strncpy(opts.key, key, ENCRYPT_KEY_LEN);
opts.len = len;
opts.source = content;
opts.result = result;
opts.unitLen = 16;
CBC_Decrypt(&opts);
if(strcmp(opts.result, DM_KEY_INDICATOR) != 0) {
terrno = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED;
dError("failed to compare decrypted result");
goto _OVER;
}
dInfo("succeed to compare checkCode file:%s", file);
code = 0;
_OVER:
if(result != NULL) taosMemoryFree(result);
if(content != NULL) taosMemoryFree(content);
if(pFile != NULL) taosCloseFile(&pFile);
return code;
}
int32_t dmUpdateEncryptKey(char *key) {
#ifdef TD_ENTERPRISE
int32_t code = -1;
char *machineId = NULL;
char *encryptCode = NULL;
char folder[PATH_MAX] = {0};
char encryptFile[PATH_MAX] = {0};
char realEncryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0};
char realCheckFile[PATH_MAX] = {0};
snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP);
snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
terrno = 0;
if (taosMkDir(folder) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", folder, terrstr());
goto _OVER;
}
if(taosCheckExistFile(realCheckFile)){
if(dmCompareEncryptKey(realCheckFile, key) != 0){
goto _OVER;
}
}
if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (generateEncryptCode(key, machineId, &encryptCode) != 0) {
goto _OVER;
}
if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode) != 0){
goto _OVER;
}
if(dmWriteCheckCodeFile(checkFile, realCheckFile, key) != 0){
goto _OVER;
}
code = 0;
_OVER:
taosMemoryFree(encryptCode);
taosMemoryFree(machineId);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to update encrypt key since %s", terrstr());
}
return code;
#else
return 0;
#endif
}
extern int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);
static int32_t dmReadEncryptCodeFile(char* file, char** output){
TdFilePtr pFile = NULL;
int32_t code = -1;
char *content = NULL;
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content[size] = '\0';
*output = content;
dInfo("succeed to read encryptCode file:%s", file);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
return code;
}
int32_t dmGetEncryptKey(){
#ifdef TD_ENTERPRISE
int32_t code = -1;
char encryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0};
char *machineId = NULL;
char *encryptKey = NULL;
char *content = NULL;
snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE);
if(!taosCheckExistFile(encryptFile)){
dInfo("no exist, checkCode file:%s", encryptFile);
return 0;
}
if(dmReadEncryptCodeFile(encryptFile, &content) != 0){
goto _OVER;
}
if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if(checkAndGetCryptKey(content, machineId, &encryptKey) != 0){
goto _OVER;
}
taosMemoryFreeClear(machineId);
taosMemoryFreeClear(content);
if(encryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("failed to read key since %s", terrstr());
goto _OVER;
}
if(dmCompareEncryptKey(checkFile, encryptKey) != 0){
goto _OVER;
}
strncpy(tsEncryptKey, encryptKey, ENCRYPT_KEY_LEN);
taosMemoryFreeClear(encryptKey);
tsEncryptionKeyChksum = taosCalcChecksum(0, tsEncryptKey, strlen(tsEncryptKey));
tsEncryptionKeyStat = ENCRYPT_KEY_STAT_LOADED;
code = 0;
_OVER:
if (content != NULL) taosMemoryFree(content);
if (encryptKey != NULL) taosMemoryFree(encryptKey);
if (machineId != NULL) taosMemoryFree(machineId);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to get encrypt key since %s", terrstr());
}
return code;
#else
return 0;
#endif
}

4
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -141,6 +141,7 @@ typedef enum {
DND_REASON_CHARSET_NOT_MATCH,
DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH,
DND_REASON_ENABLE_WHITELIST_NOT_MATCH,
DND_REASON_ENCRYPTION_KEY_NOT_MATCH,
DND_REASON_OTHERS
} EDndReason;
@ -215,6 +216,8 @@ typedef struct {
int64_t memAvail;
int64_t memUsed;
EDndReason offlineReason;
uint32_t encryptionKeyChksum;
int8_t encryptionKeyStat;
uint16_t port;
char fqdn[TSDB_FQDN_LEN];
char ep[TSDB_EP_LEN];
@ -383,6 +386,7 @@ typedef struct {
int32_t s3KeepLocal;
int8_t s3Compact;
int8_t withArbitrator;
int8_t encryptAlgorithm;
} SDbCfg;
typedef struct {

8
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -104,6 +104,13 @@ typedef struct {
int64_t timeseriesAllowed;
} SGrantInfo;
typedef struct {
int8_t encrypting;
int16_t nEncrypt;
int16_t nSuccess;
int16_t nFailed;
} SEncryptMgmt;
typedef struct SMnode {
int32_t selfDnodeId;
int64_t clusterId;
@ -127,6 +134,7 @@ typedef struct SMnode {
SProfileMgmt profileMgmt;
STelemMgmt telemMgmt;
SSyncMgmt syncMgmt;
SEncryptMgmt encryptMgmt;
SGrantInfo grant;
MndMsgFp msgFp[TDMT_MAX];
SMsgCb msgCb;

73
source/dnode/mnode/impl/src/mndDb.c
View File

@ -36,7 +36,7 @@
#include "tjson.h"
#define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 28
#define DB_RESERVE_SIZE 27
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -146,6 +146,7 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3KeepLocal, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.encryptAlgorithm, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->tsmaVersion, _OVER);
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
@ -242,6 +243,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3KeepLocal, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.encryptAlgorithm, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->tsmaVersion, _OVER);
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
@ -425,9 +427,12 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1;
#ifdef TD_ENTERPRISE
if ((pCfg->replications == 2) ^ (pCfg->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) return -1;
if (pCfg->encryptAlgorithm < TSDB_MIN_ENCRYPT_ALGO || pCfg->encryptAlgorithm > TSDB_MAX_ENCRYPT_ALGO) return -1;
#else
if (pCfg->replications != 1 && pCfg->replications != 3) return -1;
if (pCfg->encryptAlgorithm != TSDB_DEFAULT_ENCRYPT_ALGO) return -1;
#endif
if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1;
if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1;
if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1;
@ -544,6 +549,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (pCfg->encryptAlgorithm < 0) pCfg->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
}
static int32_t mndSetCreateDbPrepareAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
@ -721,6 +727,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
.s3Compact = pCreate->s3Compact,
.tsdbPageSize = pCreate->tsdbPageSize,
.withArbitrator = pCreate->withArbitrator,
.encryptAlgorithm = pCreate->encryptAlgorithm,
};
dbObj.cfg.numOfRetensions = pCreate->numOfRetensions;
@ -806,6 +813,46 @@ static void mndBuildAuditDetailInt64(char *detail, char *tmp, char *format, int6
}
}
static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
SDnodeObj *pDnode = NULL;
void *pIter = NULL;
#ifdef TD_ENTERPRISE
if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit;
if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) {
code = TSDB_CODE_MND_INVALID_ENCRYPT_KEY;
mError("db:%s, failed to check encryption key:%" PRIi8 " in mnode leader since it's not loaded", pReq->db,
tsEncryptionKeyStat);
goto _exit;
}
int64_t curMs = taosGetTimestampMs();
while ((pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode))) {
bool online = false;
if ((pDnode->encryptionKeyStat != tsEncryptionKeyStat || pDnode->encryptionKeyChksum != tsEncryptionKeyChksum) &&
(online = mndIsDnodeOnline(pDnode, curMs))) {
code = TSDB_CODE_MND_INVALID_ENCRYPT_KEY;
mError("db:%s, failed to check encryption key:%" PRIi8
"-%u in dnode:%d since it's inconsitent with mnode leader:%" PRIi8 "-%u",
pReq->db, pDnode->encryptionKeyStat, pDnode->encryptionKeyChksum, pDnode->id, tsEncryptionKeyStat,
tsEncryptionKeyChksum);
sdbRelease(pSdb, pDnode);
break;
}
sdbRelease(pSdb, pDnode);
}
#else
if (pReq->encryptAlgorithm != TSDB_ENCRYPT_ALGO_NONE) {
code = TSDB_CODE_MND_INVALID_DB_OPTION;
goto _exit;
}
#endif
_exit:
return code;
}
static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
@ -856,6 +903,11 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
}
}
if ((code = mndCheckDbEncryptKey(pMnode, &createReq)) != 0) {
terrno = code;
goto _OVER;
}
pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
if (pUser == NULL) {
goto _OVER;
@ -1207,6 +1259,7 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) {
cfgRsp->s3KeepLocal = pDb->cfg.s3KeepLocal;
cfgRsp->s3Compact = pDb->cfg.s3Compact;
cfgRsp->withArbitrator = pDb->cfg.withArbitrator;
cfgRsp->encryptAlgorithm = pDb->cfg.encryptAlgorithm;
}
static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) {
@ -2028,6 +2081,18 @@ static const char *getCacheModelStr(int8_t cacheModel) {
return "unknown";
}
static const char *getEncryptAlgorithmStr(int8_t encryptAlgorithm) {
switch (encryptAlgorithm) {
case TSDB_ENCRYPT_ALGO_NONE:
return TSDB_ENCRYPT_ALGO_NONE_STR;
case TSDB_ENCRYPT_ALGO_SM4:
return TSDB_ENCRYPT_ALGO_SM4_STR;
default:
break;
}
return "unknown";
}
bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb) {
if (pDb->cfg.replications == 1) return true;
@ -2248,6 +2313,12 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false);
const char *encryptAlgorithmStr = getEncryptAlgorithmStr(pDb->cfg.encryptAlgorithm);
char encryptAlgorithmVStr[24] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(encryptAlgorithmVStr, encryptAlgorithmStr, 24);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)encryptAlgorithmVStr, false);
}
taosMemoryFree(buf);

169
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -42,11 +42,13 @@ static const char *offlineReason[] = {
"version not match",
"dnodeId not match",
"clusterId not match",
"interval not match",
"statusInterval not match",
"timezone not match",
"locale not match",
"charset not match",
"ttlChangeOnWrite not match",
"enableWhiteList not match",
"encryptionKey not match",
"unknown",
};
@ -78,6 +80,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq);
static int32_t mndProcessNotifyReq(SRpcMsg *pReq);
static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq);
static int32_t mndProcessStatisReq(SRpcMsg *pReq);
static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pRsp);
static int32_t mndProcessCreateEncryptKeyRsp(SRpcMsg *pRsp);
static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter);
@ -114,6 +118,8 @@ int32_t mndInitDnode(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq);
mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq);
mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq);
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_ENCRYPT_KEY, mndProcessCreateEncryptKeyReq);
mndSetMsgHandle(pMnode, TDMT_DND_CREATE_ENCRYPT_KEY_RSP, mndProcessCreateEncryptKeyRsp);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig);
@ -372,7 +378,7 @@ int32_t mndGetDbSize(SMnode *pMnode) {
bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) {
int64_t interval = TABS(pDnode->lastAccessTime - curMs);
if (interval > 5000 * (int64_t)tsStatusInterval) {
if (pDnode->rebootTime > 0) {
if (pDnode->rebootTime > 0 && pDnode->offlineReason == DND_REASON_ONLINE) {
pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT;
}
return false;
@ -436,37 +442,51 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S
if (pCfg->statusInterval != tsStatusInterval) {
mError("dnode:%d, statusInterval:%d inconsistent with cluster:%d", pDnode->id, pCfg->statusInterval,
tsStatusInterval);
terrno = TSDB_CODE_DNODE_INVALID_STATUS_INTERVAL;
return DND_REASON_STATUS_INTERVAL_NOT_MATCH;
}
if ((0 != strcasecmp(pCfg->timezone, tsTimezoneStr)) && (pMnode->checkTime != pCfg->checkTime)) {
mError("dnode:%d, timezone:%s checkTime:%" PRId64 " inconsistent with cluster %s %" PRId64, pDnode->id,
pCfg->timezone, pCfg->checkTime, tsTimezoneStr, pMnode->checkTime);
terrno = TSDB_CODE_DNODE_INVALID_TIMEZONE;
return DND_REASON_TIME_ZONE_NOT_MATCH;
}
if (0 != strcasecmp(pCfg->locale, tsLocale)) {
mError("dnode:%d, locale:%s inconsistent with cluster:%s", pDnode->id, pCfg->locale, tsLocale);
terrno = TSDB_CODE_DNODE_INVALID_LOCALE;
return DND_REASON_LOCALE_NOT_MATCH;
}
if (0 != strcasecmp(pCfg->charset, tsCharset)) {
mError("dnode:%d, charset:%s inconsistent with cluster:%s", pDnode->id, pCfg->charset, tsCharset);
terrno = TSDB_CODE_DNODE_INVALID_CHARSET;
return DND_REASON_CHARSET_NOT_MATCH;
}
if (pCfg->ttlChangeOnWrite != tsTtlChangeOnWrite) {
mError("dnode:%d, ttlChangeOnWrite:%d inconsistent with cluster:%d", pDnode->id, pCfg->ttlChangeOnWrite,
tsTtlChangeOnWrite);
terrno = TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR;
return DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH;
}
int8_t enable = tsEnableWhiteList ? 1 : 0;
if (pCfg->enableWhiteList != enable) {
mError("dnode:%d, enable :%d inconsistent with cluster:%d", pDnode->id, pCfg->enableWhiteList, enable);
mError("dnode:%d, enableWhiteList:%d inconsistent with cluster:%d", pDnode->id, pCfg->enableWhiteList, enable);
terrno = TSDB_CODE_DNODE_INVALID_EN_WHITELIST;
return DND_REASON_ENABLE_WHITELIST_NOT_MATCH;
}
return 0;
if (!atomic_load_8(&pMnode->encryptMgmt.encrypting) &&
(pCfg->encryptionKeyStat != tsEncryptionKeyStat || pCfg->encryptionKeyChksum != tsEncryptionKeyChksum)) {
mError("dnode:%d, encryptionKey:%" PRIi8 "-%u inconsistent with cluster:%" PRIi8 "-%u", pDnode->id,
pCfg->encryptionKeyStat, pCfg->encryptionKeyChksum, tsEncryptionKeyStat, tsEncryptionKeyChksum);
terrno = pCfg->encryptionKeyChksum ? TSDB_CODE_DNODE_INVALID_ENCRYPTKEY : TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
return DND_REASON_ENCRYPTION_KEY_NOT_MATCH;
}
return DND_REASON_ONLINE;
}
static bool mndUpdateVnodeState(int32_t vgId, SVnodeGid *pGid, SVnodeLoad *pVload) {
@ -795,8 +815,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer);
bool reboot = (pDnode->rebootTime != statusReq.rebootTime);
bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes;
bool needCheck =
!online || dnodeChanged || reboot || supportVnodesChanged || pMnode->ipWhiteVer != statusReq.ipWhiteVer;
bool encryptKeyChanged = pDnode->encryptionKeyChksum != statusReq.clusterCfg.encryptionKeyChksum;
bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged ||
pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged;
const STraceId *trace = &pReq->info.traceId;
mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id,
@ -891,7 +912,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pDnode->offlineReason = mndCheckClusterCfgPara(pMnode, pDnode, &statusReq.clusterCfg);
if (pDnode->offlineReason != 0) {
mError("dnode:%d, cluster cfg inconsistent since:%s", pDnode->id, offlineReason[pDnode->offlineReason]);
terrno = TSDB_CODE_MND_INVALID_CLUSTER_CFG;
if (terrno == 0) terrno = TSDB_CODE_MND_INVALID_CLUSTER_CFG;
goto _OVER;
}
@ -909,6 +930,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pDnode->numOfDiskCfg = statusReq.numOfDiskCfg;
pDnode->memAvail = statusReq.memAvail;
pDnode->memTotal = statusReq.memTotal;
pDnode->encryptionKeyStat = statusReq.clusterCfg.encryptionKeyStat;
pDnode->encryptionKeyChksum = statusReq.clusterCfg.encryptionKeyChksum;
if (memcmp(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN) != 0) {
tstrncpy(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN + 1);
mndUpdateDnodeObj(pMnode, pDnode);
@ -1494,6 +1517,136 @@ static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) {
return 0;
}
static int32_t mndProcessCreateEncryptKeyReqImpl(SRpcMsg *pReq, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
int8_t encrypting = 0;
const STraceId *trace = &pReq->info.traceId;
int32_t klen = strlen(pDcfgReq->value);
if (klen > ENCRYPT_KEY_LEN || klen < ENCRYPT_KEY_LEN_MIN) {
code = TSDB_CODE_DNODE_INVALID_ENCRYPT_KLEN;
mGError("msg:%p, failed to create encrypt_key since invalid key length:%d, valid range:[%d, %d]", pReq, klen,
ENCRYPT_KEY_LEN_MIN, ENCRYPT_KEY_LEN);
goto _exit;
}
if (0 != (encrypting = atomic_val_compare_exchange_8(&pMnode->encryptMgmt.encrypting, 0, 1))) {
code = TSDB_CODE_QRY_DUPLICATED_OPERATION;
mGWarn("msg:%p, failed to create encrypt key since %s, encrypting:%" PRIi8, pReq, tstrerror(code), encrypting);
goto _exit;
}
if (tsEncryptionKeyStat == ENCRYPT_KEY_STAT_SET || tsEncryptionKeyStat == ENCRYPT_KEY_STAT_LOADED) {
atomic_store_8(&pMnode->encryptMgmt.encrypting, 0);
code = TSDB_CODE_QRY_DUPLICATED_OPERATION;
mGWarn("msg:%p, failed to create encrypt key since %s, stat:%" PRIi8 ", checksum:%u", pReq, tstrerror(code),
tsEncryptionKeyStat, tsEncryptionKeyChksum);
goto _exit;
}
atomic_store_16(&pMnode->encryptMgmt.nEncrypt, 0);
atomic_store_16(&pMnode->encryptMgmt.nSuccess, 0);
atomic_store_16(&pMnode->encryptMgmt.nFailed, 0);
while (1) {
SDnodeObj *pDnode = NULL;
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode);
if (pIter == NULL) break;
if (pDnode->offlineReason != DND_REASON_ONLINE) {
mGWarn("msg:%p, don't send create encrypt_key req since dnode:%d in offline state:%s", pReq, pDnode->id,
offlineReason[pDnode->offlineReason]);
sdbRelease(pSdb, pDnode);
continue;
}
if (dnodeId == -1 || pDnode->id == dnodeId || dnodeId == 0) {
SEpSet epSet = mndGetDnodeEpset(pDnode);
int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, pDcfgReq);
void *pBuf = rpcMallocCont(bufLen);
if (pBuf != NULL) {
tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq);
SRpcMsg rpcMsg = {.msgType = TDMT_DND_CREATE_ENCRYPT_KEY, .pCont = pBuf, .contLen = bufLen};
if (0 == tmsgSendReq(&epSet, &rpcMsg)) {
atomic_add_fetch_16(&pMnode->encryptMgmt.nEncrypt, 1);
}
}
}
sdbRelease(pSdb, pDnode);
}
if (atomic_load_16(&pMnode->encryptMgmt.nEncrypt) <= 0) {
atomic_store_8(&pMnode->encryptMgmt.encrypting, 0);
}
_exit:
if (code != 0) {
if (terrno == 0) terrno = code;
}
return code;
}
static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) {
#ifdef TD_ENTERPRISE
SMnode *pMnode = pReq->info.node;
SMCfgDnodeReq cfgReq = {0};
if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) {
tFreeSMCfgDnodeReq(&cfgReq);
return -1;
}
const STraceId *trace = &pReq->info.traceId;
SDCfgDnodeReq dcfgReq = {0};
if (strncasecmp(cfgReq.config, "encrypt_key", 12) == 0) {
strcpy(dcfgReq.config, cfgReq.config);
strcpy(dcfgReq.value, cfgReq.value);
tFreeSMCfgDnodeReq(&cfgReq);
return mndProcessCreateEncryptKeyReqImpl(pReq, cfgReq.dnodeId, &dcfgReq);
} else {
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
tFreeSMCfgDnodeReq(&cfgReq);
return -1;
}
#else
return 0;
#endif
}
static int32_t mndProcessCreateEncryptKeyRsp(SRpcMsg *pRsp) {
SMnode *pMnode = pRsp->info.node;
int16_t nSuccess = 0;
int16_t nFailed = 0;
if (0 == pRsp->code) {
nSuccess = atomic_add_fetch_16(&pMnode->encryptMgmt.nSuccess, 1);
} else {
nFailed = atomic_add_fetch_16(&pMnode->encryptMgmt.nFailed, 1);
}
int16_t nReq = atomic_load_16(&pMnode->encryptMgmt.nEncrypt);
bool finished = nSuccess + nFailed >= nReq;
if (finished) {
atomic_store_8(&pMnode->encryptMgmt.encrypting, 0);
}
const STraceId *trace = &pRsp->info.traceId;
mGInfo("msg:%p, create encrypt key rsp, nReq:%" PRIi16 ", nSucess:%" PRIi16 ", nFailed:%" PRIi16 ", %s", pRsp, nReq,
nSuccess, nFailed, finished ? "encrypt done" : "in encrypting");
return 0;
}
static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
int32_t totalRows = 0;
@ -1550,7 +1703,7 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
ESdbStatus objStatus = 0;
SDnodeObj *pDnode = NULL;
int64_t curMs = taosGetTimestampMs();
char buf[TSDB_CONN_ACTIVE_KEY_LEN + VARSTR_HEADER_SIZE]; // make sure TSDB_CONN_ACTIVE_KEY_LEN >= TSDB_EP_LEN
char buf[TSDB_EP_LEN + VARSTR_HEADER_SIZE];
while (numOfRows < rows) {
pShow->pIter = sdbFetchAll(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode, &objStatus, true);

1
source/dnode/mnode/impl/src/mndDump.c
View File

@ -104,6 +104,7 @@ void dumpDb(SSdb *pSdb, SJson *json) {
tjsonAddStringToObject(item, "maxRows", i642str(pObj->cfg.maxRows));
tjsonAddStringToObject(item, "precision", i642str(pObj->cfg.precision));
tjsonAddStringToObject(item, "compression", i642str(pObj->cfg.compression));
tjsonAddStringToObject(item, "encryptAlgorithm", i642str(pObj->cfg.encryptAlgorithm));
tjsonAddStringToObject(item, "replications", i642str(pObj->cfg.replications));
tjsonAddStringToObject(item, "strict", i642str(pObj->cfg.strict));
tjsonAddStringToObject(item, "cacheLast", i642str(pObj->cfg.cacheLast));

2
source/dnode/mnode/impl/src/mndGrant.c
View File

@ -58,6 +58,7 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
static int32_t mndRetrieveGrantFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndRetrieveGrantLogs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndRetrieveMachines(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndRetrieveEncryptions(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndProcessGrantHB(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; }
@ -66,6 +67,7 @@ int32_t mndInitGrant(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS_FULL, mndRetrieveGrantFull);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS_LOGS, mndRetrieveGrantLogs);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_MACHINES, mndRetrieveMachines);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ENCRYPTIONS, mndRetrieveEncryptions);
mndSetMsgHandle(pMnode, TDMT_MND_GRANT_HB_TIMER, mndProcessGrantHB);
return 0;
}

15
source/dnode/mnode/impl/src/mndMain.c
View File

@ -485,8 +485,23 @@ static int32_t mndInitWal(SMnode *pMnode) {
.retentionPeriod = 0,
.retentionSize = 0,
.level = TAOS_WAL_FSYNC,
.encryptAlgorithm = 0,
.encryptKey = {0}
};
#if defined(TD_ENTERPRISE)
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){
cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0;
if(tsEncryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
pMnode->pWal = walOpen(path, &cfg);
if (pMnode->pWal == NULL) {
mError("failed to open wal since %s. wal:%s", terrstr(), path);

2
source/dnode/mnode/impl/src/mndShow.c
View File

@ -131,6 +131,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_GRANTS_LOGS;
} else if (strncasecmp(name, TSDB_INS_TABLE_MACHINES, len) == 0) {
type = TSDB_MGMT_TABLE_MACHINES;
} else if (strncasecmp(name, TSDB_INS_TABLE_ENCRYPTIONS, len) == 0) {
type = TSDB_MGMT_TABLE_ENCRYPTIONS;
} else if (strncasecmp(name, TSDB_INS_TABLE_TSMAS, len) == 0) {
type = TSDB_MGMT_TABLE_TSMAS;
} else {

1
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -307,6 +307,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
createReq.hashSuffix = pDb->cfg.hashSuffix;
createReq.tsdbPageSize = pDb->cfg.tsdbPageSize;
createReq.changeVersion = ++(pVgroup->syncConfChangeVer);
createReq.encryptAlgorithm = pDb->cfg.encryptAlgorithm;
for (int32_t v = 0; v < pVgroup->replica; ++v) {
SReplica *pReplica = NULL;

2
source/dnode/mnode/sdb/CMakeLists.txt
View File

@ -5,5 +5,5 @@ target_include_directories(
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
sdb os common util wal sync
sdb os common util wal sync crypt
)

66
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -18,6 +18,8 @@
#include "sync.h"
#include "tchecksum.h"
#include "wal.h"
#include "tglobal.h"
#include "crypt.h"
#define SDB_TABLE_SIZE 24
#define SDB_RESERVE_SIZE 512
@ -276,6 +278,9 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
}
readLen = pRaw->dataLen + sizeof(int32_t);
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
readLen = ENCRYPTED_LEN(pRaw->dataLen) + sizeof(int32_t);
}
if (readLen >= bufLen) {
bufLen = pRaw->dataLen * 2;
SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100);
@ -303,6 +308,27 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
goto _OVER;
}
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
int32_t count = 0;
char *plantContent = taosMemoryMalloc(ENCRYPTED_LEN(pRaw->dataLen));
SCryptOpts opts;
opts.len = ENCRYPTED_LEN(pRaw->dataLen);
opts.source = pRaw->pData;
opts.result = plantContent;
opts.unitLen = 16;
strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN);
count = CBC_Decrypt(&opts);
//mDebug("read sdb, CBC_Decrypt dataLen:%d, descrypted len:%d, %s", pRaw->dataLen, count, __FUNCTION__);
memcpy(pRaw->pData, plantContent, pRaw->dataLen);
taosMemoryFree(plantContent);
memcpy(pRaw->pData + pRaw->dataLen, &pRaw->pData[ENCRYPTED_LEN(pRaw->dataLen)], sizeof(int32_t));
}
int32_t totalLen = sizeof(SSdbRaw) + pRaw->dataLen + sizeof(int32_t);
if ((!taosCheckChecksumWhole((const uint8_t *)pRaw, totalLen)) != 0) {
code = TSDB_CODE_CHECKSUM_ERROR;
@ -401,14 +427,50 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
SSdbRaw *pRaw = (*encodeFp)(pRow->pObj);
if (pRaw != NULL) {
pRaw->status = pRow->status;
int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen;
if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) {
if (taosWriteFile(pFile, pRaw, sizeof(SSdbRaw)) != sizeof(SSdbRaw)) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
break;
}
int32_t newDataLen = pRaw->dataLen;
char* newData = pRaw->pData;
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
newDataLen = ENCRYPTED_LEN(pRaw->dataLen);
newData = taosMemoryMalloc(newDataLen);
if (newData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
break;
}
SCryptOpts opts;
opts.len = newDataLen;
opts.source = pRaw->pData;
opts.result = newData;
opts.unitLen = 16;
strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN);
int32_t count = CBC_Encrypt(&opts);
//mDebug("write sdb, CBC_Encrypt encryptedDataLen:%d, dataLen:%d, %s",
// newDataLen, pRaw->dataLen, __FUNCTION__);
}
if (taosWriteFile(pFile, newData, newDataLen) != newDataLen) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
break;
}
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
taosMemoryFree(newData);
}
int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen);
if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
code = TAOS_SYSTEM_ERROR(errno);

3
source/dnode/vnode/CMakeLists.txt
View File

@ -121,6 +121,7 @@ if (${BUILD_CONTRIB})
PUBLIC "inc"
PUBLIC "src/inc"
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include"
)
@ -130,6 +131,7 @@ else()
PUBLIC "inc"
PUBLIC "src/inc"
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
)
if (${TD_LINUX})
@ -162,6 +164,7 @@ target_link_libraries(
PUBLIC scheduler
PUBLIC tdb
PUBLIC audit
PUBLIC crypt
# PUBLIC bdb
# PUBLIC scalar

4
source/dnode/vnode/inc/vnode.h
View File

@ -278,6 +278,8 @@ struct STsdbCfg {
int32_t keep2; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keepTimeOffset; // just for save config, use STsdbKeepCfg in STsdb instead
SRetention retentions[TSDB_RETENTION_MAX];
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
};
typedef struct {
@ -319,6 +321,8 @@ struct SVnodeCfg {
int16_t hashPrefix;
int16_t hashSuffix;
int32_t tsdbPageSize;
int32_t tdbEncryptAlgorithm;
char tdbEncryptKey[ENCRYPT_KEY_LEN];
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;

3
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -69,7 +69,8 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
taosMkDir(pMeta->path);
// open env
ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback);
ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback,
pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey);
if (ret < 0) {
metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;

2
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -76,7 +76,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
}
int32_t tqMetaOpen(STQ* pTq) {
if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0) < 0) {
if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL) < 0) {
return -1;
}

15
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
View File

@ -73,7 +73,9 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo
pBlock->file.maxVer = reader->config->file.maxVer;
pBlock->file.stt->level = reader->config->file.stt->level;
code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -127,8 +129,11 @@ static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFil
code = TARRAY2_APPEND(opArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
if (writer->fd) {
code = tsdbFsyncFile(writer->fd);
code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd);
}
@ -206,11 +211,13 @@ _exit:
return code;
}
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock) {
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength);
code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->offset += pDataBlock->dataLength;

3
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h
View File

@ -105,7 +105,8 @@ int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataF
int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr);
int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer);
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr);
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer);
#ifdef __cplusplus

143
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

@ -46,8 +46,10 @@ static int32_t tsdbDataFileReadHeadFooter(SDataFileReader *reader) {
int32_t ftype = TSDB_FTYPE_HEAD;
if (reader->fd[ftype]) {
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(SHeadFooter),
(uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0);
(uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -68,8 +70,10 @@ static int32_t tsdbDataFileReadTombFooter(SDataFileReader *reader) {
int32_t ftype = TSDB_FTYPE_TOMB;
if (reader->fd[ftype]) {
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(STombFooter),
(uint8_t *)reader->tombFooter, sizeof(STombFooter), 0);
(uint8_t *)reader->tombFooter, sizeof(STombFooter), 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
reader->ctx->tombFooterLoaded = true;
@ -164,8 +168,10 @@ int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **b
TSDB_CHECK_CODE(code, lino, _exit);
}
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], reader->headFooter->brinBlkPtr->offset, data,
reader->headFooter->brinBlkPtr->size, 0);
reader->headFooter->brinBlkPtr->size, 0, encryptAlgorithm, encryptKey);
if (code) {
taosMemoryFree(data);
TSDB_CHECK_CODE(code, lino, _exit);
@ -195,9 +201,12 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB
SBuffer *buffer = reader->buffers + 0;
SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data
tBufferClear(buffer);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, brinBlk->dp->size, buffer, 0);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, brinBlk->dp->size, buffer, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// decode brin block
@ -279,9 +288,12 @@ int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *re
SBuffer *buffer = reader->buffers + 0;
SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data
tBufferClear(buffer);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockSize, buffer, 0);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockSize, buffer, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// decompress
@ -307,9 +319,12 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
SBuffer *buffer1 = reader->buffers + 1;
SBuffer *assist = reader->buffers + 2;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load key part
tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockKeySize, buffer0, 0);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockKeySize, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// SDiskDataHdr
@ -340,11 +355,11 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
if (extraColIdx < 0) {
goto _exit;
}
// load SBlockCol part
tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize, hdr.szBlkCol,
buffer0, 0);
buffer0, 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// calc szHint
@ -454,11 +469,14 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
code = tBlockDataDecompressColData(&hdr, &none, &br, bData, assist);
TSDB_CHECK_CODE(code, lino, _exit);
} else if (cid == blockCol.cid) {
// load from file
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load from file
tBufferClear(buffer1);
code = tsdbReadFileToBuffer(
reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset,
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, firstRead ? szHint : 0);
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, firstRead ? szHint : 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
firstRead = false;
@ -486,7 +504,10 @@ int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *rec
TARRAY2_CLEAR(columnDataAggArray, NULL);
if (record->smaSize > 0) {
tBufferClear(buffer);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, record->smaSize, buffer, 0);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, record->smaSize, buffer, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// decode sma data
@ -525,8 +546,10 @@ int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **t
TSDB_CHECK_CODE(code, lino, _exit);
}
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], reader->tombFooter->tombBlkPtr->offset, data,
reader->tombFooter->tombBlkPtr->size, 0);
reader->tombFooter->tombBlkPtr->size, 0, encryptAlgorithm, encryptKey);
if (code) {
taosMemoryFree(data);
TSDB_CHECK_CODE(code, lino, _exit);
@ -557,7 +580,10 @@ int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombB
SBuffer *assist = reader->buffers + 1;
tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t size = 0;
@ -785,7 +811,8 @@ int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxV
}
int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmprAlg, int64_t *fileSize,
TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range) {
TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey) {
if (brinBlock->numOfRecords == 0) return 0;
int32_t code;
@ -838,7 +865,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr
tBufferClear(buffer0);
code = tCompressDataToBuffer(brinBlock->buffers[i].data, &info, buffer0, assist);
if (code) return code;
code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size);
code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey);
if (code) return code;
brinBlk.size[i] = info.compressedSize;
brinBlk.dp->size += info.compressedSize;
@ -854,7 +881,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr
tBufferClear(buffer0);
code = tCompressDataToBuffer(brinBlock->buffers[i].data, &info, buffer0, assist);
if (code) return code;
code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size);
code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey);
if (code) return code;
brinBlk.size[i] = info.compressedSize;
brinBlk.dp->size += info.compressedSize;
@ -879,10 +906,10 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr
}
// write to file
if ((code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size))) return code;
if ((code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey))) return code;
*fileSize += buffer0->size;
brinBlk.dp->size += buffer0->size;
if ((code = tsdbWriteFile(fd, *fileSize, buffer1->data, buffer1->size))) return code;
if ((code = tsdbWriteFile(fd, *fileSize, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey))) return code;
*fileSize += buffer1->size;
brinBlk.dp->size += buffer1->size;
}
@ -902,9 +929,12 @@ static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg,
&writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->buffers,
&writer->ctx->range);
&writer->ctx->range, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -995,9 +1025,11 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData
record->blockKeySize = buffers[0].size + buffers[1].size;
record->blockSize = record->blockKeySize + buffers[2].size + buffers[3].size;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
for (int i = 0; i < 4; i++) {
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_DATA], writer->files[TSDB_FTYPE_DATA].size, buffers[i].data,
buffers[i].size);
buffers[i].size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->files[TSDB_FTYPE_DATA].size += buffers[i].size;
}
@ -1017,7 +1049,8 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData
record->smaSize = buffers[0].size;
if (record->smaSize > 0) {
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, buffers[0].data, record->smaSize);
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, buffers[0].data, record->smaSize,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->files[TSDB_FTYPE_SMA].size += record->smaSize;
}
@ -1288,15 +1321,17 @@ _exit:
return code;
}
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer));
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey);
if (code) return code;
*fileSize += sizeof(*footer);
return 0;
}
int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range) {
TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code;
if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0;
@ -1348,7 +1383,7 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl
code = tCompressDataToBuffer(tombBlock->buffers[i].data, &cinfo, buffer0, assist);
if (code) return code;
code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size);
code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey);
if (code) return code;
tombBlk.size[i] = cinfo.compressedSize;
@ -1367,7 +1402,11 @@ static int32_t tsdbDataFileWriteHeadFooter(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -1383,9 +1422,12 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg,
&writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->buffers,
&writer->ctx->tombRange);
&writer->ctx->tombRange, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -1395,12 +1437,14 @@ _exit:
return code;
}
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) {
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey) {
ptr->size = TARRAY2_DATA_LEN(tombBlkArray);
if (ptr->size > 0) {
ptr->offset = *fileSize;
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size);
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size,
encryptAlgorithm, encryptKey);
if (code) {
return code;
}
@ -1416,8 +1460,11 @@ static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlk(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlkArray, writer->tombFooter->tombBlkPtr,
&writer->files[TSDB_FTYPE_TOMB].size);
&writer->files[TSDB_FTYPE_TOMB].size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -1427,8 +1474,9 @@ _exit:
return code;
}
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer));
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey);
if (code) return code;
*fileSize += sizeof(*footer);
return 0;
@ -1438,7 +1486,11 @@ static int32_t tsdbDataFileWriteTombFooter(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -1515,12 +1567,14 @@ _exit:
return code;
}
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize) {
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey) {
ASSERT(TARRAY2_SIZE(brinBlkArray) > 0);
ptr->offset = *fileSize;
ptr->size = TARRAY2_DATA_LEN(brinBlkArray);
int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size);
int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size, encryptAlgorithm,
encryptKey);
if (code) return code;
*fileSize += ptr->size;
@ -1531,8 +1585,11 @@ static int32_t tsdbDataFileWriteBrinBlk(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteBrinBlk(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlkArray, writer->headFooter->brinBlkPtr,
&writer->files[TSDB_FTYPE_HEAD].size);
&writer->files[TSDB_FTYPE_HEAD].size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -1689,10 +1746,11 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
code = TARRAY2_APPEND(opArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
}
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) {
if (writer->fd[i]) {
code = tsdbFsyncFile(writer->fd[i]);
code = tsdbFsyncFile(writer->fd[i], encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd[i]);
}
@ -1729,7 +1787,10 @@ static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) {
if (writer->files[ftype].size == 0) {
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->files[ftype].size += TSDB_FHDR_SIZE;
@ -1895,7 +1956,9 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) {
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->files[ftype].size += TSDB_FHDR_SIZE;

18
source/dnode/vnode/src/tsdb/tsdbDataFileRW.h
View File

@ -97,16 +97,22 @@ int32_t tsdbDataFileFlush(SDataFileWriter *writer);
// head
int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmprAlg, int64_t *fileSize,
TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range);
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize);
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer);
TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer,
int32_t encryptAlgorithm, char* encryptKey);
// tomb
int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record);
int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range);
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize);
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize);
TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey);
// utils
int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer);

12
source/dnode/vnode/src/tsdb/tsdbDef.h
View File

@ -33,10 +33,14 @@ typedef struct SFDataPtr {
extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn);
extern void tsdbCloseFile(STsdbFD **ppFD);
extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size);
extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint);
extern int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint);
extern int32_t tsdbFsyncFile(STsdbFD *pFD);
extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size,
int32_t encryptAlgorithm, char* encryptKey);
extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey);
extern int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey);
extern int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey);
typedef struct SColCompressInfo SColCompressInfo;
struct SColCompressInfo {

5
source/dnode/vnode/src/tsdb/tsdbFSetRAW.c
View File

@ -152,7 +152,8 @@ _exit:
return code;
}
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) {
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = 0;
int32_t lino = 0;
@ -166,7 +167,7 @@ int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeade
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr);
code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->offset += bHdr->dataLength;

3
source/dnode/vnode/src/tsdb/tsdbFSetRAW.h
View File

@ -36,7 +36,8 @@ typedef struct SFSetRAWWriter SFSetRAWWriter;
int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer);
int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr);
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr);
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm,
char* encryptKey);
#ifdef __cplusplus
}

98
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -15,6 +15,7 @@
#include "cos.h"
#include "tsdb.h"
#include "crypt.h"
#include "vnd.h"
static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
@ -169,7 +170,7 @@ void tsdbCloseFile(STsdbFD **ppFD) {
}
}
static int32_t tsdbWriteFilePage(STsdbFD *pFD) {
static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0;
if (!pFD->pFD) {
@ -197,6 +198,28 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD) {
}
taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage);
if(encryptAlgorithm == DND_CA_SM4){
//if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
unsigned char PacketData[128];
int NewLen;
int32_t count = 0;
while (count < pFD->szPage) {
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pFD->pBuf + count;
opts.result = PacketData;
opts.unitLen = 128;
//strncpy(opts.key, tsEncryptKey, 16);
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Encrypt(&opts);
memcpy(pFD->pBuf + count, PacketData, NewLen);
count += NewLen;
}
//tsdbDebug("CBC_Encrypt count:%d %s", count, __FUNCTION__);
}
n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) {
@ -214,7 +237,7 @@ _exit:
return code;
}
static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0;
// ASSERT(pgno <= pFD->szFile);
@ -274,6 +297,30 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
}
//}
if(encryptAlgorithm == DND_CA_SM4){
//if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
unsigned char PacketData[128];
int NewLen;
int32_t count = 0;
while(count < pFD->szPage)
{
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pFD->pBuf + count;
opts.result = PacketData;
opts.unitLen = 128;
//strncpy(opts.key, tsEncryptKey, 16);
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Decrypt(&opts);
memcpy(pFD->pBuf + count, PacketData, NewLen);
count += NewLen;
}
//tsdbDebug("CBC_Decrypt count:%d %s", count, __FUNCTION__);
}
// check
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED;
@ -286,7 +333,8 @@ _exit:
return code;
}
int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) {
int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
@ -295,11 +343,11 @@ int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t
do {
if (pFD->pgno != pgno) {
code = tsdbWriteFilePage(pFD);
code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
if (code) goto _exit;
if (pgno <= pFD->szFile) {
code = tsdbReadFilePage(pFD, pgno);
code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey);
if (code) goto _exit;
} else {
pFD->pgno = pgno;
@ -318,7 +366,8 @@ _exit:
return code;
}
static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) {
static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = 0;
int64_t n = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
@ -331,7 +380,7 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6
while (n < size) {
if (pFD->pgno != pgno) {
code = tsdbReadFilePage(pFD, pgno);
code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey);
if (code) goto _exit;
}
@ -523,7 +572,8 @@ _exit:
return code;
}
int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) {
int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0;
if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD);
@ -535,26 +585,28 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size,
if (pFD->lcn > 1 /*pFD->s3File && tsS3BlockSize < 0*/) {
return tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
} else {
return tsdbReadFileImp(pFD, offset, pBuf, size);
return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey);
}
_exit:
return code;
}
int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint) {
int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code;
code = tBufferEnsureCapacity(buffer, buffer->size + size);
if (code) return code;
code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint);
code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint,
encryptAlgorithm, encryptKey);
if (code) return code;
buffer->size += size;
return code;
}
int32_t tsdbFsyncFile(STsdbFD *pFD) {
int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0;
/*
if (pFD->s3File) {
@ -562,7 +614,7 @@ int32_t tsdbFsyncFile(STsdbFD *pFD) {
return code;
}
*/
code = tsdbWriteFilePage(pFD);
code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
if (code) goto _exit;
if (taosFsyncFile(pFD->pFD) < 0) {
@ -673,7 +725,9 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
if (code) goto _err;
// read
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
// decode
@ -710,7 +764,9 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
if (code) goto _err;
// read
code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
// decode
@ -743,7 +799,9 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
if (code) goto _err;
// read
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
// decode
@ -836,7 +894,9 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
if (code) goto _err;
// read
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
// // decode
@ -876,7 +936,9 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
if (code) goto _err;
// read
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
// decode

5
source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
View File

@ -545,7 +545,10 @@ static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbD
int32_t code = 0;
int32_t lino = 0;
code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr);
int32_t encryptAlgorithm = writer->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:

112
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -63,7 +63,10 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
int64_t offset = config->file->size - sizeof(SSttFooter);
ASSERT(offset >= TSDB_FHDR_SIZE);
code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0);
int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, encryptAlgoirthm,
encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -99,8 +102,11 @@ int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray *
void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code =
tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0);
tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0,
encryptAlgorithm, encryptKey);
if (code) {
taosMemoryFree(data);
return code;
@ -127,8 +133,11 @@ int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tom
void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code =
tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0);
tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0,
encryptAlgorithm, encryptKey);
if (code) {
taosMemoryFree(data);
return code;
@ -155,8 +164,11 @@ int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBl
void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code =
tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0);
tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0,
encryptAlgorithm, encryptKey);
if (code) {
taosMemoryFree(data);
return code;
@ -181,9 +193,12 @@ int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk,
SBuffer *buffer0 = reader->buffers + 0;
SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data
tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
@ -207,9 +222,12 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
SBuffer *buffer1 = reader->buffers + 1;
SBuffer *assist = reader->buffers + 2;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load key part
tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// decode header
@ -244,7 +262,8 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
// load SBlockCol part
tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// load each column
@ -290,7 +309,8 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
tBufferClear(buffer1);
code =
tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset,
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0);
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// decode the buffer
@ -314,9 +334,12 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk
SBuffer *buffer0 = reader->buffers + 0;
SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load
tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0);
code = tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// decode
@ -351,9 +374,12 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
SBuffer *buffer0 = reader->buffers + 0;
SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data
tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0);
code = tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// decode data
@ -442,7 +468,7 @@ struct SSttFileWriter {
static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, SColCompressInfo *info,
int64_t *fileSize, TSttBlkArray *sttBlkArray, SBuffer *buffers,
SVersionRange *range) {
SVersionRange *range, int32_t encryptAlgorithm, char* encryptKey) {
if (blockData->nRow == 0) return 0;
int32_t code = 0;
@ -474,7 +500,7 @@ static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, S
sttBlk->bInfo.szBlock = buffers[2].size + buffers[3].size + sttBlk->bInfo.szKey;
for (int i = 0; i < 4; i++) {
if (buffers[i].size) {
code = tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size);
code = tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size, encryptAlgorithm, encryptKey);
if (code) return code;
*fileSize += buffers[i].size;
}
@ -498,8 +524,11 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) {
SColCompressInfo info = {.defaultCmprAlg = writer->config->cmprAlg, .pColCmpr = NULL};
code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr));
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size, writer->sttBlkArray,
writer->buffers, &writer->ctx->range);
writer->buffers, &writer->ctx->range,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -538,6 +567,9 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
statisBlk.maxTbid.suid = record.suid;
statisBlk.maxTbid.uid = record.uid;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// compress each column
for (int32_t i = 0; i < ARRAY_SIZE(statisBlk.size); i++) {
SCompressInfo info = {
@ -550,7 +582,8 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
code = tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize);
code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
statisBlk.size[i] = info.compressedSize;
@ -579,12 +612,12 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size);
code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += buffer0->size;
statisBlk.dp->size += buffer0->size;
code = tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size);
code = tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += buffer1->size;
statisBlk.dp->size += buffer1->size;
@ -608,8 +641,12 @@ static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size,
writer->tombBlkArray, writer->buffers, &writer->ctx->range);
writer->tombBlkArray, writer->buffers, &writer->ctx->range,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -619,12 +656,14 @@ _exit:
return code;
}
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize) {
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey) {
ptr->size = TARRAY2_DATA_LEN(sttBlkArray);
if (ptr->size > 0) {
ptr->offset = *fileSize;
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size);
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size, encryptAlgorithm,
encryptKey);
if (code) {
return code;
}
@ -638,7 +677,11 @@ static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) {
int32_t code = 0;
int32_t lino;
code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -651,12 +694,14 @@ _exit:
static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) {
int32_t code = 0;
int32_t lino;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray);
if (writer->footer->statisBlkPtr->size) {
writer->footer->statisBlkPtr->offset = writer->file->size;
code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray),
writer->footer->statisBlkPtr->size);
writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += writer->footer->statisBlkPtr->size;
}
@ -672,7 +717,11 @@ static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -682,15 +731,19 @@ _exit:
return code;
}
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer));
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey);
if (code) return code;
*fileSize += sizeof(*footer);
return 0;
}
static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) {
return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size, encryptAlgorithm, encryptKey);
}
static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
@ -728,7 +781,10 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr));
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += sizeof(hdr);
@ -793,7 +849,9 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o
code = tsdbSttFileDoUpdateHeader(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFsyncFile(writer->fd);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd);

6
source/dnode/vnode/src/tsdb/tsdbSttFileRW.h
View File

@ -71,8 +71,10 @@ int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *pBlockData
int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record);
bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer);
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize);
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize);
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
char* encryptKey);
struct SSttFileWriterConfig {
STsdb *tsdb;

53
source/dnode/vnode/src/tsdb/tsdbUpgrade.c
View File

@ -37,6 +37,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
int8_t cmprAlg;
int32_t szPage;
SBuffer buffers[10];
int32_t encryptAlgorithm;
char* encryptKey;
// reader
SArray *aBlockIdx;
SMapData mDataBlk[1];
@ -50,6 +52,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
.maxRow = tsdb->pVnode->config.tsdbCfg.maxRows,
.cmprAlg = tsdb->pVnode->config.tsdbCfg.compression,
.szPage = tsdb->pVnode->config.tsdbPageSize,
.encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm,
.encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey,
}};
// read SBlockIdx array
@ -136,7 +140,7 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
if (ctx->brinBlock->numOfRecords >= ctx->maxRow) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
ctx->brinBlkArray, ctx->buffers, &range);
ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -145,18 +149,19 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
if (ctx->brinBlock->numOfRecords > 0) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
ctx->brinBlkArray, ctx->buffers, &range);
ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr,
&fset->farr[TSDB_FTYPE_HEAD]->f->size);
&fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer);
code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFsyncFile(ctx->fd);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&ctx->fd);
@ -252,12 +257,16 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
STFileObj *fobj;
struct {
int32_t szPage;
int32_t encryptAlgorithm;
char* encryptKey;
// writer
STsdbFD *fd;
TSttBlkArray sttBlkArray[1];
SSttFooter footer[1];
} ctx[1] = {{
.szPage = tsdb->pVnode->config.tsdbPageSize,
.encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm,
.encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey,
}};
STFile file = {
@ -280,13 +289,14 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
TSDB_CHECK_CODE(code, lino, _exit1);
}
code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size);
code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbFsyncFile(ctx->fd);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1);
tsdbCloseFile(&ctx->fd);
@ -431,7 +441,10 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE);
int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
fobj[0]->f->size += TSDB_FHDR_SIZE;
@ -454,6 +467,8 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
int64_t minKey;
int64_t maxKey;
SBuffer buffers[10];
int32_t encryptAlgorithm;
char* encryptKey;
// reader
SArray *aDelData;
// writer
@ -466,6 +481,8 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
} ctx[1] = {{
.maxRow = tsdb->pVnode->config.tsdbCfg.maxRows,
.cmprAlg = tsdb->pVnode->config.tsdbCfg.compression,
.encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm,
.encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey,
}};
tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey);
@ -502,7 +519,7 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
}
SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
ctx->buffers, &tombRange);
ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -515,26 +532,30 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
}
SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
ctx->buffers, &tombRange);
ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
if (ctx->fd != NULL) {
if (ctx->toStt) {
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size);
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size,
ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size);
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size,
ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size);
code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbFsyncFile(ctx->fd);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&ctx->fd);

43
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -15,6 +15,7 @@
#include "tutil.h"
#include "vnd.h"
#include "tglobal.h"
const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.dbname = "",
@ -133,6 +134,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
tjsonAddItemToArray(pNodeRetentions, pNodeRetention);
}
}
if (tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;
@ -140,6 +142,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
@ -242,6 +246,19 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code);
tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code);
}
tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code);
@ -256,6 +273,32 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code);
if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER;
tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);

2
source/libs/CMakeLists.txt
View File

@ -9,6 +9,8 @@ add_subdirectory(nodes)
add_subdirectory(catalog)
add_subdirectory(audit)
add_subdirectory(monitorfw)
add_subdirectory(sm4)
add_subdirectory(crypt)
add_subdirectory(scalar)
add_subdirectory(function)

16
source/libs/command/src/command.c
View File

@ -315,6 +315,18 @@ static const char* cacheModelStr(int8_t cacheModel) {
return TSDB_CACHE_MODEL_NONE_STR;
}
static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) {
switch (encryptAlgorithm) {
case TSDB_ENCRYPT_ALGO_NONE:
return TSDB_ENCRYPT_ALGO_NONE_STR;
case TSDB_ENCRYPT_ALGO_SM4:
return TSDB_ENCRYPT_ALGO_SM4_STR;
default:
break;
}
return TSDB_CACHE_MODEL_NONE_STR;
}
static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) {
blockDataEnsureCapacity(pBlock, 1);
pBlock->info.rows = 1;
@ -360,12 +372,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
"CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm "
"WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
"WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d "
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d",
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d",
dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize,
pCfg->keepTimeOffset, pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact);
pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact);
if (retentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);

14
source/libs/crypt/CMakeLists.txt Normal file
View File

@ -0,0 +1,14 @@
aux_source_directory(src CRYPT_SRC)
IF (TD_ENTERPRISE)
LIST(APPEND CRYPT_SRC ${TD_ENTERPRISE_DIR}/src/plugins/crypt/cryptImpl.c)
ENDIF ()
add_library(crypt STATIC ${CRYPT_SRC})
target_include_directories(
crypt
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(crypt common sm4)

36
source/libs/crypt/src/crypt.c Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "crypt.h"
extern int32_t CBC_DecryptImpl(SCryptOpts *opts);
extern int32_t CBC_EncryptImpl(SCryptOpts *opts);
int32_t CBC_Encrypt(SCryptOpts *opts) {
return CBC_EncryptImpl(opts);
}
int32_t CBC_Decrypt(SCryptOpts *opts) {
return CBC_DecryptImpl(opts);
}
#ifndef TD_ENTERPRISE
int32_t CBC_EncryptImpl(SCryptOpts *opts) {
memcpy(opts->result, opts->source, opts->len);
return opts->len;
}
int32_t CBC_DecryptImpl(SCryptOpts *opts) {
memcpy(opts->result, opts->source, opts->len);
return opts->len;
}
#endif

10
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -281,6 +281,8 @@ const char* nodesNodeName(ENodeType type) {
return "ShowGrantsLogsStmt";
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return "ShowClusterMachinesStmt";
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return "ShowEncryptionsStmt";
case QUERY_NODE_DELETE_STMT:
return "DeleteStmt";
case QUERY_NODE_INSERT_STMT:
@ -7047,10 +7049,14 @@ static int32_t showGrantsLogsStmtToJson(const void* pObj, SJson* pJson) { return
static int32_t showClusterMachinesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t showEncryptionsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t jsonToShowGrantsLogsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t jsonToShowClusterMachinesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t jsonToShowEncryptionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static const char* jkShowDnodeVariablesStmtDnodeId = "DnodeId";
static const char* jkShowDnodeVariablesStmtLikePattern = "LikePattern";
@ -7651,6 +7657,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return showGrantsLogsStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return showClusterMachinesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return showEncryptionsStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT:
return showDnodeVariablesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
@ -7996,6 +8004,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToShowGrantsLogsStmt(pJson, pObj);
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return jsonToShowClusterMachinesStmt(pJson, pObj);
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return jsonToShowEncryptionsStmt(pJson, pObj);
case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT:
return jsonToShowDnodeVariablesStmt(pJson, pObj);
case QUERY_NODE_SHOW_TRANSACTIONS_STMT:

2
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -521,6 +521,7 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT:
return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
@ -1210,6 +1211,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT: {
SShowStmt* pStmt = (SShowStmt*)pNode;
nodesDestroyNode(pStmt->pDbName);

4
source/libs/parser/inc/parAst.h
View File

@ -67,7 +67,8 @@ typedef enum EDatabaseOptionType {
DB_OPTION_S3_CHUNKSIZE,
DB_OPTION_S3_KEEPLOCAL,
DB_OPTION_S3_COMPACT,
DB_OPTION_KEEP_TIME_OFFSET
DB_OPTION_KEEP_TIME_OFFSET,
DB_OPTION_ENCRYPT_ALGORITHM,
} EDatabaseOptionType;
typedef enum ETableOptionType {
@ -232,6 +233,7 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force, bool unsafe);
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue);
SNode* createEncryptKeyStmt(SAstCreateContext* pCxt, const SToken* pValue);
SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName);
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName,
SNode* pRealTable, SNodeList* pCols, SNode* pOptions);

9
source/libs/parser/inc/sql.y
View File

@ -144,6 +144,9 @@ priv_level(A) ::= topic_name(B).
with_opt(A) ::= . { A = NULL; }
with_opt(A) ::= WITH search_condition(B). { A = B; }
/************************************************ create encrypt_key *********************************************/
cmd ::= CREATE ENCRYPT_KEY NK_STRING(A). { pCxt->pRootNode = createEncryptKeyStmt(pCxt, &A); }
/************************************************ create/drop/alter/restore dnode *********************************************/
cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); }
cmd ::= CREATE DNODE dnode_endpoint(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); }
@ -266,6 +269,7 @@ db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_INTEGER(C).
db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_COMPACT, &C); }
db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); }
db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_ENCRYPT_ALGORITHM, &C); }
alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); }
@ -300,6 +304,7 @@ alter_db_option(A) ::= S3_KEEPLOCAL NK_INTEGER(B).
alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; }
alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; }
alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; }
alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; }
%type integer_list { SNodeList* }
%destructor integer_list { nodesDestroyList($$); }
@ -516,7 +521,9 @@ cmd ::= SHOW GRANTS LOGS.
cmd ::= SHOW CLUSTER MACHINES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT); }
cmd ::= SHOW CREATE DATABASE db_name(A). { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &A); }
cmd ::= SHOW CREATE TABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, A); }
cmd ::= SHOW CREATE STABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, A); }
cmd ::= SHOW CREATE STABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT,
A); }
cmd ::= SHOW ENCRYPTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_ENCRYPTIONS_STMT); }
cmd ::= SHOW QUERIES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); }
cmd ::= SHOW SCORES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); }
cmd ::= SHOW TOPICS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); }

14
source/libs/parser/src/parAstCreater.c
View File

@ -1375,6 +1375,7 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
pOptions->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pOptions->s3Compact = TSDB_DEFAULT_S3_COMPACT;
pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
pOptions->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
return (SNode*)pOptions;
}
@ -1414,6 +1415,7 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
pOptions->s3KeepLocal = -1;
pOptions->s3Compact = -1;
pOptions->withArbitrator = -1;
pOptions->encryptAlgorithm = -1;
return (SNode*)pOptions;
}
@ -1547,6 +1549,10 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
case DB_OPTION_KEEP_TIME_OFFSET: {
pDbOptions->keepTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_ENCRYPT_ALGORITHM:
COPY_STRING_FORM_STR_TOKEN(pDbOptions->encryptAlgorithmStr, (SToken*)pVal);
pDbOptions->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
break;
}
default:
break;
@ -2334,6 +2340,14 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const
return (SNode*)pStmt;
}
SNode* createEncryptKeyStmt(SAstCreateContext* pCxt, const SToken* pValue) {
SToken config;
config.type = TK_NK_STRING;
config.z = "\"encrypt_key\"";
config.n = strlen(config.z);
return createAlterDnodeStmt(pCxt, NULL, &config, pValue);
}
SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName) {
if (!checkIndexName(pCxt, pIndexName)) {
return NULL;

7
source/libs/parser/src/parAstParser.c
View File

@ -644,6 +644,11 @@ static int32_t collectMetaKeyFromShowClusterMachines(SCollectMetaKeyCxt* pCxt, S
pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromShowEncryptions(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_ENCRYPTIONS,
pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) {
return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
}
@ -916,6 +921,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromShowGrantsLogs(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return collectMetaKeyFromShowClusterMachines(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return collectMetaKeyFromShowEncryptions(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_TABLE_STMT:

1
source/libs/parser/src/parAuthenticator.c
View File

@ -356,6 +356,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ARBGROUPS_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return !pCxt->pParseCxt->enableSysInfo ? TSDB_CODE_PAR_PERMISSION_DENIED : TSDB_CODE_SUCCESS;
case QUERY_NODE_SHOW_TABLES_STMT:
case QUERY_NODE_SHOW_STABLES_STMT:

3
source/libs/parser/src/parTokenizer.c
View File

@ -94,6 +94,9 @@ static SKeyword keywordTable[] = {
{"DURATION", TK_DURATION},
{"ELSE", TK_ELSE},
{"ENABLE", TK_ENABLE},
{"ENCRYPTIONS", TK_ENCRYPTIONS},
{"ENCRYPT_ALGORITHM", TK_ENCRYPT_ALGORITHM},
{"ENCRYPT_KEY", TK_ENCRYPT_KEY},
{"END", TK_END},
{"EXISTS", TK_EXISTS},
{"EXPIRED", TK_EXPIRED},

59
source/libs/parser/src/parTranslater.c
View File

@ -305,7 +305,15 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
{ .showType = QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_TABLE_MACHINES,
},
.numOfShowCols = 1,
.pShowCols = {"*"}
},
{ .showType = QUERY_NODE_SHOW_ENCRYPTIONS_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_TABLE_ENCRYPTIONS,
.numOfShowCols = 1,
.pShowCols = {"*"}
},
{ .showType = QUERY_NODE_SHOW_TSMAS_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_TABLE_TSMAS,
@ -6501,6 +6509,7 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
pReq->s3Compact = pStmt->pOptions->s3Compact;
pReq->ignoreExist = pStmt->ignoreExists;
pReq->withArbitrator = pStmt->pOptions->withArbitrator;
pReq->encryptAlgorithm = pStmt->pOptions->encryptAlgorithm;
return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
}
@ -6645,6 +6654,21 @@ static int32_t checkDbCacheModelOption(STranslateContext* pCxt, SDatabaseOptions
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbEncryptAlgorithmOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if ('\0' != pOptions->encryptAlgorithmStr[0]) {
if (0 == strcasecmp(pOptions->encryptAlgorithmStr, TSDB_ENCRYPT_ALGO_NONE_STR)) {
pOptions->encryptAlgorithm = TSDB_ENCRYPT_ALGO_NONE;
} else if (0 == strcasecmp(pOptions->encryptAlgorithmStr, TSDB_ENCRYPT_ALGO_SM4_STR)) {
pOptions->encryptAlgorithm = TSDB_ENCRYPT_ALGO_SM4;
} else {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option encrypt_algorithm: %s", pOptions->encryptAlgorithmStr);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if ('\0' != pOptions->precisionStr[0]) {
if (0 == strcasecmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MILLI_STR)) {
@ -6838,6 +6862,9 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
if (TSDB_CODE_SUCCESS == code) {
code = checkDbCacheModelOption(pCxt, pOptions);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEncryptAlgorithmOption(pCxt, pOptions);
}
if (TSDB_CODE_SUCCESS == code) {
code =
checkDbRangeOption(pCxt, "cacheSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_SIZE, TSDB_MAX_DB_CACHE_SIZE);
@ -6924,6 +6951,10 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
code = checkDbEnumOption(pCxt, "withArbitrator", pOptions->withArbitrator, TSDB_MIN_DB_WITH_ARBITRATOR,
TSDB_MAX_DB_WITH_ARBITRATOR);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "encryptAlgorithm", pOptions->encryptAlgorithm, TSDB_MIN_ENCRYPT_ALGO,
TSDB_MAX_ENCRYPT_ALGO);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix);
}
@ -7070,6 +7101,11 @@ static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq)
FILL_CMD_SQL(sql, sqlLen, pCmdReq, SMCfgClusterReq, pReq);
break;
}
case TDMT_MND_CREATE_ENCRYPT_KEY: {
FILL_CMD_SQL(sql, sqlLen, pCmdReq, SMCfgDnodeReq, pReq);
break;
}
default: {
break;
}
@ -7157,6 +7193,11 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm
return code;
}
if (pStmt->pOptions->encryptAlgorithm != -1) {
code = TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE;
return code;
}
SAlterDbReq alterReq = {0};
buildAlterDbReq(pCxt, pStmt, &alterReq);
@ -8378,7 +8419,20 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt
strcpy(cfgReq.config, pStmt->config);
strcpy(cfgReq.value, pStmt->value);
int32_t code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq);
int32_t code = 0;
if (0 == strncasecmp(cfgReq.config, "encrypt_key", 12)) {
int32_t klen = strlen(cfgReq.value);
if (klen > ENCRYPT_KEY_LEN || klen < ENCRYPT_KEY_LEN_MIN) {
tFreeSMCfgDnodeReq(&cfgReq);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_ENCRYPT_KLEN,
"Invalid encryption key length: %d, valid range [%d,%d]", klen,
ENCRYPT_KEY_LEN_MIN, ENCRYPT_KEY_LEN);
}
code = buildCmdMsg(pCxt, TDMT_MND_CREATE_ENCRYPT_KEY, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq);
} else {
code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq);
}
tFreeSMCfgDnodeReq(&cfgReq);
return code;
}
@ -13117,6 +13171,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ARBGROUPS_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT:
code = rewriteShow(pCxt, pQuery);
break;

12022
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

9
source/libs/sm4/CMakeLists.txt Normal file
View File

@ -0,0 +1,9 @@
aux_source_directory(src SM4_SRC)
add_library(sm4 STATIC ${SM4_SRC})
target_include_directories(
sm4
PUBLIC "${TD_SOURCE_DIR}/include/libs/sm4"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(sm4 common)

544
source/libs/sm4/src/sm4.c Normal file
View File

@ -0,0 +1,544 @@
#include "sm4.h"
#define SM4_ROUND 32
static unsigned int FK[4]={
0xA3B1BAC6,0x56AA3350,0x677D9197,0xB27022DC
};
static unsigned int CK[SM4_ROUND]={
0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269,
0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9,
0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249,
0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9,
0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229,
0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299,
0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209,
0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279
};
static unsigned char Sbox[256]={
0xd6,0x90,0xe9,0xfe,0xcc,0xe1,0x3d,0xb7,0x16,0xb6,0x14,0xc2,0x28,0xfb,0x2c,0x05,
0x2b,0x67,0x9a,0x76,0x2a,0xbe,0x04,0xc3,0xaa,0x44,0x13,0x26,0x49,0x86,0x06,0x99,
0x9c,0x42,0x50,0xf4,0x91,0xef,0x98,0x7a,0x33,0x54,0x0b,0x43,0xed,0xcf,0xac,0x62,
0xe4,0xb3,0x1c,0xa9,0xc9,0x08,0xe8,0x95,0x80,0xdf,0x94,0xfa,0x75,0x8f,0x3f,0xa6,
0x47,0x07,0xa7,0xfc,0xf3,0x73,0x17,0xba,0x83,0x59,0x3c,0x19,0xe6,0x85,0x4f,0xa8,
0x68,0x6b,0x81,0xb2,0x71,0x64,0xda,0x8b,0xf8,0xeb,0x0f,0x4b,0x70,0x56,0x9d,0x35,
0x1e,0x24,0x0e,0x5e,0x63,0x58,0xd1,0xa2,0x25,0x22,0x7c,0x3b,0x01,0x21,0x78,0x87,
0xd4,0x00,0x46,0x57,0x9f,0xd3,0x27,0x52,0x4c,0x36,0x02,0xe7,0xa0,0xc4,0xc8,0x9e,
0xea,0xbf,0x8a,0xd2,0x40,0xc7,0x38,0xb5,0xa3,0xf7,0xf2,0xce,0xf9,0x61,0x15,0xa1,
0xe0,0xae,0x5d,0xa4,0x9b,0x34,0x1a,0x55,0xad,0x93,0x32,0x30,0xf5,0x8c,0xb1,0xe3,
0x1d,0xf6,0xe2,0x2e,0x82,0x66,0xca,0x60,0xc0,0x29,0x23,0xab,0x0d,0x53,0x4e,0x6f,
0xd5,0xdb,0x37,0x45,0xde,0xfd,0x8e,0x2f,0x03,0xff,0x6a,0x72,0x6d,0x6c,0x5b,0x51,
0x8d,0x1b,0xaf,0x92,0xbb,0xdd,0xbc,0x7f,0x11,0xd9,0x5c,0x41,0x1f,0x10,0x5a,0xd8,
0x0a,0xc1,0x31,0x88,0xa5,0xcd,0x7b,0xbd,0x2d,0x74,0xd0,0x12,0xb8,0xe5,0xb4,0xb0,
0x89,0x69,0x97,0x4a,0x0c,0x96,0x77,0x7e,0x65,0xb9,0xf1,0x09,0xc5,0x6e,0xc6,0x84,
0x18,0xf0,0x7d,0xec,0x3a,0xdc,0x4d,0x20,0x79,0xee,0x5f,0x3e,0xd7,0xcb,0x39,0x48
};
#define ROL(x,y) ((x)<<(y) | (x)>>(32-(y)))
unsigned int SMS4_T1(unsigned int dwA)
{
unsigned char a0[4]={0};
unsigned char b0[4]={0};
unsigned int dwB=0;
unsigned int dwC=0;
int i=0;
/*
for (i=0;i<4;i++)
{
a0[i] = (unsigned char)((dwA>>(i*8)) & 0xff);
b0[i] = Sbox[a0[i]];
dwB |= (b0[i]<<(i*8));
}
*/
a0[0] = (unsigned char)((dwA) & 0xff);
b0[0] = Sbox[a0[0]];
dwB |= (b0[0]);
a0[1] = (unsigned char)((dwA>>(8)) & 0xff);
b0[1] = Sbox[a0[1]];
dwB |= (b0[1]<<(8));
a0[2] = (unsigned char)((dwA>>(16)) & 0xff);
b0[2] = Sbox[a0[2]];
dwB |= (b0[2]<<(16));
a0[3] = (unsigned char)((dwA>>(24)) & 0xff);
b0[3] = Sbox[a0[3]];
dwB |= (b0[3]<<(24)); // 2013-08-13
dwC=dwB^ROL(dwB,2)^ROL(dwB,10)^ROL(dwB,18)^ROL(dwB,24);
return dwC;
}
unsigned int SMS4_T2(unsigned int dwA)
{
unsigned char a0[4]={0};
unsigned char b0[4]={0};
unsigned int dwB=0;
unsigned int dwC=0;
int i=0;
/*
for (i=0;i<4;i++)
{
a0[i] = (unsigned char)((dwA>>(i*8)) & 0xff);
b0[i] = Sbox[a0[i]];
dwB |= (b0[i]<<(i*8));
}
*/
a0[0] = (unsigned char)((dwA) & 0xff);
b0[0] = Sbox[a0[0]];
dwB |= (b0[0]);
a0[1] = (unsigned char)((dwA>>(8)) & 0xff);
b0[1] = Sbox[a0[1]];
dwB |= (b0[1]<<(8));
a0[2] = (unsigned char)((dwA>>(16)) & 0xff);
b0[2] = Sbox[a0[2]];
dwB |= (b0[2]<<(16));
a0[3] = (unsigned char)((dwA>>(24)) & 0xff);
b0[3] = Sbox[a0[3]];
dwB |= (b0[3]<<(24)); // 2013-08-13
dwC=dwB^ROL(dwB,13)^ROL(dwB,23);
return dwC;
}
/* MK[4] is the Encrypt Key, rk[32] is Round Key */
void SMS4_Key_Expansion(unsigned int MK[], unsigned int rk[])
{
unsigned int K[4]={0};
int i=0;
for (i=0;i<4;i++)
{
K[i] = MK[i] ^ FK[i];
}
for (i=0;i<SM4_ROUND;i++)
{
K[i%4]^=SMS4_T2(K[(i+1)%4]^K[(i+2)%4]^K[(i+3)%4]^CK[i]);
rk[i]=K[i%4];
}
}
/* X[4] is PlainText, rk[32] is round Key, Y[4] is CipherText */
void SMS4_ECB_Encryption_Core(unsigned int X[], unsigned int rk[], unsigned int Y[])
{
unsigned int tempX[4]={0};
int i=0;
/*
for (i=0;i<4;i++)
{
tempX[i]=X[i];
}
*/
tempX[0]=X[0];
tempX[1]=X[1];
tempX[2]=X[2];
tempX[3]=X[3];
/*
for (i=0;i<SM4_ROUND;i++)
{
tempX[i%4]^=SMS4_T1(tempX[(i+1)%4]^tempX[(i+2)%4]^tempX[(i+3)%4]^rk[i]);
}
*/
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[0]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[1]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[2]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[3]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[4]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[5]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[6]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[7]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[8]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[9]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[10]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[11]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[12]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[13]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[14]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[15]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[16]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[17]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[18]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[19]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[20]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[21]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[22]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[23]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[24]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[25]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[26]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[27]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[28]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[29]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[30]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[31]);
/* for (i=0;i<4;i++)
{
Y[i]=tempX[3-i];
}*/
Y[0]=tempX[3];
Y[1]=tempX[2];
Y[2]=tempX[1];
Y[3]=tempX[0]; // 2013-08-19
}
/* X[4] is PlainText, rk[32] is round Key, Y[4] is CipherText */
void SMS4_ECB_Decryption_Core(unsigned int X[], unsigned int rk[], unsigned int Y[])
{
unsigned int tempX[4]={0};
int i=0;
/* for (i=0;i<4;i++)
{
tempX[i]=X[i];
}
*/
tempX[0]=X[0];
tempX[1]=X[1];
tempX[2]=X[2];
tempX[3]=X[3]; // 2013-08-19
/* for (i=0;i<SM4_ROUND;i++)
{
tempX[i%4]^=SMS4_T1(tempX[(i+1)%4]^tempX[(i+2)%4]^tempX[(i+3)%4]^rk[(31-i)]);
}
*/
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[31]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[30]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[29]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[28]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[27]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[26]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[25]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[24]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[23]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[22]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[21]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[20]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[19]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[18]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[17]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[16]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[15]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[14]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[13]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[12]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[11]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[10]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[9]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[8]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[7]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[6]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[5]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[4]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[3]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[2]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[1]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[0]);// 2013-08-19
/* for (i=0;i<4;i++)
{
Y[i]=tempX[3-i];
}*/
Y[0]=tempX[3];
Y[1]=tempX[2];
Y[2]=tempX[1];
Y[3]=tempX[0]; // 2013-08-19
}
void SMS4_convert_to_network_order(unsigned int* src,unsigned int* dst,int count)
{
int i=0;
for ( ; i<count; i++ )
{
unsigned char* ps = (unsigned char*)(src+i);
unsigned char* pd = (unsigned char*)(dst+i);
pd[0] = ps[3];
pd[1] = ps[2];
pd[2] = ps[1];
pd[3] = ps[0];
}
}
void SMS4_convert_to_host_order(unsigned int* src,unsigned int* dst,int count)
{
SMS4_convert_to_network_order(src,dst,count);
}
void SMS4_ECB_Encryption(unsigned char plaintext[16], unsigned char key[16], unsigned char ciphertext[16])
{
unsigned int _pt[4];
unsigned int _ky[4];
unsigned int _ct[4];
unsigned int _rk[32];
SMS4_convert_to_network_order((unsigned int*)plaintext,_pt,4);
SMS4_convert_to_network_order((unsigned int*)key,_ky,4);
SMS4_Key_Expansion(_ky,_rk);
SMS4_ECB_Encryption_Core(_pt,_rk,_ct);
SMS4_convert_to_host_order(_ct,(unsigned int*)ciphertext,4);
}
void Key_Expansion_init( unsigned char key[16], unsigned int rk[32])
{
unsigned int _ky[4];
SMS4_convert_to_network_order((unsigned int*)key,_ky,4);
SMS4_Key_Expansion (_ky,rk);
}
void SMS4_ECB_EncryptionEx(unsigned char plaintext[16], unsigned int key[32], unsigned char ciphertext[16])
{
unsigned int _pt[4];
unsigned int _ct[4];
SMS4_convert_to_network_order((unsigned int*)plaintext,_pt,4);
SMS4_ECB_Encryption_Core(_pt,key,_ct);
SMS4_convert_to_host_order(_ct,(unsigned int*)ciphertext,4);
}
void SMS4_ECB_Decryption(unsigned char ciphertext[16], unsigned char key[16], unsigned char plaintext[16])
{
unsigned int _ct[4];
unsigned int _ky[4];
unsigned int _pt[4];
unsigned int _rk[32];
SMS4_convert_to_network_order((unsigned int*)ciphertext,_ct,4);
SMS4_convert_to_network_order((unsigned int*)key,_ky,4);
SMS4_Key_Expansion(_ky,_rk);
SMS4_ECB_Decryption_Core(_ct,_rk,_pt);
SMS4_convert_to_host_order(_pt,(unsigned int*)plaintext,4);
}
void SMS4_ECB_DecryptionEx(unsigned char ciphertext[16], unsigned int key[32], unsigned char plaintext[16])
{
unsigned int _ct[4];
unsigned int _pt[4];
SMS4_convert_to_network_order((unsigned int*)ciphertext,_ct,4);
SMS4_ECB_Decryption_Core(_ct,key,_pt);
SMS4_convert_to_host_order(_pt,(unsigned int*)plaintext,4);
}
void SMS4_CBC_Encryption(unsigned char plaintext[16], unsigned char key[16], unsigned char iv[16], unsigned char ciphertext[16])
{
unsigned char plaintextNew[16];
int i = 0;
for (i = 0; i < 16; i ++)
{
plaintextNew[i] = plaintext[i] ^ iv[i];
}
SMS4_ECB_Encryption(plaintextNew, key, ciphertext);
}
void SMS4_CBC_Decryption(unsigned char ciphertext[16], unsigned char key[16], unsigned char iv[16], unsigned char plaintext[16])
{
unsigned char plaintextTemp[16];
int i = 0;
SMS4_ECB_Decryption(ciphertext, key, plaintextTemp);
for (i = 0; i < 16; i ++)
{
plaintext[i] = plaintextTemp[i] ^ iv[i];
}
}
void SMS4_CBC_EncryptionEx(unsigned char plaintext[16], unsigned int key[32], unsigned char iv[16], unsigned char ciphertext[16])
{
unsigned char plaintextNew[16];
int i = 0;
for (i = 0; i < 16; i ++)
{
plaintextNew[i] = plaintext[i] ^ iv[i];
}
SMS4_ECB_EncryptionEx(plaintextNew, key, ciphertext);
}
void SMS4_CBC_DecryptionEx(unsigned char ciphertext[16], unsigned int key[32], unsigned char iv[16], unsigned char plaintext[16])
{
unsigned char plaintextTemp[16];
int i = 0;
SMS4_ECB_DecryptionEx(ciphertext, key, plaintextTemp);
for (i = 0; i < 16; i ++)
{
plaintext[i] = plaintextTemp[i] ^ iv[i];
}
}
/*
int SM4_ECB_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
//int rv = 0;
int loop = 0;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
for (i = 0; i < loop; i ++)
{
//SMS4_ECB_Encryption(pInData + i * 16, pKey, pOutData + i * 16);
SMS4_ECB_EncryptionEx(pInData + i * 16, rk, pOutData + i * 16);
}
*pOutDataLen = inDataLen;
return 0;
}
int SM4_ECB_Decrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
// int rv = 0;
int loop = 0;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
for (i = 0; i < loop; i ++)
{
//SMS4_ECB_Decryption(pInData + i * 16, pKey, pOutData + i * 16);
SMS4_ECB_DecryptionEx(pInData + i * 16, rk, pOutData + i * 16);
}
*pOutDataLen = inDataLen;
return 0;
}
*/
int SM4_CBC_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
// int rv = 0;
int loop = 0;
unsigned char *pIVTemp = NULL;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
if(ivLen != 16)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
pIVTemp = pIV;
for (i = 0; i < loop; i ++)
{
SMS4_CBC_EncryptionEx(pInData + i * 16, rk, pIVTemp, pOutData + i * 16);
pIVTemp = pOutData + i * 16;
}
*pOutDataLen = inDataLen;
return 0;
}
int SM4_CBC_Decrypt(unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
// int rv = 0;
int loop = 0;
unsigned char *pIVTemp = NULL;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
if(ivLen != 16)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
pIVTemp = pIV;
for (i = 0; i < loop; i ++)
{
SMS4_CBC_DecryptionEx(pInData + i * 16, rk, pIVTemp, pOutData + i * 16);
pIVTemp = pInData + i * 16;
}
*pOutDataLen = inDataLen;
return 0;
}

2
source/libs/stream/src/streamMeta.c
View File

@ -130,7 +130,7 @@ typedef struct {
} StreamMetaTaskState;
int32_t streamMetaOpenTdb(SStreamMeta* pMeta) {
if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0) < 0) {
if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0, 0, NULL) < 0) {
return -1;
// goto _err;
}

2
source/libs/stream/src/streamState.c
View File

@ -158,7 +158,7 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz
}
taosCloseFile(&pCfgFile);
if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 1) < 0) {
if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 1, 0, NULL) < 0) {
goto _err;
}

1
source/libs/tdb/CMakeLists.txt
View File

@ -22,6 +22,7 @@ target_link_libraries(
tdb
PUBLIC os
PUBLIC util
PUBLIC crypt
)
# for test

3
source/libs/tdb/inc/tdb.h
View File

@ -32,7 +32,8 @@ typedef struct STBC TBC;
typedef struct STxn TXN;
// TDB
int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback);
int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm,
char *encryptKey);
int32_t tdbClose(TDB *pDb);
int32_t tdbBegin(TDB *pDb, TXN **pTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
int flags);

8
source/libs/tdb/src/db/tdbDb.c
View File

@ -15,7 +15,8 @@
#include "tdbInt.h"
int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback) {
int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm,
char *encryptKey) {
TDB *pDb;
int dsize;
int zsize;
@ -49,6 +50,11 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
pDb->jfd = -1;
pDb->encryptAlgorithm = encryptAlgorithm;
if(encryptKey != NULL){
strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN);
}
ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache));
if (ret < 0) {
return -1;

100
source/libs/tdb/src/db/tdbPager.c
View File

@ -14,6 +14,8 @@
*/
#include "tdbInt.h"
#include "crypt.h"
#include "tglobal.h"
/*
#pragma pack(push, 1)
typedef struct {
@ -457,6 +459,49 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
return 0;
}
static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, const char* function,
int64_t offset){
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey;
char* buf = pPageData;
if(encryptAlgorithm == DND_CA_SM4){
//tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
//ASSERT(strlen(encryptKey) > 0);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]);
buf = taosMemoryMalloc(pageSize);
unsigned char packetData[128];
int32_t count = 0;
while (count < pageSize) {
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pPageData + count;
opts.result = packetData;
opts.unitLen = 128;
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
int32_t newLen = CBC_Encrypt(&opts);
memcpy(buf + count, packetData, newLen);
count += newLen;
}
//tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]);
}
return buf;
}
void tdbFreeEncryptBuf(SPager *pPager, char* buf){
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf);
}
// recovery dirty pages
int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
SPage *pPage;
@ -525,14 +570,19 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
return -1;
}
ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize);
char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize);
if (ret < 0) {
tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
pPager->pageSize);
tdbFreeEncryptBuf(pPager, buf);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf);
return -1;
}
tdbFreeEncryptBuf(pPager, buf);
}
if (tdbOsFSync(pPager->fd) < 0) {
@ -872,10 +922,44 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
TDB_UNLOCK_PAGE(pPage);
return -1;
}
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey;
if(encryptAlgorithm == DND_CA_SM4){
//tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
//ASSERT(strlen(encryptKey) > 0);
//uint8_t flags = pPage->pData[0];
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags);
unsigned char packetData[128];
int32_t count = 0;
while(count < pPage->pageSize)
{
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pPage->pData + count;
opts.result = packetData;
opts.unitLen = 128;
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
int newLen = CBC_Decrypt(&opts);
memcpy(pPage->pData + count, packetData, newLen);
count += newLen;
}
//tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count, __FUNCTION__);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__);
}
} else {
init = 0;
}
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__);
ret = (*initPage)(pPage, arg, init);
if (ret < 0) {
tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " init page failed.", pPager, pgno, nRead,
@ -960,14 +1044,19 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) {
offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
ret = tdbOsPWrite(pPager->fd, pPage->pData, pPage->pageSize, offset);
char* buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset);
ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset);
if (ret < 0) {
tdbFreeEncryptBuf(pPager, buf);
tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName,
pPage->pageSize);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
tdbFreeEncryptBuf(pPager, buf);
return 0;
}
@ -1025,14 +1114,19 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
return -1;
}
ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize);
char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize);
if (ret < 0) {
tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
pPager->pageSize);
tdbFreeEncryptBuf(pPager, buf);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf);
return -1;
}
tdbFreeEncryptBuf(pPager, buf);
}
if (tdbOsFSync(pPager->fd) < 0) {

3
source/libs/tdb/src/inc/tdbInt.h
View File

@ -18,6 +18,7 @@
#include "tdb.h"
#include "tdef.h"
#include "tlog.h"
#include "trbtree.h"
@ -392,6 +393,8 @@ struct STDB {
TTB *pFreeDb;
#endif
int64_t txnId;
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
};
struct SPager {

4
source/libs/tdb/test/tdbExOVFLTest.cpp
View File

@ -141,7 +141,7 @@ static void generateBigVal(char *val, int valLen) {
static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
TDB *pEnv = NULL;
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0);
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL);
if (ret) {
pEnv = NULL;
}
@ -374,7 +374,7 @@ TEST(tdb_test, simple_insert1) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database

10
source/libs/tdb/test/tdbPageDefragmentTest.cpp
View File

@ -141,7 +141,7 @@ static void generateBigVal(char *val, int valLen) {
static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
TDB *pEnv = NULL;
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0);
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL);
if (ret) {
pEnv = NULL;
}
@ -352,7 +352,7 @@ TEST(TdbPageDefragmentTest, DISABLED_simple_insert1) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -485,7 +485,7 @@ TEST(TdbPageDefragmentTest, seq_insert) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -566,7 +566,7 @@ TEST(TdbPageDefragmentTest, seq_delete) {
int const pageSize = 1 * 1024 * 1024;
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -650,7 +650,7 @@ TEST(TdbPageDefragmentTest, defragment_insert) {
int const pageSize = 1 * 1024 * 1024;
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database

8
source/libs/tdb/test/tdbPageRecycleTest.cpp
View File

@ -123,7 +123,7 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in
static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
TDB *pEnv = NULL;
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0);
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0 , 0, NULL);
if (ret) {
pEnv = NULL;
}
@ -355,7 +355,7 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -484,7 +484,7 @@ static void insertDb(int nData) {
int const pageSize = 4 * 1024;
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -552,7 +552,7 @@ static void deleteDb(int nData) {
int const pageSize = 4 * 1024;
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database

12
source/libs/tdb/test/tdbTest.cpp
View File

@ -131,7 +131,7 @@ TEST(tdb_test, DISABLED_simple_insert1) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", 4096, 64, &pEnv, 0);
ret = tdbOpen("tdb", 4096, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -246,7 +246,7 @@ TEST(tdb_test, DISABLED_simple_insert2) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", 1024, 10, &pEnv, 0);
ret = tdbOpen("tdb", 1024, 10, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -339,7 +339,7 @@ TEST(tdb_test, DISABLED_simple_delete1) {
pPool = openPool();
// open env
ret = tdbOpen("tdb", 1024, 256, &pEnv, 0);
ret = tdbOpen("tdb", 1024, 256, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// open database
@ -428,7 +428,7 @@ TEST(tdb_test, DISABLED_simple_upsert1) {
taosRemoveDir("tdb");
// open env
ret = tdbOpen("tdb", 4096, 64, &pEnv, 0);
ret = tdbOpen("tdb", 4096, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// open database
@ -490,7 +490,7 @@ TEST(tdb_test, multi_thread_query) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", 4096, 10, &pEnv, 0);
ret = tdbOpen("tdb", 4096, 10, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -599,7 +599,7 @@ TEST(tdb_test, DISABLED_multi_thread1) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", 512, 1, &pDb, 0);
ret = tdbOpen("tdb", 512, 1, &pDb, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
ret = tdbTbOpen("db.db", -1, -1, NULL, pDb, &pTb, 0);

1
source/libs/wal/CMakeLists.txt
View File

@ -12,6 +12,7 @@ target_link_libraries(
PUBLIC os
PUBLIC util
PUBLIC common
PUBLIC crypt
)
if(${BUILD_TEST})

15
source/libs/wal/src/walMeta.c
View File

@ -153,7 +153,11 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
}
// validate body
recordLen = walCkHeadSz + logContent->head.bodyLen;
int32_t cryptedBodyLen = logContent->head.bodyLen;
if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
recordLen = walCkHeadSz + cryptedBodyLen;
if (len < recordLen) {
int64_t extraSize = recordLen - len;
if (capacity < readSize + extraSize + sizeof(magic)) {
@ -181,6 +185,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
}
logContent = (SWalCkHead*)(buf + pos);
decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__);
if (walValidBodyCksum(logContent) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
@ -618,7 +623,13 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
/*A(idxEntry.ver == ckHead.head.version);*/
idxEntry.ver += 1;
idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen;
int32_t plainBodyLen = ckHead.head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
idxEntry.offset += sizeof(SWalCkHead) + cryptedBodyLen;
if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s",

75
source/libs/wal/src/walRead.c
View File

@ -13,7 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "crypt.h"
#include "taoserror.h"
#include "wal.h"
#include "walInt.h"
SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) {
@ -307,7 +309,13 @@ int32_t walSkipFetchBody(SWalReader *pRead) {
pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer,
pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId);
int64_t code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR);
int32_t plainBodyLen = pRead->pHead->head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pRead->pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
int64_t code = taosLSeekFile(pRead->pLogFile, cryptedBodyLen, SEEK_CUR);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -328,19 +336,27 @@ int32_t walFetchBody(SWalReader *pRead) {
", 0x%" PRIx64,
vgId, ver, pVer->firstVer, pVer->commitVer, pVer->lastVer, pVer->appliedVer, id);
if (pRead->capacity < pReadHead->bodyLen) {
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen);
int32_t plainBodyLen = pReadHead->bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pRead->pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
if (pRead->capacity < cryptedBodyLen) {
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pRead->pHead = ptr;
pReadHead = &pRead->pHead->head;
pRead->capacity = pReadHead->bodyLen;
pRead->capacity = cryptedBodyLen;
}
if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) {
if (pReadHead->bodyLen < 0) {
if (cryptedBodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, cryptedBodyLen)) {
if (plainBodyLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%"PRIx64,
vgId, pReadHead->version, ver, tstrerror(terrno), id);
@ -359,6 +375,8 @@ int32_t walFetchBody(SWalReader *pRead) {
return -1;
}
decryptBody(&pRead->pWal->cfg, pRead->pHead, plainBodyLen, __FUNCTION__);
if (walValidBodyCksum(pRead->pHead) != 0) {
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver, id);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
@ -431,20 +449,28 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
return -1;
}
if (pReader->capacity < pReader->pHead->head.bodyLen) {
int32_t plainBodyLen = pReader->pHead->head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pReader->pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
if (pReader->capacity < cryptedBodyLen) {
SWalCkHead *ptr =
(SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReader->pHead->head.bodyLen);
(SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosThreadMutexUnlock(&pReader->mutex);
return -1;
}
pReader->pHead = ptr;
pReader->capacity = pReader->pHead->head.bodyLen;
pReader->capacity = cryptedBodyLen;
}
if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, pReader->pHead->head.bodyLen)) !=
pReader->pHead->head.bodyLen) {
if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) !=
cryptedBodyLen) {
if (contLen < 0)
terrno = TAOS_SYSTEM_ERROR(errno);
else {
@ -465,11 +491,13 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
return -1;
}
decryptBody(&pReader->pWal->cfg, pReader->pHead, plainBodyLen, __FUNCTION__);
code = walValidBodyCksum(pReader->pHead);
if (code != 0) {
wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId,
ver);
uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, pReader->pHead->head.bodyLen);
uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, plainBodyLen);
uint32_t logCkSum = pReader->pHead->cksumBody;
wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum);
// pReader->curInvalid = 1;
@ -484,6 +512,29 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
return 0;
}
void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func) {
//TODO: dmchen emun
if (cfg->encryptAlgorithm == 1) {
int32_t cryptedBodyLen = ENCRYPTED_LEN(plainBodyLen);
char *newBody = taosMemoryMalloc(cryptedBodyLen);
SCryptOpts opts;
opts.len = cryptedBodyLen;
opts.source = pHead->head.body;
opts.result = newBody;
opts.unitLen = 16;
strncpy(opts.key, cfg->encryptKey, 16);
int32_t count = CBC_Decrypt(&opts);
//wDebug("CBC_Decrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", count, plainBodyLen, func);
memcpy(pHead->head.body, newBody, plainBodyLen);
taosMemoryFree(newBody);
}
}
void walReadReset(SWalReader *pReader) {
taosThreadMutexLock(&pReader->mutex);
taosCloseFile(&pReader->pIdxFile);

65
source/libs/wal/src/walWrite.c
View File

@ -18,6 +18,7 @@
#include "tchecksum.h"
#include "tglobal.h"
#include "walInt.h"
#include "crypt.h"
int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
taosThreadMutexLock(&pWal->mutex);
@ -491,12 +492,13 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta,
const void *body, int32_t bodyLen) {
int64_t code = 0;
int32_t plainBodyLen = bodyLen;
int64_t offset = walGetCurFileOffset(pWal);
SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal);
pWal->writeHead.head.version = index;
pWal->writeHead.head.bodyLen = bodyLen;
pWal->writeHead.head.bodyLen = plainBodyLen;
pWal->writeHead.head.msgType = msgType;
pWal->writeHead.head.ingestTs = taosGetTimestampUs();
@ -504,7 +506,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
pWal->writeHead.head.syncMeta = syncMeta;
pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead);
pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen);
pWal->writeHead.cksumBody = walCalcBodyCksum(body, plainBodyLen);
wDebug("vgId:%d, wal write log %" PRId64 ", msgType: %s, cksum head %u cksum body %u", pWal->cfg.vgId, index,
TMSG_INFO(msgType), pWal->writeHead.cksumHead, pWal->writeHead.cksumBody);
@ -521,22 +523,75 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
goto END;
}
if (taosWriteFile(pWal->pLogFile, (char *)body, bodyLen) != bodyLen) {
int32_t cyptedBodyLen = plainBodyLen;
char* buf = (char*)body;
char* newBody = NULL;
char* newBodyEncrypted = NULL;
if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){
cyptedBodyLen = ENCRYPTED_LEN(cyptedBodyLen);
newBody = taosMemoryMalloc(cyptedBodyLen);
if(newBody == NULL){
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
goto END;
}
memset(newBody, 0, cyptedBodyLen);
memcpy(newBody, body, plainBodyLen);
newBodyEncrypted = taosMemoryMalloc(cyptedBodyLen);
if(newBodyEncrypted == NULL){
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
if(newBody != NULL) taosMemoryFreeClear(newBody);
goto END;
}
SCryptOpts opts;
opts.len = cyptedBodyLen;
opts.source = newBody;
opts.result = newBodyEncrypted;
opts.unitLen = 16;
strncpy(opts.key, pWal->cfg.encryptKey, ENCRYPT_KEY_LEN);
int32_t count = CBC_Encrypt(&opts);
//wDebug("vgId:%d, file:%" PRId64 ".log, index:%" PRId64 ", CBC_Encrypt cryptedBodyLen:%d, plainBodyLen:%d, %s",
// pWal->cfg.vgId, walGetLastFileFirstVer(pWal), index, count, plainBodyLen, __FUNCTION__);
buf = newBodyEncrypted;
}
if (taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){
taosMemoryFreeClear(newBody);
taosMemoryFreeClear(newBodyEncrypted);
}
goto END;
}
if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){
taosMemoryFreeClear(newBody);
taosMemoryFreeClear(newBodyEncrypted);
//wInfo("vgId:%d, free newBody newBodyEncrypted %s",
// pWal->cfg.vgId, __FUNCTION__);
}
// set status
if (pWal->vers.firstVer == -1) {
pWal->vers.firstVer = 0;
}
pWal->vers.lastVer = index;
pWal->totSize += sizeof(SWalCkHead) + bodyLen;
pWal->totSize += sizeof(SWalCkHead) + cyptedBodyLen;
pFileInfo->lastVer = index;
pFileInfo->fileSize += sizeof(SWalCkHead) + bodyLen;
pFileInfo->fileSize += sizeof(SWalCkHead) + cyptedBodyLen;
return 0;

36
source/util/src/tconfig.c
View File

@ -1046,16 +1046,38 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
paGetToken(name + olen + 1, &value, &vlen);
if (vlen == 0) continue;
value[vlen] = 0;
if (strcasecmp(name, "encryptScope") == 0) {
char* tmp = NULL;
int32_t len = 0;
char newValue[1024] = {0};
paGetToken(value + vlen + 1, &value2, &vlen2);
if (vlen2 != 0) {
value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0;
strcpy(newValue, value);
int32_t count = 1;
while(vlen < 1024){
paGetToken(value + vlen + 1 * count, &tmp, &len);
if(len == 0) break;
tmp[len] = 0;
strcpy(newValue + vlen, tmp);
vlen += len;
count++;
}
code = cfgSetItem(pConfig, name, newValue, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
}
else{
paGetToken(value + vlen + 1, &value2, &vlen2);
if (vlen2 != 0) {
value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0;
}
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
}
if (strcasecmp(name, "dataDir") == 0) {
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE);

14
source/util/src/terror.c
View File

@ -247,6 +247,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST, "Index not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating status")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "Encryption is not allowed to be changed after database is created")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY, "Inconsistent encryption key")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ENCRYPT_KEY, "The cluster has not been set properly for database encryption")
// mnode-node
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists")
@ -358,6 +361,17 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ONLY_TWO_MNODE, "Only two mnodes exist
TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NO_NEED_RESTORE, "No need to restore on this dnode")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE, "Please use this command when the dnode is offline")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NO_MACHINE_CODE, "Dnode can not get machine code")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NO_ENCRYPT_KEY, "no encryption key exists")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG, "invalid encryption configuration")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_ENCRYPTKEY, "invalid encryption key")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED, "encryption key was changed")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_ENCRYPT_KLEN, "Invalid encryption key length")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_STATUS_INTERVAL,"statusInterval not match")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_TIMEZONE, "timezone not match")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_CHARSET, "charset not match")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_LOCALE, "locale not match")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR, "ttlChangeOnWrite not match")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_EN_WHITELIST, "enableWhiteList not match")
// vnode
TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed")

13
tests/develop-test/2-query/show_create_db.py
View File

@ -42,17 +42,17 @@ class TDTestCase:
tdSql.query('show create database scd;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd')
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd2')
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd4')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd4')
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
self.restartTaosd(1, dbname='scd')
@ -60,17 +60,16 @@ class TDTestCase:
tdSql.query('show create database scd;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd')
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd2;')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd2')
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.query('show create database scd4')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd4')
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0")
tdSql.execute('drop database scd')

16
tests/develop-test/2-query/table_count_scan.py
View File

@ -65,7 +65,7 @@ class TDTestCase:
tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;')
tdSql.checkRows(3)
tdSql.checkData(0, 0, 31)
tdSql.checkData(0, 0, 32)
tdSql.checkData(0, 1, 'information_schema')
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 0, 3)
@ -77,7 +77,7 @@ class TDTestCase:
tdSql.query('select count(1) v,db_name, stable_name from information_schema.ins_tables group by db_name, stable_name order by v desc;')
tdSql.checkRows(3)
tdSql.checkData(0, 0, 31)
tdSql.checkData(0, 0, 32)
tdSql.checkData(0, 1, 'information_schema')
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 0, 5)
@ -93,7 +93,7 @@ class TDTestCase:
tdSql.checkData(1, 1, 'performance_schema')
tdSql.checkData(0, 0, 3)
tdSql.checkData(0, 1, 'tbl_count')
tdSql.checkData(2, 0, 31)
tdSql.checkData(2, 0, 32)
tdSql.checkData(2, 1, 'information_schema')
tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'")
@ -106,7 +106,7 @@ class TDTestCase:
tdSql.query('select count(*) from information_schema.ins_tables')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 39)
tdSql.checkData(0, 0, 40)
tdSql.execute('create table stba (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);')
@ -189,7 +189,7 @@ class TDTestCase:
tdSql.checkData(2, 0, 5)
tdSql.checkData(2, 1, 'performance_schema')
tdSql.checkData(2, 2, None)
tdSql.checkData(3, 0, 31)
tdSql.checkData(3, 0, 32)
tdSql.checkData(3, 1, 'information_schema')
tdSql.checkData(3, 2, None)
@ -204,7 +204,7 @@ class TDTestCase:
tdSql.checkData(2, 0, 5)
tdSql.checkData(2, 1, 'performance_schema')
tdSql.checkData(2, 2, None)
tdSql.checkData(3, 0, 31)
tdSql.checkData(3, 0, 32)
tdSql.checkData(3, 1, 'information_schema')
tdSql.checkData(3, 2, None)
@ -215,7 +215,7 @@ class TDTestCase:
tdSql.checkData(0, 1, 'tbl_count')
tdSql.checkData(1, 0, 5)
tdSql.checkData(1, 1, 'performance_schema')
tdSql.checkData(2, 0, 31)
tdSql.checkData(2, 0, 32)
tdSql.checkData(2, 1, 'information_schema')
tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'")
@ -228,7 +228,7 @@ class TDTestCase:
tdSql.query('select count(*) from information_schema.ins_tables')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 40)
tdSql.checkData(0, 0, 41)
tdSql.execute('drop database tbl_count')

2
tests/script/tsim/query/sys_tbname.sim
View File

@ -58,7 +58,7 @@ endi
sql select tbname from information_schema.ins_tables;
print $rows $data00
if $rows != 40 then
if $rows != 41 then
return -1
endi
if $data00 != @ins_tables@ then

6
tests/script/tsim/query/tableCount.sim
View File

@ -53,7 +53,7 @@ sql select stable_name,count(table_name) from information_schema.ins_tables grou
if $rows != 3 then
return -1
endi
if $data01 != 37 then
if $data01 != 38 then
return -1
endi
if $data11 != 10 then
@ -72,7 +72,7 @@ endi
if $data11 != 5 then
return -1
endi
if $data21 != 31 then
if $data21 != 32 then
return -1
endi
if $data31 != 5 then
@ -97,7 +97,7 @@ endi
if $data42 != 3 then
return -1
endi
if $data52 != 31 then
if $data52 != 32 then
return -1
endi
if $data62 != 5 then

53
tests/system-test/0-others/information_schema.py
View File

@ -11,7 +11,7 @@
# -*- coding: utf-8 -*-
import time
from util.log import *
from util.cases import *
from util.sql import *
@ -61,7 +61,7 @@ class TDTestCase:
self.ins_list = ['ins_dnodes','ins_mnodes','ins_qnodes','ins_snodes','ins_cluster','ins_databases','ins_functions',\
'ins_indexes','ins_stables','ins_tables','ins_tags','ins_columns','ins_users','ins_grants','ins_vgroups','ins_configs','ins_dnode_variables',\
'ins_topics','ins_subscriptions','ins_streams','ins_stream_tasks','ins_vnodes','ins_user_privileges','ins_views',
'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas']
'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions"]
self.perf_list = ['perf_connections','perf_queries','perf_consumers','perf_trans','perf_apps']
def insert_data(self,column_dict,tbname,row_num):
insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str)
@ -221,7 +221,8 @@ class TDTestCase:
tdSql.checkEqual(20470,len(tdSql.queryResult))
tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
tdSql.checkEqual(True, len(tdSql.queryResult) in range(226, 250))
tdLog.info(len(tdSql.queryResult))
tdSql.checkEqual(True, len(tdSql.queryResult) in range(251, 252))
tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
tdSql.checkEqual(54, len(tdSql.queryResult))
@ -314,11 +315,56 @@ class TDTestCase:
tdSql.error('alter cluster "activeCode" ""')
tdSql.execute('alter cluster "activeCode" "revoked"')
def ins_encryptions_check(self):
key_status_list = ['unknown', 'unset', 'set', 'loaded']
# unset/none
tdSql.execute('drop database if exists db2')
tdSql.execute('create database if not exists db2 vgroups 1 replica 1')
time.sleep(2)
tdSql.query(f'select * from information_schema.ins_encryptions')
result = tdSql.queryResult
index = 0
for i in range(0, len(result)):
tdSql.checkEqual(True, result[i][1] in key_status_list[1])
index += 1
tdSql.checkEqual(True, index > 0)
tdSql.query(f'show encryptions')
result = tdSql.queryResult
index = 0
for i in range(0, len(result)):
tdSql.checkEqual(True, result[i][1] in key_status_list[1])
index += 1
tdSql.checkEqual(True, index > 0)
# loaded/sm4
tdSql.execute('drop database if exists db2')
tdSql.execute('create encrypt_key \'12345678\'')
time.sleep(3)
tdSql.execute('create database if not exists db2 vgroups 1 replica 1 encrypt_algorithm \'sm4\'')
tdSql.query(f'select * from information_schema.ins_encryptions')
result = tdSql.queryResult
index = 0
for i in range(0, len(result)):
tdSql.checkEqual(True, result[i][1] in key_status_list[3])
index += 1
tdSql.checkEqual(True, index > 0)
tdSql.query(f'show encryptions')
result = tdSql.queryResult
index = 0
for i in range(0, len(result)):
tdSql.checkEqual(True, result[i][1] in key_status_list[3])
index += 1
tdSql.checkEqual(True, index > 0)
def test_query_ins_tags(self):
sql = f'select tag_name, tag_value from information_schema.ins_tags where table_name = "{self.stbname}_0"'
tdSql.query(sql)
tdSql.checkRows(2)
def run(self):
self.prepare_data()
self.count_check()
@ -328,6 +374,7 @@ class TDTestCase:
self.ins_stable_check2()
self.ins_dnodes_check()
self.ins_grants_check()
self.ins_encryptions_check()
self.test_query_ins_tags()

9
tests/system-test/1-insert/alter_database.py
View File

@ -64,6 +64,14 @@ class TDTestCase:
tdSql.error(f'alter database db pages {pages}')
tdSql.execute('drop database db')
def alter_encrypt_alrogithm(self):
tdSql.execute('create database db')
tdSql.checkEqual("Encryption is not allowed to be changed after database is created", tdSql.error('alter database db encrypt_algorithm \'sM4\''))
tdSql.checkEqual("Encryption is not allowed to be changed after database is created", tdSql.error('alter database db encrypt_algorithm \'noNe\''))
tdSql.checkEqual("Encryption is not allowed to be changed after database is created", tdSql.error('alter database db encrypt_algorithm \'\''))
tdSql.checkEqual("Invalid option encrypt_algorithm: none ", tdSql.error('alter database db encrypt_algorithm \'none \''))
tdSql.execute('drop database db')
def alter_same_options(self):
tdSql.execute('drop database if exists db')
tdSql.execute('create database db')
@ -92,6 +100,7 @@ class TDTestCase:
self.alter_buffer()
self.alter_pages()
self.alter_encrypt_alrogithm()
self.alter_same_options()
def stop(self):