xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/3_liaohj

This commit is contained in:
Haojun Liao 2024-04-19 16:24:53 +08:00
parent 3baa47f266 06bd826e6c
commit adfc1fb927
111 changed files with 8532 additions and 7405 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/common/systable.h
View File

@ -56,6 +56,7 @@ extern "C" {
#define TSDB_INS_TABLE_GRANTS_FULL "ins_grants_full" #define TSDB_INS_TABLE_GRANTS_FULL "ins_grants_full"
#define TSDB_INS_TABLE_GRANTS_LOGS "ins_grants_logs" #define TSDB_INS_TABLE_GRANTS_LOGS "ins_grants_logs"
#define TSDB_INS_TABLE_MACHINES "ins_machines" #define TSDB_INS_TABLE_MACHINES "ins_machines"
#define TSDB_INS_TABLE_ENCRYPTIONS "ins_encryptions"
#define TSDB_INS_TABLE_TSMAS "ins_tsmas" #define TSDB_INS_TABLE_TSMAS "ins_tsmas"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"

24
include/common/tglobal.h
View File

@ -29,6 +29,17 @@ extern "C" {
#define SLOW_LOG_TYPE_OTHERS 0x4 #define SLOW_LOG_TYPE_OTHERS 0x4
#define SLOW_LOG_TYPE_ALL 0xFFFFFFFF #define SLOW_LOG_TYPE_ALL 0xFFFFFFFF
typedef enum {
DND_CA_SM4 = 1,
} EEncryptAlgor;
typedef enum {
DND_CS_TSDB = 1,
DND_CS_VNODE_WAL = 2,
DND_CS_SDB = 4,
DND_CS_MNODE_WAL = 8,
} EEncryptScope;
// cluster // cluster
extern char tsFirst[]; extern char tsFirst[];
extern char tsSecond[]; extern char tsSecond[];
@ -39,6 +50,12 @@ extern uint16_t tsServerPort;
extern int32_t tsVersion; extern int32_t tsVersion;
extern int32_t tsStatusInterval; extern int32_t tsStatusInterval;
extern int32_t tsNumOfSupportVnodes; extern int32_t tsNumOfSupportVnodes;
extern char tsEncryptAlgorithm[];
extern char tsEncryptScope[];
extern EEncryptAlgor tsiEncryptAlgorithm;
extern EEncryptScope tsiEncryptScope;
//extern char tsAuthCode[];
extern char tsEncryptKey[];
// common // common
extern int32_t tsMaxShellConns; extern int32_t tsMaxShellConns;
@ -94,8 +111,6 @@ extern char tsSnodeAddress[]; // 127.0.0.1:873
// mnode // mnode
extern int64_t tsMndSdbWriteDelta; extern int64_t tsMndSdbWriteDelta;
extern int64_t tsMndLogRetention; extern int64_t tsMndLogRetention;
extern int8_t tsGrant;
extern int32_t tsMndGrantMode;
extern bool tsMndSkipGrant; extern bool tsMndSkipGrant;
extern bool tsEnableWhiteList; extern bool tsEnableWhiteList;
@ -104,6 +119,11 @@ extern int64_t tsDndStart;
extern int64_t tsDndStartOsUptime; extern int64_t tsDndStartOsUptime;
extern int64_t tsDndUpTime; extern int64_t tsDndUpTime;
// dnode misc
extern uint32_t tsEncryptionKeyChksum;
extern int8_t tsEncryptionKeyStat;
extern int8_t tsGrant;
// monitor // monitor
extern bool tsEnableMonitor; extern bool tsEnableMonitor;
extern int32_t tsMonitorInterval; extern int32_t tsMonitorInterval;

4
include/common/tgrant.h
View File

@ -58,9 +58,11 @@ typedef enum {
TSDB_GRANT_BACKUP_RESTORE, TSDB_GRANT_BACKUP_RESTORE,
} EGrantType; } EGrantType;
int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);
int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode);
int32_t grantCheck(EGrantType grant); int32_t grantCheck(EGrantType grant);
int32_t grantCheckExpire(EGrantType grant); int32_t grantCheckExpire(EGrantType grant);
char* tGetMachineId(); char *tGetMachineId();
// #ifndef GRANTS_CFG // #ifndef GRANTS_CFG
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE

21
include/common/tmsg.h
View File

@ -154,6 +154,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_GRANTS_LOGS, TSDB_MGMT_TABLE_GRANTS_LOGS,
TSDB_MGMT_TABLE_MACHINES, TSDB_MGMT_TABLE_MACHINES,
TSDB_MGMT_TABLE_ARBGROUP, TSDB_MGMT_TABLE_ARBGROUP,
TSDB_MGMT_TABLE_ENCRYPTIONS,
TSDB_MGMT_TABLE_MAX, TSDB_MGMT_TABLE_MAX,
} EShowType; } EShowType;
@ -374,6 +375,7 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_GRANTS_FULL_STMT, QUERY_NODE_SHOW_GRANTS_FULL_STMT,
QUERY_NODE_SHOW_GRANTS_LOGS_STMT, QUERY_NODE_SHOW_GRANTS_LOGS_STMT,
QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT, QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT,
QUERY_NODE_SHOW_ENCRYPTIONS_STMT,
QUERY_NODE_SHOW_TSMAS_STMT, QUERY_NODE_SHOW_TSMAS_STMT,
QUERY_NODE_CREATE_TSMA_STMT, QUERY_NODE_CREATE_TSMA_STMT,
QUERY_NODE_SHOW_CREATE_TSMA_STMT, QUERY_NODE_SHOW_CREATE_TSMA_STMT,
@ -1247,6 +1249,7 @@ typedef struct {
int32_t sqlLen; int32_t sqlLen;
char* sql; char* sql;
int8_t withArbitrator; int8_t withArbitrator;
int8_t encryptAlgorithm;
} SCreateDbReq; } SCreateDbReq;
int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq);
@ -1405,6 +1408,7 @@ typedef struct {
int8_t replications; int8_t replications;
int8_t strict; int8_t strict;
int8_t cacheLast; int8_t cacheLast;
int8_t encryptAlgorithm;
int32_t s3ChunkSize; int32_t s3ChunkSize;
int32_t s3KeepLocal; int32_t s3KeepLocal;
int8_t s3Compact; int8_t s3Compact;
@ -1605,13 +1609,15 @@ void tFreeSFuncInfo(SFuncInfo* pInfo);
void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp); void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp);
typedef struct { typedef struct {
int32_t statusInterval; int32_t statusInterval;
int64_t checkTime; // 1970-01-01 00:00:00.000 int64_t checkTime; // 1970-01-01 00:00:00.000
char timezone[TD_TIMEZONE_LEN]; // tsTimezone char timezone[TD_TIMEZONE_LEN]; // tsTimezone
char locale[TD_LOCALE_LEN]; // tsLocale char locale[TD_LOCALE_LEN]; // tsLocale
char charset[TD_LOCALE_LEN]; // tsCharset char charset[TD_LOCALE_LEN]; // tsCharset
int8_t ttlChangeOnWrite; int8_t ttlChangeOnWrite;
int8_t enableWhiteList; int8_t enableWhiteList;
int8_t encryptionKeyStat;
uint32_t encryptionKeyChksum;
} SClusterCfg; } SClusterCfg;
typedef struct { typedef struct {
@ -1823,6 +1829,7 @@ typedef struct {
int8_t learnerSelfIndex; int8_t learnerSelfIndex;
SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA];
int32_t changeVersion; int32_t changeVersion;
int8_t encryptAlgorithm;
} SCreateVnodeReq; } SCreateVnodeReq;
int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);

2
include/common/tmsgdef.h
View File

@ -106,6 +106,7 @@
TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE_TYPE, "dnode-alter-mnode-type", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE_TYPE, "dnode-alter-mnode-type", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CREATE_ENCRYPT_KEY, "create-encrypt-key", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_DND_MSG) TD_CLOSE_MSG_SEG(TDMT_END_DND_MSG)
@ -220,6 +221,7 @@
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ENCRYPT_KEY, "create-encrypt-key", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)

747
include/common/ttokendef.h
View File

@ -16,378 +16,381 @@
#ifndef _TD_COMMON_TOKEN_H_ #ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_
#define TK_OR 1 #define TK_OR 1
#define TK_AND 2 #define TK_AND 2
#define TK_UNION 3 #define TK_UNION 3
#define TK_ALL 4 #define TK_ALL 4
#define TK_MINUS 5 #define TK_MINUS 5
#define TK_EXCEPT 6 #define TK_EXCEPT 6
#define TK_INTERSECT 7 #define TK_INTERSECT 7
#define TK_NK_BITAND 8 #define TK_NK_BITAND 8
#define TK_NK_BITOR 9 #define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10 #define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11 #define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12 #define TK_NK_PLUS 12
#define TK_NK_MINUS 13 #define TK_NK_MINUS 13
#define TK_NK_STAR 14 #define TK_NK_STAR 14
#define TK_NK_SLASH 15 #define TK_NK_SLASH 15
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_ACCOUNT 19 #define TK_ACCOUNT 19
#define TK_NK_ID 20 #define TK_NK_ID 20
#define TK_PASS 21 #define TK_PASS 21
#define TK_NK_STRING 22 #define TK_NK_STRING 22
#define TK_ALTER 23 #define TK_ALTER 23
#define TK_PPS 24 #define TK_PPS 24
#define TK_TSERIES 25 #define TK_TSERIES 25
#define TK_STORAGE 26 #define TK_STORAGE 26
#define TK_STREAMS 27 #define TK_STREAMS 27
#define TK_QTIME 28 #define TK_QTIME 28
#define TK_DBS 29 #define TK_DBS 29
#define TK_USERS 30 #define TK_USERS 30
#define TK_CONNS 31 #define TK_CONNS 31
#define TK_STATE 32 #define TK_STATE 32
#define TK_NK_COMMA 33 #define TK_NK_COMMA 33
#define TK_HOST 34 #define TK_HOST 34
#define TK_USER 35 #define TK_USER 35
#define TK_ENABLE 36 #define TK_ENABLE 36
#define TK_NK_INTEGER 37 #define TK_NK_INTEGER 37
#define TK_SYSINFO 38 #define TK_SYSINFO 38
#define TK_ADD 39 #define TK_ADD 39
#define TK_DROP 40 #define TK_DROP 40
#define TK_GRANT 41 #define TK_GRANT 41
#define TK_ON 42 #define TK_ON 42
#define TK_TO 43 #define TK_TO 43
#define TK_REVOKE 44 #define TK_REVOKE 44
#define TK_FROM 45 #define TK_FROM 45
#define TK_SUBSCRIBE 46 #define TK_SUBSCRIBE 46
#define TK_READ 47 #define TK_READ 47
#define TK_WRITE 48 #define TK_WRITE 48
#define TK_NK_DOT 49 #define TK_NK_DOT 49
#define TK_WITH 50 #define TK_WITH 50
#define TK_DNODE 51 #define TK_ENCRYPT_KEY 51
#define TK_PORT 52 #define TK_DNODE 52
#define TK_DNODES 53 #define TK_PORT 53
#define TK_RESTORE 54 #define TK_DNODES 54
#define TK_NK_IPTOKEN 55 #define TK_RESTORE 55
#define TK_FORCE 56 #define TK_NK_IPTOKEN 56
#define TK_UNSAFE 57 #define TK_FORCE 57
#define TK_CLUSTER 58 #define TK_UNSAFE 58
#define TK_LOCAL 59 #define TK_CLUSTER 59
#define TK_QNODE 60 #define TK_LOCAL 60
#define TK_BNODE 61 #define TK_QNODE 61
#define TK_SNODE 62 #define TK_BNODE 62
#define TK_MNODE 63 #define TK_SNODE 63
#define TK_VNODE 64 #define TK_MNODE 64
#define TK_DATABASE 65 #define TK_VNODE 65
#define TK_USE 66 #define TK_DATABASE 66
#define TK_FLUSH 67 #define TK_USE 67
#define TK_TRIM 68 #define TK_FLUSH 68
#define TK_S3MIGRATE 69 #define TK_TRIM 69
#define TK_COMPACT 70 #define TK_S3MIGRATE 70
#define TK_IF 71 #define TK_COMPACT 71
#define TK_NOT 72 #define TK_IF 72
#define TK_EXISTS 73 #define TK_NOT 73
#define TK_BUFFER 74 #define TK_EXISTS 74
#define TK_CACHEMODEL 75 #define TK_BUFFER 75
#define TK_CACHESIZE 76 #define TK_CACHEMODEL 76
#define TK_COMP 77 #define TK_CACHESIZE 77
#define TK_DURATION 78 #define TK_COMP 78
#define TK_NK_VARIABLE 79 #define TK_DURATION 79
#define TK_MAXROWS 80 #define TK_NK_VARIABLE 80
#define TK_MINROWS 81 #define TK_MAXROWS 81
#define TK_KEEP 82 #define TK_MINROWS 82
#define TK_PAGES 83 #define TK_KEEP 83
#define TK_PAGESIZE 84 #define TK_PAGES 84
#define TK_TSDB_PAGESIZE 85 #define TK_PAGESIZE 85
#define TK_PRECISION 86 #define TK_TSDB_PAGESIZE 86
#define TK_REPLICA 87 #define TK_PRECISION 87
#define TK_VGROUPS 88 #define TK_REPLICA 88
#define TK_SINGLE_STABLE 89 #define TK_VGROUPS 89
#define TK_RETENTIONS 90 #define TK_SINGLE_STABLE 90
#define TK_SCHEMALESS 91 #define TK_RETENTIONS 91
#define TK_WAL_LEVEL 92 #define TK_SCHEMALESS 92
#define TK_WAL_FSYNC_PERIOD 93 #define TK_WAL_LEVEL 93
#define TK_WAL_RETENTION_PERIOD 94 #define TK_WAL_FSYNC_PERIOD 94
#define TK_WAL_RETENTION_SIZE 95 #define TK_WAL_RETENTION_PERIOD 95
#define TK_WAL_ROLL_PERIOD 96 #define TK_WAL_RETENTION_SIZE 96
#define TK_WAL_SEGMENT_SIZE 97 #define TK_WAL_ROLL_PERIOD 97
#define TK_STT_TRIGGER 98 #define TK_WAL_SEGMENT_SIZE 98
#define TK_TABLE_PREFIX 99 #define TK_STT_TRIGGER 99
#define TK_TABLE_SUFFIX 100 #define TK_TABLE_PREFIX 100
#define TK_S3_CHUNKSIZE 101 #define TK_TABLE_SUFFIX 101
#define TK_S3_KEEPLOCAL 102 #define TK_S3_CHUNKSIZE 102
#define TK_S3_COMPACT 103 #define TK_S3_KEEPLOCAL 103
#define TK_KEEP_TIME_OFFSET 104 #define TK_S3_COMPACT 104
#define TK_NK_COLON 105 #define TK_KEEP_TIME_OFFSET 105
#define TK_BWLIMIT 106 #define TK_ENCRYPT_ALGORITHM 106
#define TK_START 107 #define TK_NK_COLON 107
#define TK_TIMESTAMP 108 #define TK_BWLIMIT 108
#define TK_END 109 #define TK_START 109
#define TK_TABLE 110 #define TK_TIMESTAMP 110
#define TK_NK_LP 111 #define TK_END 111
#define TK_NK_RP 112 #define TK_TABLE 112
#define TK_STABLE 113 #define TK_NK_LP 113
#define TK_COLUMN 114 #define TK_NK_RP 114
#define TK_MODIFY 115 #define TK_STABLE 115
#define TK_RENAME 116 #define TK_COLUMN 116
#define TK_TAG 117 #define TK_MODIFY 117
#define TK_SET 118 #define TK_RENAME 118
#define TK_NK_EQ 119 #define TK_TAG 119
#define TK_USING 120 #define TK_SET 120
#define TK_TAGS 121 #define TK_NK_EQ 121
#define TK_BOOL 122 #define TK_USING 122
#define TK_TINYINT 123 #define TK_TAGS 123
#define TK_SMALLINT 124 #define TK_BOOL 124
#define TK_INT 125 #define TK_TINYINT 125
#define TK_INTEGER 126 #define TK_SMALLINT 126
#define TK_BIGINT 127 #define TK_INT 127
#define TK_FLOAT 128 #define TK_INTEGER 128
#define TK_DOUBLE 129 #define TK_BIGINT 129
#define TK_BINARY 130 #define TK_FLOAT 130
#define TK_NCHAR 131 #define TK_DOUBLE 131
#define TK_UNSIGNED 132 #define TK_BINARY 132
#define TK_JSON 133 #define TK_NCHAR 133
#define TK_VARCHAR 134 #define TK_UNSIGNED 134
#define TK_MEDIUMBLOB 135 #define TK_JSON 135
#define TK_BLOB 136 #define TK_VARCHAR 136
#define TK_VARBINARY 137 #define TK_MEDIUMBLOB 137
#define TK_GEOMETRY 138 #define TK_BLOB 138
#define TK_DECIMAL 139 #define TK_VARBINARY 139
#define TK_COMMENT 140 #define TK_GEOMETRY 140
#define TK_MAX_DELAY 141 #define TK_DECIMAL 141
#define TK_WATERMARK 142 #define TK_COMMENT 142
#define TK_ROLLUP 143 #define TK_MAX_DELAY 143
#define TK_TTL 144 #define TK_WATERMARK 144
#define TK_SMA 145 #define TK_ROLLUP 145
#define TK_DELETE_MARK 146 #define TK_TTL 146
#define TK_FIRST 147 #define TK_SMA 147
#define TK_LAST 148 #define TK_DELETE_MARK 148
#define TK_SHOW 149 #define TK_FIRST 149
#define TK_PRIVILEGES 150 #define TK_LAST 150
#define TK_DATABASES 151 #define TK_SHOW 151
#define TK_TABLES 152 #define TK_PRIVILEGES 152
#define TK_STABLES 153 #define TK_DATABASES 153
#define TK_MNODES 154 #define TK_TABLES 154
#define TK_QNODES 155 #define TK_STABLES 155
#define TK_ARBGROUPS 156 #define TK_MNODES 156
#define TK_FUNCTIONS 157 #define TK_QNODES 157
#define TK_INDEXES 158 #define TK_ARBGROUPS 158
#define TK_ACCOUNTS 159 #define TK_FUNCTIONS 159
#define TK_APPS 160 #define TK_INDEXES 160
#define TK_CONNECTIONS 161 #define TK_ACCOUNTS 161
#define TK_LICENCES 162 #define TK_APPS 162
#define TK_GRANTS 163 #define TK_CONNECTIONS 163
#define TK_FULL 164 #define TK_LICENCES 164
#define TK_LOGS 165 #define TK_GRANTS 165
#define TK_MACHINES 166 #define TK_FULL 166
#define TK_QUERIES 167 #define TK_LOGS 167
#define TK_SCORES 168 #define TK_MACHINES 168
#define TK_TOPICS 169 #define TK_ENCRYPTIONS 169
#define TK_VARIABLES 170 #define TK_QUERIES 170
#define TK_BNODES 171 #define TK_SCORES 171
#define TK_SNODES 172 #define TK_TOPICS 172
#define TK_TRANSACTIONS 173 #define TK_VARIABLES 173
#define TK_DISTRIBUTED 174 #define TK_BNODES 174
#define TK_CONSUMERS 175 #define TK_SNODES 175
#define TK_SUBSCRIPTIONS 176 #define TK_TRANSACTIONS 176
#define TK_VNODES 177 #define TK_DISTRIBUTED 177
#define TK_ALIVE 178 #define TK_CONSUMERS 178
#define TK_VIEWS 179 #define TK_SUBSCRIPTIONS 179
#define TK_VIEW 180 #define TK_VNODES 180
#define TK_COMPACTS 181 #define TK_ALIVE 181
#define TK_NORMAL 182 #define TK_VIEWS 182
#define TK_CHILD 183 #define TK_VIEW 183
#define TK_LIKE 184 #define TK_COMPACTS 184
#define TK_TBNAME 185 #define TK_NORMAL 185
#define TK_QTAGS 186 #define TK_CHILD 186
#define TK_AS 187 #define TK_LIKE 187
#define TK_SYSTEM 188 #define TK_TBNAME 188
#define TK_TSMA 189 #define TK_QTAGS 189
#define TK_INTERVAL 190 #define TK_AS 190
#define TK_RECURSIVE 191 #define TK_SYSTEM 191
#define TK_TSMAS 192 #define TK_TSMA 192
#define TK_FUNCTION 193 #define TK_INTERVAL 193
#define TK_INDEX 194 #define TK_RECURSIVE 194
#define TK_COUNT 195 #define TK_TSMAS 195
#define TK_LAST_ROW 196 #define TK_FUNCTION 196
#define TK_META 197 #define TK_INDEX 197
#define TK_ONLY 198 #define TK_COUNT 198
#define TK_TOPIC 199 #define TK_LAST_ROW 199
#define TK_CONSUMER 200 #define TK_META 200
#define TK_GROUP 201 #define TK_ONLY 201
#define TK_DESC 202 #define TK_TOPIC 202
#define TK_DESCRIBE 203 #define TK_CONSUMER 203
#define TK_RESET 204 #define TK_GROUP 204
#define TK_QUERY 205 #define TK_DESC 205
#define TK_CACHE 206 #define TK_DESCRIBE 206
#define TK_EXPLAIN 207 #define TK_RESET 207
#define TK_ANALYZE 208 #define TK_QUERY 208
#define TK_VERBOSE 209 #define TK_CACHE 209
#define TK_NK_BOOL 210 #define TK_EXPLAIN 210
#define TK_RATIO 211 #define TK_ANALYZE 211
#define TK_NK_FLOAT 212 #define TK_VERBOSE 212
#define TK_OUTPUTTYPE 213 #define TK_NK_BOOL 213
#define TK_AGGREGATE 214 #define TK_RATIO 214
#define TK_BUFSIZE 215 #define TK_NK_FLOAT 215
#define TK_LANGUAGE 216 #define TK_OUTPUTTYPE 216
#define TK_REPLACE 217 #define TK_AGGREGATE 217
#define TK_STREAM 218 #define TK_BUFSIZE 218
#define TK_INTO 219 #define TK_LANGUAGE 219
#define TK_PAUSE 220 #define TK_REPLACE 220
#define TK_RESUME 221 #define TK_STREAM 221
#define TK_PRIMARY 222 #define TK_INTO 222
#define TK_KEY 223 #define TK_PAUSE 223
#define TK_TRIGGER 224 #define TK_RESUME 224
#define TK_AT_ONCE 225 #define TK_PRIMARY 225
#define TK_WINDOW_CLOSE 226 #define TK_KEY 226
#define TK_IGNORE 227 #define TK_TRIGGER 227
#define TK_EXPIRED 228 #define TK_AT_ONCE 228
#define TK_FILL_HISTORY 229 #define TK_WINDOW_CLOSE 229
#define TK_UPDATE 230 #define TK_IGNORE 230
#define TK_SUBTABLE 231 #define TK_EXPIRED 231
#define TK_UNTREATED 232 #define TK_FILL_HISTORY 232
#define TK_KILL 233 #define TK_UPDATE 233
#define TK_CONNECTION 234 #define TK_SUBTABLE 234
#define TK_TRANSACTION 235 #define TK_UNTREATED 235
#define TK_BALANCE 236 #define TK_KILL 236
#define TK_VGROUP 237 #define TK_CONNECTION 237
#define TK_LEADER 238 #define TK_TRANSACTION 238
#define TK_MERGE 239 #define TK_BALANCE 239
#define TK_REDISTRIBUTE 240 #define TK_VGROUP 240
#define TK_SPLIT 241 #define TK_LEADER 241
#define TK_DELETE 242 #define TK_MERGE 242
#define TK_INSERT 243 #define TK_REDISTRIBUTE 243
#define TK_NK_BIN 244 #define TK_SPLIT 244
#define TK_NK_HEX 245 #define TK_DELETE 245
#define TK_NULL 246 #define TK_INSERT 246
#define TK_NK_QUESTION 247 #define TK_NK_BIN 247
#define TK_NK_ALIAS 248 #define TK_NK_HEX 248
#define TK_NK_ARROW 249 #define TK_NULL 249
#define TK_ROWTS 250 #define TK_NK_QUESTION 250
#define TK_QSTART 251 #define TK_NK_ALIAS 251
#define TK_QEND 252 #define TK_NK_ARROW 252
#define TK_QDURATION 253 #define TK_ROWTS 253
#define TK_WSTART 254 #define TK_QSTART 254
#define TK_WEND 255 #define TK_QEND 255
#define TK_WDURATION 256 #define TK_QDURATION 256
#define TK_IROWTS 257 #define TK_WSTART 257
#define TK_ISFILLED 258 #define TK_WEND 258
#define TK_CAST 259 #define TK_WDURATION 259
#define TK_NOW 260 #define TK_IROWTS 260
#define TK_TODAY 261 #define TK_ISFILLED 261
#define TK_TIMEZONE 262 #define TK_CAST 262
#define TK_CLIENT_VERSION 263 #define TK_NOW 263
#define TK_SERVER_VERSION 264 #define TK_TODAY 264
#define TK_SERVER_STATUS 265 #define TK_TIMEZONE 265
#define TK_CURRENT_USER 266 #define TK_CLIENT_VERSION 266
#define TK_CASE 267 #define TK_SERVER_VERSION 267
#define TK_WHEN 268 #define TK_SERVER_STATUS 268
#define TK_THEN 269 #define TK_CURRENT_USER 269
#define TK_ELSE 270 #define TK_CASE 270
#define TK_BETWEEN 271 #define TK_WHEN 271
#define TK_IS 272 #define TK_THEN 272
#define TK_NK_LT 273 #define TK_ELSE 273
#define TK_NK_GT 274 #define TK_BETWEEN 274
#define TK_NK_LE 275 #define TK_IS 275
#define TK_NK_GE 276 #define TK_NK_LT 276
#define TK_NK_NE 277 #define TK_NK_GT 277
#define TK_MATCH 278 #define TK_NK_LE 278
#define TK_NMATCH 279 #define TK_NK_GE 279
#define TK_CONTAINS 280 #define TK_NK_NE 280
#define TK_IN 281 #define TK_MATCH 281
#define TK_JOIN 282 #define TK_NMATCH 282
#define TK_INNER 283 #define TK_CONTAINS 283
#define TK_LEFT 284 #define TK_IN 284
#define TK_RIGHT 285 #define TK_JOIN 285
#define TK_OUTER 286 #define TK_INNER 286
#define TK_SEMI 287 #define TK_LEFT 287
#define TK_ANTI 288 #define TK_RIGHT 288
#define TK_ASOF 289 #define TK_OUTER 289
#define TK_WINDOW 290 #define TK_SEMI 290
#define TK_WINDOW_OFFSET 291 #define TK_ANTI 291
#define TK_JLIMIT 292 #define TK_ASOF 292
#define TK_SELECT 293 #define TK_WINDOW 293
#define TK_NK_HINT 294 #define TK_WINDOW_OFFSET 294
#define TK_DISTINCT 295 #define TK_JLIMIT 295
#define TK_WHERE 296 #define TK_SELECT 296
#define TK_PARTITION 297 #define TK_NK_HINT 297
#define TK_BY 298 #define TK_DISTINCT 298
#define TK_SESSION 299 #define TK_WHERE 299
#define TK_STATE_WINDOW 300 #define TK_PARTITION 300
#define TK_EVENT_WINDOW 301 #define TK_BY 301
#define TK_COUNT_WINDOW 302 #define TK_SESSION 302
#define TK_SLIDING 303 #define TK_STATE_WINDOW 303
#define TK_FILL 304 #define TK_EVENT_WINDOW 304
#define TK_VALUE 305 #define TK_COUNT_WINDOW 305
#define TK_VALUE_F 306 #define TK_SLIDING 306
#define TK_NONE 307 #define TK_FILL 307
#define TK_PREV 308 #define TK_VALUE 308
#define TK_NULL_F 309 #define TK_VALUE_F 309
#define TK_LINEAR 310 #define TK_NONE 310
#define TK_NEXT 311 #define TK_PREV 311
#define TK_HAVING 312 #define TK_NULL_F 312
#define TK_RANGE 313 #define TK_LINEAR 313
#define TK_EVERY 314 #define TK_NEXT 314
#define TK_ORDER 315 #define TK_HAVING 315
#define TK_SLIMIT 316 #define TK_RANGE 316
#define TK_SOFFSET 317 #define TK_EVERY 317
#define TK_LIMIT 318 #define TK_ORDER 318
#define TK_OFFSET 319 #define TK_SLIMIT 319
#define TK_ASC 320 #define TK_SOFFSET 320
#define TK_NULLS 321 #define TK_LIMIT 321
#define TK_ABORT 322 #define TK_OFFSET 322
#define TK_AFTER 323 #define TK_ASC 323
#define TK_ATTACH 324 #define TK_NULLS 324
#define TK_BEFORE 325 #define TK_ABORT 325
#define TK_BEGIN 326 #define TK_AFTER 326
#define TK_BITAND 327 #define TK_ATTACH 327
#define TK_BITNOT 328 #define TK_BEFORE 328
#define TK_BITOR 329 #define TK_BEGIN 329
#define TK_BLOCKS 330 #define TK_BITAND 330
#define TK_CHANGE 331 #define TK_BITNOT 331
#define TK_COMMA 332 #define TK_BITOR 332
#define TK_CONCAT 333 #define TK_BLOCKS 333
#define TK_CONFLICT 334 #define TK_CHANGE 334
#define TK_COPY 335 #define TK_COMMA 335
#define TK_DEFERRED 336 #define TK_CONCAT 336
#define TK_DELIMITERS 337 #define TK_CONFLICT 337
#define TK_DETACH 338 #define TK_COPY 338
#define TK_DIVIDE 339 #define TK_DEFERRED 339
#define TK_DOT 340 #define TK_DELIMITERS 340
#define TK_EACH 341 #define TK_DETACH 341
#define TK_FAIL 342 #define TK_DIVIDE 342
#define TK_FILE 343 #define TK_DOT 343
#define TK_FOR 344 #define TK_EACH 344
#define TK_GLOB 345 #define TK_FAIL 345
#define TK_ID 346 #define TK_FILE 346
#define TK_IMMEDIATE 347 #define TK_FOR 347
#define TK_IMPORT 348 #define TK_GLOB 348
#define TK_INITIALLY 349 #define TK_ID 349
#define TK_INSTEAD 350 #define TK_IMMEDIATE 350
#define TK_ISNULL 351 #define TK_IMPORT 351
#define TK_MODULES 352 #define TK_INITIALLY 352
#define TK_NK_BITNOT 353 #define TK_INSTEAD 353
#define TK_NK_SEMI 354 #define TK_ISNULL 354
#define TK_NOTNULL 355 #define TK_MODULES 355
#define TK_OF 356 #define TK_NK_BITNOT 356
#define TK_PLUS 357 #define TK_NK_SEMI 357
#define TK_PRIVILEGE 358 #define TK_NOTNULL 358
#define TK_RAISE 359 #define TK_OF 359
#define TK_RESTRICT 360 #define TK_PLUS 360
#define TK_ROW 361 #define TK_PRIVILEGE 361
#define TK_STAR 362 #define TK_RAISE 362
#define TK_STATEMENT 363 #define TK_RESTRICT 363
#define TK_STRICT 364 #define TK_ROW 364
#define TK_STRING 365 #define TK_STAR 365
#define TK_TIMES 366 #define TK_STATEMENT 366
#define TK_VALUES 367 #define TK_STRICT 367
#define TK_VARIABLE 368 #define TK_STRING 368
#define TK_WAL 369 #define TK_TIMES 369
#define TK_ENCODE 370 #define TK_VALUES 370
#define TK_COMPRESS 371 #define TK_VARIABLE 371
#define TK_LEVEL 372 #define TK_WAL 372
#define TK_ENCODE 373
#define TK_COMPRESS 374
#define TK_LEVEL 375
#define TK_NK_SPACE 600 #define TK_NK_SPACE 600
#define TK_NK_COMMENT 601 #define TK_NK_COMMENT 601

39
include/libs/crypt/crypt.h Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _CRYPT_H_
#define _CRYPT_H_
#include "tdef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SCryptOpts{
int32_t len;
char* source;
char* result;
int32_t unitLen;
unsigned char key[17];
}SCryptOpts;
int32_t CBC_Decrypt(SCryptOpts *opts);
int32_t CBC_Encrypt(SCryptOpts *opts);
#ifdef __cplusplus
}
#endif
#endif // _CRYPT_H_

2
include/libs/nodes/cmdnodes.h
View File

@ -70,7 +70,9 @@ typedef struct SDatabaseOptions {
int8_t cacheModel; int8_t cacheModel;
int32_t cacheLastSize; int32_t cacheLastSize;
int8_t compressionLevel; int8_t compressionLevel;
int8_t encryptAlgorithm;
int32_t daysPerFile; int32_t daysPerFile;
char encryptAlgorithmStr[TSDB_ENCRYPT_ALGO_STR_LEN];
SValueNode* pDaysPerFile; SValueNode* pDaysPerFile;
int32_t fsyncPeriod; int32_t fsyncPeriod;
int32_t maxRowsPerBlock; int32_t maxRowsPerBlock;

63
include/libs/sm4/sm4.h Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <memory.h>
#include <time.h>
#ifndef _SM4_H_
#define _SM4_H_
#ifdef __cplusplus
extern "C" {
#endif
int SM4_ECB_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
int SM4_ECB_Decrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
int SM4_CBC_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
int SM4_CBC_Decrypt(unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
#ifdef __cplusplus
}
#endif
#endif // _SM4_H_

3
include/libs/wal/wal.h
View File

@ -57,6 +57,8 @@ typedef struct {
int64_t retentionSize; int64_t retentionSize;
int64_t segSize; int64_t segSize;
EWalType level; // wal level EWalType level; // wal level
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
} SWalCfg; } SWalCfg;
typedef struct { typedef struct {
@ -196,6 +198,7 @@ SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id);
void walCloseReader(SWalReader *pRead); void walCloseReader(SWalReader *pRead);
void walReadReset(SWalReader *pReader); void walReadReset(SWalReader *pReader);
int32_t walReadVer(SWalReader *pRead, int64_t ver); int32_t walReadVer(SWalReader *pRead, int64_t ver);
void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func);
int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver); int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver);
int32_t walNextValidMsg(SWalReader *pRead); int32_t walNextValidMsg(SWalReader *pRead);
int64_t walReaderGetCurrentVer(const SWalReader *pReader); int64_t walReaderGetCurrentVer(const SWalReader *pReader);

5
include/os/osString.h
View File

@ -22,6 +22,8 @@ extern "C" {
typedef wchar_t TdWchar; typedef wchar_t TdWchar;
typedef int32_t TdUcs4; typedef int32_t TdUcs4;
typedef void *iconv_t;
typedef enum { M2C = 0, C2M } ConvType;
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section. // When you want to use this feature, you should find or add the same function in the following section.
@ -59,7 +61,10 @@ int64_t taosStr2int64(const char *str);
int32_t taosConvInit(void); int32_t taosConvInit(void);
void taosConvDestroy(); void taosConvDestroy();
iconv_t taosAcquireConv(int32_t *idx, ConvType type);
void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type);
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs); int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs);
int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len);
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
TdUcs4 *tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); TdUcs4 *tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);

14
include/util/taoserror.h
View File

@ -306,6 +306,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) // #define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) //
#define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B) #define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B)
#define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C) #define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C)
#define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D)
#define TSDB_CODE_MND_INVALID_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038E)
// #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x // #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x
// #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x // #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x
// #define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x0392) // 2.x // #define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x0392) // 2.x
@ -314,6 +316,7 @@ int32_t* taosGetErrno();
// #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) // 2.x // #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) // 2.x
#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) // #define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) //
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A) #define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
// mnode-node // mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)
@ -421,6 +424,17 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal #define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal
#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416) #define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416)
#define TSDB_CODE_DNODE_NO_MACHINE_CODE TAOS_DEF_ERROR_CODE(0, 0x0417) #define TSDB_CODE_DNODE_NO_MACHINE_CODE TAOS_DEF_ERROR_CODE(0, 0x0417)
#define TSDB_CODE_DNODE_NO_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x0418)
#define TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0419)
#define TSDB_CODE_DNODE_INVALID_ENCRYPTKEY TAOS_DEF_ERROR_CODE(0, 0x0420)
#define TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED TAOS_DEF_ERROR_CODE(0, 0x0421)
#define TSDB_CODE_DNODE_INVALID_ENCRYPT_KLEN TAOS_DEF_ERROR_CODE(0, 0x0422)
#define TSDB_CODE_DNODE_INVALID_STATUS_INTERVAL TAOS_DEF_ERROR_CODE(0, 0x0423)
#define TSDB_CODE_DNODE_INVALID_TIMEZONE TAOS_DEF_ERROR_CODE(0, 0x0424)
#define TSDB_CODE_DNODE_INVALID_CHARSET TAOS_DEF_ERROR_CODE(0, 0x0425)
#define TSDB_CODE_DNODE_INVALID_LOCALE TAOS_DEF_ERROR_CODE(0, 0x0426)
#define TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR TAOS_DEF_ERROR_CODE(0, 0x0427)
#define TSDB_CODE_DNODE_INVALID_EN_WHITELIST TAOS_DEF_ERROR_CODE(0, 0x0428)
// mnode-sma // mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)

1
include/util/tcompare.h
View File

@ -83,6 +83,7 @@ int32_t compareLenBinaryVal(const void *pLeft, const void *pRight);
int32_t comparestrRegexMatch(const void *pLeft, const void *pRight); int32_t comparestrRegexMatch(const void *pLeft, const void *pRight);
int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight); int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight);
void DestoryThreadLocalRegComp();
int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight); int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight);
int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight); int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight);

13
include/util/tdef.h
View File

@ -188,6 +188,10 @@ typedef enum ELogicConditionType {
LOGIC_COND_TYPE_NOT, LOGIC_COND_TYPE_NOT,
} ELogicConditionType; } ELogicConditionType;
#define ENCRYPTED_LEN(len) (len/16) * 16 + (len%16?1:0) * 16
#define ENCRYPT_KEY_LEN 16
#define ENCRYPT_KEY_LEN_MIN 8
#define TSDB_INT32_ID_LEN 11 #define TSDB_INT32_ID_LEN 11
#define TSDB_NAME_DELIMITER_LEN 1 #define TSDB_NAME_DELIMITER_LEN 1
@ -388,6 +392,14 @@ typedef enum ELogicConditionType {
#define TSDB_CACHE_MODEL_LAST_ROW 1 #define TSDB_CACHE_MODEL_LAST_ROW 1
#define TSDB_CACHE_MODEL_LAST_VALUE 2 #define TSDB_CACHE_MODEL_LAST_VALUE 2
#define TSDB_CACHE_MODEL_BOTH 3 #define TSDB_CACHE_MODEL_BOTH 3
#define TSDB_ENCRYPT_ALGO_STR_LEN 16
#define TSDB_ENCRYPT_ALGO_NONE_STR "none"
#define TSDB_ENCRYPT_ALGO_SM4_STR "sm4"
#define TSDB_ENCRYPT_ALGO_NONE 0
#define TSDB_ENCRYPT_ALGO_SM4 1
#define TSDB_DEFAULT_ENCRYPT_ALGO TSDB_ENCRYPT_ALGO_NONE
#define TSDB_MIN_ENCRYPT_ALGO TSDB_ENCRYPT_ALGO_NONE
#define TSDB_MAX_ENCRYPT_ALGO TSDB_ENCRYPT_ALGO_SM4
#define TSDB_DEFAULT_CACHE_MODEL TSDB_CACHE_MODEL_NONE #define TSDB_DEFAULT_CACHE_MODEL TSDB_CACHE_MODEL_NONE
#define TSDB_MIN_DB_CACHE_SIZE 1 // MB #define TSDB_MIN_DB_CACHE_SIZE 1 // MB
#define TSDB_MAX_DB_CACHE_SIZE 65536 #define TSDB_MAX_DB_CACHE_SIZE 65536
@ -523,6 +535,7 @@ typedef enum ELogicConditionType {
enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED }; enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED };
enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK }; enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK };
enum { ENCRYPT_KEY_STAT_UNKNOWN = 0, ENCRYPT_KEY_STAT_UNSET, ENCRYPT_KEY_STAT_SET, ENCRYPT_KEY_STAT_LOADED};
typedef struct { typedef struct {
char dir[TSDB_FILENAME_LEN]; char dir[TSDB_FILENAME_LEN];

15
source/client/src/clientImpl.c
View File

@ -1804,6 +1804,10 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
} }
static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) {
int32_t idx = -1;
iconv_t conv = taosAcquireConv(&idx, C2M);
if (!conv) return TSDB_CODE_TSC_INTERNAL_ERROR;
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
int32_t type = pResultInfo->fields[i].type; int32_t type = pResultInfo->fields[i].type;
int32_t bytes = pResultInfo->fields[i].bytes; int32_t bytes = pResultInfo->fields[i].bytes;
@ -1811,6 +1815,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) { if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
if (p == NULL) { if (p == NULL) {
taosReleaseConv(idx, conv, C2M);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -1821,12 +1826,13 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
if (pCol->offset[j] != -1) { if (pCol->offset[j] != -1) {
char* pStart = pCol->offset[j] + pCol->pData; char* pStart = pCol->offset[j] + pCol->pData;
int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p)); int32_t len = taosUcs4ToMbsEx((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p), conv);
if (len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) { if (len < 0 || len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) {
tscError( tscError(
"doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + " "doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + "
"colLength[i]):%p", "colLength[i]):%p",
len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i]));
taosReleaseConv(idx, conv, C2M);
return TSDB_CODE_TSC_INTERNAL_ERROR; return TSDB_CODE_TSC_INTERNAL_ERROR;
} }
@ -1840,7 +1846,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
pResultInfo->row[i] = pResultInfo->pCol[i].pData; pResultInfo->row[i] = pResultInfo->pCol[i].pData;
} }
} }
taosReleaseConv(idx, conv, C2M);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2620,8 +2626,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4); setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4);
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0; pResultInfo->numOfRows = 0;
pRequest->code = code; tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code),
tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId); pRequest->requestId);
} else { } else {
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,

9
source/common/src/systable.c
View File

@ -121,6 +121,7 @@ static const SSysDbTableSchema userDBSchema[] = {
{.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "encrypt_algorithm", .bytes = TSDB_ENCRYPT_ALGO_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
}; };
static const SSysDbTableSchema userFuncSchema[] = { static const SSysDbTableSchema userFuncSchema[] = {
@ -400,12 +401,15 @@ static const SSysDbTableSchema userGrantsLogsSchema[] = {
static const SSysDbTableSchema userMachinesSchema[] = { static const SSysDbTableSchema userMachinesSchema[] = {
{.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
#ifndef TD_UNIQ_GRANT
{.name = "dnode_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "dnode_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
#endif
{.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
}; };
static const SSysDbTableSchema encryptionsSchema[] = {
{.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "key_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysTableMeta infosMeta[] = { static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true}, {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true}, {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
@ -439,6 +443,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_GRANTS_LOGS, userGrantsLogsSchema, tListLen(userGrantsLogsSchema), true}, {TSDB_INS_TABLE_GRANTS_LOGS, userGrantsLogsSchema, tListLen(userGrantsLogsSchema), true},
{TSDB_INS_TABLE_MACHINES, userMachinesSchema, tListLen(userMachinesSchema), true}, {TSDB_INS_TABLE_MACHINES, userMachinesSchema, tListLen(userMachinesSchema), true},
{TSDB_INS_TABLE_ARBGROUPS, arbGroupsSchema, tListLen(arbGroupsSchema), true}, {TSDB_INS_TABLE_ARBGROUPS, arbGroupsSchema, tListLen(arbGroupsSchema), true},
{TSDB_INS_TABLE_ENCRYPTIONS, encryptionsSchema, tListLen(encryptionsSchema), true},
{TSDB_INS_TABLE_TSMAS, tsmaSchema, tListLen(tsmaSchema), false}, {TSDB_INS_TABLE_TSMAS, tsmaSchema, tListLen(tsmaSchema), false},
}; };

4
source/common/src/tcol.c
View File

@ -67,6 +67,8 @@ uint8_t getDefaultEncode(uint8_t type) {
return TSDB_COLVAL_ENCODE_SIMPLE8B; return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_MEDIUMBLOB: case TSDB_DATA_TYPE_MEDIUMBLOB:
case TSDB_DATA_TYPE_GEOMETRY: case TSDB_DATA_TYPE_GEOMETRY:
return TSDB_COLVAL_ENCODE_DISABLED;
case TSDB_DATA_TYPE_MAX: case TSDB_DATA_TYPE_MAX:
return TSDB_COLVAL_ENCODE_SIMPLE8B; return TSDB_COLVAL_ENCODE_SIMPLE8B;
@ -365,7 +367,7 @@ int8_t validColEncode(uint8_t type, uint8_t l1) {
} else if (type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) { } else if (type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) {
return TSDB_COLVAL_ENCODE_DELTAD == l1 ? 1 : 0; return TSDB_COLVAL_ENCODE_DELTAD == l1 ? 1 : 0;
} else if ((type == TSDB_DATA_TYPE_VARCHAR || type == TSDB_DATA_TYPE_NCHAR) || type == TSDB_DATA_TYPE_JSON || } else if ((type == TSDB_DATA_TYPE_VARCHAR || type == TSDB_DATA_TYPE_NCHAR) || type == TSDB_DATA_TYPE_JSON ||
type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_BINARY) { type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) {
return l1 == TSDB_COLVAL_ENCODE_DISABLED ? 1 : 0; return l1 == TSDB_COLVAL_ENCODE_DISABLED ? 1 : 0;
// if (l1 >= TSDB_COLVAL_ENCODE_NOCHANGE || l1 <= TSDB_COLVAL_ENCODE_DELTAD) { // if (l1 >= TSDB_COLVAL_ENCODE_NOCHANGE || l1 <= TSDB_COLVAL_ENCODE_DELTAD) {
// return 1; // return 1;

2
source/common/src/tdataformat.c
View File

@ -2450,7 +2450,7 @@ static int32_t tColDataUpdateValue72(SColData *pColData, uint8_t *pData, uint32_
pColData->numOfValue--; pColData->numOfValue--;
pColData->nVal--; pColData->nVal--;
if (pColData->numOfValue) { if (pColData->numOfValue) {
if (IS_STR_DATA_TYPE(pColData->type)) { if (IS_VAR_DATA_TYPE(pColData->type)) {
pColData->nData = pColData->aOffset[pColData->nVal]; pColData->nData = pColData->aOffset[pColData->nVal];
} else { } else {
pColData->nData -= TYPE_BYTES[pColData->type]; pColData->nData -= TYPE_BYTES[pColData->type];

23
source/common/src/tglobal.c
View File

@ -41,6 +41,13 @@ uint16_t tsServerPort = 6030;
int32_t tsVersion = 30000000; int32_t tsVersion = 30000000;
int32_t tsStatusInterval = 1; // second int32_t tsStatusInterval = 1; // second
int32_t tsNumOfSupportVnodes = 256; int32_t tsNumOfSupportVnodes = 256;
char tsEncryptAlgorithm[16] = {0};
char tsEncryptScope[100] = {0};
EEncryptAlgor tsiEncryptAlgorithm = 0;
EEncryptScope tsiEncryptScope = 0;
//char tsAuthCode[500] = {0};
//char tsEncryptKey[17] = {0};
char tsEncryptKey[17] = {0};
// common // common
int32_t tsMaxShellConns = 50000; int32_t tsMaxShellConns = 50000;
@ -77,8 +84,6 @@ int32_t tsSnapReplMaxWaitN = 128;
// mnode // mnode
int64_t tsMndSdbWriteDelta = 200; int64_t tsMndSdbWriteDelta = 200;
int64_t tsMndLogRetention = 2000; int64_t tsMndLogRetention = 2000;
int8_t tsGrant = 1;
int32_t tsMndGrantMode = 0;
bool tsMndSkipGrant = false; bool tsMndSkipGrant = false;
bool tsEnableWhiteList = false; // ip white list cfg bool tsEnableWhiteList = false; // ip white list cfg
@ -92,6 +97,11 @@ int64_t tsDndStart = 0;
int64_t tsDndStartOsUptime = 0; int64_t tsDndStartOsUptime = 0;
int64_t tsDndUpTime = 0; int64_t tsDndUpTime = 0;
// dnode misc
uint32_t tsEncryptionKeyChksum = 0;
int8_t tsEncryptionKeyStat = ENCRYPT_KEY_STAT_UNSET;
int8_t tsGrant = 1;
// monitor // monitor
bool tsEnableMonitor = true; bool tsEnableMonitor = true;
int32_t tsMonitorInterval = 30; int32_t tsMonitorInterval = 30;
@ -609,6 +619,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
//if (cfgAddString(pCfg, "authCode", tsAuthCode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
return -1; return -1;
@ -705,7 +719,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
return -1; return -1;
if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -1163,6 +1176,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32;
tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32;
tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32;
tstrncpy(tsEncryptAlgorithm, cfgGetItem(pCfg, "encryptAlgorithm")->str, 16);
tstrncpy(tsEncryptScope, cfgGetItem(pCfg, "encryptScope")->str, 100);
//tstrncpy(tsAuthCode, cfgGetItem(pCfg, "authCode")->str, 100);
tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32;
tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32; tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32;
@ -1238,7 +1254,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64; tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64;
tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64; tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64;
tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval; tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval;
tsMndGrantMode = cfgGetItem(pCfg, "grantMode")->i32;
tsEnableWhiteList = cfgGetItem(pCfg, "enableWhiteList")->bval; tsEnableWhiteList = cfgGetItem(pCfg, "enableWhiteList")->bval;
tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tsStartUdfd = cfgGetItem(pCfg, "udf")->bval;

36
source/common/src/tmsg.c
View File

@ -1216,6 +1216,8 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tEncodeCStr(&encoder, pReq->clusterCfg.locale) < 0) return -1; if (tEncodeCStr(&encoder, pReq->clusterCfg.locale) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->clusterCfg.charset) < 0) return -1; if (tEncodeCStr(&encoder, pReq->clusterCfg.charset) < 0) return -1;
if (tEncodeI8(&encoder, pReq->clusterCfg.enableWhiteList) < 0) return -1; if (tEncodeI8(&encoder, pReq->clusterCfg.enableWhiteList) < 0) return -1;
if (tEncodeI8(&encoder, pReq->clusterCfg.encryptionKeyStat) < 0) return -1;
if (tEncodeU32(&encoder, pReq->clusterCfg.encryptionKeyChksum) < 0) return -1;
// vnode loads // vnode loads
int32_t vlen = (int32_t)taosArrayGetSize(pReq->pVloads); int32_t vlen = (int32_t)taosArrayGetSize(pReq->pVloads);
@ -1308,6 +1310,8 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tDecodeCStrTo(&decoder, pReq->clusterCfg.locale) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->clusterCfg.locale) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->clusterCfg.charset) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->clusterCfg.charset) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->clusterCfg.enableWhiteList) < 0) return -1; if (tDecodeI8(&decoder, &pReq->clusterCfg.enableWhiteList) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->clusterCfg.encryptionKeyStat) < 0) return -1;
if (tDecodeU32(&decoder, &pReq->clusterCfg.encryptionKeyChksum) < 0) return -1;
// vnode loads // vnode loads
int32_t vlen = 0; int32_t vlen = 0;
@ -3080,13 +3084,14 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
} }
if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1; if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
ENCODESQL(); ENCODESQL();
if (tEncodeI8(&encoder, pReq->withArbitrator) < 0) return -1; if (tEncodeI8(&encoder, pReq->withArbitrator) < 0) return -1;
if (tEncodeI8(&encoder, pReq->encryptAlgorithm) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
@ -3155,22 +3160,21 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1; if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
} }
DECODESQL();
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) { if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->withArbitrator) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->encryptAlgorithm) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1; if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1; if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
} }
DECODESQL();
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->withArbitrator) < 0) return -1;
}
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
return 0; return 0;
@ -4137,6 +4141,7 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) {
if (tEncodeI16(encoder, pRsp->sstTrigger) < 0) return -1; if (tEncodeI16(encoder, pRsp->sstTrigger) < 0) return -1;
if (tEncodeI32(encoder, pRsp->keepTimeOffset) < 0) return -1; if (tEncodeI32(encoder, pRsp->keepTimeOffset) < 0) return -1;
if (tEncodeI8(encoder, pRsp->withArbitrator) < 0) return -1; if (tEncodeI8(encoder, pRsp->withArbitrator) < 0) return -1;
if (tEncodeI8(encoder, pRsp->encryptAlgorithm) < 0) return -1;
if (tEncodeI32(encoder, pRsp->s3ChunkSize) < 0) return -1; if (tEncodeI32(encoder, pRsp->s3ChunkSize) < 0) return -1;
if (tEncodeI32(encoder, pRsp->s3KeepLocal) < 0) return -1; if (tEncodeI32(encoder, pRsp->s3KeepLocal) < 0) return -1;
if (tEncodeI8(encoder, pRsp->s3Compact) < 0) return -1; if (tEncodeI8(encoder, pRsp->s3Compact) < 0) return -1;
@ -4213,14 +4218,13 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) {
if (tDecodeI32(decoder, &pRsp->keepTimeOffset) < 0) return -1; if (tDecodeI32(decoder, &pRsp->keepTimeOffset) < 0) return -1;
} }
pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; pRsp->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (!tDecodeIsEnd(decoder)) { pRsp->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
if (tDecodeI8(decoder, &pRsp->withArbitrator) < 0) return -1;
}
pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT; pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(decoder)) { if (!tDecodeIsEnd(decoder)) {
if (tDecodeI8(decoder, &pRsp->withArbitrator) < 0) return -1;
if (tDecodeI8(decoder, &pRsp->encryptAlgorithm) < 0) return -1;
if (tDecodeI32(decoder, &pRsp->s3ChunkSize) < 0) return -1; if (tDecodeI32(decoder, &pRsp->s3ChunkSize) < 0) return -1;
if (tDecodeI32(decoder, &pRsp->s3KeepLocal) < 0) return -1; if (tDecodeI32(decoder, &pRsp->s3KeepLocal) < 0) return -1;
if (tDecodeI8(decoder, &pRsp->s3Compact) < 0) return -1; if (tDecodeI8(decoder, &pRsp->s3Compact) < 0) return -1;
@ -5282,6 +5286,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
} }
if (tEncodeI32(&encoder, pReq->changeVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->changeVersion) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1; if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI8(&encoder, pReq->encryptAlgorithm) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1; if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1; if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
@ -5378,11 +5383,12 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (!tDecodeIsEnd(&decoder)) { if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1; if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
} }
pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) { if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->encryptAlgorithm) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1; if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1; if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;

4
source/dnode/mgmt/CMakeLists.txt
View File

@ -31,8 +31,8 @@ ENDIF ()
IF (TD_LINUX_64 AND JEMALLOC_ENABLED) IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEPENDENCIES(taosd jemalloc) ADD_DEPENDENCIES(taosd jemalloc)
target_link_libraries(taosd dnode ${LINK_JEMALLOC}) target_link_libraries(taosd dnode crypt ${LINK_JEMALLOC})
ELSE () ELSE ()
target_link_libraries(taosd dnode) target_link_libraries(taosd dnode crypt)
ENDIF () ENDIF ()

41
source/dnode/mgmt/exe/dmMain.c
View File

@ -22,6 +22,7 @@
#ifdef TD_JEMALLOC_ENABLED #ifdef TD_JEMALLOC_ENABLED
#include "jemalloc/jemalloc.h" #include "jemalloc/jemalloc.h"
#endif #endif
#include "dmUtil.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h" #include "cus_name.h"
@ -49,6 +50,8 @@
#define DM_VERSION "Print program version." #define DM_VERSION "Print program version."
#define DM_EMAIL "<support@taosdata.com>" #define DM_EMAIL "<support@taosdata.com>"
#define DM_MEM_DBG "Enable memory debug" #define DM_MEM_DBG "Enable memory debug"
#define DM_SET_ENCRYPTKEY "Set encrypt key. such as: -y 1234567890abcdef, the length should be less or equal to 16."
// clang-format on // clang-format on
static struct { static struct {
#ifdef WINDOWS #ifdef WINDOWS
@ -67,6 +70,8 @@ static struct {
const char **envCmd; const char **envCmd;
SArray *pArgs; // SConfigPair SArray *pArgs; // SConfigPair
int64_t startTime; int64_t startTime;
bool generateCode;
char encryptKey[ENCRYPT_KEY_LEN + 1];
} global = {0}; } global = {0};
static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetGlobalDebugFlag(143); } static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetGlobalDebugFlag(143); }
@ -195,6 +200,23 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
} }
} else if (strcmp(argv[i], "-k") == 0) { } else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true; global.generateGrant = true;
} else if (strcmp(argv[i], "-y") == 0) {
global.generateCode = true;
if(i < argc - 1) {
int32_t len = strlen(argv[++i]);
if (len < ENCRYPT_KEY_LEN_MIN) {
printf("encrypt key is too short, it should be great or equal to %d\n", ENCRYPT_KEY_LEN_MIN);
return -1;
}
if (len > ENCRYPT_KEY_LEN) {
printf("encrypt key overflow, it should be less or equal to %d\n", ENCRYPT_KEY_LEN);
return -1;
}
tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
} else {
printf("'-y' requires a parameter\n");
return -1;
}
} else if (strcmp(argv[i], "-C") == 0) { } else if (strcmp(argv[i], "-C") == 0) {
global.dumpConfig = true; global.dumpConfig = true;
} else if (strcmp(argv[i], "-V") == 0) { } else if (strcmp(argv[i], "-V") == 0) {
@ -259,6 +281,7 @@ static void dmPrintHelp() {
printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD); printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE); printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE); printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE);
printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY);
printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG); printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG);
printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION); printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION);
@ -364,6 +387,24 @@ int mainWindows(int argc, char **argv) {
return -1; return -1;
} }
if(global.generateCode) {
if(dmCheckRunning(tsDataDir) == NULL) {
dError("failed to generate encrypt code since taosd is running, please stop it first");
return -1;
}
int ret = dmUpdateEncryptKey(global.encryptKey);
taosCloseLog();
taosCleanupArgs();
return ret;
}
if(dmGetEncryptKey() != 0){
dError("failed to start since failed to get encrypt key");
taosCloseLog();
taosCleanupArgs();
return -1;
};
if (taosConvInit() != 0) { if (taosConvInit() != 0) {
dError("failed to init conv"); dError("failed to init conv");
taosCloseLog(); taosCloseLog();

1
source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
View File

@ -57,6 +57,7 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg); int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg);
int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg); int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg);
int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
// dmWorker.c // dmWorker.c
int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);

30
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dmInt.h" #include "dmInt.h"
#include "systable.h" #include "systable.h"
#include "tchecksum.h"
extern SConfig *tsCfg; extern SConfig *tsCfg;
@ -113,6 +114,8 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.clusterCfg.checkTime = 0; req.clusterCfg.checkTime = 0;
req.clusterCfg.ttlChangeOnWrite = tsTtlChangeOnWrite; req.clusterCfg.ttlChangeOnWrite = tsTtlChangeOnWrite;
req.clusterCfg.enableWhiteList = tsEnableWhiteList ? 1 : 0; req.clusterCfg.enableWhiteList = tsEnableWhiteList ? 1 : 0;
req.clusterCfg.encryptionKeyStat = tsEncryptionKeyStat;
req.clusterCfg.encryptionKeyChksum = tsEncryptionKeyChksum;
char timestr[32] = "1970-01-01 00:00:00.00"; char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN);
@ -221,6 +224,32 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return 0; return 0;
} }
int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
#ifdef TD_ENTERPRISE
int32_t code = 0;
SDCfgDnodeReq cfgReq = {0};
if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
code = dmUpdateEncryptKey(cfgReq.value);
if (code == 0) {
tsEncryptionKeyChksum = taosCalcChecksum(0, cfgReq.value, strlen(cfgReq.value));
tsEncryptionKeyStat = ENCRYPT_KEY_STAT_LOADED;
tstrncpy(tsEncryptKey, cfgReq.value, ENCRYPT_KEY_LEN + 1);
}
_exit:
pMsg->code = code;
pMsg->info.rsp = NULL;
pMsg->info.rspLen = 0;
return code;
#else
return 0;
#endif
}
static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK; pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK;
pStatus->details[0] = 0; pStatus->details[0] = 0;
@ -421,6 +450,7 @@ SArray *dmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_ENCRYPT_KEY, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
// Requests handled by MNODE // Requests handled by MNODE
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;

3
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -359,6 +359,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
case TDMT_MND_GRANT_NOTIFY: case TDMT_MND_GRANT_NOTIFY:
code = dmProcessGrantNotify(NULL, pMsg); code = dmProcessGrantNotify(NULL, pMsg);
break; break;
case TDMT_DND_CREATE_ENCRYPT_KEY:
code = dmProcessCreateEncryptKeyReq(pMgmt, pMsg);
break;
default: default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED; terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed in mgmt queue", pMsg); dGError("msg:%p, not processed in mgmt queue", pMsg);

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -113,6 +113,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CONFIG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CONFIG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_ENCRYPT_KEY_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -204,6 +205,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ENCRYPT_KEY, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;

38
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -142,6 +142,14 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
if ((pRetention->freq >= 0 && pRetention->keep > 0)) pCfg->isRsma = 1; if ((pRetention->freq >= 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
} }
} }
#if defined(TD_ENTERPRISE)
pCfg->tsdbCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
pCfg->tsdbCfg.encryptAlgorithm = 0;
#endif
pCfg->walCfg.vgId = pCreate->vgId; pCfg->walCfg.vgId = pCreate->vgId;
pCfg->walCfg.fsyncPeriod = pCreate->walFsyncPeriod; pCfg->walCfg.fsyncPeriod = pCreate->walFsyncPeriod;
@ -150,6 +158,23 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->walCfg.retentionSize = pCreate->walRetentionSize; pCfg->walCfg.retentionSize = pCreate->walRetentionSize;
pCfg->walCfg.segSize = pCreate->walSegmentSize; pCfg->walCfg.segSize = pCreate->walSegmentSize;
pCfg->walCfg.level = pCreate->walLevel; pCfg->walCfg.level = pCreate->walLevel;
#if defined(TD_ENTERPRISE)
pCfg->walCfg.encryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
pCfg->walCfg.encryptAlgorithm = 0;
#endif
#if defined(TD_ENTERPRISE)
pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm;
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
pCfg->tdbEncryptAlgorithm = 0;
#endif
pCfg->sttTrigger = pCreate->sstTrigger; pCfg->sttTrigger = pCreate->sstTrigger;
pCfg->hashBegin = pCreate->hashBegin; pCfg->hashBegin = pCreate->hashBegin;
@ -246,14 +271,15 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
"precision:%d compression:%d minRows:%d maxRows:%d" "precision:%d compression:%d minRows:%d maxRows:%d"
", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64 ", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64
", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d " ", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d "
"learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d", "learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d encryptAlgorithm:%d",
req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024,
(uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, (uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize,
req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression, req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression,
req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion); req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion,
req.encryptAlgorithm);
for (int32_t i = 0; i < req.replica; ++i) { for (int32_t i = 0; i < req.replica; ++i) {
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port, dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port,
@ -278,6 +304,14 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1; return -1;
} }
if(req.encryptAlgorithm == DND_CA_SM4){
if(strlen(tsEncryptKey) == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId);
return -1;
}
}
vmGenerateVnodeCfg(&req, &vnodeCfg); vmGenerateVnodeCfg(&req, &vnodeCfg);
if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) { if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) {

40
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -161,6 +161,46 @@ int32_t dmInitVars(SDnode *pDnode) {
return -1; return -1;
} }
#if defined(TD_ENTERPRISE)
tsiEncryptAlgorithm = pData->encryptAlgorigthm;
tsiEncryptScope = pData->encryptScope;
/*
if(tsiEncryptAlgorithm != 0) {
if(pData->machineId != NULL && strlen(pData->machineId) > 0){
dInfo("get crypt key at startup, machineId:%s", pData->machineId);
int32_t code = 0;
//code = taosGetCryptKey(tsAuthCode, pData->machineId, tsCryptKey);
code = 0;
strncpy(tsEncryptKey, tsAuthCode, 16);
if(code != 0) {
if(code == -1){
terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
dError("machine code changed, can't get crypt key");
}
if(code == -2){
terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
dError("failed to get crypt key");
}
return -1;
}
if(strlen(tsEncryptKey) == 0){
terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
dError("failed to get crypt key at startup since key is null, machineId:%s", pData->machineId);
return -1;
}
}
else{
terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
dError("failed to get crypt key at startup, machineId:%s", pData->machineId);
return -1;
}
}
*/
#endif
if (pData->dropped) { if (pData->dropped) {
dError("dnode will not start since its already dropped"); dError("dnode will not start since its already dropped");
return -1; return -1;

5
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -110,6 +110,8 @@ typedef struct {
bool validMnodeEps; bool validMnodeEps;
int64_t ipWhiteVer; int64_t ipWhiteVer;
char machineId[TSDB_MACHINE_ID_LEN + 1]; char machineId[TSDB_MACHINE_ID_LEN + 1];
EEncryptAlgor encryptAlgorigthm;
EEncryptScope encryptScope;
} SDnodeData; } SDnodeData;
typedef struct { typedef struct {
@ -191,7 +193,8 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
bool dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port); bool dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port);
void dmRemoveDnodePairs(SDnodeData *pData); void dmRemoveDnodePairs(SDnodeData *pData);
void dmGetDnodeEp(void *pData, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort); void dmGetDnodeEp(void *pData, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort);
int32_t dmUpdateEncryptKey(char *key);
int32_t dmGetEncryptKey();
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

99
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -64,7 +64,12 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) {
if (code < 0) return -1; if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "dropped", pData->dropped, code); tjsonGetInt32ValueFromDouble(pJson, "dropped", pData->dropped, code);
if (code < 0) return -1; if (code < 0) return -1;
#ifdef TD_ENTERPRISE
tjsonGetInt32ValueFromDouble(pJson, "encryptAlgor", pData->encryptAlgorigthm, code);
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "encryptScope", pData->encryptScope, code);
if (code < 0) return -1;
#endif
SJson *dnodes = tjsonGetObjectItem(pJson, "dnodes"); SJson *dnodes = tjsonGetObjectItem(pJson, "dnodes");
if (dnodes == NULL) return 0; if (dnodes == NULL) return 0;
int32_t numOfDnodes = tjsonGetArraySize(dnodes); int32_t numOfDnodes = tjsonGetArraySize(dnodes);
@ -89,6 +94,25 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) {
return 0; return 0;
} }
int dmOccurrences(char *str, char *toSearch) {
int count = 0;
char *ptr = str;
while ((ptr = strstr(ptr, toSearch)) != NULL) {
count++;
ptr++;
}
return count;
}
void dmSplitStr(char** arr, char* str, const char* del) {
char *lasts;
char* s = strsep(&str, del);
while (s != NULL) {
*arr++ = s;
s = strsep(&str, del);
}
}
int32_t dmReadEps(SDnodeData *pData) { int32_t dmReadEps(SDnodeData *pData) {
int32_t code = -1; int32_t code = -1;
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
@ -106,6 +130,74 @@ int32_t dmReadEps(SDnodeData *pData) {
if (taosStatFile(file, NULL, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("dnode file:%s not exist", file); dInfo("dnode file:%s not exist", file);
#ifdef TD_ENTERPRISE
if(strlen(tsEncryptAlgorithm) > 0){
if(strcmp(tsEncryptAlgorithm, "sm4") == 0) {
pData->encryptAlgorigthm = DND_CA_SM4;
}
else{
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptAlgorithm:%s", tsEncryptAlgorithm);
goto _OVER;
}
dInfo("start to parse encryptScope:%s", tsEncryptScope);
int32_t scopeLen = strlen(tsEncryptScope);
if(scopeLen == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptScope:%s", tsEncryptScope);
goto _OVER;
}
char* tmp = taosMemoryMalloc(scopeLen + 1);
memset(tmp, 0, scopeLen + 1);
memcpy(tmp, tsEncryptScope, scopeLen);
int32_t count = dmOccurrences(tmp, ",");
char** array = taosMemoryMalloc(sizeof(char*) * (count + 1));
memset(array, 0, sizeof(char*) * (count + 1));
dmSplitStr(array, tmp, ",");
for(int32_t i = 0; i < count + 1; i++){
char* str = *(array + i);
bool success = false;
if(strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_TSDB;
success = true;
}
if(strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_VNODE_WAL;
success = true;
}
if(strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_SDB;
success = true;
}
if(strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_MNODE_WAL;
success = true;
}
if(!success){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
taosMemoryFree(tmp);
taosMemoryFree(array);
dError("invalid tsEncryptScope:%s", tsEncryptScope);
goto _OVER;
}
}
taosMemoryFree(tmp);
taosMemoryFree(array);
dInfo("set tsCryptAlgorithm:%s, tsCryptScope:%s from cfg", tsEncryptAlgorithm, tsEncryptScope);
}
#endif
code = 0; code = 0;
goto _OVER; goto _OVER;
} }
@ -191,7 +283,10 @@ static int32_t dmEncodeEps(SJson *pJson, SDnodeData *pData) {
if (tjsonAddIntegerToObject(pJson, "engineVer", pData->engineVer) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "engineVer", pData->engineVer) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "clusterId", pData->clusterId) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "clusterId", pData->clusterId) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "dropped", pData->dropped) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "dropped", pData->dropped) < 0) return -1;
#ifdef TD_ENTERPRISE
if (tjsonAddDoubleToObject(pJson, "encryptAlgor", pData->encryptAlgorigthm) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "encryptScope", pData->encryptScope) < 0) return -1;
#endif
SJson *dnodes = tjsonCreateArray(); SJson *dnodes = tjsonCreateArray();
if (dnodes == NULL) return -1; if (dnodes == NULL) return -1;
if (tjsonAddItemToObject(pJson, "dnodes", dnodes) < 0) return -1; if (tjsonAddItemToObject(pJson, "dnodes", dnodes) < 0) return -1;

299
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -16,8 +16,14 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dmUtil.h" #include "dmUtil.h"
#include "tjson.h" #include "tjson.h"
#include "tgrant.h"
#include "crypt.h"
#include "tchecksum.h"
#define MAXLEN 1024 #define MAXLEN 1024
#define DM_KEY_INDICATOR "this indicator!"
#define DM_ENCRYPT_CODE_FILE "encryptCode.cfg"
#define DM_CHECK_CODE_FILE "checkCode.bin"
static int32_t dmDecodeFile(SJson *pJson, bool *deployed) { static int32_t dmDecodeFile(SJson *pJson, bool *deployed) {
int32_t code = 0; int32_t code = 0;
@ -177,3 +183,296 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
dDebug("lock file:%s to prevent repeated starts", filepath); dDebug("lock file:%s to prevent repeated starts", filepath);
return pFile; return pFile;
} }
extern int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode);
static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key){
TdFilePtr pFile = NULL;
char *result = NULL;
int32_t code = -1;
int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR));
result = taosMemoryMalloc(len);
SCryptOpts opts;
strncpy(opts.key, key, ENCRYPT_KEY_LEN);
opts.len = len;
opts.source = DM_KEY_INDICATOR;
opts.result = result;
opts.unitLen = 16;
CBC_Encrypt(&opts);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (taosWriteFile(pFile, opts.result, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
dInfo("succeed to write checkCode file:%s", realfile);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
if(result != NULL) taosMemoryFree(result);
return code;
}
static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptCode){
TdFilePtr pFile = NULL;
int32_t code = -1;
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
int32_t len = strlen(encryptCode);
if (taosWriteFile(pFile, encryptCode, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
dInfo("succeed to write encryptCode file:%s", realfile);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
return code;
}
static int32_t dmCompareEncryptKey(char* file, char* key){
char *content = NULL;
int64_t size = 0;
TdFilePtr pFile = NULL;
char *result = NULL;
int32_t code = -1;
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content = taosMemoryMalloc(size);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER;
}
dInfo("succeed to read checkCode file:%s", file);
int len = ENCRYPTED_LEN(size);
result = taosMemoryMalloc(len);
SCryptOpts opts = {0};
strncpy(opts.key, key, ENCRYPT_KEY_LEN);
opts.len = len;
opts.source = content;
opts.result = result;
opts.unitLen = 16;
CBC_Decrypt(&opts);
if(strcmp(opts.result, DM_KEY_INDICATOR) != 0) {
terrno = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED;
dError("failed to compare decrypted result");
goto _OVER;
}
dInfo("succeed to compare checkCode file:%s", file);
code = 0;
_OVER:
if(result != NULL) taosMemoryFree(result);
if(content != NULL) taosMemoryFree(content);
if(pFile != NULL) taosCloseFile(&pFile);
return code;
}
int32_t dmUpdateEncryptKey(char *key) {
#ifdef TD_ENTERPRISE
int32_t code = -1;
char *machineId = NULL;
char *encryptCode = NULL;
char folder[PATH_MAX] = {0};
char encryptFile[PATH_MAX] = {0};
char realEncryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0};
char realCheckFile[PATH_MAX] = {0};
snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP);
snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
terrno = 0;
if (taosMkDir(folder) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", folder, terrstr());
goto _OVER;
}
if(taosCheckExistFile(realCheckFile)){
if(dmCompareEncryptKey(realCheckFile, key) != 0){
goto _OVER;
}
}
if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (generateEncryptCode(key, machineId, &encryptCode) != 0) {
goto _OVER;
}
if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode) != 0){
goto _OVER;
}
if(dmWriteCheckCodeFile(checkFile, realCheckFile, key) != 0){
goto _OVER;
}
code = 0;
_OVER:
taosMemoryFree(encryptCode);
taosMemoryFree(machineId);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to update encrypt key since %s", terrstr());
}
return code;
#else
return 0;
#endif
}
extern int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);
static int32_t dmReadEncryptCodeFile(char* file, char** output){
TdFilePtr pFile = NULL;
int32_t code = -1;
char *content = NULL;
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content[size] = '\0';
*output = content;
dInfo("succeed to read encryptCode file:%s", file);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
return code;
}
int32_t dmGetEncryptKey(){
#ifdef TD_ENTERPRISE
int32_t code = -1;
char encryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0};
char *machineId = NULL;
char *encryptKey = NULL;
char *content = NULL;
snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE);
if(!taosCheckExistFile(encryptFile)){
dInfo("no exist, checkCode file:%s", encryptFile);
return 0;
}
if(dmReadEncryptCodeFile(encryptFile, &content) != 0){
goto _OVER;
}
if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if(checkAndGetCryptKey(content, machineId, &encryptKey) != 0){
goto _OVER;
}
taosMemoryFreeClear(machineId);
taosMemoryFreeClear(content);
if(encryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("failed to read key since %s", terrstr());
goto _OVER;
}
if(dmCompareEncryptKey(checkFile, encryptKey) != 0){
goto _OVER;
}
strncpy(tsEncryptKey, encryptKey, ENCRYPT_KEY_LEN);
taosMemoryFreeClear(encryptKey);
tsEncryptionKeyChksum = taosCalcChecksum(0, tsEncryptKey, strlen(tsEncryptKey));
tsEncryptionKeyStat = ENCRYPT_KEY_STAT_LOADED;
code = 0;
_OVER:
if (content != NULL) taosMemoryFree(content);
if (encryptKey != NULL) taosMemoryFree(encryptKey);
if (machineId != NULL) taosMemoryFree(machineId);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to get encrypt key since %s", terrstr());
}
return code;
#else
return 0;
#endif
}

4
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -141,6 +141,7 @@ typedef enum {
DND_REASON_CHARSET_NOT_MATCH, DND_REASON_CHARSET_NOT_MATCH,
DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH, DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH,
DND_REASON_ENABLE_WHITELIST_NOT_MATCH, DND_REASON_ENABLE_WHITELIST_NOT_MATCH,
DND_REASON_ENCRYPTION_KEY_NOT_MATCH,
DND_REASON_OTHERS DND_REASON_OTHERS
} EDndReason; } EDndReason;
@ -215,6 +216,8 @@ typedef struct {
int64_t memAvail; int64_t memAvail;
int64_t memUsed; int64_t memUsed;
EDndReason offlineReason; EDndReason offlineReason;
uint32_t encryptionKeyChksum;
int8_t encryptionKeyStat;
uint16_t port; uint16_t port;
char fqdn[TSDB_FQDN_LEN]; char fqdn[TSDB_FQDN_LEN];
char ep[TSDB_EP_LEN]; char ep[TSDB_EP_LEN];
@ -383,6 +386,7 @@ typedef struct {
int32_t s3KeepLocal; int32_t s3KeepLocal;
int8_t s3Compact; int8_t s3Compact;
int8_t withArbitrator; int8_t withArbitrator;
int8_t encryptAlgorithm;
} SDbCfg; } SDbCfg;
typedef struct { typedef struct {

8
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -104,6 +104,13 @@ typedef struct {
int64_t timeseriesAllowed; int64_t timeseriesAllowed;
} SGrantInfo; } SGrantInfo;
typedef struct {
int8_t encrypting;
int16_t nEncrypt;
int16_t nSuccess;
int16_t nFailed;
} SEncryptMgmt;
typedef struct SMnode { typedef struct SMnode {
int32_t selfDnodeId; int32_t selfDnodeId;
int64_t clusterId; int64_t clusterId;
@ -127,6 +134,7 @@ typedef struct SMnode {
SProfileMgmt profileMgmt; SProfileMgmt profileMgmt;
STelemMgmt telemMgmt; STelemMgmt telemMgmt;
SSyncMgmt syncMgmt; SSyncMgmt syncMgmt;
SEncryptMgmt encryptMgmt;
SGrantInfo grant; SGrantInfo grant;
MndMsgFp msgFp[TDMT_MAX]; MndMsgFp msgFp[TDMT_MAX];
SMsgCb msgCb; SMsgCb msgCb;

73
source/dnode/mnode/impl/src/mndDb.c
View File

@ -36,7 +36,7 @@
#include "tjson.h" #include "tjson.h"
#define DB_VER_NUMBER 1 #define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 28 #define DB_RESERVE_SIZE 27
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -146,6 +146,7 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3KeepLocal, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3KeepLocal, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.encryptAlgorithm, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->tsmaVersion, _OVER); SDB_SET_INT32(pRaw, dataPos, pDb->tsmaVersion, _OVER);
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
@ -242,6 +243,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3KeepLocal, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3KeepLocal, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.encryptAlgorithm, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->tsmaVersion, _OVER); SDB_GET_INT32(pRaw, dataPos, &pDb->tsmaVersion, _OVER);
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
@ -425,9 +427,12 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1;
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
if ((pCfg->replications == 2) ^ (pCfg->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) return -1; if ((pCfg->replications == 2) ^ (pCfg->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) return -1;
if (pCfg->encryptAlgorithm < TSDB_MIN_ENCRYPT_ALGO || pCfg->encryptAlgorithm > TSDB_MAX_ENCRYPT_ALGO) return -1;
#else #else
if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->replications != 1 && pCfg->replications != 3) return -1;
if (pCfg->encryptAlgorithm != TSDB_DEFAULT_ENCRYPT_ALGO) return -1;
#endif #endif
if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1;
if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1; if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1;
if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1; if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1;
@ -544,6 +549,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT; if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (pCfg->encryptAlgorithm < 0) pCfg->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
} }
static int32_t mndSetCreateDbPrepareAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { static int32_t mndSetCreateDbPrepareAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
@ -721,6 +727,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
.s3Compact = pCreate->s3Compact, .s3Compact = pCreate->s3Compact,
.tsdbPageSize = pCreate->tsdbPageSize, .tsdbPageSize = pCreate->tsdbPageSize,
.withArbitrator = pCreate->withArbitrator, .withArbitrator = pCreate->withArbitrator,
.encryptAlgorithm = pCreate->encryptAlgorithm,
}; };
dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; dbObj.cfg.numOfRetensions = pCreate->numOfRetensions;
@ -806,6 +813,46 @@ static void mndBuildAuditDetailInt64(char *detail, char *tmp, char *format, int6
} }
} }
static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
SDnodeObj *pDnode = NULL;
void *pIter = NULL;
#ifdef TD_ENTERPRISE
if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit;
if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) {
code = TSDB_CODE_MND_INVALID_ENCRYPT_KEY;
mError("db:%s, failed to check encryption key:%" PRIi8 " in mnode leader since it's not loaded", pReq->db,
tsEncryptionKeyStat);
goto _exit;
}
int64_t curMs = taosGetTimestampMs();
while ((pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode))) {
bool online = false;
if ((pDnode->encryptionKeyStat != tsEncryptionKeyStat || pDnode->encryptionKeyChksum != tsEncryptionKeyChksum) &&
(online = mndIsDnodeOnline(pDnode, curMs))) {
code = TSDB_CODE_MND_INVALID_ENCRYPT_KEY;
mError("db:%s, failed to check encryption key:%" PRIi8
"-%u in dnode:%d since it's inconsitent with mnode leader:%" PRIi8 "-%u",
pReq->db, pDnode->encryptionKeyStat, pDnode->encryptionKeyChksum, pDnode->id, tsEncryptionKeyStat,
tsEncryptionKeyChksum);
sdbRelease(pSdb, pDnode);
break;
}
sdbRelease(pSdb, pDnode);
}
#else
if (pReq->encryptAlgorithm != TSDB_ENCRYPT_ALGO_NONE) {
code = TSDB_CODE_MND_INVALID_DB_OPTION;
goto _exit;
}
#endif
_exit:
return code;
}
static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
int32_t code = -1; int32_t code = -1;
@ -856,6 +903,11 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
} }
} }
if ((code = mndCheckDbEncryptKey(pMnode, &createReq)) != 0) {
terrno = code;
goto _OVER;
}
pUser = mndAcquireUser(pMnode, pReq->info.conn.user); pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
if (pUser == NULL) { if (pUser == NULL) {
goto _OVER; goto _OVER;
@ -1207,6 +1259,7 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) {
cfgRsp->s3KeepLocal = pDb->cfg.s3KeepLocal; cfgRsp->s3KeepLocal = pDb->cfg.s3KeepLocal;
cfgRsp->s3Compact = pDb->cfg.s3Compact; cfgRsp->s3Compact = pDb->cfg.s3Compact;
cfgRsp->withArbitrator = pDb->cfg.withArbitrator; cfgRsp->withArbitrator = pDb->cfg.withArbitrator;
cfgRsp->encryptAlgorithm = pDb->cfg.encryptAlgorithm;
} }
static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) {
@ -2028,6 +2081,18 @@ static const char *getCacheModelStr(int8_t cacheModel) {
return "unknown"; return "unknown";
} }
static const char *getEncryptAlgorithmStr(int8_t encryptAlgorithm) {
switch (encryptAlgorithm) {
case TSDB_ENCRYPT_ALGO_NONE:
return TSDB_ENCRYPT_ALGO_NONE_STR;
case TSDB_ENCRYPT_ALGO_SM4:
return TSDB_ENCRYPT_ALGO_SM4_STR;
default:
break;
}
return "unknown";
}
bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb) { bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb) {
if (pDb->cfg.replications == 1) return true; if (pDb->cfg.replications == 1) return true;
@ -2248,6 +2313,12 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false);
const char *encryptAlgorithmStr = getEncryptAlgorithmStr(pDb->cfg.encryptAlgorithm);
char encryptAlgorithmVStr[24] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(encryptAlgorithmVStr, encryptAlgorithmStr, 24);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)encryptAlgorithmVStr, false);
} }
taosMemoryFree(buf); taosMemoryFree(buf);

169
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -42,11 +42,13 @@ static const char *offlineReason[] = {
"version not match", "version not match",
"dnodeId not match", "dnodeId not match",
"clusterId not match", "clusterId not match",
"interval not match", "statusInterval not match",
"timezone not match", "timezone not match",
"locale not match", "locale not match",
"charset not match", "charset not match",
"ttlChangeOnWrite not match", "ttlChangeOnWrite not match",
"enableWhiteList not match",
"encryptionKey not match",
"unknown", "unknown",
}; };
@ -78,6 +80,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq);
static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessNotifyReq(SRpcMsg *pReq);
static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq);
static int32_t mndProcessStatisReq(SRpcMsg *pReq); static int32_t mndProcessStatisReq(SRpcMsg *pReq);
static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pRsp);
static int32_t mndProcessCreateEncryptKeyRsp(SRpcMsg *pRsp);
static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter);
@ -114,6 +118,8 @@ int32_t mndInitDnode(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq);
mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq);
mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq); mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq);
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_ENCRYPT_KEY, mndProcessCreateEncryptKeyReq);
mndSetMsgHandle(pMnode, TDMT_DND_CREATE_ENCRYPT_KEY_RSP, mndProcessCreateEncryptKeyRsp);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig);
@ -372,7 +378,7 @@ int32_t mndGetDbSize(SMnode *pMnode) {
bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) { bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) {
int64_t interval = TABS(pDnode->lastAccessTime - curMs); int64_t interval = TABS(pDnode->lastAccessTime - curMs);
if (interval > 5000 * (int64_t)tsStatusInterval) { if (interval > 5000 * (int64_t)tsStatusInterval) {
if (pDnode->rebootTime > 0) { if (pDnode->rebootTime > 0 && pDnode->offlineReason == DND_REASON_ONLINE) {
pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT;
} }
return false; return false;
@ -436,37 +442,51 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S
if (pCfg->statusInterval != tsStatusInterval) { if (pCfg->statusInterval != tsStatusInterval) {
mError("dnode:%d, statusInterval:%d inconsistent with cluster:%d", pDnode->id, pCfg->statusInterval, mError("dnode:%d, statusInterval:%d inconsistent with cluster:%d", pDnode->id, pCfg->statusInterval,
tsStatusInterval); tsStatusInterval);
terrno = TSDB_CODE_DNODE_INVALID_STATUS_INTERVAL;
return DND_REASON_STATUS_INTERVAL_NOT_MATCH; return DND_REASON_STATUS_INTERVAL_NOT_MATCH;
} }
if ((0 != strcasecmp(pCfg->timezone, tsTimezoneStr)) && (pMnode->checkTime != pCfg->checkTime)) { if ((0 != strcasecmp(pCfg->timezone, tsTimezoneStr)) && (pMnode->checkTime != pCfg->checkTime)) {
mError("dnode:%d, timezone:%s checkTime:%" PRId64 " inconsistent with cluster %s %" PRId64, pDnode->id, mError("dnode:%d, timezone:%s checkTime:%" PRId64 " inconsistent with cluster %s %" PRId64, pDnode->id,
pCfg->timezone, pCfg->checkTime, tsTimezoneStr, pMnode->checkTime); pCfg->timezone, pCfg->checkTime, tsTimezoneStr, pMnode->checkTime);
terrno = TSDB_CODE_DNODE_INVALID_TIMEZONE;
return DND_REASON_TIME_ZONE_NOT_MATCH; return DND_REASON_TIME_ZONE_NOT_MATCH;
} }
if (0 != strcasecmp(pCfg->locale, tsLocale)) { if (0 != strcasecmp(pCfg->locale, tsLocale)) {
mError("dnode:%d, locale:%s inconsistent with cluster:%s", pDnode->id, pCfg->locale, tsLocale); mError("dnode:%d, locale:%s inconsistent with cluster:%s", pDnode->id, pCfg->locale, tsLocale);
terrno = TSDB_CODE_DNODE_INVALID_LOCALE;
return DND_REASON_LOCALE_NOT_MATCH; return DND_REASON_LOCALE_NOT_MATCH;
} }
if (0 != strcasecmp(pCfg->charset, tsCharset)) { if (0 != strcasecmp(pCfg->charset, tsCharset)) {
mError("dnode:%d, charset:%s inconsistent with cluster:%s", pDnode->id, pCfg->charset, tsCharset); mError("dnode:%d, charset:%s inconsistent with cluster:%s", pDnode->id, pCfg->charset, tsCharset);
terrno = TSDB_CODE_DNODE_INVALID_CHARSET;
return DND_REASON_CHARSET_NOT_MATCH; return DND_REASON_CHARSET_NOT_MATCH;
} }
if (pCfg->ttlChangeOnWrite != tsTtlChangeOnWrite) { if (pCfg->ttlChangeOnWrite != tsTtlChangeOnWrite) {
mError("dnode:%d, ttlChangeOnWrite:%d inconsistent with cluster:%d", pDnode->id, pCfg->ttlChangeOnWrite, mError("dnode:%d, ttlChangeOnWrite:%d inconsistent with cluster:%d", pDnode->id, pCfg->ttlChangeOnWrite,
tsTtlChangeOnWrite); tsTtlChangeOnWrite);
terrno = TSDB_CODE_DNODE_INVALID_TTL_CHG_ON_WR;
return DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH; return DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH;
} }
int8_t enable = tsEnableWhiteList ? 1 : 0; int8_t enable = tsEnableWhiteList ? 1 : 0;
if (pCfg->enableWhiteList != enable) { if (pCfg->enableWhiteList != enable) {
mError("dnode:%d, enable :%d inconsistent with cluster:%d", pDnode->id, pCfg->enableWhiteList, enable); mError("dnode:%d, enableWhiteList:%d inconsistent with cluster:%d", pDnode->id, pCfg->enableWhiteList, enable);
terrno = TSDB_CODE_DNODE_INVALID_EN_WHITELIST;
return DND_REASON_ENABLE_WHITELIST_NOT_MATCH; return DND_REASON_ENABLE_WHITELIST_NOT_MATCH;
} }
return 0; if (!atomic_load_8(&pMnode->encryptMgmt.encrypting) &&
(pCfg->encryptionKeyStat != tsEncryptionKeyStat || pCfg->encryptionKeyChksum != tsEncryptionKeyChksum)) {
mError("dnode:%d, encryptionKey:%" PRIi8 "-%u inconsistent with cluster:%" PRIi8 "-%u", pDnode->id,
pCfg->encryptionKeyStat, pCfg->encryptionKeyChksum, tsEncryptionKeyStat, tsEncryptionKeyChksum);
terrno = pCfg->encryptionKeyChksum ? TSDB_CODE_DNODE_INVALID_ENCRYPTKEY : TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
return DND_REASON_ENCRYPTION_KEY_NOT_MATCH;
}
return DND_REASON_ONLINE;
} }
static bool mndUpdateVnodeState(int32_t vgId, SVnodeGid *pGid, SVnodeLoad *pVload) { static bool mndUpdateVnodeState(int32_t vgId, SVnodeGid *pGid, SVnodeLoad *pVload) {
@ -795,8 +815,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer); bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer);
bool reboot = (pDnode->rebootTime != statusReq.rebootTime); bool reboot = (pDnode->rebootTime != statusReq.rebootTime);
bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes; bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes;
bool needCheck = bool encryptKeyChanged = pDnode->encryptionKeyChksum != statusReq.clusterCfg.encryptionKeyChksum;
!online || dnodeChanged || reboot || supportVnodesChanged || pMnode->ipWhiteVer != statusReq.ipWhiteVer; bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged ||
pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged;
const STraceId *trace = &pReq->info.traceId; const STraceId *trace = &pReq->info.traceId;
mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id,
@ -891,7 +912,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pDnode->offlineReason = mndCheckClusterCfgPara(pMnode, pDnode, &statusReq.clusterCfg); pDnode->offlineReason = mndCheckClusterCfgPara(pMnode, pDnode, &statusReq.clusterCfg);
if (pDnode->offlineReason != 0) { if (pDnode->offlineReason != 0) {
mError("dnode:%d, cluster cfg inconsistent since:%s", pDnode->id, offlineReason[pDnode->offlineReason]); mError("dnode:%d, cluster cfg inconsistent since:%s", pDnode->id, offlineReason[pDnode->offlineReason]);
terrno = TSDB_CODE_MND_INVALID_CLUSTER_CFG; if (terrno == 0) terrno = TSDB_CODE_MND_INVALID_CLUSTER_CFG;
goto _OVER; goto _OVER;
} }
@ -909,6 +930,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pDnode->numOfDiskCfg = statusReq.numOfDiskCfg; pDnode->numOfDiskCfg = statusReq.numOfDiskCfg;
pDnode->memAvail = statusReq.memAvail; pDnode->memAvail = statusReq.memAvail;
pDnode->memTotal = statusReq.memTotal; pDnode->memTotal = statusReq.memTotal;
pDnode->encryptionKeyStat = statusReq.clusterCfg.encryptionKeyStat;
pDnode->encryptionKeyChksum = statusReq.clusterCfg.encryptionKeyChksum;
if (memcmp(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN) != 0) { if (memcmp(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN) != 0) {
tstrncpy(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN + 1); tstrncpy(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN + 1);
mndUpdateDnodeObj(pMnode, pDnode); mndUpdateDnodeObj(pMnode, pDnode);
@ -1494,6 +1517,136 @@ static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) {
return 0; return 0;
} }
static int32_t mndProcessCreateEncryptKeyReqImpl(SRpcMsg *pReq, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
int8_t encrypting = 0;
const STraceId *trace = &pReq->info.traceId;
int32_t klen = strlen(pDcfgReq->value);
if (klen > ENCRYPT_KEY_LEN || klen < ENCRYPT_KEY_LEN_MIN) {
code = TSDB_CODE_DNODE_INVALID_ENCRYPT_KLEN;
mGError("msg:%p, failed to create encrypt_key since invalid key length:%d, valid range:[%d, %d]", pReq, klen,
ENCRYPT_KEY_LEN_MIN, ENCRYPT_KEY_LEN);
goto _exit;
}
if (0 != (encrypting = atomic_val_compare_exchange_8(&pMnode->encryptMgmt.encrypting, 0, 1))) {
code = TSDB_CODE_QRY_DUPLICATED_OPERATION;
mGWarn("msg:%p, failed to create encrypt key since %s, encrypting:%" PRIi8, pReq, tstrerror(code), encrypting);
goto _exit;
}
if (tsEncryptionKeyStat == ENCRYPT_KEY_STAT_SET || tsEncryptionKeyStat == ENCRYPT_KEY_STAT_LOADED) {
atomic_store_8(&pMnode->encryptMgmt.encrypting, 0);
code = TSDB_CODE_QRY_DUPLICATED_OPERATION;
mGWarn("msg:%p, failed to create encrypt key since %s, stat:%" PRIi8 ", checksum:%u", pReq, tstrerror(code),
tsEncryptionKeyStat, tsEncryptionKeyChksum);
goto _exit;
}
atomic_store_16(&pMnode->encryptMgmt.nEncrypt, 0);
atomic_store_16(&pMnode->encryptMgmt.nSuccess, 0);
atomic_store_16(&pMnode->encryptMgmt.nFailed, 0);
while (1) {
SDnodeObj *pDnode = NULL;
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode);
if (pIter == NULL) break;
if (pDnode->offlineReason != DND_REASON_ONLINE) {
mGWarn("msg:%p, don't send create encrypt_key req since dnode:%d in offline state:%s", pReq, pDnode->id,
offlineReason[pDnode->offlineReason]);
sdbRelease(pSdb, pDnode);
continue;
}
if (dnodeId == -1 || pDnode->id == dnodeId || dnodeId == 0) {
SEpSet epSet = mndGetDnodeEpset(pDnode);
int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, pDcfgReq);
void *pBuf = rpcMallocCont(bufLen);
if (pBuf != NULL) {
tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq);
SRpcMsg rpcMsg = {.msgType = TDMT_DND_CREATE_ENCRYPT_KEY, .pCont = pBuf, .contLen = bufLen};
if (0 == tmsgSendReq(&epSet, &rpcMsg)) {
atomic_add_fetch_16(&pMnode->encryptMgmt.nEncrypt, 1);
}
}
}
sdbRelease(pSdb, pDnode);
}
if (atomic_load_16(&pMnode->encryptMgmt.nEncrypt) <= 0) {
atomic_store_8(&pMnode->encryptMgmt.encrypting, 0);
}
_exit:
if (code != 0) {
if (terrno == 0) terrno = code;
}
return code;
}
static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) {
#ifdef TD_ENTERPRISE
SMnode *pMnode = pReq->info.node;
SMCfgDnodeReq cfgReq = {0};
if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) {
tFreeSMCfgDnodeReq(&cfgReq);
return -1;
}
const STraceId *trace = &pReq->info.traceId;
SDCfgDnodeReq dcfgReq = {0};
if (strncasecmp(cfgReq.config, "encrypt_key", 12) == 0) {
strcpy(dcfgReq.config, cfgReq.config);
strcpy(dcfgReq.value, cfgReq.value);
tFreeSMCfgDnodeReq(&cfgReq);
return mndProcessCreateEncryptKeyReqImpl(pReq, cfgReq.dnodeId, &dcfgReq);
} else {
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
tFreeSMCfgDnodeReq(&cfgReq);
return -1;
}
#else
return 0;
#endif
}
static int32_t mndProcessCreateEncryptKeyRsp(SRpcMsg *pRsp) {
SMnode *pMnode = pRsp->info.node;
int16_t nSuccess = 0;
int16_t nFailed = 0;
if (0 == pRsp->code) {
nSuccess = atomic_add_fetch_16(&pMnode->encryptMgmt.nSuccess, 1);
} else {
nFailed = atomic_add_fetch_16(&pMnode->encryptMgmt.nFailed, 1);
}
int16_t nReq = atomic_load_16(&pMnode->encryptMgmt.nEncrypt);
bool finished = nSuccess + nFailed >= nReq;
if (finished) {
atomic_store_8(&pMnode->encryptMgmt.encrypting, 0);
}
const STraceId *trace = &pRsp->info.traceId;
mGInfo("msg:%p, create encrypt key rsp, nReq:%" PRIi16 ", nSucess:%" PRIi16 ", nFailed:%" PRIi16 ", %s", pRsp, nReq,
nSuccess, nFailed, finished ? "encrypt done" : "in encrypting");
return 0;
}
static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
int32_t totalRows = 0; int32_t totalRows = 0;
@ -1550,7 +1703,7 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
ESdbStatus objStatus = 0; ESdbStatus objStatus = 0;
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
int64_t curMs = taosGetTimestampMs(); int64_t curMs = taosGetTimestampMs();
char buf[TSDB_CONN_ACTIVE_KEY_LEN + VARSTR_HEADER_SIZE]; // make sure TSDB_CONN_ACTIVE_KEY_LEN >= TSDB_EP_LEN char buf[TSDB_EP_LEN + VARSTR_HEADER_SIZE];
while (numOfRows < rows) { while (numOfRows < rows) {
pShow->pIter = sdbFetchAll(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode, &objStatus, true); pShow->pIter = sdbFetchAll(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode, &objStatus, true);

1
source/dnode/mnode/impl/src/mndDump.c
View File

@ -104,6 +104,7 @@ void dumpDb(SSdb *pSdb, SJson *json) {
tjsonAddStringToObject(item, "maxRows", i642str(pObj->cfg.maxRows)); tjsonAddStringToObject(item, "maxRows", i642str(pObj->cfg.maxRows));
tjsonAddStringToObject(item, "precision", i642str(pObj->cfg.precision)); tjsonAddStringToObject(item, "precision", i642str(pObj->cfg.precision));
tjsonAddStringToObject(item, "compression", i642str(pObj->cfg.compression)); tjsonAddStringToObject(item, "compression", i642str(pObj->cfg.compression));
tjsonAddStringToObject(item, "encryptAlgorithm", i642str(pObj->cfg.encryptAlgorithm));
tjsonAddStringToObject(item, "replications", i642str(pObj->cfg.replications)); tjsonAddStringToObject(item, "replications", i642str(pObj->cfg.replications));
tjsonAddStringToObject(item, "strict", i642str(pObj->cfg.strict)); tjsonAddStringToObject(item, "strict", i642str(pObj->cfg.strict));
tjsonAddStringToObject(item, "cacheLast", i642str(pObj->cfg.cacheLast)); tjsonAddStringToObject(item, "cacheLast", i642str(pObj->cfg.cacheLast));

2
source/dnode/mnode/impl/src/mndGrant.c
View File

@ -58,6 +58,7 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
static int32_t mndRetrieveGrantFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; } static int32_t mndRetrieveGrantFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndRetrieveGrantLogs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; } static int32_t mndRetrieveGrantLogs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndRetrieveMachines(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; } static int32_t mndRetrieveMachines(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndRetrieveEncryptions(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndProcessGrantHB(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; } static int32_t mndProcessGrantHB(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; }
@ -66,6 +67,7 @@ int32_t mndInitGrant(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS_FULL, mndRetrieveGrantFull); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS_FULL, mndRetrieveGrantFull);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS_LOGS, mndRetrieveGrantLogs); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS_LOGS, mndRetrieveGrantLogs);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_MACHINES, mndRetrieveMachines); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_MACHINES, mndRetrieveMachines);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ENCRYPTIONS, mndRetrieveEncryptions);
mndSetMsgHandle(pMnode, TDMT_MND_GRANT_HB_TIMER, mndProcessGrantHB); mndSetMsgHandle(pMnode, TDMT_MND_GRANT_HB_TIMER, mndProcessGrantHB);
return 0; return 0;
} }

15
source/dnode/mnode/impl/src/mndMain.c
View File

@ -485,8 +485,23 @@ static int32_t mndInitWal(SMnode *pMnode) {
.retentionPeriod = 0, .retentionPeriod = 0,
.retentionSize = 0, .retentionSize = 0,
.level = TAOS_WAL_FSYNC, .level = TAOS_WAL_FSYNC,
.encryptAlgorithm = 0,
.encryptKey = {0}
}; };
#if defined(TD_ENTERPRISE)
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){
cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0;
if(tsEncryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
pMnode->pWal = walOpen(path, &cfg); pMnode->pWal = walOpen(path, &cfg);
if (pMnode->pWal == NULL) { if (pMnode->pWal == NULL) {
mError("failed to open wal since %s. wal:%s", terrstr(), path); mError("failed to open wal since %s. wal:%s", terrstr(), path);

2
source/dnode/mnode/impl/src/mndShow.c
View File

@ -131,6 +131,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_GRANTS_LOGS; type = TSDB_MGMT_TABLE_GRANTS_LOGS;
} else if (strncasecmp(name, TSDB_INS_TABLE_MACHINES, len) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_MACHINES, len) == 0) {
type = TSDB_MGMT_TABLE_MACHINES; type = TSDB_MGMT_TABLE_MACHINES;
} else if (strncasecmp(name, TSDB_INS_TABLE_ENCRYPTIONS, len) == 0) {
type = TSDB_MGMT_TABLE_ENCRYPTIONS;
} else if (strncasecmp(name, TSDB_INS_TABLE_TSMAS, len) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_TSMAS, len) == 0) {
type = TSDB_MGMT_TABLE_TSMAS; type = TSDB_MGMT_TABLE_TSMAS;
} else { } else {

1
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -307,6 +307,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
createReq.hashSuffix = pDb->cfg.hashSuffix; createReq.hashSuffix = pDb->cfg.hashSuffix;
createReq.tsdbPageSize = pDb->cfg.tsdbPageSize; createReq.tsdbPageSize = pDb->cfg.tsdbPageSize;
createReq.changeVersion = ++(pVgroup->syncConfChangeVer); createReq.changeVersion = ++(pVgroup->syncConfChangeVer);
createReq.encryptAlgorithm = pDb->cfg.encryptAlgorithm;
for (int32_t v = 0; v < pVgroup->replica; ++v) { for (int32_t v = 0; v < pVgroup->replica; ++v) {
SReplica *pReplica = NULL; SReplica *pReplica = NULL;

2
source/dnode/mnode/sdb/CMakeLists.txt
View File

@ -5,5 +5,5 @@ target_include_directories(
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
target_link_libraries( target_link_libraries(
sdb os common util wal sync sdb os common util wal sync crypt
) )

66
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -18,6 +18,8 @@
#include "sync.h" #include "sync.h"
#include "tchecksum.h" #include "tchecksum.h"
#include "wal.h" #include "wal.h"
#include "tglobal.h"
#include "crypt.h"
#define SDB_TABLE_SIZE 24 #define SDB_TABLE_SIZE 24
#define SDB_RESERVE_SIZE 512 #define SDB_RESERVE_SIZE 512
@ -276,6 +278,9 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
} }
readLen = pRaw->dataLen + sizeof(int32_t); readLen = pRaw->dataLen + sizeof(int32_t);
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
readLen = ENCRYPTED_LEN(pRaw->dataLen) + sizeof(int32_t);
}
if (readLen >= bufLen) { if (readLen >= bufLen) {
bufLen = pRaw->dataLen * 2; bufLen = pRaw->dataLen * 2;
SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100); SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100);
@ -303,6 +308,27 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
goto _OVER; goto _OVER;
} }
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
int32_t count = 0;
char *plantContent = taosMemoryMalloc(ENCRYPTED_LEN(pRaw->dataLen));
SCryptOpts opts;
opts.len = ENCRYPTED_LEN(pRaw->dataLen);
opts.source = pRaw->pData;
opts.result = plantContent;
opts.unitLen = 16;
strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN);
count = CBC_Decrypt(&opts);
//mDebug("read sdb, CBC_Decrypt dataLen:%d, descrypted len:%d, %s", pRaw->dataLen, count, __FUNCTION__);
memcpy(pRaw->pData, plantContent, pRaw->dataLen);
taosMemoryFree(plantContent);
memcpy(pRaw->pData + pRaw->dataLen, &pRaw->pData[ENCRYPTED_LEN(pRaw->dataLen)], sizeof(int32_t));
}
int32_t totalLen = sizeof(SSdbRaw) + pRaw->dataLen + sizeof(int32_t); int32_t totalLen = sizeof(SSdbRaw) + pRaw->dataLen + sizeof(int32_t);
if ((!taosCheckChecksumWhole((const uint8_t *)pRaw, totalLen)) != 0) { if ((!taosCheckChecksumWhole((const uint8_t *)pRaw, totalLen)) != 0) {
code = TSDB_CODE_CHECKSUM_ERROR; code = TSDB_CODE_CHECKSUM_ERROR;
@ -401,14 +427,50 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
SSdbRaw *pRaw = (*encodeFp)(pRow->pObj); SSdbRaw *pRaw = (*encodeFp)(pRow->pObj);
if (pRaw != NULL) { if (pRaw != NULL) {
pRaw->status = pRow->status; pRaw->status = pRow->status;
int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen;
if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) { if (taosWriteFile(pFile, pRaw, sizeof(SSdbRaw)) != sizeof(SSdbRaw)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow); taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
break; break;
} }
int32_t newDataLen = pRaw->dataLen;
char* newData = pRaw->pData;
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
newDataLen = ENCRYPTED_LEN(pRaw->dataLen);
newData = taosMemoryMalloc(newDataLen);
if (newData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
break;
}
SCryptOpts opts;
opts.len = newDataLen;
opts.source = pRaw->pData;
opts.result = newData;
opts.unitLen = 16;
strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN);
int32_t count = CBC_Encrypt(&opts);
//mDebug("write sdb, CBC_Encrypt encryptedDataLen:%d, dataLen:%d, %s",
// newDataLen, pRaw->dataLen, __FUNCTION__);
}
if (taosWriteFile(pFile, newData, newDataLen) != newDataLen) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
break;
}
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
taosMemoryFree(newData);
}
int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen); int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen);
if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);

3
source/dnode/vnode/CMakeLists.txt
View File

@ -121,6 +121,7 @@ if (${BUILD_CONTRIB})
PUBLIC "inc" PUBLIC "inc"
PUBLIC "src/inc" PUBLIC "src/inc"
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include" PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include"
) )
@ -130,6 +131,7 @@ else()
PUBLIC "inc" PUBLIC "inc"
PUBLIC "src/inc" PUBLIC "src/inc"
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
) )
if (${TD_LINUX}) if (${TD_LINUX})
@ -162,6 +164,7 @@ target_link_libraries(
PUBLIC scheduler PUBLIC scheduler
PUBLIC tdb PUBLIC tdb
PUBLIC audit PUBLIC audit
PUBLIC crypt
# PUBLIC bdb # PUBLIC bdb
# PUBLIC scalar # PUBLIC scalar

4
source/dnode/vnode/inc/vnode.h
View File

@ -278,6 +278,8 @@ struct STsdbCfg {
int32_t keep2; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead int32_t keep2; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keepTimeOffset; // just for save config, use STsdbKeepCfg in STsdb instead int32_t keepTimeOffset; // just for save config, use STsdbKeepCfg in STsdb instead
SRetention retentions[TSDB_RETENTION_MAX]; SRetention retentions[TSDB_RETENTION_MAX];
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
}; };
typedef struct { typedef struct {
@ -319,6 +321,8 @@ struct SVnodeCfg {
int16_t hashPrefix; int16_t hashPrefix;
int16_t hashSuffix; int16_t hashSuffix;
int32_t tsdbPageSize; int32_t tsdbPageSize;
int32_t tdbEncryptAlgorithm;
char tdbEncryptKey[ENCRYPT_KEY_LEN];
int32_t s3ChunkSize; int32_t s3ChunkSize;
int32_t s3KeepLocal; int32_t s3KeepLocal;
int8_t s3Compact; int8_t s3Compact;

21
source/dnode/vnode/src/meta/metaEntry.c
View File

@ -149,17 +149,22 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type); metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type);
return -1; return -1;
} }
if (!tDecodeIsEnd(pCoder)) { if (pME->type == TSDB_SUPER_TABLE) {
uDebug("set type: %d, tableName:%s", pME->type, pME->name); if (TABLE_IS_COL_COMPRESSED(pME->flags)) {
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1; if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1;
TABLE_SET_COL_COMPRESSED(pME->flags); } else {
} else {
uDebug("set default type: %d, tableName:%s", pME->type, pME->name);
if (pME->type == TSDB_SUPER_TABLE) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow); metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow);
} else if (pME->type == TSDB_NORMAL_TABLE) { TABLE_SET_COL_COMPRESSED(pME->flags);
}
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (!tDecodeIsEnd(pCoder)) {
uDebug("set type: %d, tableName:%s", pME->type, pME->name);
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1;
} else {
uDebug("set default type: %d, tableName:%s", pME->type, pME->name);
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow); metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow);
} }
TABLE_SET_COL_COMPRESSED(pME->flags);
} }
tEndDecode(pCoder); tEndDecode(pCoder);

3
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -69,7 +69,8 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
taosMkDir(pMeta->path); taosMkDir(pMeta->path);
// open env // open env
ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback); ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback,
pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;

92
source/dnode/vnode/src/meta/metaTable.c
View File

@ -109,7 +109,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
return -1; return -1;
} }
void * data = pCtbEntry->ctbEntry.pTags; void *data = pCtbEntry->ctbEntry.pTags;
const char *tagName = pSchema->name; const char *tagName = pSchema->name;
tb_uid_t suid = pCtbEntry->ctbEntry.suid; tb_uid_t suid = pCtbEntry->ctbEntry.suid;
@ -128,7 +128,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
char type = pTagVal->type; char type = pTagVal->type;
char * key = pTagVal->pKey; char *key = pTagVal->pKey;
int32_t nKey = strlen(key); int32_t nKey = strlen(key);
SIndexTerm *term = NULL; SIndexTerm *term = NULL;
@ -136,7 +136,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
} else if (type == TSDB_DATA_TYPE_NCHAR) { } else if (type == TSDB_DATA_TYPE_NCHAR) {
if (pTagVal->nData > 0) { if (pTagVal->nData > 0) {
char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
type = TSDB_DATA_TYPE_VARCHAR; type = TSDB_DATA_TYPE_VARCHAR;
@ -170,7 +170,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
return -1; return -1;
} }
void * data = pCtbEntry->ctbEntry.pTags; void *data = pCtbEntry->ctbEntry.pTags;
const char *tagName = pSchema->name; const char *tagName = pSchema->name;
tb_uid_t suid = pCtbEntry->ctbEntry.suid; tb_uid_t suid = pCtbEntry->ctbEntry.suid;
@ -189,7 +189,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
char type = pTagVal->type; char type = pTagVal->type;
char * key = pTagVal->pKey; char *key = pTagVal->pKey;
int32_t nKey = strlen(key); int32_t nKey = strlen(key);
SIndexTerm *term = NULL; SIndexTerm *term = NULL;
@ -197,7 +197,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
} else if (type == TSDB_DATA_TYPE_NCHAR) { } else if (type == TSDB_DATA_TYPE_NCHAR) {
if (pTagVal->nData > 0) { if (pTagVal->nData > 0) {
char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
type = TSDB_DATA_TYPE_VARCHAR; type = TSDB_DATA_TYPE_VARCHAR;
@ -244,9 +244,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
int vLen = 0; int vLen = 0;
const void *pKey = NULL; const void *pKey = NULL;
const void *pVal = NULL; const void *pVal = NULL;
void * pBuf = NULL; void *pBuf = NULL;
int32_t szBuf = 0; int32_t szBuf = 0;
void * p = NULL; void *p = NULL;
// validate req // validate req
void *pData = NULL; void *pData = NULL;
@ -385,7 +385,7 @@ static void metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) {
int c = 0; int c = 0;
void *pKey = NULL; void *pKey = NULL;
int nKey = 0; int nKey = 0;
TBC * pCtbIdxc = NULL; TBC *pCtbIdxc = NULL;
tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL);
int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c);
@ -416,8 +416,8 @@ static void metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) {
int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SMetaEntry oStbEntry = {0}; SMetaEntry oStbEntry = {0};
SMetaEntry nStbEntry = {0}; SMetaEntry nStbEntry = {0};
TBC * pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TBC * pTbDbc = NULL; TBC *pTbDbc = NULL;
const void *pData; const void *pData;
int nData; int nData;
int64_t oversion; int64_t oversion;
@ -484,7 +484,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1); bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1);
if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { if (!TSDB_CACHE_NO(pMeta->pVnode->config)) {
STsdb * pTsdb = pMeta->pVnode->pTsdb; STsdb *pTsdb = pMeta->pVnode->pTsdb;
SArray *uids = taosArrayInit(8, sizeof(int64_t)); SArray *uids = taosArrayInit(8, sizeof(int64_t));
if (deltaCol == 1) { if (deltaCol == 1) {
int16_t cid = pReq->schemaRow.pSchema[nCols - 1].colId; int16_t cid = pReq->schemaRow.pSchema[nCols - 1].colId;
@ -554,9 +554,9 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
STbDbKey tbDbKey = {0}; STbDbKey tbDbKey = {0};
TBC * pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TBC * pTbDbc = NULL; TBC *pTbDbc = NULL;
void * pData = NULL; void *pData = NULL;
int nData = 0; int nData = 0;
int64_t oversion; int64_t oversion;
SDecoder dc = {0}; SDecoder dc = {0};
@ -710,11 +710,11 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
SMetaEntry nStbEntry = {0}; SMetaEntry nStbEntry = {0};
STbDbKey tbDbKey = {0}; STbDbKey tbDbKey = {0};
TBC * pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TBC * pTbDbc = NULL; TBC *pTbDbc = NULL;
int ret = 0; int ret = 0;
int c = -2; int c = -2;
void * pData = NULL; void *pData = NULL;
int nData = 0; int nData = 0;
int64_t oversion; int64_t oversion;
SDecoder dc = {0}; SDecoder dc = {0};
@ -817,8 +817,8 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
nStbEntry.uid = oStbEntry.uid; nStbEntry.uid = oStbEntry.uid;
nStbEntry.name = oStbEntry.name; nStbEntry.name = oStbEntry.name;
SSchemaWrapper * row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow); SSchemaWrapper *row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow);
SSchemaWrapper * tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag); SSchemaWrapper *tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag);
SColCmprWrapper *cmpr = tCloneSColCmprWrapper(&oStbEntry.colCmpr); SColCmprWrapper *cmpr = tCloneSColCmprWrapper(&oStbEntry.colCmpr);
nStbEntry.stbEntry.schemaRow = *row; nStbEntry.stbEntry.schemaRow = *row;
@ -999,7 +999,7 @@ _err:
} }
int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) {
void * pData = NULL; void *pData = NULL;
int nData = 0; int nData = 0;
int rc = 0; int rc = 0;
tb_uid_t uid = 0; tb_uid_t uid = 0;
@ -1078,10 +1078,10 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) {
metaULock(pMeta); metaULock(pMeta);
// update timeseries // update timeseries
void * pCtbDropped = NULL; void *pCtbDropped = NULL;
int32_t iter = 0; int32_t iter = 0;
while ((pCtbDropped = tSimpleHashIterate(suidHash, pCtbDropped, &iter))) { while ((pCtbDropped = tSimpleHashIterate(suidHash, pCtbDropped, &iter))) {
tb_uid_t * pSuid = tSimpleHashGetKey(pCtbDropped, NULL); tb_uid_t *pSuid = tSimpleHashGetKey(pCtbDropped, NULL);
int32_t nCols = 0; int32_t nCols = 0;
SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
if (metaGetStbStats(pMeta->pVnode, *pSuid, NULL, &nCols) == 0) { if (metaGetStbStats(pMeta->pVnode, *pSuid, NULL, &nCols) == 0) {
@ -1221,7 +1221,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) {
} }
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl) { static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl) {
void * pData = NULL; void *pData = NULL;
int nData = 0; int nData = 0;
int rc = 0; int rc = 0;
SMetaEntry e = {0}; SMetaEntry e = {0};
@ -1260,7 +1260,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p
if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta, stbEntry.name, 1) ? 1 : 0; if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta, stbEntry.name, 1) ? 1 : 0;
SSchema * pTagColumn = NULL; SSchema *pTagColumn = NULL;
SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) {
pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0];
@ -1384,14 +1384,14 @@ int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
} }
static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) {
void * pVal = NULL; void *pVal = NULL;
int nVal = 0; int nVal = 0;
const void * pData = NULL; const void *pData = NULL;
int nData = 0; int nData = 0;
int ret = 0; int ret = 0;
tb_uid_t uid; tb_uid_t uid;
int64_t oversion; int64_t oversion;
SSchema * pColumn = NULL; SSchema *pColumn = NULL;
SMetaEntry entry = {0}; SMetaEntry entry = {0};
SSchemaWrapper *pSchema; SSchemaWrapper *pSchema;
int c; int c;
@ -1652,7 +1652,7 @@ _err:
static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
SMetaEntry ctbEntry = {0}; SMetaEntry ctbEntry = {0};
SMetaEntry stbEntry = {0}; SMetaEntry stbEntry = {0};
void * pVal = NULL; void *pVal = NULL;
int nVal = 0; int nVal = 0;
int ret; int ret;
int c; int c;
@ -1693,7 +1693,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
oversion = ((SUidIdxVal *)pData)[0].version; oversion = ((SUidIdxVal *)pData)[0].version;
// search table.db // search table.db
TBC * pTbDbc = NULL; TBC *pTbDbc = NULL;
SDecoder dc1 = {0}; SDecoder dc1 = {0};
SDecoder dc2 = {0}; SDecoder dc2 = {0};
@ -1729,7 +1729,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
metaDecodeEntry(&dc2, &stbEntry); metaDecodeEntry(&dc2, &stbEntry);
SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
SSchema * pColumn = NULL; SSchema *pColumn = NULL;
int32_t iCol = 0; int32_t iCol = 0;
for (;;) { for (;;) {
pColumn = NULL; pColumn = NULL;
@ -1756,8 +1756,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
} else { } else {
const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags;
STag * pNewTag = NULL; STag *pNewTag = NULL;
SArray * pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
if (!pTagArray) { if (!pTagArray) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
@ -1839,7 +1839,7 @@ _err:
} }
static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
void * pVal = NULL; void *pVal = NULL;
int nVal = 0; int nVal = 0;
const void *pData = NULL; const void *pData = NULL;
int nData = 0; int nData = 0;
@ -1943,7 +1943,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
SMetaEntry stbEntry = {0}; SMetaEntry stbEntry = {0};
void * pVal = NULL; void *pVal = NULL;
int nVal = 0; int nVal = 0;
int ret; int ret;
int c; int c;
@ -2076,7 +2076,7 @@ typedef struct SMetaPair {
static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
SMetaEntry stbEntry = {0}; SMetaEntry stbEntry = {0};
void * pVal = NULL; void *pVal = NULL;
int nVal = 0; int nVal = 0;
int ret; int ret;
int c; int c;
@ -2186,7 +2186,7 @@ _err:
int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) {
// impl later // impl later
SMetaEntry tbEntry = {0}; SMetaEntry tbEntry = {0};
void * pVal = NULL; void *pVal = NULL;
int nVal = 0; int nVal = 0;
int ret; int ret;
int c; int c;
@ -2296,8 +2296,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
STbDbKey tbDbKey; STbDbKey tbDbKey;
void * pKey = NULL; void *pKey = NULL;
void * pVal = NULL; void *pVal = NULL;
int kLen = 0; int kLen = 0;
int vLen = 0; int vLen = 0;
SEncoder coder = {0}; SEncoder coder = {0};
@ -2448,14 +2448,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) {
} }
static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
void * pData = NULL; void *pData = NULL;
int nData = 0; int nData = 0;
STbDbKey tbDbKey = {0}; STbDbKey tbDbKey = {0};
SMetaEntry stbEntry = {0}; SMetaEntry stbEntry = {0};
STagIdxKey * pTagIdxKey = NULL; STagIdxKey *pTagIdxKey = NULL;
int32_t nTagIdxKey; int32_t nTagIdxKey;
const SSchema *pTagColumn; const SSchema *pTagColumn;
const void * pTagData = NULL; const void *pTagData = NULL;
int32_t nTagData = 0; int32_t nTagData = 0;
SDecoder dc = {0}; SDecoder dc = {0};
int32_t ret = 0; int32_t ret = 0;
@ -2524,7 +2524,7 @@ end:
static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
SEncoder coder = {0}; SEncoder coder = {0};
void * pVal = NULL; void *pVal = NULL;
int vLen = 0; int vLen = 0;
int rcode = 0; int rcode = 0;
SSkmDbKey skmDbKey = {0}; SSkmDbKey skmDbKey = {0};
@ -2652,15 +2652,15 @@ int32_t colCompressDebug(SHashObj *pColCmprObj) {
const char *l1str = columnEncodeStr(l1); const char *l1str = columnEncodeStr(l1);
const char *l2str = columnCompressStr(l2); const char *l2str = columnCompressStr(l2);
const char *lvlstr = columnLevelStr(lvl); const char *lvlstr = columnLevelStr(lvl);
metaInfo("colId: %d, encode:%s, compress:%s,level:%s", colId, l1str, l2str, lvlstr); metaDebug("colId: %d, encode:%s, compress:%s,level:%s", colId, l1str, l2str, lvlstr);
} }
return 0; return 0;
} }
int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) {
int rc = 0; int rc = 0;
SHashObj * pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); SHashObj *pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK);
void * pData = NULL; void *pData = NULL;
int nData = 0; int nData = 0;
SMetaEntry e = {0}; SMetaEntry e = {0};
SDecoder dc = {0}; SDecoder dc = {0};

2
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -76,7 +76,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
} }
int32_t tqMetaOpen(STQ* pTq) { int32_t tqMetaOpen(STQ* pTq) {
if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0) < 0) { if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL) < 0) {
return -1; return -1;
} }

2
source/dnode/vnode/src/tq/tqRead.c
View File

@ -592,7 +592,7 @@ static int32_t buildResSDataBlock(SSDataBlock* pBlock, SSchemaWrapper* pSchema,
static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SColVal* pColVal) { static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SColVal* pColVal) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (IS_STR_DATA_TYPE(pColVal->value.type)) { if (IS_VAR_DATA_TYPE(pColVal->value.type)) {
char val[65535 + 2] = {0}; char val[65535 + 2] = {0};
if (COL_VAL_IS_VALUE(pColVal)) { if (COL_VAL_IS_VALUE(pColVal)) {
if (pColVal->value.pData != NULL) { if (pColVal->value.pData != NULL) {

2
source/dnode/vnode/src/tq/tqSink.c
View File

@ -603,7 +603,7 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
dataIndex++; dataIndex++;
} else { } else {
void* colData = colDataGetData(pColData, j); void* colData = colDataGetData(pColData, j);
if (IS_STR_DATA_TYPE(pCol->type)) { if (IS_VAR_DATA_TYPE(pCol->type)) {
// address copy, no value // address copy, no value
SValue sv = SValue sv =
(SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)}; (SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)};

15
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
View File

@ -73,7 +73,9 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo
pBlock->file.maxVer = reader->config->file.maxVer; pBlock->file.maxVer = reader->config->file.maxVer;
pBlock->file.stt->level = reader->config->file.stt->level; pBlock->file.stt->level = reader->config->file.stt->level;
code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0); int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -127,8 +129,11 @@ static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFil
code = TARRAY2_APPEND(opArr, op); code = TARRAY2_APPEND(opArr, op);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
if (writer->fd) { if (writer->fd) {
code = tsdbFsyncFile(writer->fd); code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd); tsdbCloseFile(&writer->fd);
} }
@ -206,11 +211,13 @@ _exit:
return code; return code;
} }
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock) { int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength); code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->offset += pDataBlock->dataLength; writer->ctx->offset += pDataBlock->dataLength;

3
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h
View File

@ -105,7 +105,8 @@ int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataF
int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr); int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr);
int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer); int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer);
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr); int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer); int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer);
#ifdef __cplusplus #ifdef __cplusplus

143
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

@ -46,8 +46,10 @@ static int32_t tsdbDataFileReadHeadFooter(SDataFileReader *reader) {
int32_t ftype = TSDB_FTYPE_HEAD; int32_t ftype = TSDB_FTYPE_HEAD;
if (reader->fd[ftype]) { if (reader->fd[ftype]) {
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(SHeadFooter), code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(SHeadFooter),
(uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0); (uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -68,8 +70,10 @@ static int32_t tsdbDataFileReadTombFooter(SDataFileReader *reader) {
int32_t ftype = TSDB_FTYPE_TOMB; int32_t ftype = TSDB_FTYPE_TOMB;
if (reader->fd[ftype]) { if (reader->fd[ftype]) {
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(STombFooter), code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(STombFooter),
(uint8_t *)reader->tombFooter, sizeof(STombFooter), 0); (uint8_t *)reader->tombFooter, sizeof(STombFooter), 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
reader->ctx->tombFooterLoaded = true; reader->ctx->tombFooterLoaded = true;
@ -164,8 +168,10 @@ int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **b
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], reader->headFooter->brinBlkPtr->offset, data, code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], reader->headFooter->brinBlkPtr->offset, data,
reader->headFooter->brinBlkPtr->size, 0); reader->headFooter->brinBlkPtr->size, 0, encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -195,9 +201,12 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB
SBuffer *buffer = reader->buffers + 0; SBuffer *buffer = reader->buffers + 0;
SBuffer *assist = reader->buffers + 1; SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data // load data
tBufferClear(buffer); tBufferClear(buffer);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, brinBlk->dp->size, buffer, 0); code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, brinBlk->dp->size, buffer, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// decode brin block // decode brin block
@ -279,9 +288,12 @@ int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *re
SBuffer *buffer = reader->buffers + 0; SBuffer *buffer = reader->buffers + 0;
SBuffer *assist = reader->buffers + 1; SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data // load data
tBufferClear(buffer); tBufferClear(buffer);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockSize, buffer, 0); code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockSize, buffer, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// decompress // decompress
@ -307,9 +319,12 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
SBuffer *buffer1 = reader->buffers + 1; SBuffer *buffer1 = reader->buffers + 1;
SBuffer *assist = reader->buffers + 2; SBuffer *assist = reader->buffers + 2;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load key part // load key part
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockKeySize, buffer0, 0); code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, record->blockKeySize, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// SDiskDataHdr // SDiskDataHdr
@ -340,11 +355,11 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
if (extraColIdx < 0) { if (extraColIdx < 0) {
goto _exit; goto _exit;
} }
// load SBlockCol part // load SBlockCol part
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize, hdr.szBlkCol, code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize, hdr.szBlkCol,
buffer0, 0); buffer0, 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// calc szHint // calc szHint
@ -454,11 +469,14 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
code = tBlockDataDecompressColData(&hdr, &none, &br, bData, assist); code = tBlockDataDecompressColData(&hdr, &none, &br, bData, assist);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} else if (cid == blockCol.cid) { } else if (cid == blockCol.cid) {
// load from file int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load from file
tBufferClear(buffer1); tBufferClear(buffer1);
code = tsdbReadFileToBuffer( code = tsdbReadFileToBuffer(
reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset, reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset,
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, firstRead ? szHint : 0); blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, firstRead ? szHint : 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
firstRead = false; firstRead = false;
@ -486,7 +504,10 @@ int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *rec
TARRAY2_CLEAR(columnDataAggArray, NULL); TARRAY2_CLEAR(columnDataAggArray, NULL);
if (record->smaSize > 0) { if (record->smaSize > 0) {
tBufferClear(buffer); tBufferClear(buffer);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, record->smaSize, buffer, 0); int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, record->smaSize, buffer, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// decode sma data // decode sma data
@ -525,8 +546,10 @@ int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **t
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], reader->tombFooter->tombBlkPtr->offset, data, code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], reader->tombFooter->tombBlkPtr->offset, data,
reader->tombFooter->tombBlkPtr->size, 0); reader->tombFooter->tombBlkPtr->size, 0, encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -557,7 +580,10 @@ int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombB
SBuffer *assist = reader->buffers + 1; SBuffer *assist = reader->buffers + 1;
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0); int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFileToBuffer(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
int32_t size = 0; int32_t size = 0;
@ -785,7 +811,8 @@ int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxV
} }
int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmprAlg, int64_t *fileSize, int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmprAlg, int64_t *fileSize,
TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range) { TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey) {
if (brinBlock->numOfRecords == 0) return 0; if (brinBlock->numOfRecords == 0) return 0;
int32_t code; int32_t code;
@ -838,7 +865,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr
tBufferClear(buffer0); tBufferClear(buffer0);
code = tCompressDataToBuffer(brinBlock->buffers[i].data, &info, buffer0, assist); code = tCompressDataToBuffer(brinBlock->buffers[i].data, &info, buffer0, assist);
if (code) return code; if (code) return code;
code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size); code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey);
if (code) return code; if (code) return code;
brinBlk.size[i] = info.compressedSize; brinBlk.size[i] = info.compressedSize;
brinBlk.dp->size += info.compressedSize; brinBlk.dp->size += info.compressedSize;
@ -854,7 +881,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr
tBufferClear(buffer0); tBufferClear(buffer0);
code = tCompressDataToBuffer(brinBlock->buffers[i].data, &info, buffer0, assist); code = tCompressDataToBuffer(brinBlock->buffers[i].data, &info, buffer0, assist);
if (code) return code; if (code) return code;
code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size); code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey);
if (code) return code; if (code) return code;
brinBlk.size[i] = info.compressedSize; brinBlk.size[i] = info.compressedSize;
brinBlk.dp->size += info.compressedSize; brinBlk.dp->size += info.compressedSize;
@ -879,10 +906,10 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr
} }
// write to file // write to file
if ((code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size))) return code; if ((code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey))) return code;
*fileSize += buffer0->size; *fileSize += buffer0->size;
brinBlk.dp->size += buffer0->size; brinBlk.dp->size += buffer0->size;
if ((code = tsdbWriteFile(fd, *fileSize, buffer1->data, buffer1->size))) return code; if ((code = tsdbWriteFile(fd, *fileSize, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey))) return code;
*fileSize += buffer1->size; *fileSize += buffer1->size;
brinBlk.dp->size += buffer1->size; brinBlk.dp->size += buffer1->size;
} }
@ -902,9 +929,12 @@ static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg, code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg,
&writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->buffers, &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->buffers,
&writer->ctx->range); &writer->ctx->range, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -995,9 +1025,11 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData
record->blockKeySize = buffers[0].size + buffers[1].size; record->blockKeySize = buffers[0].size + buffers[1].size;
record->blockSize = record->blockKeySize + buffers[2].size + buffers[3].size; record->blockSize = record->blockKeySize + buffers[2].size + buffers[3].size;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
for (int i = 0; i < 4; i++) { for (int i = 0; i < 4; i++) {
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_DATA], writer->files[TSDB_FTYPE_DATA].size, buffers[i].data, code = tsdbWriteFile(writer->fd[TSDB_FTYPE_DATA], writer->files[TSDB_FTYPE_DATA].size, buffers[i].data,
buffers[i].size); buffers[i].size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->files[TSDB_FTYPE_DATA].size += buffers[i].size; writer->files[TSDB_FTYPE_DATA].size += buffers[i].size;
} }
@ -1017,7 +1049,8 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData
record->smaSize = buffers[0].size; record->smaSize = buffers[0].size;
if (record->smaSize > 0) { if (record->smaSize > 0) {
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, buffers[0].data, record->smaSize); code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, buffers[0].data, record->smaSize,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->files[TSDB_FTYPE_SMA].size += record->smaSize; writer->files[TSDB_FTYPE_SMA].size += record->smaSize;
} }
@ -1288,15 +1321,17 @@ _exit:
return code; return code;
} }
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer) { int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm,
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); char* encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey);
if (code) return code; if (code) return code;
*fileSize += sizeof(*footer); *fileSize += sizeof(*footer);
return 0; return 0;
} }
int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range) { TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code; int32_t code;
if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0; if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0;
@ -1348,7 +1383,7 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl
code = tCompressDataToBuffer(tombBlock->buffers[i].data, &cinfo, buffer0, assist); code = tCompressDataToBuffer(tombBlock->buffers[i].data, &cinfo, buffer0, assist);
if (code) return code; if (code) return code;
code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size); code = tsdbWriteFile(fd, *fileSize, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey);
if (code) return code; if (code) return code;
tombBlk.size[i] = cinfo.compressedSize; tombBlk.size[i] = cinfo.compressedSize;
@ -1367,7 +1402,11 @@ static int32_t tsdbDataFileWriteHeadFooter(SDataFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -1383,9 +1422,12 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg,
&writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->buffers, &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->buffers,
&writer->ctx->tombRange); &writer->ctx->tombRange, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -1395,12 +1437,14 @@ _exit:
return code; return code;
} }
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) { int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey) {
ptr->size = TARRAY2_DATA_LEN(tombBlkArray); ptr->size = TARRAY2_DATA_LEN(tombBlkArray);
if (ptr->size > 0) { if (ptr->size > 0) {
ptr->offset = *fileSize; ptr->offset = *fileSize;
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size); int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size,
encryptAlgorithm, encryptKey);
if (code) { if (code) {
return code; return code;
} }
@ -1416,8 +1460,11 @@ static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlk(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlkArray, writer->tombFooter->tombBlkPtr, code = tsdbFileWriteTombBlk(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlkArray, writer->tombFooter->tombBlkPtr,
&writer->files[TSDB_FTYPE_TOMB].size); &writer->files[TSDB_FTYPE_TOMB].size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -1427,8 +1474,9 @@ _exit:
return code; return code;
} }
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize) { int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); char* encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey);
if (code) return code; if (code) return code;
*fileSize += sizeof(*footer); *fileSize += sizeof(*footer);
return 0; return 0;
@ -1438,7 +1486,11 @@ static int32_t tsdbDataFileWriteTombFooter(SDataFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -1515,12 +1567,14 @@ _exit:
return code; return code;
} }
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize) { int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey) {
ASSERT(TARRAY2_SIZE(brinBlkArray) > 0); ASSERT(TARRAY2_SIZE(brinBlkArray) > 0);
ptr->offset = *fileSize; ptr->offset = *fileSize;
ptr->size = TARRAY2_DATA_LEN(brinBlkArray); ptr->size = TARRAY2_DATA_LEN(brinBlkArray);
int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size); int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size, encryptAlgorithm,
encryptKey);
if (code) return code; if (code) return code;
*fileSize += ptr->size; *fileSize += ptr->size;
@ -1531,8 +1585,11 @@ static int32_t tsdbDataFileWriteBrinBlk(SDataFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteBrinBlk(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlkArray, writer->headFooter->brinBlkPtr, code = tsdbFileWriteBrinBlk(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlkArray, writer->headFooter->brinBlkPtr,
&writer->files[TSDB_FTYPE_HEAD].size); &writer->files[TSDB_FTYPE_HEAD].size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -1689,10 +1746,11 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
code = TARRAY2_APPEND(opArr, op); code = TARRAY2_APPEND(opArr, op);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) {
if (writer->fd[i]) { if (writer->fd[i]) {
code = tsdbFsyncFile(writer->fd[i]); code = tsdbFsyncFile(writer->fd[i], encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd[i]); tsdbCloseFile(&writer->fd[i]);
} }
@ -1729,7 +1787,10 @@ static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) {
if (writer->files[ftype].size == 0) { if (writer->files[ftype].size == 0) {
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->files[ftype].size += TSDB_FHDR_SIZE; writer->files[ftype].size += TSDB_FHDR_SIZE;
@ -1895,7 +1956,9 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->files[ftype].size += TSDB_FHDR_SIZE; writer->files[ftype].size += TSDB_FHDR_SIZE;

18
source/dnode/vnode/src/tsdb/tsdbDataFileRW.h
View File

@ -97,16 +97,22 @@ int32_t tsdbDataFileFlush(SDataFileWriter *writer);
// head // head
int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmprAlg, int64_t *fileSize, int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmprAlg, int64_t *fileSize,
TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range); TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range,
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize); int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer); int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer,
int32_t encryptAlgorithm, char* encryptKey);
// tomb // tomb
int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record); int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record);
int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range); TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range,
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize); int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey);
// utils // utils
int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer); int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer);

12
source/dnode/vnode/src/tsdb/tsdbDef.h
View File

@ -33,10 +33,14 @@ typedef struct SFDataPtr {
extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn); extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn);
extern void tsdbCloseFile(STsdbFD **ppFD); extern void tsdbCloseFile(STsdbFD **ppFD);
extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size);
extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint); extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size,
extern int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint); int32_t encryptAlgorithm, char* encryptKey);
extern int32_t tsdbFsyncFile(STsdbFD *pFD); extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey);
extern int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey);
extern int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey);
typedef struct SColCompressInfo SColCompressInfo; typedef struct SColCompressInfo SColCompressInfo;
struct SColCompressInfo { struct SColCompressInfo {

5
source/dnode/vnode/src/tsdb/tsdbFSetRAW.c
View File

@ -152,7 +152,8 @@ _exit:
return code; return code;
} }
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) { int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
@ -166,7 +167,7 @@ int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeade
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr); code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->offset += bHdr->dataLength; writer->ctx->offset += bHdr->dataLength;

3
source/dnode/vnode/src/tsdb/tsdbFSetRAW.h
View File

@ -36,7 +36,8 @@ typedef struct SFSetRAWWriter SFSetRAWWriter;
int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer); int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer);
int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr); int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr);
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr); int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm,
char* encryptKey);
#ifdef __cplusplus #ifdef __cplusplus
} }

17
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -828,17 +828,22 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTre
SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node)); SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node));
SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node)); SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node));
TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row); SRowKey rkey1, rkey2;
TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row); tRowGetKeyEx(&pIter1->rInfo.row, &rkey1);
tRowGetKeyEx(&pIter2->rInfo.row, &rkey2);
if (key1.ts < key2.ts) { int32_t ret = tRowKeyCompare(&rkey1, &rkey2);
if (ret < 0) {
return -1; return -1;
} else if (key1.ts > key2.ts) { } else if (ret > 0) {
return 1; return 1;
} else { } else {
if (key1.version < key2.version) { int64_t ver1 = TSDBROW_VERSION(&pIter1->rInfo.row);
int64_t ver2 = TSDBROW_VERSION(&pIter2->rInfo.row);
if (ver1 < ver2) {
return -1; return -1;
} else if (key1.version > key2.version) { } else if (ver1 > ver2) {
return 1; return 1;
} else { } else {
return 0; return 0;

39
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -25,15 +25,6 @@
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
#define getCurrentKeyInSttBlock(_r) (&((_r)->currentKey)) #define getCurrentKeyInSttBlock(_r) (&((_r)->currentKey))
#define tRowGetKeyEx(_pRow, _pKey) \
do { \
if ((_pRow)->type == TSDBROW_ROW_FMT) { \
tRowGetKey((_pRow)->pTSRow, (_pKey)); \
} else { \
tColRowGetKey((_pRow)->pBlockData, (_pRow)->iRow, (_pKey)); \
} \
} while (0)
typedef struct { typedef struct {
bool overlapWithNeighborBlock; bool overlapWithNeighborBlock;
bool hasDupTs; bool hasDupTs;
@ -103,26 +94,10 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
return 1; return 1;
} }
ASSERT(p1->numOfPKs == p2->numOfPKs);
if (p1->numOfPKs == 0) { if (p1->numOfPKs == 0) {
return 0; return 0;
} else { } else {
if (IS_VAR_DATA_TYPE(p1->pks[0].type)) { return tRowKeyCompare(p1, p2);
int32_t len = TMIN(p1->pks[0].nData, p2->pks[0].nData);
int32_t ret = strncmp((char*)p1->pks[0].pData, (char*)p2->pks[0].pData, len);
if (ret == 0) {
if (p1->pks[0].nData == p2->pks[0].nData) {
return 0;
} else {
return p1->pks[0].nData > p2->pks[0].nData ? 1 : -1;
}
} else {
return ret > 0 ? 1 : -1;
}
} else {
return tValueCompare(&p1->pks[0], &p2->pks[0]);
}
} }
} }
@ -1635,6 +1610,11 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttB
doUnpinSttBlock(pSttBlockReader); doUnpinSttBlock(pSttBlockReader);
if (hasVal) { if (hasVal) {
SRowKey* pNext = getCurrentKeyInSttBlock(pSttBlockReader); SRowKey* pNext = getCurrentKeyInSttBlock(pSttBlockReader);
if (IS_VAR_DATA_TYPE(pSttKey->pks[0].type)) {
tsdbInfo("current pk:%s, next pk:%s", pSttKey->pks[0].pData, pNext->pks[0].pData);
}
if (pkCompEx(pSttKey, pNext) != 0) { if (pkCompEx(pSttKey, pNext) != 0) {
code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow);
*copied = (code == TSDB_CODE_SUCCESS); *copied = (code == TSDB_CODE_SUCCESS);
@ -2327,8 +2307,11 @@ int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanIn
TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData};
tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid,
fRow.pBlockData->aTSKEY[fRow.iRow], pReader->idStr); if (IS_VAR_DATA_TYPE(pScanInfo->lastProcKey.pks[0].type)) {
tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " pk:%s %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid,
fRow.pBlockData->aTSKEY[fRow.iRow], pScanInfo->lastProcKey.pks[0].pData, pReader->idStr);
}
int32_t code = int32_t code =
tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pScanInfo, &pScanInfo->lastProcKey, pReader, &copied); tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pScanInfo, &pScanInfo->lastProcKey, pReader, &copied);

4
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -191,6 +191,10 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno; return terrno;
} }
if (!asc) {
pKey->numOfPKs = 2;
}
} }
} }

9
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -38,6 +38,15 @@ extern "C" {
(_k)->ekey.ts = INT64_MIN; \ (_k)->ekey.ts = INT64_MIN; \
} while (0); } while (0);
#define tRowGetKeyEx(_pRow, _pKey) \
do { \
if ((_pRow)->type == TSDBROW_ROW_FMT) { \
tRowGetKey((_pRow)->pTSRow, (_pKey)); \
} else { \
tColRowGetKey((_pRow)->pBlockData, (_pRow)->iRow, (_pKey)); \
} \
} while (0)
typedef enum { typedef enum {
READER_STATUS_SUSPEND = 0x1, READER_STATUS_SUSPEND = 0x1,
READER_STATUS_NORMAL = 0x2, READER_STATUS_NORMAL = 0x2,

98
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -15,6 +15,7 @@
#include "cos.h" #include "cos.h"
#include "tsdb.h" #include "tsdb.h"
#include "crypt.h"
#include "vnd.h" #include "vnd.h"
static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
@ -169,7 +170,7 @@ void tsdbCloseFile(STsdbFD **ppFD) {
} }
} }
static int32_t tsdbWriteFilePage(STsdbFD *pFD) { static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0; int32_t code = 0;
if (!pFD->pFD) { if (!pFD->pFD) {
@ -197,6 +198,28 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD) {
} }
taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage);
if(encryptAlgorithm == DND_CA_SM4){
//if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
unsigned char PacketData[128];
int NewLen;
int32_t count = 0;
while (count < pFD->szPage) {
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pFD->pBuf + count;
opts.result = PacketData;
opts.unitLen = 128;
//strncpy(opts.key, tsEncryptKey, 16);
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Encrypt(&opts);
memcpy(pFD->pBuf + count, PacketData, NewLen);
count += NewLen;
}
//tsdbDebug("CBC_Encrypt count:%d %s", count, __FUNCTION__);
}
n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) { if (n < 0) {
@ -214,7 +237,7 @@ _exit:
return code; return code;
} }
static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0; int32_t code = 0;
// ASSERT(pgno <= pFD->szFile); // ASSERT(pgno <= pFD->szFile);
@ -274,6 +297,30 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
} }
//} //}
if(encryptAlgorithm == DND_CA_SM4){
//if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
unsigned char PacketData[128];
int NewLen;
int32_t count = 0;
while(count < pFD->szPage)
{
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pFD->pBuf + count;
opts.result = PacketData;
opts.unitLen = 128;
//strncpy(opts.key, tsEncryptKey, 16);
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Decrypt(&opts);
memcpy(pFD->pBuf + count, PacketData, NewLen);
count += NewLen;
}
//tsdbDebug("CBC_Decrypt count:%d %s", count, __FUNCTION__);
}
// check // check
if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
code = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
@ -286,7 +333,8 @@ _exit:
return code; return code;
} }
int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) { int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = 0; int32_t code = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
@ -295,11 +343,11 @@ int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t
do { do {
if (pFD->pgno != pgno) { if (pFD->pgno != pgno) {
code = tsdbWriteFilePage(pFD); code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
if (code) goto _exit; if (code) goto _exit;
if (pgno <= pFD->szFile) { if (pgno <= pFD->szFile) {
code = tsdbReadFilePage(pFD, pgno); code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey);
if (code) goto _exit; if (code) goto _exit;
} else { } else {
pFD->pgno = pgno; pFD->pgno = pgno;
@ -318,7 +366,8 @@ _exit:
return code; return code;
} }
static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = 0; int32_t code = 0;
int64_t n = 0; int64_t n = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
@ -331,7 +380,7 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6
while (n < size) { while (n < size) {
if (pFD->pgno != pgno) { if (pFD->pgno != pgno) {
code = tsdbReadFilePage(pFD, pgno); code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey);
if (code) goto _exit; if (code) goto _exit;
} }
@ -523,7 +572,8 @@ _exit:
return code; return code;
} }
int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) { int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0; int32_t code = 0;
if (!pFD->pFD) { if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD); code = tsdbOpenFileImpl(pFD);
@ -535,26 +585,28 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size,
if (pFD->lcn > 1 /*pFD->s3File && tsS3BlockSize < 0*/) { if (pFD->lcn > 1 /*pFD->s3File && tsS3BlockSize < 0*/) {
return tsdbReadFileS3(pFD, offset, pBuf, size, szHint); return tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
} else { } else {
return tsdbReadFileImp(pFD, offset, pBuf, size); return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey);
} }
_exit: _exit:
return code; return code;
} }
int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint) { int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code; int32_t code;
code = tBufferEnsureCapacity(buffer, buffer->size + size); code = tBufferEnsureCapacity(buffer, buffer->size + size);
if (code) return code; if (code) return code;
code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint); code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint,
encryptAlgorithm, encryptKey);
if (code) return code; if (code) return code;
buffer->size += size; buffer->size += size;
return code; return code;
} }
int32_t tsdbFsyncFile(STsdbFD *pFD) { int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0; int32_t code = 0;
/* /*
if (pFD->s3File) { if (pFD->s3File) {
@ -562,7 +614,7 @@ int32_t tsdbFsyncFile(STsdbFD *pFD) {
return code; return code;
} }
*/ */
code = tsdbWriteFilePage(pFD); code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
if (code) goto _exit; if (code) goto _exit;
if (taosFsyncFile(pFD->pFD) < 0) { if (taosFsyncFile(pFD->pFD) < 0) {
@ -673,7 +725,9 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
if (code) goto _err; if (code) goto _err;
// read // read
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0); int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; if (code) goto _err;
// decode // decode
@ -710,7 +764,9 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
if (code) goto _err; if (code) goto _err;
// read // read
code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0); int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; if (code) goto _err;
// decode // decode
@ -743,7 +799,9 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
if (code) goto _err; if (code) goto _err;
// read // read
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0); int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; if (code) goto _err;
// decode // decode
@ -836,7 +894,9 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
if (code) goto _err; if (code) goto _err;
// read // read
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0); int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; if (code) goto _err;
// // decode // // decode
@ -876,7 +936,9 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
if (code) goto _err; if (code) goto _err;
// read // read
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0); int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err; if (code) goto _err;
// decode // decode

5
source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
View File

@ -545,7 +545,10 @@ static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbD
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr); int32_t encryptAlgorithm = writer->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:

112
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -63,7 +63,10 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
int64_t offset = config->file->size - sizeof(SSttFooter); int64_t offset = config->file->size - sizeof(SSttFooter);
ASSERT(offset >= TSDB_FHDR_SIZE); ASSERT(offset >= TSDB_FHDR_SIZE);
code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0); int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, encryptAlgoirthm,
encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -99,8 +102,11 @@ int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray *
void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size); void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY; if (!data) return TSDB_CODE_OUT_OF_MEMORY;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code = int32_t code =
tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0); tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0,
encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
return code; return code;
@ -127,8 +133,11 @@ int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tom
void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size); void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY; if (!data) return TSDB_CODE_OUT_OF_MEMORY;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code = int32_t code =
tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0); tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0,
encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
return code; return code;
@ -155,8 +164,11 @@ int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBl
void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size); void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY; if (!data) return TSDB_CODE_OUT_OF_MEMORY;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code = int32_t code =
tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0); tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0,
encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
return code; return code;
@ -181,9 +193,12 @@ int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk,
SBuffer *buffer0 = reader->buffers + 0; SBuffer *buffer0 = reader->buffers + 0;
SBuffer *assist = reader->buffers + 1; SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data // load data
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0); code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
@ -207,9 +222,12 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
SBuffer *buffer1 = reader->buffers + 1; SBuffer *buffer1 = reader->buffers + 1;
SBuffer *assist = reader->buffers + 2; SBuffer *assist = reader->buffers + 2;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load key part // load key part
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0); code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// decode header // decode header
@ -244,7 +262,8 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
// load SBlockCol part // load SBlockCol part
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0); code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// load each column // load each column
@ -290,7 +309,8 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
tBufferClear(buffer1); tBufferClear(buffer1);
code = code =
tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset, tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset,
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0); blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// decode the buffer // decode the buffer
@ -314,9 +334,12 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk
SBuffer *buffer0 = reader->buffers + 0; SBuffer *buffer0 = reader->buffers + 0;
SBuffer *assist = reader->buffers + 1; SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load // load
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0); code = tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// decode // decode
@ -351,9 +374,12 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
SBuffer *buffer0 = reader->buffers + 0; SBuffer *buffer0 = reader->buffers + 0;
SBuffer *assist = reader->buffers + 1; SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data // load data
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0); code = tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// decode data // decode data
@ -442,7 +468,7 @@ struct SSttFileWriter {
static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, SColCompressInfo *info, static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, SColCompressInfo *info,
int64_t *fileSize, TSttBlkArray *sttBlkArray, SBuffer *buffers, int64_t *fileSize, TSttBlkArray *sttBlkArray, SBuffer *buffers,
SVersionRange *range) { SVersionRange *range, int32_t encryptAlgorithm, char* encryptKey) {
if (blockData->nRow == 0) return 0; if (blockData->nRow == 0) return 0;
int32_t code = 0; int32_t code = 0;
@ -474,7 +500,7 @@ static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, S
sttBlk->bInfo.szBlock = buffers[2].size + buffers[3].size + sttBlk->bInfo.szKey; sttBlk->bInfo.szBlock = buffers[2].size + buffers[3].size + sttBlk->bInfo.szKey;
for (int i = 0; i < 4; i++) { for (int i = 0; i < 4; i++) {
if (buffers[i].size) { if (buffers[i].size) {
code = tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size); code = tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size, encryptAlgorithm, encryptKey);
if (code) return code; if (code) return code;
*fileSize += buffers[i].size; *fileSize += buffers[i].size;
} }
@ -498,8 +524,11 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) {
SColCompressInfo info = {.defaultCmprAlg = writer->config->cmprAlg, .pColCmpr = NULL}; SColCompressInfo info = {.defaultCmprAlg = writer->config->cmprAlg, .pColCmpr = NULL};
code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr)); code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr));
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size, writer->sttBlkArray, code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size, writer->sttBlkArray,
writer->buffers, &writer->ctx->range); writer->buffers, &writer->ctx->range,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -538,6 +567,9 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
statisBlk.maxTbid.suid = record.suid; statisBlk.maxTbid.suid = record.suid;
statisBlk.maxTbid.uid = record.uid; statisBlk.maxTbid.uid = record.uid;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// compress each column // compress each column
for (int32_t i = 0; i < ARRAY_SIZE(statisBlk.size); i++) { for (int32_t i = 0; i < ARRAY_SIZE(statisBlk.size); i++) {
SCompressInfo info = { SCompressInfo info = {
@ -550,7 +582,8 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
code = tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist); code = tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize); code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
statisBlk.size[i] = info.compressedSize; statisBlk.size[i] = info.compressedSize;
@ -579,12 +612,12 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size); code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += buffer0->size; writer->file->size += buffer0->size;
statisBlk.dp->size += buffer0->size; statisBlk.dp->size += buffer0->size;
code = tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size); code = tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += buffer1->size; writer->file->size += buffer1->size;
statisBlk.dp->size += buffer1->size; statisBlk.dp->size += buffer1->size;
@ -608,8 +641,12 @@ static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size,
writer->tombBlkArray, writer->buffers, &writer->ctx->range); writer->tombBlkArray, writer->buffers, &writer->ctx->range,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -619,12 +656,14 @@ _exit:
return code; return code;
} }
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize) { int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey) {
ptr->size = TARRAY2_DATA_LEN(sttBlkArray); ptr->size = TARRAY2_DATA_LEN(sttBlkArray);
if (ptr->size > 0) { if (ptr->size > 0) {
ptr->offset = *fileSize; ptr->offset = *fileSize;
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size); int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size, encryptAlgorithm,
encryptKey);
if (code) { if (code) {
return code; return code;
} }
@ -638,7 +677,11 @@ static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino; int32_t lino;
code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -651,12 +694,14 @@ _exit:
static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) { static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino; int32_t lino;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray); writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray);
if (writer->footer->statisBlkPtr->size) { if (writer->footer->statisBlkPtr->size) {
writer->footer->statisBlkPtr->offset = writer->file->size; writer->footer->statisBlkPtr->offset = writer->file->size;
code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray), code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray),
writer->footer->statisBlkPtr->size); writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += writer->footer->statisBlkPtr->size; writer->file->size += writer->footer->statisBlkPtr->size;
} }
@ -672,7 +717,11 @@ static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
@ -682,15 +731,19 @@ _exit:
return code; return code;
} }
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize) { int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); char* encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey);
if (code) return code; if (code) return code;
*fileSize += sizeof(*footer); *fileSize += sizeof(*footer);
return 0; return 0;
} }
static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) { static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) {
return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size, encryptAlgorithm, encryptKey);
} }
static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
@ -728,7 +781,10 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr)); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += sizeof(hdr); writer->file->size += sizeof(hdr);
@ -793,7 +849,9 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o
code = tsdbSttFileDoUpdateHeader(writer); code = tsdbSttFileDoUpdateHeader(writer);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFsyncFile(writer->fd); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd); tsdbCloseFile(&writer->fd);

6
source/dnode/vnode/src/tsdb/tsdbSttFileRW.h
View File

@ -71,8 +71,10 @@ int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *pBlockData
int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record); int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record);
bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer); bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer);
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize); int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize); int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
char* encryptKey);
struct SSttFileWriterConfig { struct SSttFileWriterConfig {
STsdb *tsdb; STsdb *tsdb;

53
source/dnode/vnode/src/tsdb/tsdbUpgrade.c
View File

@ -37,6 +37,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
int8_t cmprAlg; int8_t cmprAlg;
int32_t szPage; int32_t szPage;
SBuffer buffers[10]; SBuffer buffers[10];
int32_t encryptAlgorithm;
char* encryptKey;
// reader // reader
SArray *aBlockIdx; SArray *aBlockIdx;
SMapData mDataBlk[1]; SMapData mDataBlk[1];
@ -50,6 +52,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
.maxRow = tsdb->pVnode->config.tsdbCfg.maxRows, .maxRow = tsdb->pVnode->config.tsdbCfg.maxRows,
.cmprAlg = tsdb->pVnode->config.tsdbCfg.compression, .cmprAlg = tsdb->pVnode->config.tsdbCfg.compression,
.szPage = tsdb->pVnode->config.tsdbPageSize, .szPage = tsdb->pVnode->config.tsdbPageSize,
.encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm,
.encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey,
}}; }};
// read SBlockIdx array // read SBlockIdx array
@ -136,7 +140,7 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
if (ctx->brinBlock->numOfRecords >= ctx->maxRow) { if (ctx->brinBlock->numOfRecords >= ctx->maxRow) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
ctx->brinBlkArray, ctx->buffers, &range); ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
@ -145,18 +149,19 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
if (ctx->brinBlock->numOfRecords > 0) { if (ctx->brinBlock->numOfRecords > 0) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
ctx->brinBlkArray, ctx->buffers, &range); ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr, code = tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr,
&fset->farr[TSDB_FTYPE_HEAD]->f->size); &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer); code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFsyncFile(ctx->fd); code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&ctx->fd); tsdbCloseFile(&ctx->fd);
@ -252,12 +257,16 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
STFileObj *fobj; STFileObj *fobj;
struct { struct {
int32_t szPage; int32_t szPage;
int32_t encryptAlgorithm;
char* encryptKey;
// writer // writer
STsdbFD *fd; STsdbFD *fd;
TSttBlkArray sttBlkArray[1]; TSttBlkArray sttBlkArray[1];
SSttFooter footer[1]; SSttFooter footer[1];
} ctx[1] = {{ } ctx[1] = {{
.szPage = tsdb->pVnode->config.tsdbPageSize, .szPage = tsdb->pVnode->config.tsdbPageSize,
.encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm,
.encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey,
}}; }};
STFile file = { STFile file = {
@ -280,13 +289,14 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
TSDB_CHECK_CODE(code, lino, _exit1); TSDB_CHECK_CODE(code, lino, _exit1);
} }
code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size); code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1); TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size); code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1); TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbFsyncFile(ctx->fd); code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1); TSDB_CHECK_CODE(code, lino, _exit1);
tsdbCloseFile(&ctx->fd); tsdbCloseFile(&ctx->fd);
@ -431,7 +441,10 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE); int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
fobj[0]->f->size += TSDB_FHDR_SIZE; fobj[0]->f->size += TSDB_FHDR_SIZE;
@ -454,6 +467,8 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
int64_t minKey; int64_t minKey;
int64_t maxKey; int64_t maxKey;
SBuffer buffers[10]; SBuffer buffers[10];
int32_t encryptAlgorithm;
char* encryptKey;
// reader // reader
SArray *aDelData; SArray *aDelData;
// writer // writer
@ -466,6 +481,8 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
} ctx[1] = {{ } ctx[1] = {{
.maxRow = tsdb->pVnode->config.tsdbCfg.maxRows, .maxRow = tsdb->pVnode->config.tsdbCfg.maxRows,
.cmprAlg = tsdb->pVnode->config.tsdbCfg.compression, .cmprAlg = tsdb->pVnode->config.tsdbCfg.compression,
.encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm,
.encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey,
}}; }};
tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey); tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey);
@ -502,7 +519,7 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
} }
SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
ctx->buffers, &tombRange); ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
@ -515,26 +532,30 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
} }
SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
ctx->buffers, &tombRange); ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
if (ctx->fd != NULL) { if (ctx->fd != NULL) {
if (ctx->toStt) { if (ctx->toStt) {
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size); code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size,
ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size); code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size); code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size,
ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size); code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbFsyncFile(ctx->fd); code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&ctx->fd); tsdbCloseFile(&ctx->fd);

43
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -15,6 +15,7 @@
#include "tutil.h" #include "tutil.h"
#include "vnd.h" #include "vnd.h"
#include "tglobal.h"
const SVnodeCfg vnodeCfgDefault = {.vgId = -1, const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.dbname = "", .dbname = "",
@ -133,6 +134,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
tjsonAddItemToArray(pNodeRetentions, pNodeRetention); tjsonAddItemToArray(pNodeRetentions, pNodeRetention);
} }
} }
if (tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;
@ -140,6 +142,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
@ -242,6 +246,19 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code); tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code);
tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code); tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code);
} }
tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code); tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code);
if (code < 0) return -1; if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code); tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code);
@ -256,6 +273,32 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (code < 0) return -1; if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code); tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
if (code < 0) return -1; if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code); tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code);
if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER; if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER;
tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code); tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);

2
source/libs/CMakeLists.txt
View File

@ -9,6 +9,8 @@ add_subdirectory(nodes)
add_subdirectory(catalog) add_subdirectory(catalog)
add_subdirectory(audit) add_subdirectory(audit)
add_subdirectory(monitorfw) add_subdirectory(monitorfw)
add_subdirectory(sm4)
add_subdirectory(crypt)
add_subdirectory(scalar) add_subdirectory(scalar)
add_subdirectory(function) add_subdirectory(function)

16
source/libs/command/src/command.c
View File

@ -315,6 +315,18 @@ static const char* cacheModelStr(int8_t cacheModel) {
return TSDB_CACHE_MODEL_NONE_STR; return TSDB_CACHE_MODEL_NONE_STR;
} }
static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) {
switch (encryptAlgorithm) {
case TSDB_ENCRYPT_ALGO_NONE:
return TSDB_ENCRYPT_ALGO_NONE_STR;
case TSDB_ENCRYPT_ALGO_SM4:
return TSDB_ENCRYPT_ALGO_SM4_STR;
default:
break;
}
return TSDB_CACHE_MODEL_NONE_STR;
}
static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) { static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) {
blockDataEnsureCapacity(pBlock, 1); blockDataEnsureCapacity(pBlock, 1);
pBlock->info.rows = 1; pBlock->info.rows = 1;
@ -360,12 +372,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
"CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm " "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm "
"WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
"WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d "
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d", "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d",
dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize, 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize,
pCfg->keepTimeOffset, pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact); pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact);
if (retentions) { if (retentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);

14
source/libs/crypt/CMakeLists.txt Normal file
View File

@ -0,0 +1,14 @@
aux_source_directory(src CRYPT_SRC)
IF (TD_ENTERPRISE)
LIST(APPEND CRYPT_SRC ${TD_ENTERPRISE_DIR}/src/plugins/crypt/cryptImpl.c)
ENDIF ()
add_library(crypt STATIC ${CRYPT_SRC})
target_include_directories(
crypt
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(crypt common sm4)

36
source/libs/crypt/src/crypt.c Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "crypt.h"
extern int32_t CBC_DecryptImpl(SCryptOpts *opts);
extern int32_t CBC_EncryptImpl(SCryptOpts *opts);
int32_t CBC_Encrypt(SCryptOpts *opts) {
return CBC_EncryptImpl(opts);
}
int32_t CBC_Decrypt(SCryptOpts *opts) {
return CBC_DecryptImpl(opts);
}
#ifndef TD_ENTERPRISE
int32_t CBC_EncryptImpl(SCryptOpts *opts) {
memcpy(opts->result, opts->source, opts->len);
return opts->len;
}
int32_t CBC_DecryptImpl(SCryptOpts *opts) {
memcpy(opts->result, opts->source, opts->len);
return opts->len;
}
#endif

10
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -281,6 +281,8 @@ const char* nodesNodeName(ENodeType type) {
return "ShowGrantsLogsStmt"; return "ShowGrantsLogsStmt";
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return "ShowClusterMachinesStmt"; return "ShowClusterMachinesStmt";
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return "ShowEncryptionsStmt";
case QUERY_NODE_DELETE_STMT: case QUERY_NODE_DELETE_STMT:
return "DeleteStmt"; return "DeleteStmt";
case QUERY_NODE_INSERT_STMT: case QUERY_NODE_INSERT_STMT:
@ -7047,10 +7049,14 @@ static int32_t showGrantsLogsStmtToJson(const void* pObj, SJson* pJson) { return
static int32_t showClusterMachinesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } static int32_t showClusterMachinesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t showEncryptionsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t jsonToShowGrantsLogsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } static int32_t jsonToShowGrantsLogsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t jsonToShowClusterMachinesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } static int32_t jsonToShowClusterMachinesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t jsonToShowEncryptionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static const char* jkShowDnodeVariablesStmtDnodeId = "DnodeId"; static const char* jkShowDnodeVariablesStmtDnodeId = "DnodeId";
static const char* jkShowDnodeVariablesStmtLikePattern = "LikePattern"; static const char* jkShowDnodeVariablesStmtLikePattern = "LikePattern";
@ -7651,6 +7657,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return showGrantsLogsStmtToJson(pObj, pJson); return showGrantsLogsStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return showClusterMachinesStmtToJson(pObj, pJson); return showClusterMachinesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return showEncryptionsStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT:
return showDnodeVariablesStmtToJson(pObj, pJson); return showDnodeVariablesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_TRANSACTIONS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
@ -7996,6 +8004,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToShowGrantsLogsStmt(pJson, pObj); return jsonToShowGrantsLogsStmt(pJson, pObj);
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return jsonToShowClusterMachinesStmt(pJson, pObj); return jsonToShowClusterMachinesStmt(pJson, pObj);
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return jsonToShowEncryptionsStmt(pJson, pObj);
case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT:
return jsonToShowDnodeVariablesStmt(pJson, pObj); return jsonToShowDnodeVariablesStmt(pJson, pObj);
case QUERY_NODE_SHOW_TRANSACTIONS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT:

2
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -521,6 +521,7 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_SHOW_GRANTS_FULL_STMT: case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT: case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT: case QUERY_NODE_SHOW_TSMAS_STMT:
return makeNode(type, sizeof(SShowStmt)); return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_SHOW_TABLE_TAGS_STMT: case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
@ -1210,6 +1211,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_SHOW_GRANTS_FULL_STMT: case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT: case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT: { case QUERY_NODE_SHOW_TSMAS_STMT: {
SShowStmt* pStmt = (SShowStmt*)pNode; SShowStmt* pStmt = (SShowStmt*)pNode;
nodesDestroyNode(pStmt->pDbName); nodesDestroyNode(pStmt->pDbName);

4
source/libs/parser/inc/parAst.h
View File

@ -67,7 +67,8 @@ typedef enum EDatabaseOptionType {
DB_OPTION_S3_CHUNKSIZE, DB_OPTION_S3_CHUNKSIZE,
DB_OPTION_S3_KEEPLOCAL, DB_OPTION_S3_KEEPLOCAL,
DB_OPTION_S3_COMPACT, DB_OPTION_S3_COMPACT,
DB_OPTION_KEEP_TIME_OFFSET DB_OPTION_KEEP_TIME_OFFSET,
DB_OPTION_ENCRYPT_ALGORITHM,
} EDatabaseOptionType; } EDatabaseOptionType;
typedef enum ETableOptionType { typedef enum ETableOptionType {
@ -232,6 +233,7 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort); SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force, bool unsafe); SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force, bool unsafe);
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue); SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue);
SNode* createEncryptKeyStmt(SAstCreateContext* pCxt, const SToken* pValue);
SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName); SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName);
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName, SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName,
SNode* pRealTable, SNodeList* pCols, SNode* pOptions); SNode* pRealTable, SNodeList* pCols, SNode* pOptions);

9
source/libs/parser/inc/sql.y
View File

@ -144,6 +144,9 @@ priv_level(A) ::= topic_name(B).
with_opt(A) ::= . { A = NULL; } with_opt(A) ::= . { A = NULL; }
with_opt(A) ::= WITH search_condition(B). { A = B; } with_opt(A) ::= WITH search_condition(B). { A = B; }
/************************************************ create encrypt_key *********************************************/
cmd ::= CREATE ENCRYPT_KEY NK_STRING(A). { pCxt->pRootNode = createEncryptKeyStmt(pCxt, &A); }
/************************************************ create/drop/alter/restore dnode *********************************************/ /************************************************ create/drop/alter/restore dnode *********************************************/
cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); } cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); }
cmd ::= CREATE DNODE dnode_endpoint(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); } cmd ::= CREATE DNODE dnode_endpoint(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); }
@ -266,6 +269,7 @@ db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_INTEGER(C).
db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); } db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_COMPACT, &C); } db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_COMPACT, &C); }
db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); } db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); }
db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_ENCRYPT_ALGORITHM, &C); }
alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); } alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); }
@ -300,6 +304,7 @@ alter_db_option(A) ::= S3_KEEPLOCAL NK_INTEGER(B).
alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; } alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; }
alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; } alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; }
alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; } alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; }
alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; }
%type integer_list { SNodeList* } %type integer_list { SNodeList* }
%destructor integer_list { nodesDestroyList($$); } %destructor integer_list { nodesDestroyList($$); }
@ -516,7 +521,9 @@ cmd ::= SHOW GRANTS LOGS.
cmd ::= SHOW CLUSTER MACHINES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT); } cmd ::= SHOW CLUSTER MACHINES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT); }
cmd ::= SHOW CREATE DATABASE db_name(A). { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &A); } cmd ::= SHOW CREATE DATABASE db_name(A). { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &A); }
cmd ::= SHOW CREATE TABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, A); } cmd ::= SHOW CREATE TABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, A); }
cmd ::= SHOW CREATE STABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, A); } cmd ::= SHOW CREATE STABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT,
A); }
cmd ::= SHOW ENCRYPTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_ENCRYPTIONS_STMT); }
cmd ::= SHOW QUERIES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } cmd ::= SHOW QUERIES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); }
cmd ::= SHOW SCORES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } cmd ::= SHOW SCORES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); }
cmd ::= SHOW TOPICS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } cmd ::= SHOW TOPICS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); }

14
source/libs/parser/src/parAstCreater.c
View File

@ -1375,6 +1375,7 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
pOptions->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; pOptions->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pOptions->s3Compact = TSDB_DEFAULT_S3_COMPACT; pOptions->s3Compact = TSDB_DEFAULT_S3_COMPACT;
pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
pOptions->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
return (SNode*)pOptions; return (SNode*)pOptions;
} }
@ -1414,6 +1415,7 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
pOptions->s3KeepLocal = -1; pOptions->s3KeepLocal = -1;
pOptions->s3Compact = -1; pOptions->s3Compact = -1;
pOptions->withArbitrator = -1; pOptions->withArbitrator = -1;
pOptions->encryptAlgorithm = -1;
return (SNode*)pOptions; return (SNode*)pOptions;
} }
@ -1547,6 +1549,10 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
case DB_OPTION_KEEP_TIME_OFFSET: { case DB_OPTION_KEEP_TIME_OFFSET: {
pDbOptions->keepTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); pDbOptions->keepTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_ENCRYPT_ALGORITHM:
COPY_STRING_FORM_STR_TOKEN(pDbOptions->encryptAlgorithmStr, (SToken*)pVal);
pDbOptions->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
break;
} }
default: default:
break; break;
@ -2334,6 +2340,14 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createEncryptKeyStmt(SAstCreateContext* pCxt, const SToken* pValue) {
SToken config;
config.type = TK_NK_STRING;
config.z = "\"encrypt_key\"";
config.n = strlen(config.z);
return createAlterDnodeStmt(pCxt, NULL, &config, pValue);
}
SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName) { SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName) {
if (!checkIndexName(pCxt, pIndexName)) { if (!checkIndexName(pCxt, pIndexName)) {
return NULL; return NULL;

7
source/libs/parser/src/parAstParser.c
View File

@ -644,6 +644,11 @@ static int32_t collectMetaKeyFromShowClusterMachines(SCollectMetaKeyCxt* pCxt, S
pCxt->pMetaCache); pCxt->pMetaCache);
} }
static int32_t collectMetaKeyFromShowEncryptions(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_ENCRYPTIONS,
pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) { static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) {
return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
} }
@ -916,6 +921,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromShowGrantsLogs(pCxt, (SShowStmt*)pStmt); return collectMetaKeyFromShowGrantsLogs(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return collectMetaKeyFromShowClusterMachines(pCxt, (SShowStmt*)pStmt); return collectMetaKeyFromShowClusterMachines(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return collectMetaKeyFromShowEncryptions(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt); return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT:

1
source/libs/parser/src/parAuthenticator.c
View File

@ -356,6 +356,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT: case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ARBGROUPS_STMT: case QUERY_NODE_SHOW_ARBGROUPS_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
return !pCxt->pParseCxt->enableSysInfo ? TSDB_CODE_PAR_PERMISSION_DENIED : TSDB_CODE_SUCCESS; return !pCxt->pParseCxt->enableSysInfo ? TSDB_CODE_PAR_PERMISSION_DENIED : TSDB_CODE_SUCCESS;
case QUERY_NODE_SHOW_TABLES_STMT: case QUERY_NODE_SHOW_TABLES_STMT:
case QUERY_NODE_SHOW_STABLES_STMT: case QUERY_NODE_SHOW_STABLES_STMT:

6
source/libs/parser/src/parInsertSql.c
View File

@ -670,11 +670,13 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
int32_t output = 0; int32_t output = 0;
void* p = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE); int64_t realLen = pToken->n << 2;
if (realLen > pSchema->bytes - VARSTR_HEADER_SIZE) realLen = pSchema->bytes - VARSTR_HEADER_SIZE;
void* p = taosMemoryMalloc(realLen);
if (p == NULL) { if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output)) {
if (errno == E2BIG) { if (errno == E2BIG) {
taosMemoryFree(p); taosMemoryFree(p);
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);

3
source/libs/parser/src/parTokenizer.c
View File

@ -94,6 +94,9 @@ static SKeyword keywordTable[] = {
{"DURATION", TK_DURATION}, {"DURATION", TK_DURATION},
{"ELSE", TK_ELSE}, {"ELSE", TK_ELSE},
{"ENABLE", TK_ENABLE}, {"ENABLE", TK_ENABLE},
{"ENCRYPTIONS", TK_ENCRYPTIONS},
{"ENCRYPT_ALGORITHM", TK_ENCRYPT_ALGORITHM},
{"ENCRYPT_KEY", TK_ENCRYPT_KEY},
{"END", TK_END}, {"END", TK_END},
{"EXISTS", TK_EXISTS}, {"EXISTS", TK_EXISTS},
{"EXPIRED", TK_EXPIRED}, {"EXPIRED", TK_EXPIRED},

59
source/libs/parser/src/parTranslater.c
View File

@ -305,7 +305,15 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
{ .showType = QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT, { .showType = QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB, .pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_TABLE_MACHINES, .pTableName = TSDB_INS_TABLE_MACHINES,
}, .numOfShowCols = 1,
.pShowCols = {"*"}
},
{ .showType = QUERY_NODE_SHOW_ENCRYPTIONS_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_TABLE_ENCRYPTIONS,
.numOfShowCols = 1,
.pShowCols = {"*"}
},
{ .showType = QUERY_NODE_SHOW_TSMAS_STMT, { .showType = QUERY_NODE_SHOW_TSMAS_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB, .pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_TABLE_TSMAS, .pTableName = TSDB_INS_TABLE_TSMAS,
@ -6501,6 +6509,7 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
pReq->s3Compact = pStmt->pOptions->s3Compact; pReq->s3Compact = pStmt->pOptions->s3Compact;
pReq->ignoreExist = pStmt->ignoreExists; pReq->ignoreExist = pStmt->ignoreExists;
pReq->withArbitrator = pStmt->pOptions->withArbitrator; pReq->withArbitrator = pStmt->pOptions->withArbitrator;
pReq->encryptAlgorithm = pStmt->pOptions->encryptAlgorithm;
return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
} }
@ -6645,6 +6654,21 @@ static int32_t checkDbCacheModelOption(STranslateContext* pCxt, SDatabaseOptions
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t checkDbEncryptAlgorithmOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if ('\0' != pOptions->encryptAlgorithmStr[0]) {
if (0 == strcasecmp(pOptions->encryptAlgorithmStr, TSDB_ENCRYPT_ALGO_NONE_STR)) {
pOptions->encryptAlgorithm = TSDB_ENCRYPT_ALGO_NONE;
} else if (0 == strcasecmp(pOptions->encryptAlgorithmStr, TSDB_ENCRYPT_ALGO_SM4_STR)) {
pOptions->encryptAlgorithm = TSDB_ENCRYPT_ALGO_SM4;
} else {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option encrypt_algorithm: %s", pOptions->encryptAlgorithmStr);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
if ('\0' != pOptions->precisionStr[0]) { if ('\0' != pOptions->precisionStr[0]) {
if (0 == strcasecmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MILLI_STR)) { if (0 == strcasecmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MILLI_STR)) {
@ -6838,6 +6862,9 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbCacheModelOption(pCxt, pOptions); code = checkDbCacheModelOption(pCxt, pOptions);
} }
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEncryptAlgorithmOption(pCxt, pOptions);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = code =
checkDbRangeOption(pCxt, "cacheSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_SIZE, TSDB_MAX_DB_CACHE_SIZE); checkDbRangeOption(pCxt, "cacheSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_SIZE, TSDB_MAX_DB_CACHE_SIZE);
@ -6924,6 +6951,10 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
code = checkDbEnumOption(pCxt, "withArbitrator", pOptions->withArbitrator, TSDB_MIN_DB_WITH_ARBITRATOR, code = checkDbEnumOption(pCxt, "withArbitrator", pOptions->withArbitrator, TSDB_MIN_DB_WITH_ARBITRATOR,
TSDB_MAX_DB_WITH_ARBITRATOR); TSDB_MAX_DB_WITH_ARBITRATOR);
} }
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "encryptAlgorithm", pOptions->encryptAlgorithm, TSDB_MIN_ENCRYPT_ALGO,
TSDB_MAX_ENCRYPT_ALGO);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix); code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix);
} }
@ -7070,6 +7101,11 @@ static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq)
FILL_CMD_SQL(sql, sqlLen, pCmdReq, SMCfgClusterReq, pReq); FILL_CMD_SQL(sql, sqlLen, pCmdReq, SMCfgClusterReq, pReq);
break; break;
} }
case TDMT_MND_CREATE_ENCRYPT_KEY: {
FILL_CMD_SQL(sql, sqlLen, pCmdReq, SMCfgDnodeReq, pReq);
break;
}
default: { default: {
break; break;
} }
@ -7157,6 +7193,11 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm
return code; return code;
} }
if (pStmt->pOptions->encryptAlgorithm != -1) {
code = TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE;
return code;
}
SAlterDbReq alterReq = {0}; SAlterDbReq alterReq = {0};
buildAlterDbReq(pCxt, pStmt, &alterReq); buildAlterDbReq(pCxt, pStmt, &alterReq);
@ -8378,7 +8419,20 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt
strcpy(cfgReq.config, pStmt->config); strcpy(cfgReq.config, pStmt->config);
strcpy(cfgReq.value, pStmt->value); strcpy(cfgReq.value, pStmt->value);
int32_t code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); int32_t code = 0;
if (0 == strncasecmp(cfgReq.config, "encrypt_key", 12)) {
int32_t klen = strlen(cfgReq.value);
if (klen > ENCRYPT_KEY_LEN || klen < ENCRYPT_KEY_LEN_MIN) {
tFreeSMCfgDnodeReq(&cfgReq);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_ENCRYPT_KLEN,
"Invalid encryption key length: %d, valid range [%d,%d]", klen,
ENCRYPT_KEY_LEN_MIN, ENCRYPT_KEY_LEN);
}
code = buildCmdMsg(pCxt, TDMT_MND_CREATE_ENCRYPT_KEY, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq);
} else {
code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq);
}
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return code; return code;
} }
@ -13117,6 +13171,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT: case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ARBGROUPS_STMT: case QUERY_NODE_SHOW_ARBGROUPS_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT: case QUERY_NODE_SHOW_TSMAS_STMT:
code = rewriteShow(pCxt, pQuery); code = rewriteShow(pCxt, pQuery);
break; break;

12022
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

9
source/libs/sm4/CMakeLists.txt Normal file
View File

@ -0,0 +1,9 @@
aux_source_directory(src SM4_SRC)
add_library(sm4 STATIC ${SM4_SRC})
target_include_directories(
sm4
PUBLIC "${TD_SOURCE_DIR}/include/libs/sm4"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(sm4 common)

544
source/libs/sm4/src/sm4.c Normal file
View File

@ -0,0 +1,544 @@
#include "sm4.h"
#define SM4_ROUND 32
static unsigned int FK[4]={
0xA3B1BAC6,0x56AA3350,0x677D9197,0xB27022DC
};
static unsigned int CK[SM4_ROUND]={
0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269,
0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9,
0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249,
0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9,
0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229,
0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299,
0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209,
0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279
};
static unsigned char Sbox[256]={
0xd6,0x90,0xe9,0xfe,0xcc,0xe1,0x3d,0xb7,0x16,0xb6,0x14,0xc2,0x28,0xfb,0x2c,0x05,
0x2b,0x67,0x9a,0x76,0x2a,0xbe,0x04,0xc3,0xaa,0x44,0x13,0x26,0x49,0x86,0x06,0x99,
0x9c,0x42,0x50,0xf4,0x91,0xef,0x98,0x7a,0x33,0x54,0x0b,0x43,0xed,0xcf,0xac,0x62,
0xe4,0xb3,0x1c,0xa9,0xc9,0x08,0xe8,0x95,0x80,0xdf,0x94,0xfa,0x75,0x8f,0x3f,0xa6,
0x47,0x07,0xa7,0xfc,0xf3,0x73,0x17,0xba,0x83,0x59,0x3c,0x19,0xe6,0x85,0x4f,0xa8,
0x68,0x6b,0x81,0xb2,0x71,0x64,0xda,0x8b,0xf8,0xeb,0x0f,0x4b,0x70,0x56,0x9d,0x35,
0x1e,0x24,0x0e,0x5e,0x63,0x58,0xd1,0xa2,0x25,0x22,0x7c,0x3b,0x01,0x21,0x78,0x87,
0xd4,0x00,0x46,0x57,0x9f,0xd3,0x27,0x52,0x4c,0x36,0x02,0xe7,0xa0,0xc4,0xc8,0x9e,
0xea,0xbf,0x8a,0xd2,0x40,0xc7,0x38,0xb5,0xa3,0xf7,0xf2,0xce,0xf9,0x61,0x15,0xa1,
0xe0,0xae,0x5d,0xa4,0x9b,0x34,0x1a,0x55,0xad,0x93,0x32,0x30,0xf5,0x8c,0xb1,0xe3,
0x1d,0xf6,0xe2,0x2e,0x82,0x66,0xca,0x60,0xc0,0x29,0x23,0xab,0x0d,0x53,0x4e,0x6f,
0xd5,0xdb,0x37,0x45,0xde,0xfd,0x8e,0x2f,0x03,0xff,0x6a,0x72,0x6d,0x6c,0x5b,0x51,
0x8d,0x1b,0xaf,0x92,0xbb,0xdd,0xbc,0x7f,0x11,0xd9,0x5c,0x41,0x1f,0x10,0x5a,0xd8,
0x0a,0xc1,0x31,0x88,0xa5,0xcd,0x7b,0xbd,0x2d,0x74,0xd0,0x12,0xb8,0xe5,0xb4,0xb0,
0x89,0x69,0x97,0x4a,0x0c,0x96,0x77,0x7e,0x65,0xb9,0xf1,0x09,0xc5,0x6e,0xc6,0x84,
0x18,0xf0,0x7d,0xec,0x3a,0xdc,0x4d,0x20,0x79,0xee,0x5f,0x3e,0xd7,0xcb,0x39,0x48
};
#define ROL(x,y) ((x)<<(y) | (x)>>(32-(y)))
unsigned int SMS4_T1(unsigned int dwA)
{
unsigned char a0[4]={0};
unsigned char b0[4]={0};
unsigned int dwB=0;
unsigned int dwC=0;
int i=0;
/*
for (i=0;i<4;i++)
{
a0[i] = (unsigned char)((dwA>>(i*8)) & 0xff);
b0[i] = Sbox[a0[i]];
dwB |= (b0[i]<<(i*8));
}
*/
a0[0] = (unsigned char)((dwA) & 0xff);
b0[0] = Sbox[a0[0]];
dwB |= (b0[0]);
a0[1] = (unsigned char)((dwA>>(8)) & 0xff);
b0[1] = Sbox[a0[1]];
dwB |= (b0[1]<<(8));
a0[2] = (unsigned char)((dwA>>(16)) & 0xff);
b0[2] = Sbox[a0[2]];
dwB |= (b0[2]<<(16));
a0[3] = (unsigned char)((dwA>>(24)) & 0xff);
b0[3] = Sbox[a0[3]];
dwB |= (b0[3]<<(24)); // 2013-08-13
dwC=dwB^ROL(dwB,2)^ROL(dwB,10)^ROL(dwB,18)^ROL(dwB,24);
return dwC;
}
unsigned int SMS4_T2(unsigned int dwA)
{
unsigned char a0[4]={0};
unsigned char b0[4]={0};
unsigned int dwB=0;
unsigned int dwC=0;
int i=0;
/*
for (i=0;i<4;i++)
{
a0[i] = (unsigned char)((dwA>>(i*8)) & 0xff);
b0[i] = Sbox[a0[i]];
dwB |= (b0[i]<<(i*8));
}
*/
a0[0] = (unsigned char)((dwA) & 0xff);
b0[0] = Sbox[a0[0]];
dwB |= (b0[0]);
a0[1] = (unsigned char)((dwA>>(8)) & 0xff);
b0[1] = Sbox[a0[1]];
dwB |= (b0[1]<<(8));
a0[2] = (unsigned char)((dwA>>(16)) & 0xff);
b0[2] = Sbox[a0[2]];
dwB |= (b0[2]<<(16));
a0[3] = (unsigned char)((dwA>>(24)) & 0xff);
b0[3] = Sbox[a0[3]];
dwB |= (b0[3]<<(24)); // 2013-08-13
dwC=dwB^ROL(dwB,13)^ROL(dwB,23);
return dwC;
}
/* MK[4] is the Encrypt Key, rk[32] is Round Key */
void SMS4_Key_Expansion(unsigned int MK[], unsigned int rk[])
{
unsigned int K[4]={0};
int i=0;
for (i=0;i<4;i++)
{
K[i] = MK[i] ^ FK[i];
}
for (i=0;i<SM4_ROUND;i++)
{
K[i%4]^=SMS4_T2(K[(i+1)%4]^K[(i+2)%4]^K[(i+3)%4]^CK[i]);
rk[i]=K[i%4];
}
}
/* X[4] is PlainText, rk[32] is round Key, Y[4] is CipherText */
void SMS4_ECB_Encryption_Core(unsigned int X[], unsigned int rk[], unsigned int Y[])
{
unsigned int tempX[4]={0};
int i=0;
/*
for (i=0;i<4;i++)
{
tempX[i]=X[i];
}
*/
tempX[0]=X[0];
tempX[1]=X[1];
tempX[2]=X[2];
tempX[3]=X[3];
/*
for (i=0;i<SM4_ROUND;i++)
{
tempX[i%4]^=SMS4_T1(tempX[(i+1)%4]^tempX[(i+2)%4]^tempX[(i+3)%4]^rk[i]);
}
*/
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[0]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[1]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[2]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[3]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[4]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[5]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[6]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[7]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[8]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[9]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[10]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[11]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[12]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[13]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[14]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[15]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[16]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[17]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[18]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[19]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[20]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[21]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[22]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[23]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[24]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[25]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[26]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[27]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[28]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[29]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[30]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[31]);
/* for (i=0;i<4;i++)
{
Y[i]=tempX[3-i];
}*/
Y[0]=tempX[3];
Y[1]=tempX[2];
Y[2]=tempX[1];
Y[3]=tempX[0]; // 2013-08-19
}
/* X[4] is PlainText, rk[32] is round Key, Y[4] is CipherText */
void SMS4_ECB_Decryption_Core(unsigned int X[], unsigned int rk[], unsigned int Y[])
{
unsigned int tempX[4]={0};
int i=0;
/* for (i=0;i<4;i++)
{
tempX[i]=X[i];
}
*/
tempX[0]=X[0];
tempX[1]=X[1];
tempX[2]=X[2];
tempX[3]=X[3]; // 2013-08-19
/* for (i=0;i<SM4_ROUND;i++)
{
tempX[i%4]^=SMS4_T1(tempX[(i+1)%4]^tempX[(i+2)%4]^tempX[(i+3)%4]^rk[(31-i)]);
}
*/
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[31]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[30]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[29]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[28]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[27]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[26]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[25]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[24]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[23]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[22]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[21]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[20]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[19]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[18]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[17]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[16]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[15]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[14]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[13]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[12]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[11]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[10]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[9]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[8]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[7]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[6]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[5]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[4]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[3]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[2]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[1]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[0]);// 2013-08-19
/* for (i=0;i<4;i++)
{
Y[i]=tempX[3-i];
}*/
Y[0]=tempX[3];
Y[1]=tempX[2];
Y[2]=tempX[1];
Y[3]=tempX[0]; // 2013-08-19
}
void SMS4_convert_to_network_order(unsigned int* src,unsigned int* dst,int count)
{
int i=0;
for ( ; i<count; i++ )
{
unsigned char* ps = (unsigned char*)(src+i);
unsigned char* pd = (unsigned char*)(dst+i);
pd[0] = ps[3];
pd[1] = ps[2];
pd[2] = ps[1];
pd[3] = ps[0];
}
}
void SMS4_convert_to_host_order(unsigned int* src,unsigned int* dst,int count)
{
SMS4_convert_to_network_order(src,dst,count);
}
void SMS4_ECB_Encryption(unsigned char plaintext[16], unsigned char key[16], unsigned char ciphertext[16])
{
unsigned int _pt[4];
unsigned int _ky[4];
unsigned int _ct[4];
unsigned int _rk[32];
SMS4_convert_to_network_order((unsigned int*)plaintext,_pt,4);
SMS4_convert_to_network_order((unsigned int*)key,_ky,4);
SMS4_Key_Expansion(_ky,_rk);
SMS4_ECB_Encryption_Core(_pt,_rk,_ct);
SMS4_convert_to_host_order(_ct,(unsigned int*)ciphertext,4);
}
void Key_Expansion_init( unsigned char key[16], unsigned int rk[32])
{
unsigned int _ky[4];
SMS4_convert_to_network_order((unsigned int*)key,_ky,4);
SMS4_Key_Expansion (_ky,rk);
}
void SMS4_ECB_EncryptionEx(unsigned char plaintext[16], unsigned int key[32], unsigned char ciphertext[16])
{
unsigned int _pt[4];
unsigned int _ct[4];
SMS4_convert_to_network_order((unsigned int*)plaintext,_pt,4);
SMS4_ECB_Encryption_Core(_pt,key,_ct);
SMS4_convert_to_host_order(_ct,(unsigned int*)ciphertext,4);
}
void SMS4_ECB_Decryption(unsigned char ciphertext[16], unsigned char key[16], unsigned char plaintext[16])
{
unsigned int _ct[4];
unsigned int _ky[4];
unsigned int _pt[4];
unsigned int _rk[32];
SMS4_convert_to_network_order((unsigned int*)ciphertext,_ct,4);
SMS4_convert_to_network_order((unsigned int*)key,_ky,4);
SMS4_Key_Expansion(_ky,_rk);
SMS4_ECB_Decryption_Core(_ct,_rk,_pt);
SMS4_convert_to_host_order(_pt,(unsigned int*)plaintext,4);
}
void SMS4_ECB_DecryptionEx(unsigned char ciphertext[16], unsigned int key[32], unsigned char plaintext[16])
{
unsigned int _ct[4];
unsigned int _pt[4];
SMS4_convert_to_network_order((unsigned int*)ciphertext,_ct,4);
SMS4_ECB_Decryption_Core(_ct,key,_pt);
SMS4_convert_to_host_order(_pt,(unsigned int*)plaintext,4);
}
void SMS4_CBC_Encryption(unsigned char plaintext[16], unsigned char key[16], unsigned char iv[16], unsigned char ciphertext[16])
{
unsigned char plaintextNew[16];
int i = 0;
for (i = 0; i < 16; i ++)
{
plaintextNew[i] = plaintext[i] ^ iv[i];
}
SMS4_ECB_Encryption(plaintextNew, key, ciphertext);
}
void SMS4_CBC_Decryption(unsigned char ciphertext[16], unsigned char key[16], unsigned char iv[16], unsigned char plaintext[16])
{
unsigned char plaintextTemp[16];
int i = 0;
SMS4_ECB_Decryption(ciphertext, key, plaintextTemp);
for (i = 0; i < 16; i ++)
{
plaintext[i] = plaintextTemp[i] ^ iv[i];
}
}
void SMS4_CBC_EncryptionEx(unsigned char plaintext[16], unsigned int key[32], unsigned char iv[16], unsigned char ciphertext[16])
{
unsigned char plaintextNew[16];
int i = 0;
for (i = 0; i < 16; i ++)
{
plaintextNew[i] = plaintext[i] ^ iv[i];
}
SMS4_ECB_EncryptionEx(plaintextNew, key, ciphertext);
}
void SMS4_CBC_DecryptionEx(unsigned char ciphertext[16], unsigned int key[32], unsigned char iv[16], unsigned char plaintext[16])
{
unsigned char plaintextTemp[16];
int i = 0;
SMS4_ECB_DecryptionEx(ciphertext, key, plaintextTemp);
for (i = 0; i < 16; i ++)
{
plaintext[i] = plaintextTemp[i] ^ iv[i];
}
}
/*
int SM4_ECB_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
//int rv = 0;
int loop = 0;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
for (i = 0; i < loop; i ++)
{
//SMS4_ECB_Encryption(pInData + i * 16, pKey, pOutData + i * 16);
SMS4_ECB_EncryptionEx(pInData + i * 16, rk, pOutData + i * 16);
}
*pOutDataLen = inDataLen;
return 0;
}
int SM4_ECB_Decrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
// int rv = 0;
int loop = 0;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
for (i = 0; i < loop; i ++)
{
//SMS4_ECB_Decryption(pInData + i * 16, pKey, pOutData + i * 16);
SMS4_ECB_DecryptionEx(pInData + i * 16, rk, pOutData + i * 16);
}
*pOutDataLen = inDataLen;
return 0;
}
*/
int SM4_CBC_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
// int rv = 0;
int loop = 0;
unsigned char *pIVTemp = NULL;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
if(ivLen != 16)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
pIVTemp = pIV;
for (i = 0; i < loop; i ++)
{
SMS4_CBC_EncryptionEx(pInData + i * 16, rk, pIVTemp, pOutData + i * 16);
pIVTemp = pOutData + i * 16;
}
*pOutDataLen = inDataLen;
return 0;
}
int SM4_CBC_Decrypt(unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
// int rv = 0;
int loop = 0;
unsigned char *pIVTemp = NULL;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
if(ivLen != 16)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
pIVTemp = pIV;
for (i = 0; i < loop; i ++)
{
SMS4_CBC_DecryptionEx(pInData + i * 16, rk, pIVTemp, pOutData + i * 16);
pIVTemp = pInData + i * 16;
}
*pOutDataLen = inDataLen;
return 0;
}

2
source/libs/stream/src/streamMeta.c
View File

@ -130,7 +130,7 @@ typedef struct {
} StreamMetaTaskState; } StreamMetaTaskState;
int32_t streamMetaOpenTdb(SStreamMeta* pMeta) { int32_t streamMetaOpenTdb(SStreamMeta* pMeta) {
if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0) < 0) { if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0, 0, NULL) < 0) {
return -1; return -1;
// goto _err; // goto _err;
} }

2
source/libs/stream/src/streamState.c
View File

@ -158,7 +158,7 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz
} }
taosCloseFile(&pCfgFile); taosCloseFile(&pCfgFile);
if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 1) < 0) { if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 1, 0, NULL) < 0) {
goto _err; goto _err;
} }

1
source/libs/tdb/CMakeLists.txt
View File

@ -22,6 +22,7 @@ target_link_libraries(
tdb tdb
PUBLIC os PUBLIC os
PUBLIC util PUBLIC util
PUBLIC crypt
) )
# for test # for test

3
source/libs/tdb/inc/tdb.h
View File

@ -32,7 +32,8 @@ typedef struct STBC TBC;
typedef struct STxn TXN; typedef struct STxn TXN;
// TDB // TDB
int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback); int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm,
char *encryptKey);
int32_t tdbClose(TDB *pDb); int32_t tdbClose(TDB *pDb);
int32_t tdbBegin(TDB *pDb, TXN **pTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, int32_t tdbBegin(TDB *pDb, TXN **pTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
int flags); int flags);

8
source/libs/tdb/src/db/tdbDb.c
View File

@ -15,7 +15,8 @@
#include "tdbInt.h" #include "tdbInt.h"
int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback) { int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm,
char *encryptKey) {
TDB *pDb; TDB *pDb;
int dsize; int dsize;
int zsize; int zsize;
@ -49,6 +50,11 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
pDb->jfd = -1; pDb->jfd = -1;
pDb->encryptAlgorithm = encryptAlgorithm;
if(encryptKey != NULL){
strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN);
}
ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache));
if (ret < 0) { if (ret < 0) {
return -1; return -1;

100
source/libs/tdb/src/db/tdbPager.c
View File

@ -14,6 +14,8 @@
*/ */
#include "tdbInt.h" #include "tdbInt.h"
#include "crypt.h"
#include "tglobal.h"
/* /*
#pragma pack(push, 1) #pragma pack(push, 1)
typedef struct { typedef struct {
@ -457,6 +459,49 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
return 0; return 0;
} }
static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, const char* function,
int64_t offset){
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey;
char* buf = pPageData;
if(encryptAlgorithm == DND_CA_SM4){
//tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
//ASSERT(strlen(encryptKey) > 0);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]);
buf = taosMemoryMalloc(pageSize);
unsigned char packetData[128];
int32_t count = 0;
while (count < pageSize) {
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pPageData + count;
opts.result = packetData;
opts.unitLen = 128;
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
int32_t newLen = CBC_Encrypt(&opts);
memcpy(buf + count, packetData, newLen);
count += newLen;
}
//tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]);
}
return buf;
}
void tdbFreeEncryptBuf(SPager *pPager, char* buf){
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf);
}
// recovery dirty pages // recovery dirty pages
int tdbPagerAbort(SPager *pPager, TXN *pTxn) { int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
SPage *pPage; SPage *pPage;
@ -525,14 +570,19 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
return -1; return -1;
} }
ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
pPager->pageSize); pPager->pageSize);
tdbFreeEncryptBuf(pPager, buf);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return -1;
} }
tdbFreeEncryptBuf(pPager, buf);
} }
if (tdbOsFSync(pPager->fd) < 0) { if (tdbOsFSync(pPager->fd) < 0) {
@ -872,10 +922,44 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
TDB_UNLOCK_PAGE(pPage); TDB_UNLOCK_PAGE(pPage);
return -1; return -1;
} }
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey;
if(encryptAlgorithm == DND_CA_SM4){
//tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
//ASSERT(strlen(encryptKey) > 0);
//uint8_t flags = pPage->pData[0];
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags);
unsigned char packetData[128];
int32_t count = 0;
while(count < pPage->pageSize)
{
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pPage->pData + count;
opts.result = packetData;
opts.unitLen = 128;
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
int newLen = CBC_Decrypt(&opts);
memcpy(pPage->pData + count, packetData, newLen);
count += newLen;
}
//tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count, __FUNCTION__);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__);
}
} else { } else {
init = 0; init = 0;
} }
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__);
ret = (*initPage)(pPage, arg, init); ret = (*initPage)(pPage, arg, init);
if (ret < 0) { if (ret < 0) {
tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " init page failed.", pPager, pgno, nRead, tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " init page failed.", pPager, pgno, nRead,
@ -960,14 +1044,19 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) {
offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
ret = tdbOsPWrite(pPager->fd, pPage->pData, pPage->pageSize, offset); char* buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset);
ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset);
if (ret < 0) { if (ret < 0) {
tdbFreeEncryptBuf(pPager, buf);
tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName,
pPage->pageSize); pPage->pageSize);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
tdbFreeEncryptBuf(pPager, buf);
return 0; return 0;
} }
@ -1025,14 +1114,19 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
return -1; return -1;
} }
ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
pPager->pageSize); pPager->pageSize);
tdbFreeEncryptBuf(pPager, buf);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return -1;
} }
tdbFreeEncryptBuf(pPager, buf);
} }
if (tdbOsFSync(pPager->fd) < 0) { if (tdbOsFSync(pPager->fd) < 0) {

3
source/libs/tdb/src/inc/tdbInt.h
View File

@ -18,6 +18,7 @@
#include "tdb.h" #include "tdb.h"
#include "tdef.h"
#include "tlog.h" #include "tlog.h"
#include "trbtree.h" #include "trbtree.h"
@ -392,6 +393,8 @@ struct STDB {
TTB *pFreeDb; TTB *pFreeDb;
#endif #endif
int64_t txnId; int64_t txnId;
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN + 1];
}; };
struct SPager { struct SPager {

4
source/libs/tdb/test/tdbExOVFLTest.cpp
View File

@ -141,7 +141,7 @@ static void generateBigVal(char *val, int valLen) {
static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
TDB *pEnv = NULL; TDB *pEnv = NULL;
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0); int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL);
if (ret) { if (ret) {
pEnv = NULL; pEnv = NULL;
} }
@ -374,7 +374,7 @@ TEST(tdb_test, simple_insert1) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database

10
source/libs/tdb/test/tdbPageDefragmentTest.cpp
View File

@ -141,7 +141,7 @@ static void generateBigVal(char *val, int valLen) {
static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
TDB *pEnv = NULL; TDB *pEnv = NULL;
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0); int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL);
if (ret) { if (ret) {
pEnv = NULL; pEnv = NULL;
} }
@ -352,7 +352,7 @@ TEST(TdbPageDefragmentTest, DISABLED_simple_insert1) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
@ -485,7 +485,7 @@ TEST(TdbPageDefragmentTest, seq_insert) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
@ -566,7 +566,7 @@ TEST(TdbPageDefragmentTest, seq_delete) {
int const pageSize = 1 * 1024 * 1024; int const pageSize = 1 * 1024 * 1024;
// Open Env // Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
@ -650,7 +650,7 @@ TEST(TdbPageDefragmentTest, defragment_insert) {
int const pageSize = 1 * 1024 * 1024; int const pageSize = 1 * 1024 * 1024;
// Open Env // Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database

8
source/libs/tdb/test/tdbPageRecycleTest.cpp
View File

@ -123,7 +123,7 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in
static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
TDB *pEnv = NULL; TDB *pEnv = NULL;
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0); int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0 , 0, NULL);
if (ret) { if (ret) {
pEnv = NULL; pEnv = NULL;
} }
@ -355,7 +355,7 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
@ -484,7 +484,7 @@ static void insertDb(int nData) {
int const pageSize = 4 * 1024; int const pageSize = 4 * 1024;
// Open Env // Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
@ -552,7 +552,7 @@ static void deleteDb(int nData) {
int const pageSize = 4 * 1024; int const pageSize = 4 * 1024;
// Open Env // Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database

12
source/libs/tdb/test/tdbTest.cpp
View File

@ -131,7 +131,7 @@ TEST(tdb_test, DISABLED_simple_insert1) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbOpen("tdb", 4096, 64, &pEnv, 0); ret = tdbOpen("tdb", 4096, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
@ -246,7 +246,7 @@ TEST(tdb_test, DISABLED_simple_insert2) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbOpen("tdb", 1024, 10, &pEnv, 0); ret = tdbOpen("tdb", 1024, 10, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
@ -339,7 +339,7 @@ TEST(tdb_test, DISABLED_simple_delete1) {
pPool = openPool(); pPool = openPool();
// open env // open env
ret = tdbOpen("tdb", 1024, 256, &pEnv, 0); ret = tdbOpen("tdb", 1024, 256, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// open database // open database
@ -428,7 +428,7 @@ TEST(tdb_test, DISABLED_simple_upsert1) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// open env // open env
ret = tdbOpen("tdb", 4096, 64, &pEnv, 0); ret = tdbOpen("tdb", 4096, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// open database // open database
@ -490,7 +490,7 @@ TEST(tdb_test, multi_thread_query) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbOpen("tdb", 4096, 10, &pEnv, 0); ret = tdbOpen("tdb", 4096, 10, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
@ -599,7 +599,7 @@ TEST(tdb_test, DISABLED_multi_thread1) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbOpen("tdb", 512, 1, &pDb, 0); ret = tdbOpen("tdb", 512, 1, &pDb, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
ret = tdbTbOpen("db.db", -1, -1, NULL, pDb, &pTb, 0); ret = tdbTbOpen("db.db", -1, -1, NULL, pDb, &pTb, 0);

1
source/libs/wal/CMakeLists.txt
View File

@ -12,6 +12,7 @@ target_link_libraries(
PUBLIC os PUBLIC os
PUBLIC util PUBLIC util
PUBLIC common PUBLIC common
PUBLIC crypt
) )
if(${BUILD_TEST}) if(${BUILD_TEST})

15
source/libs/wal/src/walMeta.c
View File

@ -153,7 +153,11 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
} }
// validate body // validate body
recordLen = walCkHeadSz + logContent->head.bodyLen; int32_t cryptedBodyLen = logContent->head.bodyLen;
if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
recordLen = walCkHeadSz + cryptedBodyLen;
if (len < recordLen) { if (len < recordLen) {
int64_t extraSize = recordLen - len; int64_t extraSize = recordLen - len;
if (capacity < readSize + extraSize + sizeof(magic)) { if (capacity < readSize + extraSize + sizeof(magic)) {
@ -181,6 +185,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
} }
logContent = (SWalCkHead*)(buf + pos); logContent = (SWalCkHead*)(buf + pos);
decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__);
if (walValidBodyCksum(logContent) != 0) { if (walValidBodyCksum(logContent) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
@ -618,7 +623,13 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
/*A(idxEntry.ver == ckHead.head.version);*/ /*A(idxEntry.ver == ckHead.head.version);*/
idxEntry.ver += 1; idxEntry.ver += 1;
idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen;
int32_t plainBodyLen = ckHead.head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
idxEntry.offset += sizeof(SWalCkHead) + cryptedBodyLen;
if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s", wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s",

75
source/libs/wal/src/walRead.c
View File

@ -13,7 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "crypt.h"
#include "taoserror.h" #include "taoserror.h"
#include "wal.h"
#include "walInt.h" #include "walInt.h"
SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) { SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) {
@ -307,7 +309,13 @@ int32_t walSkipFetchBody(SWalReader *pRead) {
pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer,
pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId); pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId);
int64_t code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR); int32_t plainBodyLen = pRead->pHead->head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pRead->pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
int64_t code = taosLSeekFile(pRead->pLogFile, cryptedBodyLen, SEEK_CUR);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -328,19 +336,27 @@ int32_t walFetchBody(SWalReader *pRead) {
", 0x%" PRIx64, ", 0x%" PRIx64,
vgId, ver, pVer->firstVer, pVer->commitVer, pVer->lastVer, pVer->appliedVer, id); vgId, ver, pVer->firstVer, pVer->commitVer, pVer->lastVer, pVer->appliedVer, id);
if (pRead->capacity < pReadHead->bodyLen) { int32_t plainBodyLen = pReadHead->bodyLen;
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pRead->pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
if (pRead->capacity < cryptedBodyLen) {
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) { if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
pRead->pHead = ptr; pRead->pHead = ptr;
pReadHead = &pRead->pHead->head; pReadHead = &pRead->pHead->head;
pRead->capacity = pReadHead->bodyLen; pRead->capacity = cryptedBodyLen;
} }
if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (cryptedBodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, cryptedBodyLen)) {
if (pReadHead->bodyLen < 0) { if (plainBodyLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%"PRIx64, wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%"PRIx64,
vgId, pReadHead->version, ver, tstrerror(terrno), id); vgId, pReadHead->version, ver, tstrerror(terrno), id);
@ -359,6 +375,8 @@ int32_t walFetchBody(SWalReader *pRead) {
return -1; return -1;
} }
decryptBody(&pRead->pWal->cfg, pRead->pHead, plainBodyLen, __FUNCTION__);
if (walValidBodyCksum(pRead->pHead) != 0) { if (walValidBodyCksum(pRead->pHead) != 0) {
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver, id); wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver, id);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
@ -431,20 +449,28 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
return -1; return -1;
} }
if (pReader->capacity < pReader->pHead->head.bodyLen) { int32_t plainBodyLen = pReader->pHead->head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pReader->pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
if (pReader->capacity < cryptedBodyLen) {
SWalCkHead *ptr = SWalCkHead *ptr =
(SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReader->pHead->head.bodyLen); (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) { if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
return -1; return -1;
} }
pReader->pHead = ptr; pReader->pHead = ptr;
pReader->capacity = pReader->pHead->head.bodyLen; pReader->capacity = cryptedBodyLen;
} }
if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, pReader->pHead->head.bodyLen)) != if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) !=
pReader->pHead->head.bodyLen) { cryptedBodyLen) {
if (contLen < 0) if (contLen < 0)
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
else { else {
@ -465,11 +491,13 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
return -1; return -1;
} }
decryptBody(&pReader->pWal->cfg, pReader->pHead, plainBodyLen, __FUNCTION__);
code = walValidBodyCksum(pReader->pHead); code = walValidBodyCksum(pReader->pHead);
if (code != 0) { if (code != 0) {
wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId,
ver); ver);
uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, pReader->pHead->head.bodyLen); uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, plainBodyLen);
uint32_t logCkSum = pReader->pHead->cksumBody; uint32_t logCkSum = pReader->pHead->cksumBody;
wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum); wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum);
// pReader->curInvalid = 1; // pReader->curInvalid = 1;
@ -484,6 +512,29 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
return 0; return 0;
} }
void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func) {
//TODO: dmchen emun
if (cfg->encryptAlgorithm == 1) {
int32_t cryptedBodyLen = ENCRYPTED_LEN(plainBodyLen);
char *newBody = taosMemoryMalloc(cryptedBodyLen);
SCryptOpts opts;
opts.len = cryptedBodyLen;
opts.source = pHead->head.body;
opts.result = newBody;
opts.unitLen = 16;
strncpy(opts.key, cfg->encryptKey, 16);
int32_t count = CBC_Decrypt(&opts);
//wDebug("CBC_Decrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", count, plainBodyLen, func);
memcpy(pHead->head.body, newBody, plainBodyLen);
taosMemoryFree(newBody);
}
}
void walReadReset(SWalReader *pReader) { void walReadReset(SWalReader *pReader) {
taosThreadMutexLock(&pReader->mutex); taosThreadMutexLock(&pReader->mutex);
taosCloseFile(&pReader->pIdxFile); taosCloseFile(&pReader->pIdxFile);

65
source/libs/wal/src/walWrite.c
View File

@ -18,6 +18,7 @@
#include "tchecksum.h" #include "tchecksum.h"
#include "tglobal.h" #include "tglobal.h"
#include "walInt.h" #include "walInt.h"
#include "crypt.h"
int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
@ -491,12 +492,13 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta,
const void *body, int32_t bodyLen) { const void *body, int32_t bodyLen) {
int64_t code = 0; int64_t code = 0;
int32_t plainBodyLen = bodyLen;
int64_t offset = walGetCurFileOffset(pWal); int64_t offset = walGetCurFileOffset(pWal);
SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal);
pWal->writeHead.head.version = index; pWal->writeHead.head.version = index;
pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.bodyLen = plainBodyLen;
pWal->writeHead.head.msgType = msgType; pWal->writeHead.head.msgType = msgType;
pWal->writeHead.head.ingestTs = taosGetTimestampUs(); pWal->writeHead.head.ingestTs = taosGetTimestampUs();
@ -504,7 +506,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
pWal->writeHead.head.syncMeta = syncMeta; pWal->writeHead.head.syncMeta = syncMeta;
pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead);
pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); pWal->writeHead.cksumBody = walCalcBodyCksum(body, plainBodyLen);
wDebug("vgId:%d, wal write log %" PRId64 ", msgType: %s, cksum head %u cksum body %u", pWal->cfg.vgId, index, wDebug("vgId:%d, wal write log %" PRId64 ", msgType: %s, cksum head %u cksum body %u", pWal->cfg.vgId, index,
TMSG_INFO(msgType), pWal->writeHead.cksumHead, pWal->writeHead.cksumBody); TMSG_INFO(msgType), pWal->writeHead.cksumHead, pWal->writeHead.cksumBody);
@ -521,22 +523,75 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
goto END; goto END;
} }
if (taosWriteFile(pWal->pLogFile, (char *)body, bodyLen) != bodyLen) { int32_t cyptedBodyLen = plainBodyLen;
char* buf = (char*)body;
char* newBody = NULL;
char* newBodyEncrypted = NULL;
if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){
cyptedBodyLen = ENCRYPTED_LEN(cyptedBodyLen);
newBody = taosMemoryMalloc(cyptedBodyLen);
if(newBody == NULL){
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
goto END;
}
memset(newBody, 0, cyptedBodyLen);
memcpy(newBody, body, plainBodyLen);
newBodyEncrypted = taosMemoryMalloc(cyptedBodyLen);
if(newBodyEncrypted == NULL){
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
if(newBody != NULL) taosMemoryFreeClear(newBody);
goto END;
}
SCryptOpts opts;
opts.len = cyptedBodyLen;
opts.source = newBody;
opts.result = newBodyEncrypted;
opts.unitLen = 16;
strncpy(opts.key, pWal->cfg.encryptKey, ENCRYPT_KEY_LEN);
int32_t count = CBC_Encrypt(&opts);
//wDebug("vgId:%d, file:%" PRId64 ".log, index:%" PRId64 ", CBC_Encrypt cryptedBodyLen:%d, plainBodyLen:%d, %s",
// pWal->cfg.vgId, walGetLastFileFirstVer(pWal), index, count, plainBodyLen, __FUNCTION__);
buf = newBodyEncrypted;
}
if (taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno)); strerror(errno));
code = -1; code = -1;
if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){
taosMemoryFreeClear(newBody);
taosMemoryFreeClear(newBodyEncrypted);
}
goto END; goto END;
} }
if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){
taosMemoryFreeClear(newBody);
taosMemoryFreeClear(newBodyEncrypted);
//wInfo("vgId:%d, free newBody newBodyEncrypted %s",
// pWal->cfg.vgId, __FUNCTION__);
}
// set status // set status
if (pWal->vers.firstVer == -1) { if (pWal->vers.firstVer == -1) {
pWal->vers.firstVer = 0; pWal->vers.firstVer = 0;
} }
pWal->vers.lastVer = index; pWal->vers.lastVer = index;
pWal->totSize += sizeof(SWalCkHead) + bodyLen; pWal->totSize += sizeof(SWalCkHead) + cyptedBodyLen;
pFileInfo->lastVer = index; pFileInfo->lastVer = index;
pFileInfo->fileSize += sizeof(SWalCkHead) + bodyLen; pFileInfo->fileSize += sizeof(SWalCkHead) + cyptedBodyLen;
return 0; return 0;

18
source/os/src/osString.c
View File

@ -129,8 +129,6 @@ typedef struct {
int8_t inUse; int8_t inUse;
} SConv; } SConv;
typedef enum { M2C = 0, C2M } ConvType;
// 0: Mbs --> Ucs4 // 0: Mbs --> Ucs4
// 1: Ucs4--> Mbs // 1: Ucs4--> Mbs
SConv *gConv[2] = {NULL, NULL}; SConv *gConv[2] = {NULL, NULL};
@ -269,6 +267,22 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
return (int32_t)(ucs4_max_len - outLen); return (int32_t)(ucs4_max_len - outLen);
#endif #endif
} }
int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n");
return -1;
#else
size_t ucs4_input_len = ucs4_max_len;
size_t outLen = ucs4_max_len;
if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
return -1;
}
return (int32_t)(ucs4_max_len - outLen);
#endif
}
bool taosValidateEncodec(const char *encodec) { bool taosValidateEncodec(const char *encodec) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV #ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n");

Some files were not shown because too many files have changed in this diff Show More