xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

basic function

This commit is contained in:
dmchen 2024-03-29 02:48:49 +00:00
parent ca0f790f3e
commit b9deb71fc5
55 changed files with 1977 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
include/common/tglobal.h
View File

@ -29,6 +29,17 @@ extern "C" {
#define SLOW_LOG_TYPE_OTHERS 0x4
#define SLOW_LOG_TYPE_ALL 0xFFFFFFFF
typedef enum {
DND_CA_SM4 = 1,
} EEncryptAlgor;
typedef enum {
DND_CS_TSDB = 1,
DND_CS_VNODE_WAL = 2,
DND_CS_SDB = 4,
DND_CS_MNODE_WAL = 8,
} EEncryptScope;
// cluster
extern char tsFirst[];
extern char tsSecond[];
@ -39,6 +50,12 @@ extern uint16_t tsServerPort;
extern int32_t tsVersion;
extern int32_t tsStatusInterval;
extern int32_t tsNumOfSupportVnodes;
extern char tsEncryptAlgorithm[];
extern char tsEncryptScope[];
extern EEncryptAlgor tsiEncryptAlgorithm;
extern EEncryptScope tsiEncryptScope;
//extern char tsAuthCode[];
extern char tsEncryptKey[];
// common
extern int32_t tsMaxShellConns;

39
include/libs/crypt/crypt.h Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _CRYPT_H_
#define _CRYPT_H_
#include "tdef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SCryptOpts{
int32_t len;
char* source;
char* result;
int32_t unitLen;
unsigned char key[17];
}SCryptOpts;
int32_t CBC_Decrypt(SCryptOpts *opts);
int32_t CBC_Encrypt(SCryptOpts *opts);
#ifdef __cplusplus
}
#endif
#endif // _CRYPT_H_

63
include/libs/sm4/sm4.h Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <memory.h>
#include <time.h>
#ifndef _SM4_H_
#define _SM4_H_
#ifdef __cplusplus
extern "C" {
#endif
int SM4_ECB_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
int SM4_ECB_Decrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
int SM4_CBC_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
int SM4_CBC_Decrypt(unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen);
#ifdef __cplusplus
}
#endif
#endif // _SM4_H_

3
include/libs/wal/wal.h
View File

@ -57,6 +57,8 @@ typedef struct {
int64_t retentionSize;
int64_t segSize;
EWalType level; // wal level
int32_t encryptAlgorithm;
char encryptKey[17];
} SWalCfg;
typedef struct {
@ -196,6 +198,7 @@ SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id);
void walCloseReader(SWalReader *pRead);
void walReadReset(SWalReader *pReader);
int32_t walReadVer(SWalReader *pRead, int64_t ver);
void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func);
int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver);
int32_t walNextValidMsg(SWalReader *pRead);
int64_t walReaderGetCurrentVer(const SWalReader *pReader);

5
include/util/taoserror.h
View File

@ -311,6 +311,7 @@ int32_t* taosGetErrno();
// #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) // 2.x
#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) //
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
// mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)
@ -418,6 +419,10 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal
#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416)
#define TSDB_CODE_DNODE_NO_MACHINE_CODE TAOS_DEF_ERROR_CODE(0, 0x0417)
#define TSDB_CODE_DNODE_NO_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x0418)
#define TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0419)
#define TSDB_CODE_DNODE_INVALID_ENCRYPTKEY TAOS_DEF_ERROR_CODE(0, 0x0420)
#define TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED TAOS_DEF_ERROR_CODE(0, 0x0421)
// mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)

3
include/util/tdef.h
View File

@ -187,6 +187,9 @@ typedef enum ELogicConditionType {
LOGIC_COND_TYPE_NOT,
} ELogicConditionType;
#define ENCRYPTED_LEN(len) (len/16) * 16 + (len%16?1:0) * 16
#define ENCRYPT_KEY_LEN 16
#define TSDB_INT32_ID_LEN 11
#define TSDB_NAME_DELIMITER_LEN 1

13
source/common/src/tglobal.c
View File

@ -41,6 +41,12 @@ uint16_t tsServerPort = 6030;
int32_t tsVersion = 30000000;
int32_t tsStatusInterval = 1; // second
int32_t tsNumOfSupportVnodes = 256;
char tsEncryptAlgorithm[16] = {0};
char tsEncryptScope[100] = {0};
EEncryptAlgor tsiEncryptAlgorithm = 0;
EEncryptScope tsiEncryptScope = 0;
//char tsAuthCode[500] = {0};
char tsEncryptKey[17] = {0};
// common
int32_t tsMaxShellConns = 50000;
@ -593,6 +599,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
//if (cfgAddString(pCfg, "authCode", tsAuthCode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
return -1;
@ -1136,6 +1146,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32;
tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32;
tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32;
tstrncpy(tsEncryptAlgorithm, cfgGetItem(pCfg, "encryptAlgorithm")->str, 16);
tstrncpy(tsEncryptScope, cfgGetItem(pCfg, "encryptScope")->str, 100);
//tstrncpy(tsAuthCode, cfgGetItem(pCfg, "authCode")->str, 100);
tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32;
tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32;

4
source/dnode/mgmt/CMakeLists.txt
View File

@ -23,8 +23,8 @@ ENDIF ()
IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEPENDENCIES(taosd jemalloc)
target_link_libraries(taosd dnode ${LINK_JEMALLOC})
target_link_libraries(taosd dnode crypt ${LINK_JEMALLOC})
ELSE ()
target_link_libraries(taosd dnode)
target_link_libraries(taosd dnode crypt)
ENDIF ()

288
source/dnode/mgmt/exe/dmMain.c
View File

@ -22,6 +22,7 @@
#ifdef TD_JEMALLOC_ENABLED
#include "jemalloc/jemalloc.h"
#endif
#include "crypt.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@ -49,6 +50,10 @@
#define DM_VERSION "Print program version."
#define DM_EMAIL "<support@taosdata.com>"
#define DM_MEM_DBG "Enable memory debug"
#define DM_SET_ENCRYPTKEY "Set encrypt key. such as: -e 1234567890abcdef, the length should be less or equal to 16."
#define DM_KEY_INDICATOR "this indicator!"
#define DM_ENCRYPT_CODE_FILE "encryptCode.cfg"
#define DM_CHECK_CODE_FILE "checkCode.bin"
// clang-format on
static struct {
#ifdef WINDOWS
@ -66,6 +71,8 @@ static struct {
const char **envCmd;
SArray *pArgs; // SConfigPair
int64_t startTime;
bool generateCode;
char encryptKey[ENCRYPT_KEY_LEN + 1];
} global = {0};
static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetGlobalDebugFlag(143); }
@ -194,6 +201,18 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
}
} else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true;
} else if (strcmp(argv[i], "-y") == 0) {
global.generateCode = true;
if(i < argc - 1) {
if (strlen(argv[++i]) > ENCRYPT_KEY_LEN) {
printf("encrypt key overflow, it should be less or equal to %d\n", ENCRYPT_KEY_LEN);
return -1;
}
tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
} else {
printf("'-y' requires a parameter\n");
return -1;
}
} else if (strcmp(argv[i], "-C") == 0) {
global.dumpConfig = true;
} else if (strcmp(argv[i], "-V") == 0) {
@ -256,6 +275,7 @@ static void dmPrintHelp() {
printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE);
printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY);
printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG);
printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION);
@ -275,6 +295,260 @@ static void taosCleanupArgs() {
if (global.envCmd != NULL) taosMemoryFreeClear(global.envCmd);
}
static int32_t compareCheckCode(char* file, char* key){
char *content = NULL;
int64_t size = 0;
TdFilePtr pFile = NULL;
char *result = NULL;
int32_t code = -1;
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content = taosMemoryMalloc(size);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER;
}
dInfo("succeed to read checkCode file:%s", file);
int len = ENCRYPTED_LEN(size);
result = taosMemoryMalloc(len);
SCryptOpts opts = {0};
strncpy(opts.key, key, ENCRYPT_KEY_LEN);
opts.len = len;
opts.source = content;
opts.result = result;
opts.unitLen = 16;
CBC_Decrypt(&opts);
if(strcmp(opts.result, DM_KEY_INDICATOR) != 0) {
terrno = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED;
dError("failed to compare decrypted result");
goto _OVER;
}
dInfo("succeed to compare checkCode file:%s", file);
code = 0;
_OVER:
if(result != NULL) taosMemoryFree(result);
if(content != NULL) taosMemoryFree(content);
if(pFile != NULL) taosCloseFile(&pFile);
return code;
}
static int32_t writeCheckCode(char* file, char* realfile, char* key){
TdFilePtr pFile = NULL;
char *result = NULL;
int32_t code = -1;
int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR));
result = taosMemoryMalloc(len);
SCryptOpts opts;
strncpy(opts.key, key, ENCRYPT_KEY_LEN);
opts.len = len;
opts.source = DM_KEY_INDICATOR;
opts.result = result;
opts.unitLen = 16;
CBC_Encrypt(&opts);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (taosWriteFile(pFile, opts.result, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
dInfo("succeed to write checkCode file:%s", realfile);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
if(result != NULL) taosMemoryFree(result);
return code;
}
static int32_t readEncryptCode(char* file, char** output){
TdFilePtr pFile = NULL;
int32_t code = -1;
char *content = NULL;
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content[size] = '\0';
*output = content;
dInfo("succeed to read encryptCode file:%s", file);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
return code;
}
static int32_t writeEncryptCode(char* file, char* realfile, char* encryptCode){
TdFilePtr pFile = NULL;
int32_t code = -1;
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
int32_t len = strlen(encryptCode);
if (taosWriteFile(pFile, encryptCode, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
dInfo("succeed to write encryptCode file:%s", realfile);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
return code;
}
static int32_t getEncryptKey(){
int32_t code = -1;
char encryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0};
snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE);
if(!taosCheckExistFile(encryptFile)){
dInfo("no exist, checkCode file:%s", encryptFile);
return 0;
}
char *content = NULL;
if(readEncryptCode(encryptFile, &content) != 0){
goto _OVER;
}
//TODO: dmchen parse key from code
strncpy(tsEncryptKey, content, ENCRYPT_KEY_LEN);
taosMemoryFreeClear(content);
if(tsEncryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("failed to read key since %s", terrstr());
goto _OVER;
}
if(compareCheckCode(checkFile, tsEncryptKey) != 0){
goto _OVER;
}
code = 0;
_OVER:
if (content != NULL) taosMemoryFree(content);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to get encrypt key since %s", terrstr());
}
return code;
}
int32_t updateEncryptKey(char *key) {
int32_t code = -1;
char folder[PATH_MAX] = {0};
char encryptFile[PATH_MAX] = {0};
char realEncryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0};
char realCheckFile[PATH_MAX] = {0};
snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP);
snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
terrno = 0;
if (taosMkDir(folder) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", folder, terrstr());
goto _OVER;
}
if(taosCheckExistFile(realCheckFile)){
if(compareCheckCode(realCheckFile, key) != 0){
goto _OVER;
}
}
if(writeEncryptCode(encryptFile, realEncryptFile, key) != 0){
goto _OVER;
}
if(writeCheckCode(checkFile, realCheckFile, key) != 0){
goto _OVER;
}
code = 0;
_OVER:
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to update encrypt key since %s", terrstr());
}
return code;
}
int main(int argc, char const *argv[]) {
#ifdef TD_JEMALLOC_ENABLED
bool jeBackgroundThread = true;
@ -348,6 +622,20 @@ int mainWindows(int argc, char **argv) {
return -1;
}
if(global.generateCode) {
updateEncryptKey(global.encryptKey);
taosCloseLog();
taosCleanupArgs();
return 0;
}
if(getEncryptKey() != 0){
dError("failed to start since failed to get encrypt key");
taosCloseLog();
taosCleanupArgs();
return -1;
};
if (taosConvInit() != 0) {
dError("failed to init conv");
taosCloseLog();

33
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -142,6 +142,14 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
if ((pRetention->freq >= 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
}
}
#if defined(TD_ENTERPRISE)
pCfg->tsdbCfg.encryptAlgorithm = DND_CA_SM4;
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
pCfg->walCfg.cryptAlgorithm = 0;
#endif
pCfg->walCfg.vgId = pCreate->vgId;
pCfg->walCfg.fsyncPeriod = pCreate->walFsyncPeriod;
@ -150,6 +158,23 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->walCfg.retentionSize = pCreate->walRetentionSize;
pCfg->walCfg.segSize = pCreate->walSegmentSize;
pCfg->walCfg.level = pCreate->walLevel;
#if defined(TD_ENTERPRISE)
pCfg->walCfg.encryptAlgorithm = DND_CA_SM4;
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
pCfg->walCfg.cryptAlgorithm = 0;
#endif
#if defined(TD_ENTERPRISE)
pCfg->tdbEncryptAlgorithm = DND_CA_SM4;
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
#else
pCfg->tdbEncryptKey = 0;
#endif
pCfg->sttTrigger = pCreate->sstTrigger;
pCfg->hashBegin = pCreate->hashBegin;
@ -273,6 +298,14 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
//if(req.encryptAlgorithm == DND_CA_SM4){
if(strlen(tsEncryptKey) == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId);
return -1;
}
//}
vmGenerateVnodeCfg(&req, &vnodeCfg);
if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) {

40
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -161,6 +161,46 @@ int32_t dmInitVars(SDnode *pDnode) {
return -1;
}
#if defined(TD_ENTERPRISE)
tsiEncryptAlgorithm = pData->encryptAlgorigthm;
tsiEncryptScope = pData->encryptScope;
/*
if(tsiEncryptAlgorithm != 0) {
if(pData->machineId != NULL && strlen(pData->machineId) > 0){
dInfo("get crypt key at startup, machineId:%s", pData->machineId);
int32_t code = 0;
//code = taosGetCryptKey(tsAuthCode, pData->machineId, tsCryptKey);
code = 0;
strncpy(tsEncryptKey, tsAuthCode, 16);
if(code != 0) {
if(code == -1){
terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
dError("machine code changed, can't get crypt key");
}
if(code == -2){
terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
dError("failed to get crypt key");
}
return -1;
}
if(strlen(tsEncryptKey) == 0){
terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY;
dError("failed to get crypt key at startup since key is null, machineId:%s", pData->machineId);
return -1;
}
}
else{
terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
dError("failed to get crypt key at startup, machineId:%s", pData->machineId);
return -1;
}
}
*/
#endif
if (pData->dropped) {
dError("dnode will not start since its already dropped");
return -1;

2
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -110,6 +110,8 @@ typedef struct {
bool validMnodeEps;
int64_t ipWhiteVer;
char machineId[TSDB_MACHINE_ID_LEN + 1];
EEncryptAlgor encryptAlgorigthm;
EEncryptScope encryptScope;
} SDnodeData;
typedef struct {

99
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -64,7 +64,12 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) {
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "dropped", pData->dropped, code);
if (code < 0) return -1;
#ifdef TD_ENTERPRISE
tjsonGetInt32ValueFromDouble(pJson, "encryptAlgor", pData->encryptAlgorigthm, code);
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "encryptScope", pData->encryptScope, code);
if (code < 0) return -1;
#endif
SJson *dnodes = tjsonGetObjectItem(pJson, "dnodes");
if (dnodes == NULL) return 0;
int32_t numOfDnodes = tjsonGetArraySize(dnodes);
@ -89,6 +94,25 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) {
return 0;
}
int dmOccurrences(char *str, char *toSearch) {
int count = 0;
char *ptr = str;
while ((ptr = strstr(ptr, toSearch)) != NULL) {
count++;
ptr++;
}
return count;
}
void dmSplitStr(char** arr, char* str, const char* del) {
char *lasts;
char* s = strsep(&str, del);
while (s != NULL) {
*arr++ = s;
s = strsep(&str, del);
}
}
int32_t dmReadEps(SDnodeData *pData) {
int32_t code = -1;
TdFilePtr pFile = NULL;
@ -106,6 +130,74 @@ int32_t dmReadEps(SDnodeData *pData) {
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("dnode file:%s not exist", file);
#ifdef TD_ENTERPRISE
if(strlen(tsEncryptAlgorithm) > 0){
if(strcmp(tsEncryptAlgorithm, "sm4") == 0) {
pData->encryptAlgorigthm = DND_CA_SM4;
}
else{
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptAlgorithm:%s", tsEncryptAlgorithm);
goto _OVER;
}
dInfo("start to parse encryptScope:%s", tsEncryptScope);
int32_t scopeLen = strlen(tsEncryptScope);
if(scopeLen == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptScope:%s", tsEncryptScope);
goto _OVER;
}
char* tmp = taosMemoryMalloc(scopeLen + 1);
memset(tmp, 0, scopeLen + 1);
memcpy(tmp, tsEncryptScope, scopeLen);
int32_t count = dmOccurrences(tmp, ",");
char** array = taosMemoryMalloc(sizeof(char*) * (count + 1));
memset(array, 0, sizeof(char*) * (count + 1));
dmSplitStr(array, tmp, ",");
for(int32_t i = 0; i < count + 1; i++){
char* str = *(array + i);
bool success = false;
if(strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_TSDB;
success = true;
}
if(strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_VNODE_WAL;
success = true;
}
if(strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_SDB;
success = true;
}
if(strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0){
pData->encryptScope |= DND_CS_MNODE_WAL;
success = true;
}
if(!success){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
taosMemoryFree(tmp);
taosMemoryFree(array);
dError("invalid tsEncryptScope:%s", tsEncryptScope);
goto _OVER;
}
}
taosMemoryFree(tmp);
taosMemoryFree(array);
dInfo("set tsCryptAlgorithm:%s, tsCryptScope:%s from cfg", tsEncryptAlgorithm, tsEncryptScope);
}
#endif
code = 0;
goto _OVER;
}
@ -191,7 +283,10 @@ static int32_t dmEncodeEps(SJson *pJson, SDnodeData *pData) {
if (tjsonAddIntegerToObject(pJson, "engineVer", pData->engineVer) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "clusterId", pData->clusterId) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "dropped", pData->dropped) < 0) return -1;
#ifdef TD_ENTERPRISE
if (tjsonAddDoubleToObject(pJson, "encryptAlgor", pData->encryptAlgorigthm) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "encryptScope", pData->encryptScope) < 0) return -1;
#endif
SJson *dnodes = tjsonCreateArray();
if (dnodes == NULL) return -1;
if (tjsonAddItemToObject(pJson, "dnodes", dnodes) < 0) return -1;

15
source/dnode/mnode/impl/src/mndMain.c
View File

@ -472,8 +472,23 @@ static int32_t mndInitWal(SMnode *pMnode) {
.retentionPeriod = 0,
.retentionSize = 0,
.level = TAOS_WAL_FSYNC,
.encryptAlgorithm = 0,
.encryptKey = {0}
};
#if defined(TD_ENTERPRISE)
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){
cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0;
if(tsEncryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
pMnode->pWal = walOpen(path, &cfg);
if (pMnode->pWal == NULL) {
mError("failed to open wal since %s. wal:%s", terrstr(), path);

2
source/dnode/mnode/sdb/CMakeLists.txt
View File

@ -5,5 +5,5 @@ target_include_directories(
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
sdb os common util wal sync
sdb os common util wal sync crypt
)

66
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -18,6 +18,8 @@
#include "sync.h"
#include "tchecksum.h"
#include "wal.h"
#include "tglobal.h"
#include "crypt.h"
#define SDB_TABLE_SIZE 24
#define SDB_RESERVE_SIZE 512
@ -276,6 +278,9 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
}
readLen = pRaw->dataLen + sizeof(int32_t);
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
readLen = ENCRYPTED_LEN(pRaw->dataLen) + sizeof(int32_t);
}
if (readLen >= bufLen) {
bufLen = pRaw->dataLen * 2;
SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100);
@ -303,6 +308,27 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
goto _OVER;
}
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
int32_t count = 0;
char *plantContent = taosMemoryMalloc(ENCRYPTED_LEN(pRaw->dataLen));
SCryptOpts opts;
opts.len = ENCRYPTED_LEN(pRaw->dataLen);
opts.source = pRaw->pData;
opts.result = plantContent;
opts.unitLen = 16;
strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN);
count = CBC_Decrypt(&opts);
mInfo("read sdb, CBC_Decrypt dataLen:%d, descrypted len:%d, %s", pRaw->dataLen, count, __FUNCTION__);
memcpy(pRaw->pData, plantContent, pRaw->dataLen);
taosMemoryFree(plantContent);
memcpy(pRaw->pData + pRaw->dataLen, &pRaw->pData[ENCRYPTED_LEN(pRaw->dataLen)], sizeof(int32_t));
}
int32_t totalLen = sizeof(SSdbRaw) + pRaw->dataLen + sizeof(int32_t);
if ((!taosCheckChecksumWhole((const uint8_t *)pRaw, totalLen)) != 0) {
code = TSDB_CODE_CHECKSUM_ERROR;
@ -401,14 +427,50 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
SSdbRaw *pRaw = (*encodeFp)(pRow->pObj);
if (pRaw != NULL) {
pRaw->status = pRow->status;
int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen;
if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) {
if (taosWriteFile(pFile, pRaw, sizeof(SSdbRaw)) != sizeof(SSdbRaw)) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
break;
}
int32_t newDataLen = pRaw->dataLen;
char* newData = pRaw->pData;
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
newDataLen = ENCRYPTED_LEN(pRaw->dataLen);
newData = taosMemoryMalloc(newDataLen);
if (newData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
break;
}
SCryptOpts opts;
opts.len = newDataLen;
opts.source = pRaw->pData;
opts.result = newData;
opts.unitLen = 16;
strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN);
int32_t count = CBC_Encrypt(&opts);
mInfo("write sdb, CBC_Encrypt encryptedDataLen:%d, dataLen:%d, %s",
newDataLen, pRaw->dataLen, __FUNCTION__);
}
if (taosWriteFile(pFile, newData, newDataLen) != newDataLen) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
break;
}
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
taosMemoryFree(newData);
}
int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen);
if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
code = TAOS_SYSTEM_ERROR(errno);

3
source/dnode/vnode/CMakeLists.txt
View File

@ -121,6 +121,7 @@ if (${BUILD_CONTRIB})
PUBLIC "inc"
PUBLIC "src/inc"
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include"
)
@ -130,6 +131,7 @@ else()
PUBLIC "inc"
PUBLIC "src/inc"
PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
)
if (${TD_LINUX})
@ -157,6 +159,7 @@ target_link_libraries(
PUBLIC scheduler
PUBLIC tdb
PUBLIC audit
PUBLIC crypt
# PUBLIC bdb
# PUBLIC scalar

5
source/dnode/vnode/inc/vnode.h
View File

@ -272,6 +272,9 @@ struct STsdbCfg {
int32_t keep2; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keepTimeOffset; // just for save config, use STsdbKeepCfg in STsdb instead
SRetention retentions[TSDB_RETENTION_MAX];
int32_t encryptAlgorithm;
//TODO dmchen len
char encryptKey[17];
};
typedef struct {
@ -313,6 +316,8 @@ struct SVnodeCfg {
int16_t hashPrefix;
int16_t hashSuffix;
int32_t tsdbPageSize;
int32_t tdbEncryptAlgorithm;
char tdbEncryptKey[ENCRYPT_KEY_LEN];
};
#define TABLE_ROLLUP_ON ((int8_t)0x1)

3
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -69,7 +69,8 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
taosMkDir(pMeta->path);
// open env
ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback);
ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback,
pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey);
if (ret < 0) {
metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;

2
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -76,7 +76,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
}
int32_t tqMetaOpen(STQ* pTq) {
if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0) < 0) {
if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL) < 0) {
return -1;
}

15
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
View File

@ -72,7 +72,9 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo
pBlock->file.maxVer = reader->config->file.maxVer;
pBlock->file.stt->level = reader->config->file.stt->level;
code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -126,8 +128,11 @@ static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFil
code = TARRAY2_APPEND(opArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
if (writer->fd) {
code = tsdbFsyncFile(writer->fd);
code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd);
}
@ -205,11 +210,13 @@ _exit:
return code;
}
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock) {
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength);
code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->offset += pDataBlock->dataLength;

3
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h
View File

@ -105,7 +105,8 @@ int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataF
int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr);
int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer);
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr);
int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer);
#ifdef __cplusplus

142
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

@ -44,8 +44,10 @@ static int32_t tsdbDataFileReadHeadFooter(SDataFileReader *reader) {
int32_t ftype = TSDB_FTYPE_HEAD;
if (reader->fd[ftype]) {
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(SHeadFooter),
(uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0);
(uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -66,8 +68,10 @@ static int32_t tsdbDataFileReadTombFooter(SDataFileReader *reader) {
int32_t ftype = TSDB_FTYPE_TOMB;
if (reader->fd[ftype]) {
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(STombFooter),
(uint8_t *)reader->tombFooter, sizeof(STombFooter), 0);
(uint8_t *)reader->tombFooter, sizeof(STombFooter), 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
reader->ctx->tombFooterLoaded = true;
@ -160,8 +164,10 @@ int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **b
TSDB_CHECK_CODE(code, lino, _exit);
}
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], reader->headFooter->brinBlkPtr->offset, data,
reader->headFooter->brinBlkPtr->size, 0);
reader->headFooter->brinBlkPtr->size, 0, encryptAlgorithm, encryptKey);
if (code) {
taosMemoryFree(data);
TSDB_CHECK_CODE(code, lino, _exit);
@ -191,8 +197,11 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB
code = tRealloc(&reader->config->bufArr[0], brinBlk->dp->size);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code =
tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, reader->config->bufArr[0], brinBlk->dp->size, 0);
tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, reader->config->bufArr[0], brinBlk->dp->size, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t size = 0;
@ -233,8 +242,11 @@ int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *re
code = tRealloc(&reader->config->bufArr[0], record->blockSize);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code =
tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockSize, 0);
tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockSize, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tDecmprBlockData(reader->config->bufArr[0], record->blockSize, bData, &reader->config->bufArr[1]);
@ -259,8 +271,10 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
code = tRealloc(&reader->config->bufArr[0], record->blockKeySize);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockKeySize,
0);
0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// hdr
@ -297,8 +311,10 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
code = tRealloc(&reader->config->bufArr[0], hdr->szBlkCol);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize,
reader->config->bufArr[0], hdr->szBlkCol, 0);
reader->config->bufArr[0], hdr->szBlkCol, 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -374,9 +390,11 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
code = tRealloc(&reader->config->bufArr[1], size1);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA],
record->blockOffset + record->blockKeySize + hdr->szBlkCol + blockCol->offset,
reader->config->bufArr[1], size1, i > 0 ? 0 : szHint);
reader->config->bufArr[1], size1, i > 0 ? 0 : szHint, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData,
@ -404,7 +422,10 @@ int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *rec
code = tRealloc(&reader->config->bufArr[0], record->smaSize);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, reader->config->bufArr[0], record->smaSize, 0);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, reader->config->bufArr[0], record->smaSize, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// decode sma data
@ -442,8 +463,10 @@ int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **t
TSDB_CHECK_CODE(code, lino, _exit);
}
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], reader->tombFooter->tombBlkPtr->offset, data,
reader->tombFooter->tombBlkPtr->size, 0);
reader->tombFooter->tombBlkPtr->size, 0, encryptAlgorithm, encryptKey);
if (code) {
taosMemoryFree(data);
TSDB_CHECK_CODE(code, lino, _exit);
@ -473,8 +496,11 @@ int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombB
code = tRealloc(&reader->config->bufArr[0], tombBlk->dp->size);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code =
tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size, 0);
tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t size = 0;
@ -695,7 +721,8 @@ int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxV
}
int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize,
TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range) {
TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey) {
if (BRIN_BLOCK_SIZE(brinBlock) == 0) return 0;
int32_t code;
@ -742,7 +769,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAl
TSDB_DATA_TYPE_BIGINT, brinBlk->cmprAlg, &bufArr[0], 0, &brinBlk->size[i], &bufArr[1]);
if (code) return code;
code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[i]);
code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[i], encryptAlgorithm, encryptKey);
if (code) return code;
brinBlk->dp->size += brinBlk->size[i];
@ -754,7 +781,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAl
TSDB_DATA_TYPE_INT, brinBlk->cmprAlg, &bufArr[0], 0, &brinBlk->size[j], &bufArr[1]);
if (code) return code;
code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[j]);
code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[j], encryptAlgorithm, encryptKey);
if (code) return code;
brinBlk->dp->size += brinBlk->size[j];
@ -785,9 +812,12 @@ static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg,
&writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->config->bufArr,
&writer->ctx->range);
&writer->ctx->range, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -865,10 +895,13 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData
record->blockKeySize = sizeArr[3] + sizeArr[2];
record->blockSize = sizeArr[0] + sizeArr[1] + record->blockKeySize;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
for (int32_t i = 3; i >= 0; --i) {
if (sizeArr[i]) {
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_DATA], writer->files[TSDB_FTYPE_DATA].size, writer->config->bufArr[i],
sizeArr[i]);
sizeArr[i], encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->files[TSDB_FTYPE_DATA].size += sizeArr[i];
}
@ -892,7 +925,8 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData
}
if (record->smaSize > 0) {
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, writer->config->bufArr[0], record->smaSize);
code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, writer->config->bufArr[0], record->smaSize,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->files[TSDB_FTYPE_SMA].size += record->smaSize;
}
@ -924,8 +958,12 @@ static int32_t tsdbDataFileWriteDataBlk(SDataFileWriter *writer, const TDataBlkA
.size = TARRAY2_DATA_LEN(dataBlkArray),
}};
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code =
tsdbWriteFile(writer->fd[ftype], blockIdx->offset, (const uint8_t *)TARRAY2_DATA(dataBlkArray), blockIdx->size);
tsdbWriteFile(writer->fd[ftype], blockIdx->offset, (const uint8_t *)TARRAY2_DATA(dataBlkArray), blockIdx->size,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->files[ftype].size += blockIdx->size;
@ -1197,15 +1235,17 @@ _exit:
return code;
}
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer));
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey);
if (code) return code;
*fileSize += sizeof(*footer);
return 0;
}
int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range) {
TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code;
if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0;
@ -1248,7 +1288,7 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl
TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, &bufArr[0], 0, &tombBlk->size[i], &bufArr[1]);
if (code) return code;
code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i]);
code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i], encryptAlgorithm, encryptKey);
if (code) return code;
tombBlk->dp->size += tombBlk->size[i];
@ -1266,7 +1306,11 @@ static int32_t tsdbDataFileWriteHeadFooter(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -1282,9 +1326,12 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg,
&writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr,
&writer->ctx->tombRange);
&writer->ctx->tombRange, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -1294,12 +1341,14 @@ _exit:
return code;
}
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) {
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey) {
ptr->size = TARRAY2_DATA_LEN(tombBlkArray);
if (ptr->size > 0) {
ptr->offset = *fileSize;
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size);
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size,
encryptAlgorithm, encryptKey);
if (code) {
return code;
}
@ -1315,8 +1364,11 @@ static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlk(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlkArray, writer->tombFooter->tombBlkPtr,
&writer->files[TSDB_FTYPE_TOMB].size);
&writer->files[TSDB_FTYPE_TOMB].size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -1326,8 +1378,9 @@ _exit:
return code;
}
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer));
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey);
if (code) return code;
*fileSize += sizeof(*footer);
return 0;
@ -1337,7 +1390,11 @@ static int32_t tsdbDataFileWriteTombFooter(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -1414,12 +1471,14 @@ _exit:
return code;
}
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize) {
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey) {
ASSERT(TARRAY2_SIZE(brinBlkArray) > 0);
ptr->offset = *fileSize;
ptr->size = TARRAY2_DATA_LEN(brinBlkArray);
int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size);
int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size, encryptAlgorithm,
encryptKey);
if (code) return code;
*fileSize += ptr->size;
@ -1430,8 +1489,11 @@ static int32_t tsdbDataFileWriteBrinBlk(SDataFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteBrinBlk(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlkArray, writer->headFooter->brinBlkPtr,
&writer->files[TSDB_FTYPE_HEAD].size);
&writer->files[TSDB_FTYPE_HEAD].size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -1588,10 +1650,11 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
code = TARRAY2_APPEND(opArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
}
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) {
if (writer->fd[i]) {
code = tsdbFsyncFile(writer->fd[i]);
code = tsdbFsyncFile(writer->fd[i], encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd[i]);
}
@ -1627,7 +1690,10 @@ static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) {
if (writer->files[ftype].size == 0) {
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->files[ftype].size += TSDB_FHDR_SIZE;
@ -1794,7 +1860,9 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) {
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->files[ftype].size += TSDB_FHDR_SIZE;

18
source/dnode/vnode/src/tsdb/tsdbDataFileRW.h
View File

@ -97,16 +97,22 @@ int32_t tsdbDataFileFlush(SDataFileWriter *writer);
// head
int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize,
TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range);
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize);
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer);
TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm,
char* encryptKey);
// tomb
int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record);
int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range);
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize);
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize);
TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
char* encryptKey);
// utils
int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer);

8
source/dnode/vnode/src/tsdb/tsdbDef.h
View File

@ -33,9 +33,11 @@ typedef struct SFDataPtr {
extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD);
extern void tsdbCloseFile(STsdbFD **ppFD);
extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size);
extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint);
extern int32_t tsdbFsyncFile(STsdbFD *pFD);
extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey);
extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey);
extern int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey);
#ifdef __cplusplus
}

5
source/dnode/vnode/src/tsdb/tsdbFSetRAW.c
View File

@ -152,7 +152,8 @@ _exit:
return code;
}
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) {
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = 0;
int32_t lino = 0;
@ -166,7 +167,7 @@ int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeade
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr);
code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->offset += bHdr->dataLength;

3
source/dnode/vnode/src/tsdb/tsdbFSetRAW.h
View File

@ -36,7 +36,8 @@ typedef struct SFSetRAWWriter SFSetRAWWriter;
int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer);
int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr);
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr);
int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm,
char* encryptKey);
#ifdef __cplusplus
}

117
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -15,6 +15,7 @@
#include "cos.h"
#include "tsdb.h"
#include "crypt.h"
static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
int32_t code = 0;
@ -131,7 +132,7 @@ void tsdbCloseFile(STsdbFD **ppFD) {
}
}
static int32_t tsdbWriteFilePage(STsdbFD *pFD) {
static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0;
if (!pFD->pFD) {
@ -153,6 +154,28 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD) {
}
taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage);
if(encryptAlgorithm == DND_CA_SM4){
//if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
unsigned char PacketData[128];
int NewLen;
int32_t count = 0;
while (count < pFD->szPage) {
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pFD->pBuf + count;
opts.result = PacketData;
opts.unitLen = 128;
//strncpy(opts.key, tsEncryptKey, 16);
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Encrypt(&opts);
memcpy(pFD->pBuf + count, PacketData, NewLen);
count += NewLen;
}
tsdbInfo("CBC_Encrypt count:%d %s", count, __FUNCTION__);
}
n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage);
if (n < 0) {
@ -170,7 +193,7 @@ _exit:
return code;
}
static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0;
// ASSERT(pgno <= pFD->szFile);
@ -219,6 +242,30 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _exit;
}
if(encryptAlgorithm == DND_CA_SM4){
//if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){
unsigned char PacketData[128];
int NewLen;
int32_t count = 0;
while(count < pFD->szPage)
{
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pFD->pBuf + count;
opts.result = PacketData;
opts.unitLen = 128;
//strncpy(opts.key, tsEncryptKey, 16);
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Decrypt(&opts);
memcpy(pFD->pBuf + count, PacketData, NewLen);
count += NewLen;
}
tsdbInfo("CBC_Decrypt count:%d %s", count, __FUNCTION__);
}
}
// check
@ -233,7 +280,8 @@ _exit:
return code;
}
int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) {
int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
@ -242,11 +290,11 @@ int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t
do {
if (pFD->pgno != pgno) {
code = tsdbWriteFilePage(pFD);
code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
if (code) goto _exit;
if (pgno <= pFD->szFile) {
code = tsdbReadFilePage(pFD, pgno);
code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey);
if (code) goto _exit;
} else {
pFD->pgno = pgno;
@ -265,7 +313,8 @@ _exit:
return code;
}
static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) {
static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = 0;
int64_t n = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
@ -278,7 +327,7 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6
while (n < size) {
if (pFD->pgno != pgno) {
code = tsdbReadFilePage(pFD, pgno);
code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey);
if (code) goto _exit;
}
@ -395,7 +444,8 @@ _exit:
return code;
}
int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) {
int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0;
if (!pFD->pFD) {
code = tsdbOpenFileImpl(pFD);
@ -407,21 +457,21 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size,
if (pFD->s3File && tsS3BlockSize < 0) {
return tsdbReadFileS3(pFD, offset, pBuf, size, szHint);
} else {
return tsdbReadFileImp(pFD, offset, pBuf, size);
return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey);
}
_exit:
return code;
}
int32_t tsdbFsyncFile(STsdbFD *pFD) {
int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) {
int32_t code = 0;
if (pFD->s3File) {
tsdbWarn("%s file: %s", __func__, pFD->path);
return code;
}
code = tsdbWriteFilePage(pFD);
code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey);
if (code) goto _exit;
if (taosFsyncFile(pFD->pFD) < 0) {
@ -532,7 +582,9 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
if (code) goto _err;
// read
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
// decode
@ -569,7 +621,9 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
if (code) goto _err;
// read
code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
// decode
@ -602,7 +656,9 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
if (code) goto _err;
// read
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
// decode
@ -633,7 +689,10 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aCol
if (code) goto _err;
// read
code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], pSmaInfo->size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], pSmaInfo->size, 0, encryptAlgorithm,
encryptKey);
if (code) goto _err;
// decode
@ -656,7 +715,7 @@ _err:
}
static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, SBlockData *pBlockData,
int32_t iStt) {
int32_t iStt) {
int32_t code = 0;
tBlockDataClear(pBlockData);
@ -667,7 +726,9 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
code = tRealloc(&pReader->aBuf[0], pBlkInfo->szKey);
if (code) goto _err;
code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
SDiskDataHdr hdr;
@ -713,7 +774,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
code = tRealloc(&pReader->aBuf[0], hdr.szBlkCol);
if (code) goto _err;
code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol, 0);
code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
}
@ -757,7 +818,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
code = tRealloc(&pReader->aBuf[1], size);
if (code) goto _err;
code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size, 0);
code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]);
@ -783,7 +844,10 @@ int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockDat
if (code) goto _err;
// read
code = tsdbReadFile(pReader->pDataFD, pBlockInfo->offset, pReader->aBuf[0], pBlockInfo->szBlock, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pDataFD, pBlockInfo->offset, pReader->aBuf[0], pBlockInfo->szBlock, 0, encryptAlgorithm,
encryptKey);
if (code) goto _err;
// decmpr
@ -835,7 +899,10 @@ int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk
TSDB_CHECK_CODE(code, lino, _exit);
// read
code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// decmpr
@ -924,7 +991,9 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa
if (code) goto _err;
// read
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
// // decode
@ -964,7 +1033,9 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
if (code) goto _err;
// read
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0);
int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey);
if (code) goto _err;
// decode

5
source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
View File

@ -545,7 +545,10 @@ static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbD
int32_t code = 0;
int32_t lino = 0;
code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr);
int32_t encryptAlgorithm = writer->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:

112
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -60,7 +60,10 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
int64_t offset = config->file->size - sizeof(SSttFooter);
ASSERT(offset >= TSDB_FHDR_SIZE);
code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0);
int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, encryptAlgoirthm,
encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -96,8 +99,11 @@ int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray *
void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code =
tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0);
tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0,
encryptAlgorithm, encryptKey);
if (code) {
taosMemoryFree(data);
return code;
@ -124,8 +130,11 @@ int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tom
void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code =
tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0);
tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0,
encryptAlgorithm, encryptKey);
if (code) {
taosMemoryFree(data);
return code;
@ -152,8 +161,11 @@ int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBl
void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code =
tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0);
tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0,
encryptAlgorithm, encryptKey);
if (code) {
taosMemoryFree(data);
return code;
@ -178,7 +190,10 @@ int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk,
code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szBlock);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szBlock, 0);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szBlock, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tDecmprBlockData(reader->config->bufArr[0], sttBlk->bInfo.szBlock, bData, &reader->config->bufArr[1]);
@ -210,7 +225,10 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey, 0);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
// hdr
@ -255,8 +273,10 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
code = tRealloc(&reader->config->bufArr[0], hdr->szBlkCol);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, reader->config->bufArr[0],
hdr->szBlkCol, 0);
hdr->szBlkCol, 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -296,8 +316,10 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
code = tRealloc(&reader->config->bufArr[1], size1);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr->szBlkCol + blockCol->offset,
reader->config->bufArr[1], size1, 0);
reader->config->bufArr[1], size1, 0, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData,
@ -322,7 +344,10 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk
code = tRealloc(&reader->config->bufArr[0], tombBlk->dp->size);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(reader->fd, tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size, 0);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size, 0,
encryptAlgorithm, encryptKey);
if (code) TSDB_CHECK_CODE(code, lino, _exit);
int64_t size = 0;
@ -353,7 +378,10 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
code = tRealloc(&reader->config->bufArr[0], statisBlk->dp->size);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(reader->fd, statisBlk->dp->offset, reader->config->bufArr[0], statisBlk->dp->size, 0);
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, statisBlk->dp->offset, reader->config->bufArr[0], statisBlk->dp->size, 0,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
int64_t size = 0;
@ -406,7 +434,8 @@ struct SSttFileWriter {
};
static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize,
TSttBlkArray *sttBlkArray, uint8_t **bufArr, SVersionRange *range) {
TSttBlkArray *sttBlkArray, uint8_t **bufArr, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey) {
if (blockData->nRow == 0) return 0;
int32_t code = 0;
@ -441,7 +470,7 @@ static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, i
for (int32_t i = 3; i >= 0; i--) {
if (sizeArr[i]) {
code = tsdbWriteFile(fd, *fileSize, bufArr[i], sizeArr[i]);
code = tsdbWriteFile(fd, *fileSize, bufArr[i], sizeArr[i], encryptAlgorithm, encryptKey);
if (code) return code;
*fileSize += sizeArr[i];
}
@ -461,8 +490,12 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size,
writer->sttBlkArray, writer->config->bufArr, &writer->ctx->range);
writer->sttBlkArray, writer->config->bufArr, &writer->ctx->range, encryptAlgorithm,
encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -498,13 +531,17 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
.cmprAlg = writer->config->cmprAlg,
}};
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; i++) {
code = tsdbCmprData((uint8_t *)TARRAY2_DATA(writer->staticBlock->dataArr + i),
TARRAY2_DATA_LEN(&writer->staticBlock->dataArr[i]), TSDB_DATA_TYPE_BIGINT, statisBlk->cmprAlg,
&writer->config->bufArr[0], 0, &statisBlk->size[i], &writer->config->bufArr[1]);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbWriteFile(writer->fd, writer->file->size, writer->config->bufArr[0], statisBlk->size[i]);
code = tsdbWriteFile(writer->fd, writer->file->size, writer->config->bufArr[0], statisBlk->size[i],
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
statisBlk->dp->size += statisBlk->size[i];
@ -529,8 +566,12 @@ static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size,
writer->tombBlkArray, writer->config->bufArr, &writer->ctx->range);
writer->tombBlkArray, writer->config->bufArr, &writer->ctx->range, encryptAlgorithm,
encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -540,12 +581,14 @@ _exit:
return code;
}
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize) {
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey) {
ptr->size = TARRAY2_DATA_LEN(sttBlkArray);
if (ptr->size > 0) {
ptr->offset = *fileSize;
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size);
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size, encryptAlgorithm,
encryptKey);
if (code) {
return code;
}
@ -559,7 +602,11 @@ static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) {
int32_t code = 0;
int32_t lino;
code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -572,12 +619,14 @@ _exit:
static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) {
int32_t code = 0;
int32_t lino;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray);
if (writer->footer->statisBlkPtr->size) {
writer->footer->statisBlkPtr->offset = writer->file->size;
code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray),
writer->footer->statisBlkPtr->size);
writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += writer->footer->statisBlkPtr->size;
}
@ -593,7 +642,11 @@ static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) {
int32_t code = 0;
int32_t lino = 0;
code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size,
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
@ -603,15 +656,19 @@ _exit:
return code;
}
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer));
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
char* encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey);
if (code) return code;
*fileSize += sizeof(*footer);
return 0;
}
static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) {
return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size, encryptAlgorithm, encryptKey);
}
static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
@ -646,7 +703,10 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr));
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += sizeof(hdr);
@ -711,7 +771,9 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o
code = tsdbSttFileDoUpdateHeader(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFsyncFile(writer->fd);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd);

6
source/dnode/vnode/src/tsdb/tsdbSttFileRW.h
View File

@ -71,8 +71,10 @@ int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *pBlockData
int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record);
bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer);
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize);
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize);
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey);
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
char* encryptKey);
struct SSttFileWriterConfig {
STsdb *tsdb;

55
source/dnode/vnode/src/tsdb/tsdbUpgrade.c
View File

@ -37,6 +37,9 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
int8_t cmprAlg;
int32_t szPage;
uint8_t *bufArr[8];
int32_t encryptAlgorithm;
char* encryptKey;
// reader
SArray *aBlockIdx;
SMapData mDataBlk[1];
@ -50,6 +53,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
.maxRow = tsdb->pVnode->config.tsdbCfg.maxRows,
.cmprAlg = tsdb->pVnode->config.tsdbCfg.compression,
.szPage = tsdb->pVnode->config.tsdbPageSize,
.encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm,
.encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey,
}};
// read SBlockIdx array
@ -122,7 +127,7 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
if (BRIN_BLOCK_SIZE(ctx->brinBlock) >= ctx->maxRow) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
ctx->brinBlkArray, ctx->bufArr, &range);
ctx->brinBlkArray, ctx->bufArr, &range, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -131,18 +136,19 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
if (BRIN_BLOCK_SIZE(ctx->brinBlock) > 0) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
ctx->brinBlkArray, ctx->bufArr, &range);
ctx->brinBlkArray, ctx->bufArr, &range, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr,
&fset->farr[TSDB_FTYPE_HEAD]->f->size);
&fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer);
code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFsyncFile(ctx->fd);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&ctx->fd);
@ -238,12 +244,16 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
STFileObj *fobj;
struct {
int32_t szPage;
int32_t encryptAlgorithm;
char* encryptKey;
// writer
STsdbFD *fd;
TSttBlkArray sttBlkArray[1];
SSttFooter footer[1];
} ctx[1] = {{
.szPage = tsdb->pVnode->config.tsdbPageSize,
.encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm,
.encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey,
}};
STFile file = {
@ -266,13 +276,14 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
TSDB_CHECK_CODE(code, lino, _exit1);
}
code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size);
code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbFsyncFile(ctx->fd);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1);
tsdbCloseFile(&ctx->fd);
@ -417,7 +428,10 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0};
code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE);
int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
fobj[0]->f->size += TSDB_FHDR_SIZE;
@ -440,6 +454,9 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
int64_t minKey;
int64_t maxKey;
uint8_t *bufArr[8];
int32_t encryptAlgorithm;
char* encryptKey;
// reader
SArray *aDelData;
// writer
@ -452,6 +469,8 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
} ctx[1] = {{
.maxRow = tsdb->pVnode->config.tsdbCfg.maxRows,
.cmprAlg = tsdb->pVnode->config.tsdbCfg.compression,
.encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm,
.encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey,
}};
tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey);
@ -488,7 +507,7 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
}
SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
ctx->bufArr, &tombRange);
ctx->bufArr, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -501,26 +520,30 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
}
SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
ctx->bufArr, &tombRange);
ctx->bufArr, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
if (ctx->fd != NULL) {
if (ctx->toStt) {
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size);
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size,
ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size);
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size,
ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size);
code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbFsyncFile(ctx->fd);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&ctx->fd);

43
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -15,6 +15,7 @@
#include "tutil.h"
#include "vnd.h"
#include "tglobal.h"
const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.dbname = "",
@ -126,6 +127,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
tjsonAddItemToArray(pNodeRetentions, pNodeRetention);
}
}
if (tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;
@ -133,6 +135,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
@ -236,6 +240,19 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code);
tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code);
}
tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code);
@ -250,6 +267,32 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code);
if (code < 0) return -1;
#if defined(TD_ENTERPRISE)
if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){
if(tsEncryptKey[0] == 0){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code);
if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER;
tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);

2
source/libs/CMakeLists.txt
View File

@ -9,6 +9,8 @@ add_subdirectory(nodes)
add_subdirectory(catalog)
add_subdirectory(audit)
add_subdirectory(monitorfw)
add_subdirectory(sm4)
add_subdirectory(crypt)
add_subdirectory(scalar)
add_subdirectory(function)

14
source/libs/crypt/CMakeLists.txt Normal file
View File

@ -0,0 +1,14 @@
aux_source_directory(src CRYPT_SRC)
IF (TD_ENTERPRISE)
LIST(APPEND CRYPT_SRC ${TD_ENTERPRISE_DIR}/src/plugins/crypt/cryptImpl.c)
ENDIF ()
add_library(crypt STATIC ${CRYPT_SRC})
target_include_directories(
crypt
PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(crypt common sm4)

36
source/libs/crypt/src/crypt.c Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "crypt.h"
extern int32_t CBC_DecryptImpl(SCryptOpts *opts);
extern int32_t CBC_EncryptImpl(SCryptOpts *opts);
int32_t CBC_Encrypt(SCryptOpts *opts) {
return CBC_EncryptImpl(opts);
}
int32_t CBC_Decrypt(SCryptOpts *opts) {
return CBC_DecryptImpl(opts);
}
#ifndef TD_ENTERPRISE
int32_t CBC_EncryptImpl(SCryptOpts *opts) {
memcpy(opts->result, opts->source, opts->len);
return opts->len;
}
int32_t CBC_DecryptImpl(SCryptOpts *opts) {
memcpy(opts->result, opts->source, opts->len);
return opts->len;
}
#endif

9
source/libs/sm4/CMakeLists.txt Normal file
View File

@ -0,0 +1,9 @@
aux_source_directory(src SM4_SRC)
add_library(sm4 STATIC ${SM4_SRC})
target_include_directories(
sm4
PUBLIC "${TD_SOURCE_DIR}/include/libs/sm4"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(sm4 common)

543
source/libs/sm4/src/sm4.c Normal file
View File

@ -0,0 +1,543 @@
#include "sm4.h"
#define SM4_ROUND 32
static unsigned int FK[4]={
0xA3B1BAC6,0x56AA3350,0x677D9197,0xB27022DC
};
static unsigned int CK[SM4_ROUND]={
0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269,
0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9,
0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249,
0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9,
0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229,
0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299,
0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209,
0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279
};
static unsigned char Sbox[256]={
0xd6,0x90,0xe9,0xfe,0xcc,0xe1,0x3d,0xb7,0x16,0xb6,0x14,0xc2,0x28,0xfb,0x2c,0x05,
0x2b,0x67,0x9a,0x76,0x2a,0xbe,0x04,0xc3,0xaa,0x44,0x13,0x26,0x49,0x86,0x06,0x99,
0x9c,0x42,0x50,0xf4,0x91,0xef,0x98,0x7a,0x33,0x54,0x0b,0x43,0xed,0xcf,0xac,0x62,
0xe4,0xb3,0x1c,0xa9,0xc9,0x08,0xe8,0x95,0x80,0xdf,0x94,0xfa,0x75,0x8f,0x3f,0xa6,
0x47,0x07,0xa7,0xfc,0xf3,0x73,0x17,0xba,0x83,0x59,0x3c,0x19,0xe6,0x85,0x4f,0xa8,
0x68,0x6b,0x81,0xb2,0x71,0x64,0xda,0x8b,0xf8,0xeb,0x0f,0x4b,0x70,0x56,0x9d,0x35,
0x1e,0x24,0x0e,0x5e,0x63,0x58,0xd1,0xa2,0x25,0x22,0x7c,0x3b,0x01,0x21,0x78,0x87,
0xd4,0x00,0x46,0x57,0x9f,0xd3,0x27,0x52,0x4c,0x36,0x02,0xe7,0xa0,0xc4,0xc8,0x9e,
0xea,0xbf,0x8a,0xd2,0x40,0xc7,0x38,0xb5,0xa3,0xf7,0xf2,0xce,0xf9,0x61,0x15,0xa1,
0xe0,0xae,0x5d,0xa4,0x9b,0x34,0x1a,0x55,0xad,0x93,0x32,0x30,0xf5,0x8c,0xb1,0xe3,
0x1d,0xf6,0xe2,0x2e,0x82,0x66,0xca,0x60,0xc0,0x29,0x23,0xab,0x0d,0x53,0x4e,0x6f,
0xd5,0xdb,0x37,0x45,0xde,0xfd,0x8e,0x2f,0x03,0xff,0x6a,0x72,0x6d,0x6c,0x5b,0x51,
0x8d,0x1b,0xaf,0x92,0xbb,0xdd,0xbc,0x7f,0x11,0xd9,0x5c,0x41,0x1f,0x10,0x5a,0xd8,
0x0a,0xc1,0x31,0x88,0xa5,0xcd,0x7b,0xbd,0x2d,0x74,0xd0,0x12,0xb8,0xe5,0xb4,0xb0,
0x89,0x69,0x97,0x4a,0x0c,0x96,0x77,0x7e,0x65,0xb9,0xf1,0x09,0xc5,0x6e,0xc6,0x84,
0x18,0xf0,0x7d,0xec,0x3a,0xdc,0x4d,0x20,0x79,0xee,0x5f,0x3e,0xd7,0xcb,0x39,0x48
};
#define ROL(x,y) ((x)<<(y) | (x)>>(32-(y)))
unsigned int SMS4_T1(unsigned int dwA)
{
unsigned char a0[4]={0};
unsigned char b0[4]={0};
unsigned int dwB=0;
unsigned int dwC=0;
int i=0;
/*
for (i=0;i<4;i++)
{
a0[i] = (unsigned char)((dwA>>(i*8)) & 0xff);
b0[i] = Sbox[a0[i]];
dwB |= (b0[i]<<(i*8));
}
*/
a0[0] = (unsigned char)((dwA) & 0xff);
b0[0] = Sbox[a0[0]];
dwB |= (b0[0]);
a0[1] = (unsigned char)((dwA>>(8)) & 0xff);
b0[1] = Sbox[a0[1]];
dwB |= (b0[1]<<(8));
a0[2] = (unsigned char)((dwA>>(16)) & 0xff);
b0[2] = Sbox[a0[2]];
dwB |= (b0[2]<<(16));
a0[3] = (unsigned char)((dwA>>(24)) & 0xff);
b0[3] = Sbox[a0[3]];
dwB |= (b0[3]<<(24)); // 2013-08-13
dwC=dwB^ROL(dwB,2)^ROL(dwB,10)^ROL(dwB,18)^ROL(dwB,24);
return dwC;
}
unsigned int SMS4_T2(unsigned int dwA)
{
unsigned char a0[4]={0};
unsigned char b0[4]={0};
unsigned int dwB=0;
unsigned int dwC=0;
int i=0;
/*
for (i=0;i<4;i++)
{
a0[i] = (unsigned char)((dwA>>(i*8)) & 0xff);
b0[i] = Sbox[a0[i]];
dwB |= (b0[i]<<(i*8));
}
*/
a0[0] = (unsigned char)((dwA) & 0xff);
b0[0] = Sbox[a0[0]];
dwB |= (b0[0]);
a0[1] = (unsigned char)((dwA>>(8)) & 0xff);
b0[1] = Sbox[a0[1]];
dwB |= (b0[1]<<(8));
a0[2] = (unsigned char)((dwA>>(16)) & 0xff);
b0[2] = Sbox[a0[2]];
dwB |= (b0[2]<<(16));
a0[3] = (unsigned char)((dwA>>(24)) & 0xff);
b0[3] = Sbox[a0[3]];
dwB |= (b0[3]<<(24)); // 2013-08-13
dwC=dwB^ROL(dwB,13)^ROL(dwB,23);
return dwC;
}
/* MK[4] is the Encrypt Key, rk[32] is Round Key */
void SMS4_Key_Expansion(unsigned int MK[], unsigned int rk[])
{
unsigned int K[4]={0};
int i=0;
for (i=0;i<4;i++)
{
K[i] = MK[i] ^ FK[i];
}
for (i=0;i<SM4_ROUND;i++)
{
K[i%4]^=SMS4_T2(K[(i+1)%4]^K[(i+2)%4]^K[(i+3)%4]^CK[i]);
rk[i]=K[i%4];
}
}
/* X[4] is PlainText, rk[32] is round Key, Y[4] is CipherText */
void SMS4_ECB_Encryption_Core(unsigned int X[], unsigned int rk[], unsigned int Y[])
{
unsigned int tempX[4]={0};
int i=0;
/*
for (i=0;i<4;i++)
{
tempX[i]=X[i];
}
*/
tempX[0]=X[0];
tempX[1]=X[1];
tempX[2]=X[2];
tempX[3]=X[3];
/*
for (i=0;i<SM4_ROUND;i++)
{
tempX[i%4]^=SMS4_T1(tempX[(i+1)%4]^tempX[(i+2)%4]^tempX[(i+3)%4]^rk[i]);
}
*/
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[0]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[1]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[2]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[3]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[4]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[5]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[6]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[7]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[8]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[9]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[10]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[11]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[12]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[13]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[14]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[15]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[16]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[17]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[18]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[19]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[20]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[21]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[22]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[23]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[24]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[25]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[26]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[27]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[28]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[29]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[30]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[31]);
/* for (i=0;i<4;i++)
{
Y[i]=tempX[3-i];
}*/
Y[0]=tempX[3];
Y[1]=tempX[2];
Y[2]=tempX[1];
Y[3]=tempX[0]; // 2013-08-19
}
/* X[4] is PlainText, rk[32] is round Key, Y[4] is CipherText */
void SMS4_ECB_Decryption_Core(unsigned int X[], unsigned int rk[], unsigned int Y[])
{
unsigned int tempX[4]={0};
int i=0;
/* for (i=0;i<4;i++)
{
tempX[i]=X[i];
}
*/
tempX[0]=X[0];
tempX[1]=X[1];
tempX[2]=X[2];
tempX[3]=X[3]; // 2013-08-19
/* for (i=0;i<SM4_ROUND;i++)
{
tempX[i%4]^=SMS4_T1(tempX[(i+1)%4]^tempX[(i+2)%4]^tempX[(i+3)%4]^rk[(31-i)]);
}
*/
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[31]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[30]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[29]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[28]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[27]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[26]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[25]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[24]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[23]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[22]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[21]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[20]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[19]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[18]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[17]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[16]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[15]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[14]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[13]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[12]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[11]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[10]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[9]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[8]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[7]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[6]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[5]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[4]);
tempX[0]^=SMS4_T1(tempX[1]^tempX[2]^tempX[3]^rk[3]);
tempX[1]^=SMS4_T1(tempX[2]^tempX[3]^tempX[0]^rk[2]);
tempX[2]^=SMS4_T1(tempX[3]^tempX[0]^tempX[1]^rk[1]);
tempX[3]^=SMS4_T1(tempX[0]^tempX[1]^tempX[2]^rk[0]);// 2013-08-19
/* for (i=0;i<4;i++)
{
Y[i]=tempX[3-i];
}*/
Y[0]=tempX[3];
Y[1]=tempX[2];
Y[2]=tempX[1];
Y[3]=tempX[0]; // 2013-08-19
}
void SMS4_convert_to_network_order(unsigned int* src,unsigned int* dst,int count)
{
int i=0;
for ( ; i<count; i++ )
{
unsigned char* ps = (unsigned char*)(src+i);
unsigned char* pd = (unsigned char*)(dst+i);
pd[0] = ps[3];
pd[1] = ps[2];
pd[2] = ps[1];
pd[3] = ps[0];
}
}
void SMS4_convert_to_host_order(unsigned int* src,unsigned int* dst,int count)
{
SMS4_convert_to_network_order(src,dst,count);
}
void SMS4_ECB_Encryption(unsigned char plaintext[16], unsigned char key[16], unsigned char ciphertext[16])
{
unsigned int _pt[4];
unsigned int _ky[4];
unsigned int _ct[4];
unsigned int _rk[32];
SMS4_convert_to_network_order((unsigned int*)plaintext,_pt,4);
SMS4_convert_to_network_order((unsigned int*)key,_ky,4);
SMS4_Key_Expansion(_ky,_rk);
SMS4_ECB_Encryption_Core(_pt,_rk,_ct);
SMS4_convert_to_host_order(_ct,(unsigned int*)ciphertext,4);
}
void Key_Expansion_init( unsigned char key[16], unsigned int rk[32])
{
unsigned int _ky[4];
SMS4_convert_to_network_order((unsigned int*)key,_ky,4);
SMS4_Key_Expansion (_ky,rk);
}
void SMS4_ECB_EncryptionEx(unsigned char plaintext[16], unsigned int key[32], unsigned char ciphertext[16])
{
unsigned int _pt[4];
unsigned int _ct[4];
SMS4_convert_to_network_order((unsigned int*)plaintext,_pt,4);
SMS4_ECB_Encryption_Core(_pt,key,_ct);
SMS4_convert_to_host_order(_ct,(unsigned int*)ciphertext,4);
}
void SMS4_ECB_Decryption(unsigned char ciphertext[16], unsigned char key[16], unsigned char plaintext[16])
{
unsigned int _ct[4];
unsigned int _ky[4];
unsigned int _pt[4];
unsigned int _rk[32];
SMS4_convert_to_network_order((unsigned int*)ciphertext,_ct,4);
SMS4_convert_to_network_order((unsigned int*)key,_ky,4);
SMS4_Key_Expansion(_ky,_rk);
SMS4_ECB_Decryption_Core(_ct,_rk,_pt);
SMS4_convert_to_host_order(_pt,(unsigned int*)plaintext,4);
}
void SMS4_ECB_DecryptionEx(unsigned char ciphertext[16], unsigned int key[32], unsigned char plaintext[16])
{
unsigned int _ct[4];
unsigned int _pt[4];
SMS4_convert_to_network_order((unsigned int*)ciphertext,_ct,4);
SMS4_ECB_Decryption_Core(_ct,key,_pt);
SMS4_convert_to_host_order(_pt,(unsigned int*)plaintext,4);
}
void SMS4_CBC_Encryption(unsigned char plaintext[16], unsigned char key[16], unsigned char iv[16], unsigned char ciphertext[16])
{
unsigned char plaintextNew[16];
int i = 0;
for (i = 0; i < 16; i ++)
{
plaintextNew[i] = plaintext[i] ^ iv[i];
}
SMS4_ECB_Encryption(plaintextNew, key, ciphertext);
}
void SMS4_CBC_Decryption(unsigned char ciphertext[16], unsigned char key[16], unsigned char iv[16], unsigned char plaintext[16])
{
unsigned char plaintextTemp[16];
int i = 0;
SMS4_ECB_Decryption(ciphertext, key, plaintextTemp);
for (i = 0; i < 16; i ++)
{
plaintext[i] = plaintextTemp[i] ^ iv[i];
}
}
void SMS4_CBC_EncryptionEx(unsigned char plaintext[16], unsigned int key[32], unsigned char iv[16], unsigned char ciphertext[16])
{
unsigned char plaintextNew[16];
int i = 0;
for (i = 0; i < 16; i ++)
{
plaintextNew[i] = plaintext[i] ^ iv[i];
}
SMS4_ECB_EncryptionEx(plaintextNew, key, ciphertext);
}
void SMS4_CBC_DecryptionEx(unsigned char ciphertext[16], unsigned int key[32], unsigned char iv[16], unsigned char plaintext[16])
{
unsigned char plaintextTemp[16];
int i = 0;
SMS4_ECB_DecryptionEx(ciphertext, key, plaintextTemp);
for (i = 0; i < 16; i ++)
{
plaintext[i] = plaintextTemp[i] ^ iv[i];
}
}
int SM4_ECB_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
//int rv = 0;
int loop = 0;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
for (i = 0; i < loop; i ++)
{
//SMS4_ECB_Encryption(pInData + i * 16, pKey, pOutData + i * 16);
SMS4_ECB_EncryptionEx(pInData + i * 16, rk, pOutData + i * 16);
}
*pOutDataLen = inDataLen;
return 0;
}
int SM4_ECB_Decrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
// int rv = 0;
int loop = 0;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
for (i = 0; i < loop; i ++)
{
//SMS4_ECB_Decryption(pInData + i * 16, pKey, pOutData + i * 16);
SMS4_ECB_DecryptionEx(pInData + i * 16, rk, pOutData + i * 16);
}
*pOutDataLen = inDataLen;
return 0;
}
int SM4_CBC_Encrypt( unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
// int rv = 0;
int loop = 0;
unsigned char *pIVTemp = NULL;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
if(ivLen != 16)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
pIVTemp = pIV;
for (i = 0; i < loop; i ++)
{
SMS4_CBC_EncryptionEx(pInData + i * 16, rk, pIVTemp, pOutData + i * 16);
pIVTemp = pOutData + i * 16;
}
*pOutDataLen = inDataLen;
return 0;
}
int SM4_CBC_Decrypt(unsigned char *pKey,
unsigned int KeyLen,
unsigned char *pIV,
unsigned int ivLen,
unsigned char *pInData,
unsigned int inDataLen,
unsigned char *pOutData,
unsigned int *pOutDataLen)
{
int i = 0;
// int rv = 0;
int loop = 0;
unsigned char *pIVTemp = NULL;
unsigned int rk[32];
*pOutDataLen = 0;
if(KeyLen != 16)
{
return 1;
}
if(inDataLen % 16 != 0)
{
return 1;
}
if(ivLen != 16)
{
return 1;
}
Key_Expansion_init(pKey,rk);
loop = inDataLen / 16;
pIVTemp = pIV;
for (i = 0; i < loop; i ++)
{
SMS4_CBC_DecryptionEx(pInData + i * 16, rk, pIVTemp, pOutData + i * 16);
pIVTemp = pInData + i * 16;
}
*pOutDataLen = inDataLen;
return 0;
}

2
source/libs/stream/src/streamMeta.c
View File

@ -130,7 +130,7 @@ typedef struct {
} StreamMetaTaskState;
int32_t streamMetaOpenTdb(SStreamMeta* pMeta) {
if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0) < 0) {
if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0, 0, NULL) < 0) {
return -1;
// goto _err;
}

2
source/libs/stream/src/streamState.c
View File

@ -158,7 +158,7 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz
}
taosCloseFile(&pCfgFile);
if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 1) < 0) {
if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 1, 0, NULL) < 0) {
goto _err;
}

1
source/libs/tdb/CMakeLists.txt
View File

@ -22,6 +22,7 @@ target_link_libraries(
tdb
PUBLIC os
PUBLIC util
PUBLIC crypt
)
# for test

3
source/libs/tdb/inc/tdb.h
View File

@ -32,7 +32,8 @@ typedef struct STBC TBC;
typedef struct STxn TXN;
// TDB
int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback);
int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm,
char *encryptKey);
int32_t tdbClose(TDB *pDb);
int32_t tdbBegin(TDB *pDb, TXN **pTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
int flags);

8
source/libs/tdb/src/db/tdbDb.c
View File

@ -15,7 +15,8 @@
#include "tdbInt.h"
int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback) {
int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm,
char *encryptKey) {
TDB *pDb;
int dsize;
int zsize;
@ -49,6 +50,11 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
pDb->jfd = -1;
pDb->encryptAlgorithm = encryptAlgorithm;
if(encryptKey != NULL){
strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN);
}
ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache));
if (ret < 0) {
return -1;

59
source/libs/tdb/src/db/tdbPager.c
View File

@ -14,6 +14,8 @@
*/
#include "tdbInt.h"
#include "crypt.h"
#include "tglobal.h"
/*
#pragma pack(push, 1)
typedef struct {
@ -872,6 +874,31 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
TDB_UNLOCK_PAGE(pPage);
return -1;
}
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey;
if(encryptAlgorithm == DND_CA_SM4){
unsigned char PacketData[128];
int NewLen;
int32_t count = 0;
while(count < pPage->pageSize)
{
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pPage->pData + count;
opts.result = PacketData;
opts.unitLen = 128;
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Decrypt(&opts);
memcpy(pPage->pData + count, PacketData, NewLen);
count += NewLen;
}
tdbInfo("CBC_Decrypt count:%d %s", count, __FUNCTION__);
}
} else {
init = 0;
}
@ -960,14 +987,44 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) {
offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
ret = tdbOsPWrite(pPager->fd, pPage->pData, pPage->pageSize, offset);
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey;
char* buf = pPage->pData;
if(encryptAlgorithm == DND_CA_SM4){
buf = taosMemoryMalloc(pPage->pageSize);
unsigned char PacketData[128];
int NewLen;
int32_t count = 0;
while (count < pPage->pageSize) {
SCryptOpts opts = {0};
opts.len = 128;
opts.source = pPage->pData + count;
opts.result = PacketData;
opts.unitLen = 128;
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
NewLen = CBC_Encrypt(&opts);
memcpy(buf + count, PacketData, NewLen);
count += NewLen;
}
tdbInfo("CBC_Encrypt count:%d %s", count, __FUNCTION__);
}
ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset);
if (ret < 0) {
if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf);
tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName,
pPage->pageSize);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf);
return 0;
}

3
source/libs/tdb/src/inc/tdbInt.h
View File

@ -18,6 +18,7 @@
#include "tdb.h"
#include "tdef.h"
#include "tlog.h"
#include "trbtree.h"
@ -392,6 +393,8 @@ struct STDB {
TTB *pFreeDb;
#endif
int64_t txnId;
int32_t encryptAlgorithm;
char encryptKey[ENCRYPT_KEY_LEN];
};
struct SPager {

4
source/libs/tdb/test/tdbExOVFLTest.cpp
View File

@ -141,7 +141,7 @@ static void generateBigVal(char *val, int valLen) {
static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
TDB *pEnv = NULL;
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0);
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL);
if (ret) {
pEnv = NULL;
}
@ -374,7 +374,7 @@ TEST(tdb_test, simple_insert1) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database

10
source/libs/tdb/test/tdbPageDefragmentTest.cpp
View File

@ -141,7 +141,7 @@ static void generateBigVal(char *val, int valLen) {
static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
TDB *pEnv = NULL;
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0);
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL);
if (ret) {
pEnv = NULL;
}
@ -352,7 +352,7 @@ TEST(TdbPageDefragmentTest, DISABLED_simple_insert1) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -485,7 +485,7 @@ TEST(TdbPageDefragmentTest, seq_insert) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -566,7 +566,7 @@ TEST(TdbPageDefragmentTest, seq_delete) {
int const pageSize = 1 * 1024 * 1024;
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -650,7 +650,7 @@ TEST(TdbPageDefragmentTest, defragment_insert) {
int const pageSize = 1 * 1024 * 1024;
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database

8
source/libs/tdb/test/tdbPageRecycleTest.cpp
View File

@ -123,7 +123,7 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in
static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
TDB *pEnv = NULL;
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0);
int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0 , 0, NULL);
if (ret) {
pEnv = NULL;
}
@ -355,7 +355,7 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -484,7 +484,7 @@ static void insertDb(int nData) {
int const pageSize = 4 * 1024;
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -552,7 +552,7 @@ static void deleteDb(int nData) {
int const pageSize = 4 * 1024;
// Open Env
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database

12
source/libs/tdb/test/tdbTest.cpp
View File

@ -131,7 +131,7 @@ TEST(tdb_test, DISABLED_simple_insert1) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", 4096, 64, &pEnv, 0);
ret = tdbOpen("tdb", 4096, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -246,7 +246,7 @@ TEST(tdb_test, DISABLED_simple_insert2) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", 1024, 10, &pEnv, 0);
ret = tdbOpen("tdb", 1024, 10, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -339,7 +339,7 @@ TEST(tdb_test, DISABLED_simple_delete1) {
pPool = openPool();
// open env
ret = tdbOpen("tdb", 1024, 256, &pEnv, 0);
ret = tdbOpen("tdb", 1024, 256, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// open database
@ -428,7 +428,7 @@ TEST(tdb_test, DISABLED_simple_upsert1) {
taosRemoveDir("tdb");
// open env
ret = tdbOpen("tdb", 4096, 64, &pEnv, 0);
ret = tdbOpen("tdb", 4096, 64, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// open database
@ -490,7 +490,7 @@ TEST(tdb_test, multi_thread_query) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", 4096, 10, &pEnv, 0);
ret = tdbOpen("tdb", 4096, 10, &pEnv, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
@ -599,7 +599,7 @@ TEST(tdb_test, DISABLED_multi_thread1) {
taosRemoveDir("tdb");
// Open Env
ret = tdbOpen("tdb", 512, 1, &pDb, 0);
ret = tdbOpen("tdb", 512, 1, &pDb, 0, 0, NULL);
GTEST_ASSERT_EQ(ret, 0);
ret = tdbTbOpen("db.db", -1, -1, NULL, pDb, &pTb, 0);

1
source/libs/wal/CMakeLists.txt
View File

@ -12,6 +12,7 @@ target_link_libraries(
PUBLIC os
PUBLIC util
PUBLIC common
PUBLIC crypt
)
if(${BUILD_TEST})

17
source/libs/wal/src/walMeta.c
View File

@ -153,7 +153,12 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
}
// validate body
recordLen = walCkHeadSz + logContent->head.bodyLen;
int32_t cryptedBodyLen = logContent->head.bodyLen;
//TODO: dmchen enum
if(pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
recordLen = walCkHeadSz + cryptedBodyLen;
if (len < recordLen) {
int64_t extraSize = recordLen - len;
if (capacity < readSize + extraSize + sizeof(magic)) {
@ -181,6 +186,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
}
logContent = (SWalCkHead*)(buf + pos);
decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__);
if (walValidBodyCksum(logContent) != 0) {
terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH;
wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId,
@ -618,7 +624,14 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
/*A(idxEntry.ver == ckHead.head.version);*/
idxEntry.ver += 1;
idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen;
int32_t plainBodyLen = ckHead.head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen enum
if(pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
idxEntry.offset += sizeof(SWalCkHead) + cryptedBodyLen;
if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) {
wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s",

75
source/libs/wal/src/walRead.c
View File

@ -13,7 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "crypt.h"
#include "taoserror.h"
#include "wal.h"
#include "walInt.h"
SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) {
@ -307,7 +309,13 @@ int32_t walSkipFetchBody(SWalReader *pRead) {
pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer,
pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId);
int64_t code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR);
int32_t plainBodyLen = pRead->pHead->head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pRead->pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
int64_t code = taosLSeekFile(pRead->pLogFile, cryptedBodyLen, SEEK_CUR);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -328,19 +336,27 @@ int32_t walFetchBody(SWalReader *pRead) {
", 0x%" PRIx64,
vgId, ver, pVer->firstVer, pVer->commitVer, pVer->lastVer, pVer->appliedVer, id);
if (pRead->capacity < pReadHead->bodyLen) {
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen);
int32_t plainBodyLen = pReadHead->bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pRead->pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
if (pRead->capacity < cryptedBodyLen) {
SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pRead->pHead = ptr;
pReadHead = &pRead->pHead->head;
pRead->capacity = pReadHead->bodyLen;
pRead->capacity = cryptedBodyLen;
}
if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) {
if (pReadHead->bodyLen < 0) {
if (cryptedBodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, cryptedBodyLen)) {
if (plainBodyLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%"PRIx64,
vgId, pReadHead->version, ver, tstrerror(terrno), id);
@ -359,6 +375,8 @@ int32_t walFetchBody(SWalReader *pRead) {
return -1;
}
decryptBody(&pRead->pWal->cfg, pRead->pHead, plainBodyLen, __FUNCTION__);
if (walValidBodyCksum(pRead->pHead) != 0) {
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver, id);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
@ -431,20 +449,28 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
return -1;
}
if (pReader->capacity < pReader->pHead->head.bodyLen) {
int32_t plainBodyLen = pReader->pHead->head.bodyLen;
int32_t cryptedBodyLen = plainBodyLen;
//TODO: dmchen emun
if(pReader->pWal->cfg.encryptAlgorithm == 1){
cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen);
}
if (pReader->capacity < cryptedBodyLen) {
SWalCkHead *ptr =
(SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReader->pHead->head.bodyLen);
(SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosThreadMutexUnlock(&pReader->mutex);
return -1;
}
pReader->pHead = ptr;
pReader->capacity = pReader->pHead->head.bodyLen;
pReader->capacity = cryptedBodyLen;
}
if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, pReader->pHead->head.bodyLen)) !=
pReader->pHead->head.bodyLen) {
if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) !=
cryptedBodyLen) {
if (contLen < 0)
terrno = TAOS_SYSTEM_ERROR(errno);
else {
@ -465,11 +491,13 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
return -1;
}
decryptBody(&pReader->pWal->cfg, pReader->pHead, plainBodyLen, __FUNCTION__);
code = walValidBodyCksum(pReader->pHead);
if (code != 0) {
wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId,
ver);
uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, pReader->pHead->head.bodyLen);
uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, plainBodyLen);
uint32_t logCkSum = pReader->pHead->cksumBody;
wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum);
// pReader->curInvalid = 1;
@ -484,6 +512,29 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
return 0;
}
void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func) {
//TODO: dmchen emun
if (cfg->encryptAlgorithm == 1) {
int32_t cryptedBodyLen = ENCRYPTED_LEN(plainBodyLen);
char *newBody = taosMemoryMalloc(cryptedBodyLen);
SCryptOpts opts;
opts.len = cryptedBodyLen;
opts.source = pHead->head.body;
opts.result = newBody;
opts.unitLen = 16;
strncpy(opts.key, cfg->encryptKey, 16);
int32_t count = CBC_Decrypt(&opts);
wInfo("CBC_Decrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", count, plainBodyLen, func);
memcpy(pHead->head.body, newBody, plainBodyLen);
taosMemoryFree(newBody);
}
}
void walReadReset(SWalReader *pReader) {
taosThreadMutexLock(&pReader->mutex);
taosCloseFile(&pReader->pIdxFile);

68
source/libs/wal/src/walWrite.c
View File

@ -18,6 +18,7 @@
#include "tchecksum.h"
#include "tglobal.h"
#include "walInt.h"
#include "crypt.h"
int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
taosThreadMutexLock(&pWal->mutex);
@ -491,12 +492,13 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta,
const void *body, int32_t bodyLen) {
int64_t code = 0;
int32_t plainBodyLen = bodyLen;
int64_t offset = walGetCurFileOffset(pWal);
SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal);
pWal->writeHead.head.version = index;
pWal->writeHead.head.bodyLen = bodyLen;
pWal->writeHead.head.bodyLen = plainBodyLen;
pWal->writeHead.head.msgType = msgType;
pWal->writeHead.head.ingestTs = taosGetTimestampUs();
@ -504,7 +506,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
pWal->writeHead.head.syncMeta = syncMeta;
pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead);
pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen);
pWal->writeHead.cksumBody = walCalcBodyCksum(body, plainBodyLen);
wDebug("vgId:%d, wal write log %" PRId64 ", msgType: %s, cksum head %u cksum body %u", pWal->cfg.vgId, index,
TMSG_INFO(msgType), pWal->writeHead.cksumHead, pWal->writeHead.cksumBody);
@ -521,22 +523,78 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
goto END;
}
if (taosWriteFile(pWal->pLogFile, (char *)body, bodyLen) != bodyLen) {
int32_t cyptedBodyLen = plainBodyLen;
char* buf = (char*)body;
char* newBody = NULL;
char* newBodyEncrypted = NULL;
//TODO: dmchen enum
if(pWal->cfg.encryptAlgorithm == 1){
cyptedBodyLen = ENCRYPTED_LEN(cyptedBodyLen);
char* newBody = taosMemoryMalloc(cyptedBodyLen);
if(newBody == NULL){
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
goto END;
}
memset(newBody, 0, cyptedBodyLen);
memcpy(newBody, body, plainBodyLen);
char* newBodyEncrypted = taosMemoryMalloc(cyptedBodyLen);
if(newBodyEncrypted == NULL){
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
if(newBodyEncrypted != NULL) taosMemoryFreeClear(newBodyEncrypted);
if(newBody != NULL) taosMemoryFreeClear(newBody);
goto END;
}
SCryptOpts opts;
opts.len = cyptedBodyLen;
opts.source = newBody;
opts.result = newBodyEncrypted;
opts.unitLen = 16;
strncpy(opts.key, pWal->cfg.encryptKey, 16);
int32_t count = CBC_Encrypt(&opts);
wInfo("vgId:%d, file:%" PRId64 ".log, index:%" PRId64 ", CBC_Encrypt cryptedBodyLen:%d, plainBodyLen:%d, %s",
pWal->cfg.vgId, walGetLastFileFirstVer(pWal), index, count, plainBodyLen, __FUNCTION__);
buf = newBodyEncrypted;
}
if (taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
code = -1;
//TODO: dmchen enum
if(pWal->cfg.encryptAlgorithm == 1){
taosMemoryFree(newBody);
taosMemoryFree(newBodyEncrypted);
}
goto END;
}
//TODO: dmchen enum
if(pWal->cfg.encryptAlgorithm == 1){
taosMemoryFree(newBody);
taosMemoryFree(newBodyEncrypted);
wInfo("vgId:%d, free newBody newBodyEncrypted %s",
pWal->cfg.vgId, __FUNCTION__);
}
// set status
if (pWal->vers.firstVer == -1) {
pWal->vers.firstVer = 0;
}
pWal->vers.lastVer = index;
pWal->totSize += sizeof(SWalCkHead) + bodyLen;
pWal->totSize += sizeof(SWalCkHead) + cyptedBodyLen;
pFileInfo->lastVer = index;
pFileInfo->fileSize += sizeof(SWalCkHead) + bodyLen;
pFileInfo->fileSize += sizeof(SWalCkHead) + cyptedBodyLen;
return 0;

36
source/util/src/tconfig.c
View File

@ -953,16 +953,38 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
paGetToken(name + olen + 1, &value, &vlen);
if (vlen == 0) continue;
value[vlen] = 0;
if (strcasecmp(name, "encryptScope") == 0) {
char* tmp = NULL;
int32_t len = 0;
char newValue[1024] = {0};
paGetToken(value + vlen + 1, &value2, &vlen2);
if (vlen2 != 0) {
value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0;
strcpy(newValue, value);
int32_t count = 1;
while(vlen < 1024){
paGetToken(value + vlen + 1 * count, &tmp, &len);
if(len == 0) break;
tmp[len] = 0;
strcpy(newValue + vlen, tmp);
vlen += len;
count++;
}
code = cfgSetItem(pConfig, name, newValue, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
}
else{
paGetToken(value + vlen + 1, &value2, &vlen2);
if (vlen2 != 0) {
value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0;
}
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
}
if (strcasecmp(name, "dataDir") == 0) {
code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE);

5
source/util/src/terror.c
View File

@ -243,6 +243,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST, "Index not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating status")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "encryption is not allowed to be changed after database is created.")
// mnode-node
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists")
@ -351,6 +352,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ONLY_TWO_MNODE, "Only two mnodes exist
TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NO_NEED_RESTORE, "No need to restore on this dnode")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE, "Please use this command when the dnode is offline")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NO_MACHINE_CODE, "Dnode can not get machine code")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NO_ENCRYPT_KEY, "Dnode can not get encryption key")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG, "invalid encryption configuration")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_ENCRYPTKEY, "invalid encryption key")
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED, "encryption key was changed")
// vnode
TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed")