From b9deb71fc5f983f8f6d0a509b15a11d38d4fc5b3 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 29 Mar 2024 02:48:49 +0000 Subject: [PATCH] basic function --- include/common/tglobal.h | 17 + include/libs/crypt/crypt.h | 39 ++ include/libs/sm4/sm4.h | 63 ++ include/libs/wal/wal.h | 3 + include/util/taoserror.h | 5 + include/util/tdef.h | 3 + source/common/src/tglobal.c | 13 + source/dnode/mgmt/CMakeLists.txt | 4 +- source/dnode/mgmt/exe/dmMain.c | 288 ++++++++++ source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 33 ++ source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 40 ++ source/dnode/mgmt/node_util/inc/dmUtil.h | 2 + source/dnode/mgmt/node_util/src/dmEps.c | 99 +++- source/dnode/mnode/impl/src/mndMain.c | 15 + source/dnode/mnode/sdb/CMakeLists.txt | 2 +- source/dnode/mnode/sdb/src/sdbFile.c | 66 ++- source/dnode/vnode/CMakeLists.txt | 3 + source/dnode/vnode/inc/vnode.h | 5 + source/dnode/vnode/src/meta/metaOpen.c | 3 +- source/dnode/vnode/src/tq/tqMeta.c | 2 +- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 15 +- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h | 3 +- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 142 +++-- source/dnode/vnode/src/tsdb/tsdbDataFileRW.h | 18 +- source/dnode/vnode/src/tsdb/tsdbDef.h | 8 +- source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 5 +- source/dnode/vnode/src/tsdb/tsdbFSetRAW.h | 3 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 117 +++- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 5 +- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 112 +++- source/dnode/vnode/src/tsdb/tsdbSttFileRW.h | 6 +- source/dnode/vnode/src/tsdb/tsdbUpgrade.c | 55 +- source/dnode/vnode/src/vnd/vnodeCfg.c | 43 ++ source/libs/CMakeLists.txt | 2 + source/libs/crypt/CMakeLists.txt | 14 + source/libs/crypt/src/crypt.c | 36 ++ source/libs/sm4/CMakeLists.txt | 9 + source/libs/sm4/src/sm4.c | 543 ++++++++++++++++++ source/libs/stream/src/streamMeta.c | 2 +- source/libs/stream/src/streamState.c | 2 +- source/libs/tdb/CMakeLists.txt | 1 + source/libs/tdb/inc/tdb.h | 3 +- source/libs/tdb/src/db/tdbDb.c | 8 +- source/libs/tdb/src/db/tdbPager.c | 59 +- source/libs/tdb/src/inc/tdbInt.h | 3 + source/libs/tdb/test/tdbExOVFLTest.cpp | 4 +- .../libs/tdb/test/tdbPageDefragmentTest.cpp | 10 +- source/libs/tdb/test/tdbPageRecycleTest.cpp | 8 +- source/libs/tdb/test/tdbTest.cpp | 12 +- source/libs/wal/CMakeLists.txt | 1 + source/libs/wal/src/walMeta.c | 17 +- source/libs/wal/src/walRead.c | 75 ++- source/libs/wal/src/walWrite.c | 68 ++- source/util/src/tconfig.c | 36 +- source/util/src/terror.c | 5 + 55 files changed, 1977 insertions(+), 178 deletions(-) create mode 100644 include/libs/crypt/crypt.h create mode 100644 include/libs/sm4/sm4.h create mode 100644 source/libs/crypt/CMakeLists.txt create mode 100644 source/libs/crypt/src/crypt.c create mode 100644 source/libs/sm4/CMakeLists.txt create mode 100644 source/libs/sm4/src/sm4.c diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 7a7f19c3af..bc18e577ab 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -29,6 +29,17 @@ extern "C" { #define SLOW_LOG_TYPE_OTHERS 0x4 #define SLOW_LOG_TYPE_ALL 0xFFFFFFFF +typedef enum { + DND_CA_SM4 = 1, +} EEncryptAlgor; + +typedef enum { + DND_CS_TSDB = 1, + DND_CS_VNODE_WAL = 2, + DND_CS_SDB = 4, + DND_CS_MNODE_WAL = 8, +} EEncryptScope; + // cluster extern char tsFirst[]; extern char tsSecond[]; @@ -39,6 +50,12 @@ extern uint16_t tsServerPort; extern int32_t tsVersion; extern int32_t tsStatusInterval; extern int32_t tsNumOfSupportVnodes; +extern char tsEncryptAlgorithm[]; +extern char tsEncryptScope[]; +extern EEncryptAlgor tsiEncryptAlgorithm; +extern EEncryptScope tsiEncryptScope; +//extern char tsAuthCode[]; +extern char tsEncryptKey[]; // common extern int32_t tsMaxShellConns; diff --git a/include/libs/crypt/crypt.h b/include/libs/crypt/crypt.h new file mode 100644 index 0000000000..c294877a57 --- /dev/null +++ b/include/libs/crypt/crypt.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _CRYPT_H_ +#define _CRYPT_H_ +#include "tdef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SCryptOpts{ + int32_t len; + char* source; + char* result; + int32_t unitLen; + unsigned char key[17]; +}SCryptOpts; + +int32_t CBC_Decrypt(SCryptOpts *opts); +int32_t CBC_Encrypt(SCryptOpts *opts); + +#ifdef __cplusplus +} +#endif + +#endif // _CRYPT_H_ \ No newline at end of file diff --git a/include/libs/sm4/sm4.h b/include/libs/sm4/sm4.h new file mode 100644 index 0000000000..659b082418 --- /dev/null +++ b/include/libs/sm4/sm4.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#ifndef _SM4_H_ +#define _SM4_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int SM4_ECB_Encrypt( unsigned char *pKey, + unsigned int KeyLen, + unsigned char *pInData, + unsigned int inDataLen, + unsigned char *pOutData, + unsigned int *pOutDataLen); + +int SM4_ECB_Decrypt( unsigned char *pKey, + unsigned int KeyLen, + unsigned char *pInData, + unsigned int inDataLen, + unsigned char *pOutData, + unsigned int *pOutDataLen); + +int SM4_CBC_Encrypt( unsigned char *pKey, + unsigned int KeyLen, + unsigned char *pIV, + unsigned int ivLen, + unsigned char *pInData, + unsigned int inDataLen, + unsigned char *pOutData, + unsigned int *pOutDataLen); + +int SM4_CBC_Decrypt(unsigned char *pKey, + unsigned int KeyLen, + unsigned char *pIV, + unsigned int ivLen, + unsigned char *pInData, + unsigned int inDataLen, + unsigned char *pOutData, + unsigned int *pOutDataLen); +#ifdef __cplusplus +} +#endif + +#endif // _SM4_H_ \ No newline at end of file diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 7c00ff5178..e5631cb749 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -57,6 +57,8 @@ typedef struct { int64_t retentionSize; int64_t segSize; EWalType level; // wal level + int32_t encryptAlgorithm; + char encryptKey[17]; } SWalCfg; typedef struct { @@ -196,6 +198,7 @@ SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id); void walCloseReader(SWalReader *pRead); void walReadReset(SWalReader *pReader); int32_t walReadVer(SWalReader *pRead, int64_t ver); +void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func); int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); int64_t walReaderGetCurrentVer(const SWalReader *pReader); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4c3990306a..c3ce38b226 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -311,6 +311,7 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) // 2.x #define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) // #define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A) +#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B) // mnode-node #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) @@ -418,6 +419,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal #define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416) #define TSDB_CODE_DNODE_NO_MACHINE_CODE TAOS_DEF_ERROR_CODE(0, 0x0417) +#define TSDB_CODE_DNODE_NO_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x0418) +#define TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0419) +#define TSDB_CODE_DNODE_INVALID_ENCRYPTKEY TAOS_DEF_ERROR_CODE(0, 0x0420) +#define TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED TAOS_DEF_ERROR_CODE(0, 0x0421) // mnode-sma #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) diff --git a/include/util/tdef.h b/include/util/tdef.h index fe24e4e6f6..0b80b8e358 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -187,6 +187,9 @@ typedef enum ELogicConditionType { LOGIC_COND_TYPE_NOT, } ELogicConditionType; +#define ENCRYPTED_LEN(len) (len/16) * 16 + (len%16?1:0) * 16 +#define ENCRYPT_KEY_LEN 16 + #define TSDB_INT32_ID_LEN 11 #define TSDB_NAME_DELIMITER_LEN 1 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3381d52050..211a79d403 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -41,6 +41,12 @@ uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; int32_t tsStatusInterval = 1; // second int32_t tsNumOfSupportVnodes = 256; +char tsEncryptAlgorithm[16] = {0}; +char tsEncryptScope[100] = {0}; +EEncryptAlgor tsiEncryptAlgorithm = 0; +EEncryptScope tsiEncryptScope = 0; +//char tsAuthCode[500] = {0}; +char tsEncryptKey[17] = {0}; // common int32_t tsMaxShellConns = 50000; @@ -593,6 +599,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + //if (cfgAddString(pCfg, "authCode", tsAuthCode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; @@ -1136,6 +1146,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; + tstrncpy(tsEncryptAlgorithm, cfgGetItem(pCfg, "encryptAlgorithm")->str, 16); + tstrncpy(tsEncryptScope, cfgGetItem(pCfg, "encryptScope")->str, 100); + //tstrncpy(tsAuthCode, cfgGetItem(pCfg, "authCode")->str, 100); tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32; diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 762b8fd529..3ee3175a11 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -23,8 +23,8 @@ ENDIF () IF (TD_LINUX_64 AND JEMALLOC_ENABLED) ADD_DEPENDENCIES(taosd jemalloc) - target_link_libraries(taosd dnode ${LINK_JEMALLOC}) + target_link_libraries(taosd dnode crypt ${LINK_JEMALLOC}) ELSE () - target_link_libraries(taosd dnode) + target_link_libraries(taosd dnode crypt) ENDIF () diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index f9456f1729..73aace44b7 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -22,6 +22,7 @@ #ifdef TD_JEMALLOC_ENABLED #include "jemalloc/jemalloc.h" #endif +#include "crypt.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -49,6 +50,10 @@ #define DM_VERSION "Print program version." #define DM_EMAIL "" #define DM_MEM_DBG "Enable memory debug" +#define DM_SET_ENCRYPTKEY "Set encrypt key. such as: -e 1234567890abcdef, the length should be less or equal to 16." +#define DM_KEY_INDICATOR "this indicator!" +#define DM_ENCRYPT_CODE_FILE "encryptCode.cfg" +#define DM_CHECK_CODE_FILE "checkCode.bin" // clang-format on static struct { #ifdef WINDOWS @@ -66,6 +71,8 @@ static struct { const char **envCmd; SArray *pArgs; // SConfigPair int64_t startTime; + bool generateCode; + char encryptKey[ENCRYPT_KEY_LEN + 1]; } global = {0}; static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetGlobalDebugFlag(143); } @@ -194,6 +201,18 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } } else if (strcmp(argv[i], "-k") == 0) { global.generateGrant = true; + } else if (strcmp(argv[i], "-y") == 0) { + global.generateCode = true; + if(i < argc - 1) { + if (strlen(argv[++i]) > ENCRYPT_KEY_LEN) { + printf("encrypt key overflow, it should be less or equal to %d\n", ENCRYPT_KEY_LEN); + return -1; + } + tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN); + } else { + printf("'-y' requires a parameter\n"); + return -1; + } } else if (strcmp(argv[i], "-C") == 0) { global.dumpConfig = true; } else if (strcmp(argv[i], "-V") == 0) { @@ -256,6 +275,7 @@ static void dmPrintHelp() { printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD); printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE); printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE); + printf("%s%s%s%s\n", indent, "-y,", indent, DM_SET_ENCRYPTKEY); printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG); printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION); @@ -275,6 +295,260 @@ static void taosCleanupArgs() { if (global.envCmd != NULL) taosMemoryFreeClear(global.envCmd); } +static int32_t compareCheckCode(char* file, char* key){ + char *content = NULL; + int64_t size = 0; + TdFilePtr pFile = NULL; + char *result = NULL; + int32_t code = -1; + + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open dnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat dnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + content = taosMemoryMalloc(size); + if (content == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + if (taosReadFile(pFile, content, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read dnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + dInfo("succeed to read checkCode file:%s", file); + + int len = ENCRYPTED_LEN(size); + result = taosMemoryMalloc(len); + + SCryptOpts opts = {0}; + strncpy(opts.key, key, ENCRYPT_KEY_LEN); + opts.len = len; + opts.source = content; + opts.result = result; + opts.unitLen = 16; + CBC_Decrypt(&opts); + + if(strcmp(opts.result, DM_KEY_INDICATOR) != 0) { + terrno = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED; + dError("failed to compare decrypted result"); + goto _OVER; + } + + dInfo("succeed to compare checkCode file:%s", file); + code = 0; +_OVER: + if(result != NULL) taosMemoryFree(result); + if(content != NULL) taosMemoryFree(content); + if(pFile != NULL) taosCloseFile(&pFile); + + return code; +} + +static int32_t writeCheckCode(char* file, char* realfile, char* key){ + TdFilePtr pFile = NULL; + char *result = NULL; + int32_t code = -1; + + int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR)); + result = taosMemoryMalloc(len); + + SCryptOpts opts; + strncpy(opts.key, key, ENCRYPT_KEY_LEN); + opts.len = len; + opts.source = DM_KEY_INDICATOR; + opts.result = result; + opts.unitLen = 16; + CBC_Encrypt(&opts); + + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); + if (pFile == NULL) goto _OVER; + + if (taosWriteFile(pFile, opts.result, len) <= 0) goto _OVER; + if (taosFsyncFile(pFile) < 0) goto _OVER; + + taosCloseFile(&pFile); + if (taosRenameFile(file, realfile) != 0) goto _OVER; + + dInfo("succeed to write checkCode file:%s", realfile); + code = 0; +_OVER: + if(pFile != NULL) taosCloseFile(&pFile); + if(result != NULL) taosMemoryFree(result); + + return code; +} + +static int32_t readEncryptCode(char* file, char** output){ + TdFilePtr pFile = NULL; + int32_t code = -1; + char *content = NULL; + + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open dnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat dnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + content = taosMemoryMalloc(size + 1); + if (content == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + if (taosReadFile(pFile, content, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read dnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + content[size] = '\0'; + + *output = content; + + dInfo("succeed to read encryptCode file:%s", file); + code = 0; +_OVER: + if(pFile != NULL) taosCloseFile(&pFile); + + return code; +} + +static int32_t writeEncryptCode(char* file, char* realfile, char* encryptCode){ + TdFilePtr pFile = NULL; + int32_t code = -1; + + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); + if (pFile == NULL) goto _OVER; + + int32_t len = strlen(encryptCode); + if (taosWriteFile(pFile, encryptCode, len) <= 0) goto _OVER; + if (taosFsyncFile(pFile) < 0) goto _OVER; + + taosCloseFile(&pFile); + if (taosRenameFile(file, realfile) != 0) goto _OVER; + + dInfo("succeed to write encryptCode file:%s", realfile); + + code = 0; +_OVER: + if(pFile != NULL) taosCloseFile(&pFile); + + return code; +} + +static int32_t getEncryptKey(){ + int32_t code = -1; + char encryptFile[PATH_MAX] = {0}; + char checkFile[PATH_MAX] = {0}; + + snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_ENCRYPT_CODE_FILE); + snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE); + + if(!taosCheckExistFile(encryptFile)){ + dInfo("no exist, checkCode file:%s", encryptFile); + return 0; + } + + char *content = NULL; + if(readEncryptCode(encryptFile, &content) != 0){ + goto _OVER; + } + + //TODO: dmchen parse key from code + + strncpy(tsEncryptKey, content, ENCRYPT_KEY_LEN); + + taosMemoryFreeClear(content); + + if(tsEncryptKey[0] == '\0'){ + terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; + dError("failed to read key since %s", terrstr()); + goto _OVER; + } + + if(compareCheckCode(checkFile, tsEncryptKey) != 0){ + goto _OVER; + } + + code = 0; +_OVER: + if (content != NULL) taosMemoryFree(content); + if (code != 0) { + if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to get encrypt key since %s", terrstr()); + } + return code; +} + +int32_t updateEncryptKey(char *key) { + int32_t code = -1; + + char folder[PATH_MAX] = {0}; + + char encryptFile[PATH_MAX] = {0}; + char realEncryptFile[PATH_MAX] = {0}; + + char checkFile[PATH_MAX] = {0}; + char realCheckFile[PATH_MAX] = {0}; + + snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP); + snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE); + snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE); + snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE); + snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE); + + terrno = 0; + + if (taosMkDir(folder) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", folder, terrstr()); + goto _OVER; + } + + if(taosCheckExistFile(realCheckFile)){ + if(compareCheckCode(realCheckFile, key) != 0){ + goto _OVER; + } + } + + if(writeEncryptCode(encryptFile, realEncryptFile, key) != 0){ + goto _OVER; + } + + if(writeCheckCode(checkFile, realCheckFile, key) != 0){ + goto _OVER; + } + + code = 0; +_OVER: + + if (code != 0) { + if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to update encrypt key since %s", terrstr()); + } + return code; +} + int main(int argc, char const *argv[]) { #ifdef TD_JEMALLOC_ENABLED bool jeBackgroundThread = true; @@ -348,6 +622,20 @@ int mainWindows(int argc, char **argv) { return -1; } + if(global.generateCode) { + updateEncryptKey(global.encryptKey); + taosCloseLog(); + taosCleanupArgs(); + return 0; + } + + if(getEncryptKey() != 0){ + dError("failed to start since failed to get encrypt key"); + taosCloseLog(); + taosCleanupArgs(); + return -1; + }; + if (taosConvInit() != 0) { dError("failed to init conv"); taosCloseLog(); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 722eaf1fd3..f0dae8fb89 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -142,6 +142,14 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { if ((pRetention->freq >= 0 && pRetention->keep > 0)) pCfg->isRsma = 1; } } +#if defined(TD_ENTERPRISE) + pCfg->tsdbCfg.encryptAlgorithm = DND_CA_SM4; + if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){ + strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + } +#else + pCfg->walCfg.cryptAlgorithm = 0; +#endif pCfg->walCfg.vgId = pCreate->vgId; pCfg->walCfg.fsyncPeriod = pCreate->walFsyncPeriod; @@ -150,6 +158,23 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->walCfg.retentionSize = pCreate->walRetentionSize; pCfg->walCfg.segSize = pCreate->walSegmentSize; pCfg->walCfg.level = pCreate->walLevel; +#if defined(TD_ENTERPRISE) + pCfg->walCfg.encryptAlgorithm = DND_CA_SM4; + if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){ + strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + } +#else + pCfg->walCfg.cryptAlgorithm = 0; +#endif + +#if defined(TD_ENTERPRISE) + pCfg->tdbEncryptAlgorithm = DND_CA_SM4; + if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){ + strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + } +#else + pCfg->tdbEncryptKey = 0; +#endif pCfg->sttTrigger = pCreate->sstTrigger; pCfg->hashBegin = pCreate->hashBegin; @@ -273,6 +298,14 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } + //if(req.encryptAlgorithm == DND_CA_SM4){ + if(strlen(tsEncryptKey) == 0){ + terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; + dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId); + return -1; + } + //} + vmGenerateVnodeCfg(&req, &vnodeCfg); if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) { diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 049b1bdf84..11453fcfef 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -161,6 +161,46 @@ int32_t dmInitVars(SDnode *pDnode) { return -1; } +#if defined(TD_ENTERPRISE) + tsiEncryptAlgorithm = pData->encryptAlgorigthm; + tsiEncryptScope = pData->encryptScope; + /* + if(tsiEncryptAlgorithm != 0) { + if(pData->machineId != NULL && strlen(pData->machineId) > 0){ + dInfo("get crypt key at startup, machineId:%s", pData->machineId); + int32_t code = 0; + + //code = taosGetCryptKey(tsAuthCode, pData->machineId, tsCryptKey); + code = 0; + strncpy(tsEncryptKey, tsAuthCode, 16); + + if(code != 0) { + if(code == -1){ + terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY; + dError("machine code changed, can't get crypt key"); + } + if(code == -2){ + terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY; + dError("failed to get crypt key"); + } + return -1; + } + + if(strlen(tsEncryptKey) == 0){ + terrno = TSDB_CODE_DNODE_NO_ENCRYPT_KEY; + dError("failed to get crypt key at startup since key is null, machineId:%s", pData->machineId); + return -1; + } + } + else{ + terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; + dError("failed to get crypt key at startup, machineId:%s", pData->machineId); + return -1; + } + } + */ +#endif + if (pData->dropped) { dError("dnode will not start since its already dropped"); return -1; diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 0dcb77d9df..49bae11eab 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -110,6 +110,8 @@ typedef struct { bool validMnodeEps; int64_t ipWhiteVer; char machineId[TSDB_MACHINE_ID_LEN + 1]; + EEncryptAlgor encryptAlgorigthm; + EEncryptScope encryptScope; } SDnodeData; typedef struct { diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 2c341fbd4d..4b41b17cb1 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -64,7 +64,12 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) { if (code < 0) return -1; tjsonGetInt32ValueFromDouble(pJson, "dropped", pData->dropped, code); if (code < 0) return -1; - +#ifdef TD_ENTERPRISE + tjsonGetInt32ValueFromDouble(pJson, "encryptAlgor", pData->encryptAlgorigthm, code); + if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(pJson, "encryptScope", pData->encryptScope, code); + if (code < 0) return -1; +#endif SJson *dnodes = tjsonGetObjectItem(pJson, "dnodes"); if (dnodes == NULL) return 0; int32_t numOfDnodes = tjsonGetArraySize(dnodes); @@ -89,6 +94,25 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) { return 0; } +int dmOccurrences(char *str, char *toSearch) { + int count = 0; + char *ptr = str; + while ((ptr = strstr(ptr, toSearch)) != NULL) { + count++; + ptr++; + } + return count; +} + +void dmSplitStr(char** arr, char* str, const char* del) { + char *lasts; + char* s = strsep(&str, del); + while (s != NULL) { + *arr++ = s; + s = strsep(&str, del); + } +} + int32_t dmReadEps(SDnodeData *pData) { int32_t code = -1; TdFilePtr pFile = NULL; @@ -106,6 +130,74 @@ int32_t dmReadEps(SDnodeData *pData) { if (taosStatFile(file, NULL, NULL, NULL) < 0) { dInfo("dnode file:%s not exist", file); + +#ifdef TD_ENTERPRISE + if(strlen(tsEncryptAlgorithm) > 0){ + if(strcmp(tsEncryptAlgorithm, "sm4") == 0) { + pData->encryptAlgorigthm = DND_CA_SM4; + } + else{ + terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG; + dError("invalid tsEncryptAlgorithm:%s", tsEncryptAlgorithm); + goto _OVER; + } + + dInfo("start to parse encryptScope:%s", tsEncryptScope); + int32_t scopeLen = strlen(tsEncryptScope); + if(scopeLen == 0){ + terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG; + dError("invalid tsEncryptScope:%s", tsEncryptScope); + goto _OVER; + } + + char* tmp = taosMemoryMalloc(scopeLen + 1); + memset(tmp, 0, scopeLen + 1); + memcpy(tmp, tsEncryptScope, scopeLen); + + int32_t count = dmOccurrences(tmp, ","); + + char** array = taosMemoryMalloc(sizeof(char*) * (count + 1)); + memset(array, 0, sizeof(char*) * (count + 1)); + dmSplitStr(array, tmp, ","); + + for(int32_t i = 0; i < count + 1; i++){ + char* str = *(array + i); + + bool success = false; + + if(strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0){ + pData->encryptScope |= DND_CS_TSDB; + success = true; + } + if(strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0){ + pData->encryptScope |= DND_CS_VNODE_WAL; + success = true; + } + if(strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0){ + pData->encryptScope |= DND_CS_SDB; + success = true; + } + if(strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0){ + pData->encryptScope |= DND_CS_MNODE_WAL; + success = true; + } + + if(!success){ + terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG; + taosMemoryFree(tmp); + taosMemoryFree(array); + dError("invalid tsEncryptScope:%s", tsEncryptScope); + goto _OVER; + } + } + + taosMemoryFree(tmp); + taosMemoryFree(array); + + dInfo("set tsCryptAlgorithm:%s, tsCryptScope:%s from cfg", tsEncryptAlgorithm, tsEncryptScope); + } + +#endif code = 0; goto _OVER; } @@ -191,7 +283,10 @@ static int32_t dmEncodeEps(SJson *pJson, SDnodeData *pData) { if (tjsonAddIntegerToObject(pJson, "engineVer", pData->engineVer) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "clusterId", pData->clusterId) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "dropped", pData->dropped) < 0) return -1; - +#ifdef TD_ENTERPRISE + if (tjsonAddDoubleToObject(pJson, "encryptAlgor", pData->encryptAlgorigthm) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "encryptScope", pData->encryptScope) < 0) return -1; +#endif SJson *dnodes = tjsonCreateArray(); if (dnodes == NULL) return -1; if (tjsonAddItemToObject(pJson, "dnodes", dnodes) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index ee7cd83a8c..75ba0fef10 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -472,8 +472,23 @@ static int32_t mndInitWal(SMnode *pMnode) { .retentionPeriod = 0, .retentionSize = 0, .level = TAOS_WAL_FSYNC, + .encryptAlgorithm = 0, + .encryptKey = {0} }; +#if defined(TD_ENTERPRISE) + if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){ + cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0; + if(tsEncryptKey[0] == '\0'){ + terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; + return -1; + } + else{ + strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + } + } +#endif + pMnode->pWal = walOpen(path, &cfg); if (pMnode->pWal == NULL) { mError("failed to open wal since %s. wal:%s", terrstr(), path); diff --git a/source/dnode/mnode/sdb/CMakeLists.txt b/source/dnode/mnode/sdb/CMakeLists.txt index 186c85004a..63f54abfcc 100644 --- a/source/dnode/mnode/sdb/CMakeLists.txt +++ b/source/dnode/mnode/sdb/CMakeLists.txt @@ -5,5 +5,5 @@ target_include_directories( PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - sdb os common util wal sync + sdb os common util wal sync crypt ) diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 2e182ec10b..69983eac30 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -18,6 +18,8 @@ #include "sync.h" #include "tchecksum.h" #include "wal.h" +#include "tglobal.h" +#include "crypt.h" #define SDB_TABLE_SIZE 24 #define SDB_RESERVE_SIZE 512 @@ -276,6 +278,9 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { } readLen = pRaw->dataLen + sizeof(int32_t); + if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ + readLen = ENCRYPTED_LEN(pRaw->dataLen) + sizeof(int32_t); + } if (readLen >= bufLen) { bufLen = pRaw->dataLen * 2; SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100); @@ -303,6 +308,27 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { goto _OVER; } + if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ + int32_t count = 0; + + char *plantContent = taosMemoryMalloc(ENCRYPTED_LEN(pRaw->dataLen)); + + SCryptOpts opts; + opts.len = ENCRYPTED_LEN(pRaw->dataLen); + opts.source = pRaw->pData; + opts.result = plantContent; + opts.unitLen = 16; + strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN); + + count = CBC_Decrypt(&opts); + + mInfo("read sdb, CBC_Decrypt dataLen:%d, descrypted len:%d, %s", pRaw->dataLen, count, __FUNCTION__); + + memcpy(pRaw->pData, plantContent, pRaw->dataLen); + taosMemoryFree(plantContent); + memcpy(pRaw->pData + pRaw->dataLen, &pRaw->pData[ENCRYPTED_LEN(pRaw->dataLen)], sizeof(int32_t)); + } + int32_t totalLen = sizeof(SSdbRaw) + pRaw->dataLen + sizeof(int32_t); if ((!taosCheckChecksumWhole((const uint8_t *)pRaw, totalLen)) != 0) { code = TSDB_CODE_CHECKSUM_ERROR; @@ -401,14 +427,50 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { SSdbRaw *pRaw = (*encodeFp)(pRow->pObj); if (pRaw != NULL) { pRaw->status = pRow->status; - int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen; - if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) { + + if (taosWriteFile(pFile, pRaw, sizeof(SSdbRaw)) != sizeof(SSdbRaw)) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); break; } + int32_t newDataLen = pRaw->dataLen; + char* newData = pRaw->pData; + if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ + newDataLen = ENCRYPTED_LEN(pRaw->dataLen); + newData = taosMemoryMalloc(newDataLen); + if (newData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosHashCancelIterate(hash, ppRow); + sdbFreeRaw(pRaw); + break; + } + + SCryptOpts opts; + opts.len = newDataLen; + opts.source = pRaw->pData; + opts.result = newData; + opts.unitLen = 16; + strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN); + + int32_t count = CBC_Encrypt(&opts); + + mInfo("write sdb, CBC_Encrypt encryptedDataLen:%d, dataLen:%d, %s", + newDataLen, pRaw->dataLen, __FUNCTION__); + } + + if (taosWriteFile(pFile, newData, newDataLen) != newDataLen) { + code = TAOS_SYSTEM_ERROR(errno); + taosHashCancelIterate(hash, ppRow); + sdbFreeRaw(pRaw); + break; + } + + if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ + taosMemoryFree(newData); + } + int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen); if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { code = TAOS_SYSTEM_ERROR(errno); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 2c57eb3cb5..968e551f8a 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -121,6 +121,7 @@ if (${BUILD_CONTRIB}) PUBLIC "inc" PUBLIC "src/inc" PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" + PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt" PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include" ) @@ -130,6 +131,7 @@ else() PUBLIC "inc" PUBLIC "src/inc" PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" + PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt" PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" ) if (${TD_LINUX}) @@ -157,6 +159,7 @@ target_link_libraries( PUBLIC scheduler PUBLIC tdb PUBLIC audit + PUBLIC crypt # PUBLIC bdb # PUBLIC scalar diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 8ae7e9706d..5157bbcdae 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -272,6 +272,9 @@ struct STsdbCfg { int32_t keep2; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead int32_t keepTimeOffset; // just for save config, use STsdbKeepCfg in STsdb instead SRetention retentions[TSDB_RETENTION_MAX]; + int32_t encryptAlgorithm; + //TODO dmchen len + char encryptKey[17]; }; typedef struct { @@ -313,6 +316,8 @@ struct SVnodeCfg { int16_t hashPrefix; int16_t hashSuffix; int32_t tsdbPageSize; + int32_t tdbEncryptAlgorithm; + char tdbEncryptKey[ENCRYPT_KEY_LEN]; }; #define TABLE_ROLLUP_ON ((int8_t)0x1) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 4a3dcd0e51..e3b3de6cd6 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -69,7 +69,8 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { taosMkDir(pMeta->path); // open env - ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback); + ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback, + pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey); if (ret < 0) { metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index e64de9a423..82547af8d4 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -76,7 +76,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { } int32_t tqMetaOpen(STQ* pTq) { - if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0) < 0) { + if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL) < 0) { return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index d2e3cb08e5..416c03829e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -72,7 +72,9 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo pBlock->file.maxVer = reader->config->file.maxVer; pBlock->file.stt->level = reader->config->file.stt->level; - code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -126,8 +128,11 @@ static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFil code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + if (writer->fd) { - code = tsdbFsyncFile(writer->fd); + code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); tsdbCloseFile(&writer->fd); } @@ -205,11 +210,13 @@ _exit: return code; } -int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock) { +int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock, + int32_t encryptAlgorithm, char* encryptKey) { int32_t code = 0; int32_t lino = 0; - code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength); + code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); writer->ctx->offset += pDataBlock->dataLength; diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h index d765671698..c493c66460 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h @@ -105,7 +105,8 @@ int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataF int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr); int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer); -int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr); +int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr, + int32_t encryptAlgorithm, char* encryptKey); int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index e1625c9ddb..df1ad819d4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -44,8 +44,10 @@ static int32_t tsdbDataFileReadHeadFooter(SDataFileReader *reader) { int32_t ftype = TSDB_FTYPE_HEAD; if (reader->fd[ftype]) { + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(SHeadFooter), - (uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0); + (uint8_t *)reader->headFooter, sizeof(SHeadFooter), 0, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } @@ -66,8 +68,10 @@ static int32_t tsdbDataFileReadTombFooter(SDataFileReader *reader) { int32_t ftype = TSDB_FTYPE_TOMB; if (reader->fd[ftype]) { + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(STombFooter), - (uint8_t *)reader->tombFooter, sizeof(STombFooter), 0); + (uint8_t *)reader->tombFooter, sizeof(STombFooter), 0, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } reader->ctx->tombFooterLoaded = true; @@ -160,8 +164,10 @@ int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **b TSDB_CHECK_CODE(code, lino, _exit); } + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], reader->headFooter->brinBlkPtr->offset, data, - reader->headFooter->brinBlkPtr->size, 0); + reader->headFooter->brinBlkPtr->size, 0, encryptAlgorithm, encryptKey); if (code) { taosMemoryFree(data); TSDB_CHECK_CODE(code, lino, _exit); @@ -191,8 +197,11 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB code = tRealloc(&reader->config->bufArr[0], brinBlk->dp->size); TSDB_CHECK_CODE(code, lino, _exit); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = - tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, reader->config->bufArr[0], brinBlk->dp->size, 0); + tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, reader->config->bufArr[0], brinBlk->dp->size, 0, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); int32_t size = 0; @@ -233,8 +242,11 @@ int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *re code = tRealloc(&reader->config->bufArr[0], record->blockSize); TSDB_CHECK_CODE(code, lino, _exit); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = - tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockSize, 0); + tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockSize, 0, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); code = tDecmprBlockData(reader->config->bufArr[0], record->blockSize, bData, &reader->config->bufArr[1]); @@ -259,8 +271,10 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe code = tRealloc(&reader->config->bufArr[0], record->blockKeySize); TSDB_CHECK_CODE(code, lino, _exit); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockKeySize, - 0); + 0, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); // hdr @@ -297,8 +311,10 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe code = tRealloc(&reader->config->bufArr[0], hdr->szBlkCol); TSDB_CHECK_CODE(code, lino, _exit); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize, - reader->config->bufArr[0], hdr->szBlkCol, 0); + reader->config->bufArr[0], hdr->szBlkCol, 0, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } @@ -374,9 +390,11 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe code = tRealloc(&reader->config->bufArr[1], size1); TSDB_CHECK_CODE(code, lino, _exit); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize + hdr->szBlkCol + blockCol->offset, - reader->config->bufArr[1], size1, i > 0 ? 0 : szHint); + reader->config->bufArr[1], size1, i > 0 ? 0 : szHint, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData, @@ -404,7 +422,10 @@ int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *rec code = tRealloc(&reader->config->bufArr[0], record->smaSize); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbReadFile(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, reader->config->bufArr[0], record->smaSize, 0); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, reader->config->bufArr[0], record->smaSize, 0, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); // decode sma data @@ -442,8 +463,10 @@ int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **t TSDB_CHECK_CODE(code, lino, _exit); } + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], reader->tombFooter->tombBlkPtr->offset, data, - reader->tombFooter->tombBlkPtr->size, 0); + reader->tombFooter->tombBlkPtr->size, 0, encryptAlgorithm, encryptKey); if (code) { taosMemoryFree(data); TSDB_CHECK_CODE(code, lino, _exit); @@ -473,8 +496,11 @@ int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombB code = tRealloc(&reader->config->bufArr[0], tombBlk->dp->size); TSDB_CHECK_CODE(code, lino, _exit); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = - tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size, 0); + tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size, 0, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); int32_t size = 0; @@ -695,7 +721,8 @@ int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxV } int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, - TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range) { + TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range, + int32_t encryptAlgorithm, char* encryptKey) { if (BRIN_BLOCK_SIZE(brinBlock) == 0) return 0; int32_t code; @@ -742,7 +769,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAl TSDB_DATA_TYPE_BIGINT, brinBlk->cmprAlg, &bufArr[0], 0, &brinBlk->size[i], &bufArr[1]); if (code) return code; - code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[i]); + code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[i], encryptAlgorithm, encryptKey); if (code) return code; brinBlk->dp->size += brinBlk->size[i]; @@ -754,7 +781,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAl TSDB_DATA_TYPE_INT, brinBlk->cmprAlg, &bufArr[0], 0, &brinBlk->size[j], &bufArr[1]); if (code) return code; - code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[j]); + code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[j], encryptAlgorithm, encryptKey); if (code) return code; brinBlk->dp->size += brinBlk->size[j]; @@ -785,9 +812,12 @@ static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg, &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->config->bufArr, - &writer->ctx->range); + &writer->ctx->range, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -865,10 +895,13 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData record->blockKeySize = sizeArr[3] + sizeArr[2]; record->blockSize = sizeArr[0] + sizeArr[1] + record->blockKeySize; + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + for (int32_t i = 3; i >= 0; --i) { if (sizeArr[i]) { code = tsdbWriteFile(writer->fd[TSDB_FTYPE_DATA], writer->files[TSDB_FTYPE_DATA].size, writer->config->bufArr[i], - sizeArr[i]); + sizeArr[i], encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); writer->files[TSDB_FTYPE_DATA].size += sizeArr[i]; } @@ -892,7 +925,8 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData } if (record->smaSize > 0) { - code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, writer->config->bufArr[0], record->smaSize); + code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, writer->config->bufArr[0], record->smaSize, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); writer->files[TSDB_FTYPE_SMA].size += record->smaSize; } @@ -924,8 +958,12 @@ static int32_t tsdbDataFileWriteDataBlk(SDataFileWriter *writer, const TDataBlkA .size = TARRAY2_DATA_LEN(dataBlkArray), }}; + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = - tsdbWriteFile(writer->fd[ftype], blockIdx->offset, (const uint8_t *)TARRAY2_DATA(dataBlkArray), blockIdx->size); + tsdbWriteFile(writer->fd[ftype], blockIdx->offset, (const uint8_t *)TARRAY2_DATA(dataBlkArray), blockIdx->size, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); writer->files[ftype].size += blockIdx->size; @@ -1197,15 +1235,17 @@ _exit: return code; } -int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer) { - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); +int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm, + char* encryptKey) { + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey); if (code) return code; *fileSize += sizeof(*footer); return 0; } int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, - TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range) { + TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range, + int32_t encryptAlgorithm, char* encryptKey) { int32_t code; if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0; @@ -1248,7 +1288,7 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, &bufArr[0], 0, &tombBlk->size[i], &bufArr[1]); if (code) return code; - code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i]); + code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i], encryptAlgorithm, encryptKey); if (code) return code; tombBlk->dp->size += tombBlk->size[i]; @@ -1266,7 +1306,11 @@ static int32_t tsdbDataFileWriteHeadFooter(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter); + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + + code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -1282,9 +1326,12 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr, - &writer->ctx->tombRange); + &writer->ctx->tombRange, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -1294,12 +1341,14 @@ _exit: return code; } -int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) { +int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize, + int32_t encryptAlgorithm, char* encryptKey) { ptr->size = TARRAY2_DATA_LEN(tombBlkArray); if (ptr->size > 0) { ptr->offset = *fileSize; - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size); + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size, + encryptAlgorithm, encryptKey); if (code) { return code; } @@ -1315,8 +1364,11 @@ static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbFileWriteTombBlk(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlkArray, writer->tombFooter->tombBlkPtr, - &writer->files[TSDB_FTYPE_TOMB].size); + &writer->files[TSDB_FTYPE_TOMB].size, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -1326,8 +1378,9 @@ _exit: return code; } -int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize) { - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); +int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm, + char* encryptKey) { + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey); if (code) return code; *fileSize += sizeof(*footer); return 0; @@ -1337,7 +1390,11 @@ static int32_t tsdbDataFileWriteTombFooter(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size); + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + + code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -1414,12 +1471,14 @@ _exit: return code; } -int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize) { +int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize, + int32_t encryptAlgorithm, char* encryptKey) { ASSERT(TARRAY2_SIZE(brinBlkArray) > 0); ptr->offset = *fileSize; ptr->size = TARRAY2_DATA_LEN(brinBlkArray); - int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size); + int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size, encryptAlgorithm, + encryptKey); if (code) return code; *fileSize += ptr->size; @@ -1430,8 +1489,11 @@ static int32_t tsdbDataFileWriteBrinBlk(SDataFileWriter *writer) { int32_t code = 0; int32_t lino = 0; + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbFileWriteBrinBlk(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlkArray, writer->headFooter->brinBlkPtr, - &writer->files[TSDB_FTYPE_HEAD].size); + &writer->files[TSDB_FTYPE_HEAD].size, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -1588,10 +1650,11 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr code = TARRAY2_APPEND(opArr, op); TSDB_CHECK_CODE(code, lino, _exit); } - + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { if (writer->fd[i]) { - code = tsdbFsyncFile(writer->fd[i]); + code = tsdbFsyncFile(writer->fd[i], encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); tsdbCloseFile(&writer->fd[i]); } @@ -1627,7 +1690,10 @@ static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) { if (writer->files[ftype].size == 0) { uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE); + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + + code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); writer->files[ftype].size += TSDB_FHDR_SIZE; @@ -1794,7 +1860,9 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) { TSDB_CHECK_CODE(code, lino, _exit); uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE); + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); writer->files[ftype].size += TSDB_FHDR_SIZE; diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h index c4aed6e787..e62e478462 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h @@ -97,16 +97,22 @@ int32_t tsdbDataFileFlush(SDataFileWriter *writer); // head int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, - TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range); -int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize); -int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer); + TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range, + int32_t encryptAlgorithm, char* encryptKey); +int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize, + int32_t encryptAlgorithm, char* encryptKey); +int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm, + char* encryptKey); // tomb int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record); int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, - TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range); -int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); -int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize); + TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range, + int32_t encryptAlgorithm, char* encryptKey); +int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize, + int32_t encryptAlgorithm, char* encryptKey); +int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm, + char* encryptKey); // utils int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer); diff --git a/source/dnode/vnode/src/tsdb/tsdbDef.h b/source/dnode/vnode/src/tsdb/tsdbDef.h index 0f512e1306..253796e3c0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDef.h +++ b/source/dnode/vnode/src/tsdb/tsdbDef.h @@ -33,9 +33,11 @@ typedef struct SFDataPtr { extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD); extern void tsdbCloseFile(STsdbFD **ppFD); -extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size); -extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint); -extern int32_t tsdbFsyncFile(STsdbFD *pFD); +extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, + char* encryptKey); +extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint, + int32_t encryptAlgorithm, char* encryptKey); +extern int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c index ddb8e7e27e..b180f9d099 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -152,7 +152,8 @@ _exit: return code; } -int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) { +int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm, + char* encryptKey) { int32_t code = 0; int32_t lino = 0; @@ -166,7 +167,7 @@ int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeade TSDB_CHECK_CODE(code, lino, _exit); } - code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr); + code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); writer->ctx->offset += bHdr->dataLength; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h index 205c785e99..95e360b231 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h @@ -36,7 +36,8 @@ typedef struct SFSetRAWWriter SFSetRAWWriter; int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer); int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr); -int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr); +int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm, + char* encryptKey); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index babf8c75fb..46e0e01dc2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -15,6 +15,7 @@ #include "cos.h" #include "tsdb.h" +#include "crypt.h" static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { int32_t code = 0; @@ -131,7 +132,7 @@ void tsdbCloseFile(STsdbFD **ppFD) { } } -static int32_t tsdbWriteFilePage(STsdbFD *pFD) { +static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) { int32_t code = 0; if (!pFD->pFD) { @@ -153,6 +154,28 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD) { } taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); + + if(encryptAlgorithm == DND_CA_SM4){ + //if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){ + unsigned char PacketData[128]; + int NewLen; + int32_t count = 0; + while (count < pFD->szPage) { + SCryptOpts opts = {0}; + opts.len = 128; + opts.source = pFD->pBuf + count; + opts.result = PacketData; + opts.unitLen = 128; + //strncpy(opts.key, tsEncryptKey, 16); + strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + + NewLen = CBC_Encrypt(&opts); + + memcpy(pFD->pBuf + count, PacketData, NewLen); + count += NewLen; + } + tsdbInfo("CBC_Encrypt count:%d %s", count, __FUNCTION__); + } n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); if (n < 0) { @@ -170,7 +193,7 @@ _exit: return code; } -static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { +static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char* encryptKey) { int32_t code = 0; // ASSERT(pgno <= pFD->szFile); @@ -219,6 +242,30 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { code = TSDB_CODE_FILE_CORRUPTED; goto _exit; } + + if(encryptAlgorithm == DND_CA_SM4){ + //if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_TSDB) == DND_CS_TSDB){ + unsigned char PacketData[128]; + int NewLen; + + int32_t count = 0; + while(count < pFD->szPage) + { + SCryptOpts opts = {0}; + opts.len = 128; + opts.source = pFD->pBuf + count; + opts.result = PacketData; + opts.unitLen = 128; + //strncpy(opts.key, tsEncryptKey, 16); + strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + + NewLen = CBC_Decrypt(&opts); + + memcpy(pFD->pBuf + count, PacketData, NewLen); + count += NewLen; + } + tsdbInfo("CBC_Decrypt count:%d %s", count, __FUNCTION__); + } } // check @@ -233,7 +280,8 @@ _exit: return code; } -int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) { +int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, + char* encryptKey) { int32_t code = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); @@ -242,11 +290,11 @@ int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t do { if (pFD->pgno != pgno) { - code = tsdbWriteFilePage(pFD); + code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey); if (code) goto _exit; if (pgno <= pFD->szFile) { - code = tsdbReadFilePage(pFD, pgno); + code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey); if (code) goto _exit; } else { pFD->pgno = pgno; @@ -265,7 +313,8 @@ _exit: return code; } -static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { +static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, + char* encryptKey) { int32_t code = 0; int64_t n = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); @@ -278,7 +327,7 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6 while (n < size) { if (pFD->pgno != pgno) { - code = tsdbReadFilePage(pFD, pgno); + code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey); if (code) goto _exit; } @@ -395,7 +444,8 @@ _exit: return code; } -int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) { +int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint, + int32_t encryptAlgorithm, char* encryptKey) { int32_t code = 0; if (!pFD->pFD) { code = tsdbOpenFileImpl(pFD); @@ -407,21 +457,21 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, if (pFD->s3File && tsS3BlockSize < 0) { return tsdbReadFileS3(pFD, offset, pBuf, size, szHint); } else { - return tsdbReadFileImp(pFD, offset, pBuf, size); + return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey); } _exit: return code; } -int32_t tsdbFsyncFile(STsdbFD *pFD) { +int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey) { int32_t code = 0; if (pFD->s3File) { tsdbWarn("%s file: %s", __func__, pFD->path); return code; } - code = tsdbWriteFilePage(pFD); + code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey); if (code) goto _exit; if (taosFsyncFile(pFD->pFD) < 0) { @@ -532,7 +582,9 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { if (code) goto _err; // read - code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0); + int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); if (code) goto _err; // decode @@ -569,7 +621,9 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) { if (code) goto _err; // read - code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0); + int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); if (code) goto _err; // decode @@ -602,7 +656,9 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m if (code) goto _err; // read - code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0); + int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); if (code) goto _err; // decode @@ -633,7 +689,10 @@ int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aCol if (code) goto _err; // read - code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], pSmaInfo->size, 0); + int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], pSmaInfo->size, 0, encryptAlgorithm, + encryptKey); if (code) goto _err; // decode @@ -656,7 +715,7 @@ _err: } static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, SBlockData *pBlockData, - int32_t iStt) { + int32_t iStt) { int32_t code = 0; tBlockDataClear(pBlockData); @@ -667,7 +726,9 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo code = tRealloc(&pReader->aBuf[0], pBlkInfo->szKey); if (code) goto _err; - code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey, 0); + int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey, 0, encryptAlgorithm, encryptKey); if (code) goto _err; SDiskDataHdr hdr; @@ -713,7 +774,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo code = tRealloc(&pReader->aBuf[0], hdr.szBlkCol); if (code) goto _err; - code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol, 0); + code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol, 0, encryptAlgorithm, encryptKey); if (code) goto _err; } @@ -757,7 +818,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo code = tRealloc(&pReader->aBuf[1], size); if (code) goto _err; - code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size, 0); + code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size, 0, encryptAlgorithm, encryptKey); if (code) goto _err; code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]); @@ -783,7 +844,10 @@ int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockDat if (code) goto _err; // read - code = tsdbReadFile(pReader->pDataFD, pBlockInfo->offset, pReader->aBuf[0], pBlockInfo->szBlock, 0); + int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(pReader->pDataFD, pBlockInfo->offset, pReader->aBuf[0], pBlockInfo->szBlock, 0, encryptAlgorithm, + encryptKey); if (code) goto _err; // decmpr @@ -835,7 +899,10 @@ int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk TSDB_CHECK_CODE(code, lino, _exit); // read - code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock, 0); + int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock, 0, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); // decmpr @@ -924,7 +991,9 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa if (code) goto _err; // read - code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0); + int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); if (code) goto _err; // // decode @@ -964,7 +1033,9 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { if (code) goto _err; // read - code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0); + int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); if (code) goto _err; // decode diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index ae840eb6d2..ab0575cb53 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -545,7 +545,10 @@ static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbD int32_t code = 0; int32_t lino = 0; - code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr); + int32_t encryptAlgorithm = writer->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->tsdb->pVnode->config.tsdbCfg.encryptKey; + + code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index f26c6540df..cbb5702aad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -60,7 +60,10 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con int64_t offset = config->file->size - sizeof(SSttFooter); ASSERT(offset >= TSDB_FHDR_SIZE); - code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0); + int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, encryptAlgoirthm, + encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -96,8 +99,11 @@ int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray * void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size); if (!data) return TSDB_CODE_OUT_OF_MEMORY; + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; int32_t code = - tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0); + tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0, + encryptAlgorithm, encryptKey); if (code) { taosMemoryFree(data); return code; @@ -124,8 +130,11 @@ int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tom void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size); if (!data) return TSDB_CODE_OUT_OF_MEMORY; + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; int32_t code = - tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0); + tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0, + encryptAlgorithm, encryptKey); if (code) { taosMemoryFree(data); return code; @@ -152,8 +161,11 @@ int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBl void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size); if (!data) return TSDB_CODE_OUT_OF_MEMORY; + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; int32_t code = - tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0); + tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0, + encryptAlgorithm, encryptKey); if (code) { taosMemoryFree(data); return code; @@ -178,7 +190,10 @@ int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk, code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szBlock); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szBlock, 0); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szBlock, 0, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); code = tDecmprBlockData(reader->config->bufArr[0], sttBlk->bInfo.szBlock, bData, &reader->config->bufArr[1]); @@ -210,7 +225,10 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szKey); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey, 0); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey, 0, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); // hdr @@ -255,8 +273,10 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * code = tRealloc(&reader->config->bufArr[0], hdr->szBlkCol); TSDB_CHECK_CODE(code, lino, _exit); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, reader->config->bufArr[0], - hdr->szBlkCol, 0); + hdr->szBlkCol, 0, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } @@ -296,8 +316,10 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * code = tRealloc(&reader->config->bufArr[1], size1); TSDB_CHECK_CODE(code, lino, _exit); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr->szBlkCol + blockCol->offset, - reader->config->bufArr[1], size1, 0); + reader->config->bufArr[1], size1, 0, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData, @@ -322,7 +344,10 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk code = tRealloc(&reader->config->bufArr[0], tombBlk->dp->size); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbReadFile(reader->fd, tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size, 0); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(reader->fd, tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size, 0, + encryptAlgorithm, encryptKey); if (code) TSDB_CHECK_CODE(code, lino, _exit); int64_t size = 0; @@ -353,7 +378,10 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta code = tRealloc(&reader->config->bufArr[0], statisBlk->dp->size); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbReadFile(reader->fd, statisBlk->dp->offset, reader->config->bufArr[0], statisBlk->dp->size, 0); + int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbReadFile(reader->fd, statisBlk->dp->offset, reader->config->bufArr[0], statisBlk->dp->size, 0, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); int64_t size = 0; @@ -406,7 +434,8 @@ struct SSttFileWriter { }; static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize, - TSttBlkArray *sttBlkArray, uint8_t **bufArr, SVersionRange *range) { + TSttBlkArray *sttBlkArray, uint8_t **bufArr, SVersionRange *range, + int32_t encryptAlgorithm, char* encryptKey) { if (blockData->nRow == 0) return 0; int32_t code = 0; @@ -441,7 +470,7 @@ static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, i for (int32_t i = 3; i >= 0; i--) { if (sizeArr[i]) { - code = tsdbWriteFile(fd, *fileSize, bufArr[i], sizeArr[i]); + code = tsdbWriteFile(fd, *fileSize, bufArr[i], sizeArr[i], encryptAlgorithm, encryptKey); if (code) return code; *fileSize += sizeArr[i]; } @@ -461,8 +490,12 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) { int32_t code = 0; int32_t lino = 0; + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size, - writer->sttBlkArray, writer->config->bufArr, &writer->ctx->range); + writer->sttBlkArray, writer->config->bufArr, &writer->ctx->range, encryptAlgorithm, + encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -498,13 +531,17 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) { .cmprAlg = writer->config->cmprAlg, }}; + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; i++) { code = tsdbCmprData((uint8_t *)TARRAY2_DATA(writer->staticBlock->dataArr + i), TARRAY2_DATA_LEN(&writer->staticBlock->dataArr[i]), TSDB_DATA_TYPE_BIGINT, statisBlk->cmprAlg, &writer->config->bufArr[0], 0, &statisBlk->size[i], &writer->config->bufArr[1]); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbWriteFile(writer->fd, writer->file->size, writer->config->bufArr[0], statisBlk->size[i]); + code = tsdbWriteFile(writer->fd, writer->file->size, writer->config->bufArr[0], statisBlk->size[i], + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); statisBlk->dp->size += statisBlk->size[i]; @@ -529,8 +566,12 @@ static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { int32_t code = 0; int32_t lino = 0; + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, - writer->tombBlkArray, writer->config->bufArr, &writer->ctx->range); + writer->tombBlkArray, writer->config->bufArr, &writer->ctx->range, encryptAlgorithm, + encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -540,12 +581,14 @@ _exit: return code; } -int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize) { +int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize, + int32_t encryptAlgorithm, char* encryptKey) { ptr->size = TARRAY2_DATA_LEN(sttBlkArray); if (ptr->size > 0) { ptr->offset = *fileSize; - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size); + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size, encryptAlgorithm, + encryptKey); if (code) { return code; } @@ -559,7 +602,11 @@ static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) { int32_t code = 0; int32_t lino; - code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size); + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + + code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -572,12 +619,14 @@ _exit: static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) { int32_t code = 0; int32_t lino; + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray); if (writer->footer->statisBlkPtr->size) { writer->footer->statisBlkPtr->offset = writer->file->size; code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray), - writer->footer->statisBlkPtr->size); + writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); writer->file->size += writer->footer->statisBlkPtr->size; } @@ -593,7 +642,11 @@ static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) { int32_t code = 0; int32_t lino = 0; - code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size); + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + + code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size, + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -603,15 +656,19 @@ _exit: return code; } -int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize) { - int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); +int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm, + char* encryptKey) { + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey); if (code) return code; *fileSize += sizeof(*footer); return 0; } static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) { - return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size); + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + + return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size, encryptAlgorithm, encryptKey); } static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { @@ -646,7 +703,10 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { TSDB_CHECK_CODE(code, lino, _exit); uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr)); + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + + code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); writer->file->size += sizeof(hdr); @@ -711,7 +771,9 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o code = tsdbSttFileDoUpdateHeader(writer); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFsyncFile(writer->fd); + int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); tsdbCloseFile(&writer->fd); diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h index 0051a6cd92..4ec2cdcbb7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h @@ -71,8 +71,10 @@ int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *pBlockData int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record); bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer); -int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize); -int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize); +int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize, + int32_t encryptAlgorithm, char* encryptKey); +int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm, + char* encryptKey); struct SSttFileWriterConfig { STsdb *tsdb; diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c index 876c0df4a0..8e185111d9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -37,6 +37,9 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * int8_t cmprAlg; int32_t szPage; uint8_t *bufArr[8]; + int32_t encryptAlgorithm; + char* encryptKey; + // reader SArray *aBlockIdx; SMapData mDataBlk[1]; @@ -50,6 +53,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * .maxRow = tsdb->pVnode->config.tsdbCfg.maxRows, .cmprAlg = tsdb->pVnode->config.tsdbCfg.compression, .szPage = tsdb->pVnode->config.tsdbPageSize, + .encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm, + .encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey, }}; // read SBlockIdx array @@ -122,7 +127,7 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * if (BRIN_BLOCK_SIZE(ctx->brinBlock) >= ctx->maxRow) { SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, - ctx->brinBlkArray, ctx->bufArr, &range); + ctx->brinBlkArray, ctx->bufArr, &range, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -131,18 +136,19 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * if (BRIN_BLOCK_SIZE(ctx->brinBlock) > 0) { SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, - ctx->brinBlkArray, ctx->bufArr, &range); + ctx->brinBlkArray, ctx->bufArr, &range, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr, - &fset->farr[TSDB_FTYPE_HEAD]->f->size); + &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer); + code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm, + ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFsyncFile(ctx->fd); + code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); tsdbCloseFile(&ctx->fd); @@ -238,12 +244,16 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade STFileObj *fobj; struct { int32_t szPage; + int32_t encryptAlgorithm; + char* encryptKey; // writer STsdbFD *fd; TSttBlkArray sttBlkArray[1]; SSttFooter footer[1]; } ctx[1] = {{ .szPage = tsdb->pVnode->config.tsdbPageSize, + .encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm, + .encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey, }}; STFile file = { @@ -266,13 +276,14 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade TSDB_CHECK_CODE(code, lino, _exit1); } - code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size); + code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm, + ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit1); - code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size); + code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit1); - code = tsdbFsyncFile(ctx->fd); + code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit1); tsdbCloseFile(&ctx->fd); @@ -417,7 +428,10 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f TSDB_CHECK_CODE(code, lino, _exit); uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE); + int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; + char* encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey; + + code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); fobj[0]->f->size += TSDB_FHDR_SIZE; @@ -440,6 +454,9 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * int64_t minKey; int64_t maxKey; uint8_t *bufArr[8]; + int32_t encryptAlgorithm; + char* encryptKey; + // reader SArray *aDelData; // writer @@ -452,6 +469,8 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * } ctx[1] = {{ .maxRow = tsdb->pVnode->config.tsdbCfg.maxRows, .cmprAlg = tsdb->pVnode->config.tsdbCfg.compression, + .encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm, + .encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey, }}; tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey); @@ -488,7 +507,7 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * } SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, - ctx->bufArr, &tombRange); + ctx->bufArr, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -501,26 +520,30 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray * } SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, - ctx->bufArr, &tombRange); + ctx->bufArr, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } if (ctx->fd != NULL) { if (ctx->toStt) { - code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size); + code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size, + ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size); + code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, + ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } else { - code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size); + code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size, + ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size); + code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, + ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); } - code = tsdbFsyncFile(ctx->fd); + code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); TSDB_CHECK_CODE(code, lino, _exit); tsdbCloseFile(&ctx->fd); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 07bfa6c719..860daa0b31 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -15,6 +15,7 @@ #include "tutil.h" #include "vnd.h" +#include "tglobal.h" const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .dbname = "", @@ -126,6 +127,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { tjsonAddItemToArray(pNodeRetentions, pNodeRetention); } } + if (tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; @@ -133,6 +135,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; @@ -236,6 +240,19 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code); tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code); } + tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code); + if (code < 0) return -1; +#if defined(TD_ENTERPRISE) + if(pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4){ + if(tsEncryptKey[0] == 0){ + terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; + return -1; + } + else{ + strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + } + } +#endif tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code); if (code < 0) return -1; tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code); @@ -250,6 +267,32 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code); if (code < 0) return -1; + tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code); + if (code < 0) return -1; +#if defined(TD_ENTERPRISE) + if(pCfg->walCfg.encryptAlgorithm == DND_CA_SM4){ + if(tsEncryptKey[0] == 0){ + terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; + return -1; + } + else{ + strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + } + } +#endif + tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code); + if (code < 0) return -1; +#if defined(TD_ENTERPRISE) + if(pCfg->tdbEncryptAlgorithm == DND_CA_SM4){ + if(tsEncryptKey[0] == 0){ + terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; + return -1; + } + else{ + strncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + } + } +#endif tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code); if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER; tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code); diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 6196d30a58..64209572f4 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -9,6 +9,8 @@ add_subdirectory(nodes) add_subdirectory(catalog) add_subdirectory(audit) add_subdirectory(monitorfw) +add_subdirectory(sm4) +add_subdirectory(crypt) add_subdirectory(scalar) add_subdirectory(function) diff --git a/source/libs/crypt/CMakeLists.txt b/source/libs/crypt/CMakeLists.txt new file mode 100644 index 0000000000..e6d73b1882 --- /dev/null +++ b/source/libs/crypt/CMakeLists.txt @@ -0,0 +1,14 @@ +aux_source_directory(src CRYPT_SRC) + +IF (TD_ENTERPRISE) + LIST(APPEND CRYPT_SRC ${TD_ENTERPRISE_DIR}/src/plugins/crypt/cryptImpl.c) +ENDIF () + +add_library(crypt STATIC ${CRYPT_SRC}) +target_include_directories( + crypt + PUBLIC "${TD_SOURCE_DIR}/include/libs/crypt" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(crypt common sm4) diff --git a/source/libs/crypt/src/crypt.c b/source/libs/crypt/src/crypt.c new file mode 100644 index 0000000000..055c2f57cd --- /dev/null +++ b/source/libs/crypt/src/crypt.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "crypt.h" + +extern int32_t CBC_DecryptImpl(SCryptOpts *opts); +extern int32_t CBC_EncryptImpl(SCryptOpts *opts); + +int32_t CBC_Encrypt(SCryptOpts *opts) { + return CBC_EncryptImpl(opts); +} +int32_t CBC_Decrypt(SCryptOpts *opts) { + return CBC_DecryptImpl(opts); +} + +#ifndef TD_ENTERPRISE +int32_t CBC_EncryptImpl(SCryptOpts *opts) { + memcpy(opts->result, opts->source, opts->len); + return opts->len; +} +int32_t CBC_DecryptImpl(SCryptOpts *opts) { + memcpy(opts->result, opts->source, opts->len); + return opts->len; +} +#endif \ No newline at end of file diff --git a/source/libs/sm4/CMakeLists.txt b/source/libs/sm4/CMakeLists.txt new file mode 100644 index 0000000000..bd77bcd40b --- /dev/null +++ b/source/libs/sm4/CMakeLists.txt @@ -0,0 +1,9 @@ +aux_source_directory(src SM4_SRC) +add_library(sm4 STATIC ${SM4_SRC}) +target_include_directories( + sm4 + PUBLIC "${TD_SOURCE_DIR}/include/libs/sm4" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(sm4 common) diff --git a/source/libs/sm4/src/sm4.c b/source/libs/sm4/src/sm4.c new file mode 100644 index 0000000000..91dacc10b0 --- /dev/null +++ b/source/libs/sm4/src/sm4.c @@ -0,0 +1,543 @@ +#include "sm4.h" + +#define SM4_ROUND 32 + +static unsigned int FK[4]={ + 0xA3B1BAC6,0x56AA3350,0x677D9197,0xB27022DC +}; + +static unsigned int CK[SM4_ROUND]={ + 0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269, + 0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9, + 0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249, + 0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9, + 0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229, + 0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299, + 0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209, + 0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279 +}; + +static unsigned char Sbox[256]={ + 0xd6,0x90,0xe9,0xfe,0xcc,0xe1,0x3d,0xb7,0x16,0xb6,0x14,0xc2,0x28,0xfb,0x2c,0x05, + 0x2b,0x67,0x9a,0x76,0x2a,0xbe,0x04,0xc3,0xaa,0x44,0x13,0x26,0x49,0x86,0x06,0x99, + 0x9c,0x42,0x50,0xf4,0x91,0xef,0x98,0x7a,0x33,0x54,0x0b,0x43,0xed,0xcf,0xac,0x62, + 0xe4,0xb3,0x1c,0xa9,0xc9,0x08,0xe8,0x95,0x80,0xdf,0x94,0xfa,0x75,0x8f,0x3f,0xa6, + 0x47,0x07,0xa7,0xfc,0xf3,0x73,0x17,0xba,0x83,0x59,0x3c,0x19,0xe6,0x85,0x4f,0xa8, + 0x68,0x6b,0x81,0xb2,0x71,0x64,0xda,0x8b,0xf8,0xeb,0x0f,0x4b,0x70,0x56,0x9d,0x35, + 0x1e,0x24,0x0e,0x5e,0x63,0x58,0xd1,0xa2,0x25,0x22,0x7c,0x3b,0x01,0x21,0x78,0x87, + 0xd4,0x00,0x46,0x57,0x9f,0xd3,0x27,0x52,0x4c,0x36,0x02,0xe7,0xa0,0xc4,0xc8,0x9e, + 0xea,0xbf,0x8a,0xd2,0x40,0xc7,0x38,0xb5,0xa3,0xf7,0xf2,0xce,0xf9,0x61,0x15,0xa1, + 0xe0,0xae,0x5d,0xa4,0x9b,0x34,0x1a,0x55,0xad,0x93,0x32,0x30,0xf5,0x8c,0xb1,0xe3, + 0x1d,0xf6,0xe2,0x2e,0x82,0x66,0xca,0x60,0xc0,0x29,0x23,0xab,0x0d,0x53,0x4e,0x6f, + 0xd5,0xdb,0x37,0x45,0xde,0xfd,0x8e,0x2f,0x03,0xff,0x6a,0x72,0x6d,0x6c,0x5b,0x51, + 0x8d,0x1b,0xaf,0x92,0xbb,0xdd,0xbc,0x7f,0x11,0xd9,0x5c,0x41,0x1f,0x10,0x5a,0xd8, + 0x0a,0xc1,0x31,0x88,0xa5,0xcd,0x7b,0xbd,0x2d,0x74,0xd0,0x12,0xb8,0xe5,0xb4,0xb0, + 0x89,0x69,0x97,0x4a,0x0c,0x96,0x77,0x7e,0x65,0xb9,0xf1,0x09,0xc5,0x6e,0xc6,0x84, + 0x18,0xf0,0x7d,0xec,0x3a,0xdc,0x4d,0x20,0x79,0xee,0x5f,0x3e,0xd7,0xcb,0x39,0x48 +}; + +#define ROL(x,y) ((x)<<(y) | (x)>>(32-(y))) + +unsigned int SMS4_T1(unsigned int dwA) +{ + unsigned char a0[4]={0}; + unsigned char b0[4]={0}; + unsigned int dwB=0; + unsigned int dwC=0; + int i=0; +/* + for (i=0;i<4;i++) + { + a0[i] = (unsigned char)((dwA>>(i*8)) & 0xff); + b0[i] = Sbox[a0[i]]; + dwB |= (b0[i]<<(i*8)); + } +*/ + a0[0] = (unsigned char)((dwA) & 0xff); + b0[0] = Sbox[a0[0]]; + dwB |= (b0[0]); + + a0[1] = (unsigned char)((dwA>>(8)) & 0xff); + b0[1] = Sbox[a0[1]]; + dwB |= (b0[1]<<(8)); + + a0[2] = (unsigned char)((dwA>>(16)) & 0xff); + b0[2] = Sbox[a0[2]]; + dwB |= (b0[2]<<(16)); + + a0[3] = (unsigned char)((dwA>>(24)) & 0xff); + b0[3] = Sbox[a0[3]]; + dwB |= (b0[3]<<(24)); // 2013-08-13 + + dwC=dwB^ROL(dwB,2)^ROL(dwB,10)^ROL(dwB,18)^ROL(dwB,24); + + return dwC; +} + +unsigned int SMS4_T2(unsigned int dwA) +{ + unsigned char a0[4]={0}; + unsigned char b0[4]={0}; + unsigned int dwB=0; + unsigned int dwC=0; + int i=0; +/* + for (i=0;i<4;i++) + { + a0[i] = (unsigned char)((dwA>>(i*8)) & 0xff); + b0[i] = Sbox[a0[i]]; + dwB |= (b0[i]<<(i*8)); + } +*/ + a0[0] = (unsigned char)((dwA) & 0xff); + b0[0] = Sbox[a0[0]]; + dwB |= (b0[0]); + + a0[1] = (unsigned char)((dwA>>(8)) & 0xff); + b0[1] = Sbox[a0[1]]; + dwB |= (b0[1]<<(8)); + + a0[2] = (unsigned char)((dwA>>(16)) & 0xff); + b0[2] = Sbox[a0[2]]; + dwB |= (b0[2]<<(16)); + + a0[3] = (unsigned char)((dwA>>(24)) & 0xff); + b0[3] = Sbox[a0[3]]; + dwB |= (b0[3]<<(24)); // 2013-08-13 + + dwC=dwB^ROL(dwB,13)^ROL(dwB,23); + + return dwC; +} + +/* MK[4] is the Encrypt Key, rk[32] is Round Key */ +void SMS4_Key_Expansion(unsigned int MK[], unsigned int rk[]) +{ + unsigned int K[4]={0}; + int i=0; + + for (i=0;i<4;i++) + { + K[i] = MK[i] ^ FK[i]; + } + + for (i=0;ipath, 16 * 1024, 1, &pMeta->db, 0) < 0) { + if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0, 0, NULL) < 0) { return -1; // goto _err; } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index b53dc9daa6..01f7c034f1 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -158,7 +158,7 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz } taosCloseFile(&pCfgFile); - if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 1) < 0) { + if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 1, 0, NULL) < 0) { goto _err; } diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 405fb1c5a0..cc50919413 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -22,6 +22,7 @@ target_link_libraries( tdb PUBLIC os PUBLIC util + PUBLIC crypt ) # for test diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 4dd47e97ac..d3a5711352 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -32,7 +32,8 @@ typedef struct STBC TBC; typedef struct STxn TXN; // TDB -int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback); +int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm, + char *encryptKey); int32_t tdbClose(TDB *pDb); int32_t tdbBegin(TDB *pDb, TXN **pTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, int flags); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 81b306e65d..9a1d89eeac 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,7 +15,8 @@ #include "tdbInt.h" -int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback) { +int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm, + char *encryptKey) { TDB *pDb; int dsize; int zsize; @@ -49,6 +50,11 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i pDb->jfd = -1; + pDb->encryptAlgorithm = encryptAlgorithm; + if(encryptKey != NULL){ + strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN); + } + ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); if (ret < 0) { return -1; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 474e5d2270..94bed771da 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -14,6 +14,8 @@ */ #include "tdbInt.h" +#include "crypt.h" +#include "tglobal.h" /* #pragma pack(push, 1) typedef struct { @@ -872,6 +874,31 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage TDB_UNLOCK_PAGE(pPage); return -1; } + + int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; + char* encryptKey = pPager->pEnv->encryptKey; + + if(encryptAlgorithm == DND_CA_SM4){ + unsigned char PacketData[128]; + int NewLen; + + int32_t count = 0; + while(count < pPage->pageSize) + { + SCryptOpts opts = {0}; + opts.len = 128; + opts.source = pPage->pData + count; + opts.result = PacketData; + opts.unitLen = 128; + strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + + NewLen = CBC_Decrypt(&opts); + + memcpy(pPage->pData + count, PacketData, NewLen); + count += NewLen; + } + tdbInfo("CBC_Decrypt count:%d %s", count, __FUNCTION__); + } } else { init = 0; } @@ -960,14 +987,44 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) { offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); - ret = tdbOsPWrite(pPager->fd, pPage->pData, pPage->pageSize, offset); + int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; + char* encryptKey = pPager->pEnv->encryptKey; + + char* buf = pPage->pData; + + if(encryptAlgorithm == DND_CA_SM4){ + buf = taosMemoryMalloc(pPage->pageSize); + + unsigned char PacketData[128]; + int NewLen; + int32_t count = 0; + while (count < pPage->pageSize) { + SCryptOpts opts = {0}; + opts.len = 128; + opts.source = pPage->pData + count; + opts.result = PacketData; + opts.unitLen = 128; + strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); + + NewLen = CBC_Encrypt(&opts); + + memcpy(buf + count, PacketData, NewLen); + count += NewLen; + } + tdbInfo("CBC_Encrypt count:%d %s", count, __FUNCTION__); + } + + ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset); if (ret < 0) { + if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf); tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, pPage->pageSize); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf); + return 0; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 8ce294a3c6..13b091ea46 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -18,6 +18,7 @@ #include "tdb.h" +#include "tdef.h" #include "tlog.h" #include "trbtree.h" @@ -392,6 +393,8 @@ struct STDB { TTB *pFreeDb; #endif int64_t txnId; + int32_t encryptAlgorithm; + char encryptKey[ENCRYPT_KEY_LEN]; }; struct SPager { diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp index 325703c946..a2deba4696 100644 --- a/source/libs/tdb/test/tdbExOVFLTest.cpp +++ b/source/libs/tdb/test/tdbExOVFLTest.cpp @@ -141,7 +141,7 @@ static void generateBigVal(char *val, int valLen) { static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { TDB *pEnv = NULL; - int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0); + int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL); if (ret) { pEnv = NULL; } @@ -374,7 +374,7 @@ TEST(tdb_test, simple_insert1) { taosRemoveDir("tdb"); // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database diff --git a/source/libs/tdb/test/tdbPageDefragmentTest.cpp b/source/libs/tdb/test/tdbPageDefragmentTest.cpp index 7aeef94d14..b64517c787 100644 --- a/source/libs/tdb/test/tdbPageDefragmentTest.cpp +++ b/source/libs/tdb/test/tdbPageDefragmentTest.cpp @@ -141,7 +141,7 @@ static void generateBigVal(char *val, int valLen) { static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { TDB *pEnv = NULL; - int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0); + int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL); if (ret) { pEnv = NULL; } @@ -352,7 +352,7 @@ TEST(TdbPageDefragmentTest, DISABLED_simple_insert1) { taosRemoveDir("tdb"); // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -485,7 +485,7 @@ TEST(TdbPageDefragmentTest, seq_insert) { taosRemoveDir("tdb"); // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -566,7 +566,7 @@ TEST(TdbPageDefragmentTest, seq_delete) { int const pageSize = 1 * 1024 * 1024; // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -650,7 +650,7 @@ TEST(TdbPageDefragmentTest, defragment_insert) { int const pageSize = 1 * 1024 * 1024; // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp index d740bd0f94..bbbd90e73d 100644 --- a/source/libs/tdb/test/tdbPageRecycleTest.cpp +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -123,7 +123,7 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { TDB *pEnv = NULL; - int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0); + int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0 , 0, NULL); if (ret) { pEnv = NULL; } @@ -355,7 +355,7 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) { taosRemoveDir("tdb"); // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -484,7 +484,7 @@ static void insertDb(int nData) { int const pageSize = 4 * 1024; // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -552,7 +552,7 @@ static void deleteDb(int nData) { int const pageSize = 4 * 1024; // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index cd02eb8d5e..c1aa15b4d2 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -131,7 +131,7 @@ TEST(tdb_test, DISABLED_simple_insert1) { taosRemoveDir("tdb"); // Open Env - ret = tdbOpen("tdb", 4096, 64, &pEnv, 0); + ret = tdbOpen("tdb", 4096, 64, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -246,7 +246,7 @@ TEST(tdb_test, DISABLED_simple_insert2) { taosRemoveDir("tdb"); // Open Env - ret = tdbOpen("tdb", 1024, 10, &pEnv, 0); + ret = tdbOpen("tdb", 1024, 10, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -339,7 +339,7 @@ TEST(tdb_test, DISABLED_simple_delete1) { pPool = openPool(); // open env - ret = tdbOpen("tdb", 1024, 256, &pEnv, 0); + ret = tdbOpen("tdb", 1024, 256, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // open database @@ -428,7 +428,7 @@ TEST(tdb_test, DISABLED_simple_upsert1) { taosRemoveDir("tdb"); // open env - ret = tdbOpen("tdb", 4096, 64, &pEnv, 0); + ret = tdbOpen("tdb", 4096, 64, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // open database @@ -490,7 +490,7 @@ TEST(tdb_test, multi_thread_query) { taosRemoveDir("tdb"); // Open Env - ret = tdbOpen("tdb", 4096, 10, &pEnv, 0); + ret = tdbOpen("tdb", 4096, 10, &pEnv, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -599,7 +599,7 @@ TEST(tdb_test, DISABLED_multi_thread1) { taosRemoveDir("tdb"); // Open Env - ret = tdbOpen("tdb", 512, 1, &pDb, 0); + ret = tdbOpen("tdb", 512, 1, &pDb, 0, 0, NULL); GTEST_ASSERT_EQ(ret, 0); ret = tdbTbOpen("db.db", -1, -1, NULL, pDb, &pTb, 0); diff --git a/source/libs/wal/CMakeLists.txt b/source/libs/wal/CMakeLists.txt index d2ed4f1d16..7b81aff8a8 100644 --- a/source/libs/wal/CMakeLists.txt +++ b/source/libs/wal/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common + PUBLIC crypt ) if(${BUILD_TEST}) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index b897eb4922..bbca87ff9d 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -153,7 +153,12 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { } // validate body - recordLen = walCkHeadSz + logContent->head.bodyLen; + int32_t cryptedBodyLen = logContent->head.bodyLen; + //TODO: dmchen enum + if(pWal->cfg.encryptAlgorithm == 1){ + cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); + } + recordLen = walCkHeadSz + cryptedBodyLen; if (len < recordLen) { int64_t extraSize = recordLen - len; if (capacity < readSize + extraSize + sizeof(magic)) { @@ -181,6 +186,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { } logContent = (SWalCkHead*)(buf + pos); + decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__); if (walValidBodyCksum(logContent) != 0) { terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, @@ -618,7 +624,14 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { /*A(idxEntry.ver == ckHead.head.version);*/ idxEntry.ver += 1; - idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; + + int32_t plainBodyLen = ckHead.head.bodyLen; + int32_t cryptedBodyLen = plainBodyLen; + //TODO: dmchen enum + if(pWal->cfg.encryptAlgorithm == 1){ + cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); + } + idxEntry.offset += sizeof(SWalCkHead) + cryptedBodyLen; if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s", diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index d491b00e73..17983a82bc 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -13,7 +13,9 @@ * along with this program. If not, see . */ +#include "crypt.h" #include "taoserror.h" +#include "wal.h" #include "walInt.h" SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond, int64_t id) { @@ -307,7 +309,13 @@ int32_t walSkipFetchBody(SWalReader *pRead) { pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId); - int64_t code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR); + int32_t plainBodyLen = pRead->pHead->head.bodyLen; + int32_t cryptedBodyLen = plainBodyLen; + //TODO: dmchen emun + if(pRead->pWal->cfg.encryptAlgorithm == 1){ + cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); + } + int64_t code = taosLSeekFile(pRead->pLogFile, cryptedBodyLen, SEEK_CUR); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -328,19 +336,27 @@ int32_t walFetchBody(SWalReader *pRead) { ", 0x%" PRIx64, vgId, ver, pVer->firstVer, pVer->commitVer, pVer->lastVer, pVer->appliedVer, id); - if (pRead->capacity < pReadHead->bodyLen) { - SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); + int32_t plainBodyLen = pReadHead->bodyLen; + int32_t cryptedBodyLen = plainBodyLen; + + //TODO: dmchen emun + if(pRead->pWal->cfg.encryptAlgorithm == 1){ + cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); + } + + if (pRead->capacity < cryptedBodyLen) { + SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + cryptedBodyLen); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } pRead->pHead = ptr; pReadHead = &pRead->pHead->head; - pRead->capacity = pReadHead->bodyLen; + pRead->capacity = cryptedBodyLen; } - if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { - if (pReadHead->bodyLen < 0) { + if (cryptedBodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, cryptedBodyLen)) { + if (plainBodyLen < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%"PRIx64, vgId, pReadHead->version, ver, tstrerror(terrno), id); @@ -359,6 +375,8 @@ int32_t walFetchBody(SWalReader *pRead) { return -1; } + decryptBody(&pRead->pWal->cfg, pRead->pHead, plainBodyLen, __FUNCTION__); + if (walValidBodyCksum(pRead->pHead) != 0) { wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver, id); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; @@ -431,20 +449,28 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { return -1; } - if (pReader->capacity < pReader->pHead->head.bodyLen) { + int32_t plainBodyLen = pReader->pHead->head.bodyLen; + int32_t cryptedBodyLen = plainBodyLen; + + //TODO: dmchen emun + if(pReader->pWal->cfg.encryptAlgorithm == 1){ + cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); + } + + if (pReader->capacity < cryptedBodyLen) { SWalCkHead *ptr = - (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReader->pHead->head.bodyLen); + (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + cryptedBodyLen); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosThreadMutexUnlock(&pReader->mutex); return -1; } pReader->pHead = ptr; - pReader->capacity = pReader->pHead->head.bodyLen; + pReader->capacity = cryptedBodyLen; } - if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, pReader->pHead->head.bodyLen)) != - pReader->pHead->head.bodyLen) { + if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) != + cryptedBodyLen) { if (contLen < 0) terrno = TAOS_SYSTEM_ERROR(errno); else { @@ -465,11 +491,13 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { return -1; } + decryptBody(&pReader->pWal->cfg, pReader->pHead, plainBodyLen, __FUNCTION__); + code = walValidBodyCksum(pReader->pHead); if (code != 0) { wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver); - uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, pReader->pHead->head.bodyLen); + uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, plainBodyLen); uint32_t logCkSum = pReader->pHead->cksumBody; wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum); // pReader->curInvalid = 1; @@ -484,6 +512,29 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { return 0; } +void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func) { + //TODO: dmchen emun + if (cfg->encryptAlgorithm == 1) { + int32_t cryptedBodyLen = ENCRYPTED_LEN(plainBodyLen); + char *newBody = taosMemoryMalloc(cryptedBodyLen); + + SCryptOpts opts; + opts.len = cryptedBodyLen; + opts.source = pHead->head.body; + opts.result = newBody; + opts.unitLen = 16; + strncpy(opts.key, cfg->encryptKey, 16); + + int32_t count = CBC_Decrypt(&opts); + + wInfo("CBC_Decrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", count, plainBodyLen, func); + + memcpy(pHead->head.body, newBody, plainBodyLen); + + taosMemoryFree(newBody); + } +} + void walReadReset(SWalReader *pReader) { taosThreadMutexLock(&pReader->mutex); taosCloseFile(&pReader->pIdxFile); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 9783705bad..102e035425 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -18,6 +18,7 @@ #include "tchecksum.h" #include "tglobal.h" #include "walInt.h" +#include "crypt.h" int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { taosThreadMutexLock(&pWal->mutex); @@ -491,12 +492,13 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen) { int64_t code = 0; + int32_t plainBodyLen = bodyLen; int64_t offset = walGetCurFileOffset(pWal); SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); pWal->writeHead.head.version = index; - pWal->writeHead.head.bodyLen = bodyLen; + pWal->writeHead.head.bodyLen = plainBodyLen; pWal->writeHead.head.msgType = msgType; pWal->writeHead.head.ingestTs = taosGetTimestampUs(); @@ -504,7 +506,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.head.syncMeta = syncMeta; pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); - pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); + pWal->writeHead.cksumBody = walCalcBodyCksum(body, plainBodyLen); wDebug("vgId:%d, wal write log %" PRId64 ", msgType: %s, cksum head %u cksum body %u", pWal->cfg.vgId, index, TMSG_INFO(msgType), pWal->writeHead.cksumHead, pWal->writeHead.cksumBody); @@ -521,22 +523,78 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy goto END; } - if (taosWriteFile(pWal->pLogFile, (char *)body, bodyLen) != bodyLen) { + int32_t cyptedBodyLen = plainBodyLen; + char* buf = (char*)body; + char* newBody = NULL; + char* newBodyEncrypted = NULL; + + //TODO: dmchen enum + if(pWal->cfg.encryptAlgorithm == 1){ + cyptedBodyLen = ENCRYPTED_LEN(cyptedBodyLen); + char* newBody = taosMemoryMalloc(cyptedBodyLen); + if(newBody == NULL){ + wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), + strerror(errno)); + code = -1; + goto END; + } + memset(newBody, 0, cyptedBodyLen); + memcpy(newBody, body, plainBodyLen); + + char* newBodyEncrypted = taosMemoryMalloc(cyptedBodyLen); + if(newBodyEncrypted == NULL){ + wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), + strerror(errno)); + code = -1; + if(newBodyEncrypted != NULL) taosMemoryFreeClear(newBodyEncrypted); + if(newBody != NULL) taosMemoryFreeClear(newBody); + goto END; + } + + SCryptOpts opts; + opts.len = cyptedBodyLen; + opts.source = newBody; + opts.result = newBodyEncrypted; + opts.unitLen = 16; + strncpy(opts.key, pWal->cfg.encryptKey, 16); + + int32_t count = CBC_Encrypt(&opts); + + wInfo("vgId:%d, file:%" PRId64 ".log, index:%" PRId64 ", CBC_Encrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", + pWal->cfg.vgId, walGetLastFileFirstVer(pWal), index, count, plainBodyLen, __FUNCTION__); + + buf = newBodyEncrypted; + } + + if (taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); code = -1; + //TODO: dmchen enum + if(pWal->cfg.encryptAlgorithm == 1){ + taosMemoryFree(newBody); + taosMemoryFree(newBodyEncrypted); + } goto END; } + //TODO: dmchen enum + if(pWal->cfg.encryptAlgorithm == 1){ + taosMemoryFree(newBody); + taosMemoryFree(newBodyEncrypted); + wInfo("vgId:%d, free newBody newBodyEncrypted %s", + pWal->cfg.vgId, __FUNCTION__); + } + // set status if (pWal->vers.firstVer == -1) { pWal->vers.firstVer = 0; } pWal->vers.lastVer = index; - pWal->totSize += sizeof(SWalCkHead) + bodyLen; + pWal->totSize += sizeof(SWalCkHead) + cyptedBodyLen; pFileInfo->lastVer = index; - pFileInfo->fileSize += sizeof(SWalCkHead) + bodyLen; + pFileInfo->fileSize += sizeof(SWalCkHead) + cyptedBodyLen; return 0; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 8fdc2654c5..6ebafa14ce 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -953,16 +953,38 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { paGetToken(name + olen + 1, &value, &vlen); if (vlen == 0) continue; value[vlen] = 0; + + if (strcasecmp(name, "encryptScope") == 0) { + char* tmp = NULL; + int32_t len = 0; + char newValue[1024] = {0}; - paGetToken(value + vlen + 1, &value2, &vlen2); - if (vlen2 != 0) { - value2[vlen2] = 0; - paGetToken(value2 + vlen2 + 1, &value3, &vlen3); - if (vlen3 != 0) value3[vlen3] = 0; + strcpy(newValue, value); + + int32_t count = 1; + while(vlen < 1024){ + paGetToken(value + vlen + 1 * count, &tmp, &len); + if(len == 0) break; + tmp[len] = 0; + strcpy(newValue + vlen, tmp); + vlen += len; + count++; + } + + code = cfgSetItem(pConfig, name, newValue, CFG_STYPE_CFG_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + else{ + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } - code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + } if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8723f227dd..e5ef268b35 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -243,6 +243,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST, "Index not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating status") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "encryption is not allowed to be changed after database is created.") // mnode-node TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") @@ -351,6 +352,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ONLY_TWO_MNODE, "Only two mnodes exist TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NO_NEED_RESTORE, "No need to restore on this dnode") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE, "Please use this command when the dnode is offline") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NO_MACHINE_CODE, "Dnode can not get machine code") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NO_ENCRYPT_KEY, "Dnode can not get encryption key") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG, "invalid encryption configuration") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_INVALID_ENCRYPTKEY, "invalid encryption key") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED, "encryption key was changed") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed")