From 90922fd213c89c676c8ed70ee20cd4ec67346f02 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 12 Nov 2021 19:37:22 +0800 Subject: [PATCH] add checksum for sdb --- include/util/taoserror.h | 15 ++++----- source/dnode/mnode/sdb/inc/sdbInt.h | 1 - source/dnode/mnode/sdb/src/sdbFile.c | 46 +++++++++++++++++++++------- source/util/src/terror.c | 1 + 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index bcff095249..022c3e9096 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -66,13 +66,14 @@ int32_t* taosGetErrno(); #define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0103) #define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) #define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0106) -#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0107) -#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0108) -#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0109) -#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x010A) -#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x010B) -#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x010C) -#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010D) +#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0107) +#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0108) +#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0109) +#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x010A) +#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x010B) +#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x010C) +#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x010D) +#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010E) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) //"Invalid Operation") diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index fbc06775ff..30023d51ab 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -41,7 +41,6 @@ typedef struct SSdbRaw { int8_t sver; int8_t status; int8_t reserved; - int32_t cksum; int32_t dataLen; char pData[]; } SSdbRaw; diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index b01322217f..a6a6f54879 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "sdbInt.h" #include "tglobal.h" +#include "tchecksum.h" static int32_t sdbCreateDir() { mDebug("start to create mnode at %s", tsMnodeDir); @@ -56,10 +57,6 @@ static int32_t sdbRunDeployFp() { return 0; } -static int32_t sdbWriteVersion(FileFd fd) { return 0; } - -static int32_t sdbReadVersion(FileFd fd) { return 0; } - static int32_t sdbReadDataFile() { SSdbRaw *pRaw = malloc(SDB_MAX_SIZE); if (pRaw == NULL) { @@ -71,6 +68,7 @@ static int32_t sdbReadDataFile() { snprintf(file, sizeof(file), "%ssdb.data", tsSdb.currDir); FileFd fd = taosOpenFileCreateWrite(file); if (fd <= 0) { + free(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to open file:%s for read since %s", file, terrstr()); return -1; @@ -90,7 +88,27 @@ static int32_t sdbReadDataFile() { } if (ret < sizeof(SSdbRaw)) { - code = TSDB_CODE_SDB_APP_ERROR; + code = TSDB_CODE_FILE_CORRUPTED; + mError("failed to read file:%s since %s", file, tstrerror(code)); + break; + } + + ret = taosReadFile(fd, pRaw->pData, pRaw->dataLen + sizeof(int32_t)); + if (ret < 0) { + code = TAOS_SYSTEM_ERROR(errno); + mError("failed to read file:%s since %s", file, tstrerror(code)); + break; + } + + if (ret < pRaw->dataLen + sizeof(int32_t)) { + code = TSDB_CODE_FILE_CORRUPTED; + mError("failed to read file:%s since %s", file, tstrerror(code)); + break; + } + + uint32_t cksum = *(int32_t *)(pRaw->pData + pRaw->dataLen); + if (!taosCheckChecksumWhole(pRaw, sizeof(SSdbRaw) + pRaw->dataLen + sizeof(int32_t)) != 0) { + code = TSDB_CODE_CHECKSUM_ERROR; mError("failed to read file:%s since %s", file, tstrerror(code)); break; } @@ -106,6 +124,7 @@ static int32_t sdbReadDataFile() { PARSE_SDB_DATA_ERROR: taosCloseFile(fd); + sdbFreeRaw(pRaw); terrno = code; return code; } @@ -134,10 +153,19 @@ static int32_t sdbWriteDataFile() { SSdbRow *pRow = taosHashIterate(hash, NULL); while (pRow != NULL) { if (pRow->status != SDB_STATUS_READY) continue; - + SSdbRaw *pRaw = (*encodeFp)(pRow->pObj); if (pRaw != NULL) { - taosWriteFile(fd, pRaw, sizeof(SSdbRaw) + pRaw->dataLen); + int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen; + if (taosWriteFile(fd, pRaw, writeLen) != writeLen) { + code = TAOS_SYSTEM_ERROR(terrno); + break; + } + int32_t cksum = taosCalcChecksum(0, pRaw, sizeof(SSdbRaw) + pRaw->dataLen); + if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { + code = TAOS_SYSTEM_ERROR(terrno); + break; + } } else { taosHashCancelIterate(hash, pRow); code = TSDB_CODE_SDB_APP_ERROR; @@ -149,10 +177,6 @@ static int32_t sdbWriteDataFile() { taosWUnLockLatch(pLock); } - if (code == 0) { - code = sdbWriteVersion(fd); - } - taosCloseFile(fd); if (code == 0) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 0abfc89725..0b10274c00 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -76,6 +76,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RANGE, "Out of range") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PTR, "Invalid pointer") TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, "Memory corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid config message") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs")