xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #26744 from taosdata/enh/TD-30987-14 

enh: refactor error code
This commit is contained in:
Hongze Cheng 2024-07-23 16:27:43 +08:00 committed by GitHub
parent 9f0a2ac3ba 1ebec2c402
commit 9e36e33469
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 232 additions and 339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
source/libs/tdb/CMakeLists.txt
View File

@ -10,7 +10,6 @@ target_sources(tdb
"src/db/tdbTable.c" "src/db/tdbTable.c"
"src/db/tdbTxn.c" "src/db/tdbTxn.c"
"src/db/tdbPage.c" "src/db/tdbPage.c"
"src/db/tdbOs.c"
) )
target_include_directories( target_include_directories(

4
source/libs/tdb/src/db/tdbBtree.c
View File

@ -76,14 +76,14 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
if (keyLen == 0) { if (keyLen == 0) {
tdbError("tdb/btree-open: key len cannot be zero."); tdbError("tdb/btree-open: key len cannot be zero.");
return -1; return TSDB_CODE_INVALID_PARA;
} }
*ppBt = NULL; *ppBt = NULL;
pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt)); pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt));
if (pBt == NULL) { if (pBt == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
// pBt->keyLen // pBt->keyLen

37
source/libs/tdb/src/db/tdbDb.c
View File

@ -15,8 +15,8 @@
#include "tdbInt.h" #include "tdbInt.h"
int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback, int32_t encryptAlgorithm, int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback,
char *encryptKey) { int32_t encryptAlgorithm, char *encryptKey) {
TDB *pDb; TDB *pDb;
int dsize; int dsize;
int zsize; int zsize;
@ -31,7 +31,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
pPtr = (uint8_t *)tdbOsCalloc(1, zsize); pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) { if (pPtr == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pDb = (TDB *)pPtr; pDb = (TDB *)pPtr;
@ -51,38 +51,38 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
pDb->jfd = -1; pDb->jfd = -1;
pDb->encryptAlgorithm = encryptAlgorithm; pDb->encryptAlgorithm = encryptAlgorithm;
if(encryptKey != NULL){ if (encryptKey != NULL) {
strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN); strncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN);
} }
ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache));
if (ret < 0) { if (ret < 0) {
return -1; return ret;
} }
pDb->nPgrHash = 8; pDb->nPgrHash = 8;
tsize = sizeof(SPager *) * pDb->nPgrHash; tsize = sizeof(SPager *) * pDb->nPgrHash;
pDb->pgrHash = tdbOsMalloc(tsize); pDb->pgrHash = tdbOsMalloc(tsize);
if (pDb->pgrHash == NULL) { if (pDb->pgrHash == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
memset(pDb->pgrHash, 0, tsize); memset(pDb->pgrHash, 0, tsize);
ret = taosMulModeMkDir(dbname, 0755, false); ret = taosMulModeMkDir(dbname, 0755, false);
if (ret < 0) { if (ret < 0) {
return -1; return TAOS_SYSTEM_ERROR(ret);
} }
#ifdef USE_MAINDB #ifdef USE_MAINDB
// open main db // open main db
ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SBtInfo), NULL, pDb, &pDb->pMainDb, rollback); ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SBtInfo), NULL, pDb, &pDb->pMainDb, rollback);
if (ret < 0) { if (ret < 0) {
return -1; return ret;
} }
ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback); ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback);
if (ret < 0) { if (ret < 0) {
return -1; return ret;
} }
#endif #endif
@ -125,12 +125,13 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void (
TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn));
if (!pTxn) { if (!pTxn) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags) < 0) { ret = tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags);
if (ret < 0) {
tdbOsFree(pTxn); tdbOsFree(pTxn);
return -1; return ret;
} }
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
@ -139,7 +140,7 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void (
tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId); pTxn->txnId);
tdbTxnClose(pTxn); tdbTxnClose(pTxn);
return -1; return ret;
} }
} }
@ -157,7 +158,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) {
if (ret < 0) { if (ret < 0) {
tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId); pTxn->txnId);
return -1; return ret;
} }
} }
@ -173,7 +174,7 @@ int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) {
if (ret < 0) { if (ret < 0) {
tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId); pTxn->txnId);
return -1; return ret;
} }
} }
@ -191,7 +192,7 @@ int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn) {
if (ret < 0) { if (ret < 0) {
tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId); pTxn->txnId);
return -1; return ret;
} }
} }
@ -207,7 +208,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) {
if (ret < 0) { if (ret < 0) {
tdbError("failed to abort pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, tdbError("failed to abort pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
pTxn->txnId); pTxn->txnId);
return -1; return ret;
} }
} }

98
source/libs/tdb/src/db/tdbOs.c
View File

@ -1,98 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbOs.h"
#ifndef TDB_FOR_TDENGINE
// tdbOsRead
i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes) {
i64 nRead = 0;
i64 iRead = 0;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iRead = read(fd, pBuf, nBytes);
if (iRead < 0) {
if (errno == EINTR) {
continue;
} else {
return -1;
}
} else if (iRead == 0) {
break;
}
nRead += iRead;
pBuf += iRead;
nBytes -= iRead;
}
return nRead;
}
// tdbOsPRead
i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset) {
i64 nRead = 0;
i64 iRead = 0;
i64 iOffset = offset;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iRead = pread(fd, pBuf, nBytes, iOffset);
if (iRead < 0) {
if (errno == EINTR) {
continue;
} else {
return -1;
}
} else if (iRead == 0) {
break;
}
nRead += iRead;
pBuf += iRead;
iOffset += iRead;
nBytes -= iRead;
}
return nRead;
}
// tdbOsWrite
i64 tdbOsWrite(tdb_fd_t fd, const void *pData, i64 nBytes) {
i64 nWrite = 0;
i64 iWrite = 0;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iWrite = write(fd, pBuf, nBytes);
if (iWrite < 0) {
if (errno == EINTR) {
continue;
}
return -1;
}
nWrite += iWrite;
pBuf += iWrite;
nBytes -= iWrite;
}
return nWrite;
}
#endif

57
source/libs/tdb/src/db/tdbPCache.c
View File

@ -50,30 +50,36 @@ static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); }
static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); } static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); }
int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
int32_t code = 0;
int32_t lino;
SPCache *pCache; SPCache *pCache;
void *pPtr; void *pPtr;
SPage *pPgHdr; SPage *pPgHdr;
pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache) + sizeof(SPage *) * cacheSize); pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache) + sizeof(SPage *) * cacheSize);
if (pCache == NULL) { if (pCache == NULL) {
return -1; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
} }
pCache->szPage = pageSize; pCache->szPage = pageSize;
pCache->nPages = cacheSize; pCache->nPages = cacheSize;
pCache->aPage = (SPage **)tdbOsCalloc(cacheSize, sizeof(SPage *)); pCache->aPage = (SPage **)tdbOsCalloc(cacheSize, sizeof(SPage *));
if (pCache->aPage == NULL) { if (pCache->aPage == NULL) {
tdbOsFree(pCache); TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
return -1;
} }
if (tdbPCacheOpenImpl(pCache) < 0) { code = tdbPCacheOpenImpl(pCache);
tdbOsFree(pCache); TSDB_CHECK_CODE(code, lino, _exit);
return -1;
}
*ppCache = pCache; _exit:
return 0; if (code) {
tdbError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(code));
tdbPCacheClose(pCache);
*ppCache = NULL;
} else {
*ppCache = pCache;
}
return code;
} }
int tdbPCacheClose(SPCache *pCache) { int tdbPCacheClose(SPCache *pCache) {
@ -99,14 +105,14 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) {
} else if (pCache->nPages < nPage) { } else if (pCache->nPages < nPage) {
SPage **aPage = tdbOsCalloc(nPage, sizeof(SPage *)); SPage **aPage = tdbOsCalloc(nPage, sizeof(SPage *));
if (aPage == NULL) { if (aPage == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) { for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) {
if (tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL) < 0) { int32_t code = tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL);
// TODO: handle error if (code) {
tdbOsFree(aPage); tdbOsFree(aPage);
return -1; return code;
} }
// pPage->pgid = 0; // pPage->pgid = 0;
@ -156,15 +162,11 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) {
} }
int tdbPCacheAlter(SPCache *pCache, int32_t nPage) { int tdbPCacheAlter(SPCache *pCache, int32_t nPage) {
int ret = 0; int code;
tdbPCacheLock(pCache); tdbPCacheLock(pCache);
code = tdbPCacheAlterImpl(pCache, nPage);
ret = tdbPCacheAlterImpl(pCache, nPage);
tdbPCacheUnlock(pCache); tdbPCacheUnlock(pCache);
return code;
return ret;
} }
SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
@ -180,9 +182,6 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
tdbPCacheUnlock(pCache); tdbPCacheUnlock(pCache);
// printf("thread %" PRId64 " fetch page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id,
// TDB_PAGE_PGNO(pPage), pPage, nRef);
if (pPage) { if (pPage) {
tdbTrace("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); tdbTrace("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef);
} else { } else {
@ -285,6 +284,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
if (!pTxn) { if (!pTxn) {
tdbError("tdb/pcache: null ptr pTxn, fetch impl failed."); tdbError("tdb/pcache: null ptr pTxn, fetch impl failed.");
terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
@ -327,7 +327,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg);
if (ret < 0 || pPage == NULL) { if (ret < 0 || pPage == NULL) {
tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage);
// TODO: recycle other backup pages terrno = ret;
return NULL; return NULL;
} }
@ -475,10 +475,8 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
pCache->nFree = 0; pCache->nFree = 0;
pCache->pFree = NULL; pCache->pFree = NULL;
for (int i = 0; i < pCache->nPages; i++) { for (int i = 0; i < pCache->nPages; i++) {
if (tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL) < 0) { ret = tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL);
// TODO: handle error if (ret) return ret;
return -1;
}
// pPage->pgid = 0; // pPage->pgid = 0;
pPage->isAnchor = 0; pPage->isAnchor = 0;
@ -504,8 +502,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
pCache->nHash = pCache->nPages < 8 ? 8 : pCache->nPages; pCache->nHash = pCache->nPages < 8 ? 8 : pCache->nPages;
pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *)); pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *));
if (pCache->pgHash == NULL) { if (pCache->pgHash == NULL) {
// TODO return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
// Open LRU list // Open LRU list

60
source/libs/tdb/src/db/tdbPage.c
View File

@ -45,12 +45,12 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
if (!xMalloc) { if (!xMalloc) {
tdbError("tdb/page-create: null xMalloc."); tdbError("tdb/page-create: null xMalloc.");
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (!TDB_IS_PGSIZE_VLD(pageSize)) { if (!TDB_IS_PGSIZE_VLD(pageSize)) {
tdbError("tdb/page-create: invalid pageSize: %d.", pageSize); tdbError("tdb/page-create: invalid pageSize: %d.", pageSize);
return -1; return TSDB_CODE_INVALID_PARA;
} }
*ppPage = NULL; *ppPage = NULL;
@ -58,7 +58,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
ptr = (u8 *)(xMalloc(arg, size)); ptr = (u8 *)(xMalloc(arg, size));
if (ptr == NULL) { if (ptr == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
memset(ptr, 0, size); memset(ptr, 0, size);
@ -86,12 +86,12 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg)
if (pPage->isDirty) { if (pPage->isDirty) {
tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty); tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty);
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (!xFree) { if (!xFree) {
tdbError("tdb/page-destroy: null xFree."); tdbError("tdb/page-destroy: null xFree.");
return -1; return TSDB_CODE_INVALID_PARA;
} }
for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) {
@ -129,7 +129,8 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell
tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize);
pPage->pPageHdr = pPage->pData + szAmHdr; pPage->pPageHdr = pPage->pData + szAmHdr;
if (TDB_PAGE_NCELLS(pPage) == 0) { if (TDB_PAGE_NCELLS(pPage) == 0) {
return tdbPageZero(pPage, szAmHdr, xCellSize); tdbPageZero(pPage, szAmHdr, xCellSize);
return;
} }
pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage);
pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage);
@ -159,7 +160,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) { if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) {
tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell, tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell,
TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData));
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
nFree = TDB_PAGE_NFREE(pPage); nFree = TDB_PAGE_NFREE(pPage);
@ -207,7 +208,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) { if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) {
tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart, tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart,
pPage->pCellIdx, nCells); pPage->pCellIdx, nCells);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
} }
@ -234,7 +235,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) {
if (idx < 0 || idx >= nCells + pPage->nOverflow) { if (idx < 0 || idx >= nCells + pPage->nOverflow) {
tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow); tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow);
return -1; return TSDB_CODE_INVALID_PARA;
} }
iOvfl = 0; iOvfl = 0;
@ -265,14 +266,14 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) {
pPage->aiOvfl[iOvfl]--; pPage->aiOvfl[iOvfl]--;
if (pPage->aiOvfl[iOvfl] <= 0) { if (pPage->aiOvfl[iOvfl] <= 0) {
tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]); tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
} }
return 0; return 0;
} }
void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
int delta, nFree; int delta, nFree;
pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr);
@ -280,7 +281,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
if (pToPage->pFreeEnd < pToPage->pFreeStart) { if (pToPage->pFreeEnd < pToPage->pFreeStart) {
tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd); tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd);
return; return TSDB_CODE_INVALID_DATA_FMT;
} }
memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr);
@ -289,7 +290,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) { if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) {
tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage), tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage),
pToPage->pFreeEnd - pToPage->pData); pToPage->pFreeEnd - pToPage->pData);
return; return TSDB_CODE_INVALID_DATA_FMT;
} }
delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData);
@ -304,6 +305,10 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
if (deepCopyOvfl) { if (deepCopyOvfl) {
int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL); int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL);
pNewCell = (SCell *)tdbOsMalloc(szCell); pNewCell = (SCell *)tdbOsMalloc(szCell);
if (pNewCell == NULL) {
tdbError("tdb/page-copy: out of memory, size: %d", szCell);
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell); memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell);
tdbTrace("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage); tdbTrace("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage);
} }
@ -312,6 +317,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl]; pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl];
} }
pToPage->nOverflow = pFromPage->nOverflow; pToPage->nOverflow = pFromPage->nOverflow;
return 0;
} }
int tdbPageCapacity(int pageSize, int amHdrSize) { int tdbPageCapacity(int pageSize, int amHdrSize) {
@ -343,12 +349,12 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) {
tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell, tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell,
TDB_PAGE_OFFSET_SIZE(pPage)); TDB_PAGE_OFFSET_SIZE(pPage));
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) { if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) {
tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage), tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage),
pPage->pFreeEnd - pPage->pData); pPage->pFreeEnd - pPage->pData);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
// 1. Try to allocate from the free space block area // 1. Try to allocate from the free space block area
@ -363,7 +369,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
cellFree = TDB_PAGE_FCELL(pPage); cellFree = TDB_PAGE_FCELL(pPage);
if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) { if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) {
tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData); tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) {
SCell *pPrevFreeCell = NULL; SCell *pPrevFreeCell = NULL;
@ -408,19 +414,19 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
} }
// 3. Try to dfragment and allocate again // 3. Try to dfragment and allocate again
tdbPageDefragment(pPage); TAOS_CHECK_RETURN(tdbPageDefragment(pPage));
if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { if (pPage->pFreeEnd - pPage->pFreeStart != nFree) {
tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd); tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
if (TDB_PAGE_NFREE(pPage) != nFree) { if (TDB_PAGE_NFREE(pPage) != nFree) {
tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage)); tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage));
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) { if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) {
tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart, tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart,
pPage->pData); pPage->pData);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
pPage->pFreeEnd -= szCell; pPage->pFreeEnd -= szCell;
@ -430,7 +436,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
_alloc_finish: _alloc_finish:
if (NULL == pCell) { if (NULL == pCell) {
tdbError("tdb/page-allocate: null ptr pCell."); tdbError("tdb/page-allocate: null ptr pCell.");
return -1; return TSDB_CODE_OUT_OF_BUFFER;
} }
pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage);
@ -447,15 +453,15 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) {
if (pCell < pPage->pFreeEnd) { if (pCell < pPage->pFreeEnd) {
tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd); tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd);
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (pCell + szCell > (u8 *)(pPage->pPageFtr)) { if (pCell + szCell > (u8 *)(pPage->pPageFtr)) {
tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd); tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd);
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) { if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) {
tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx)); tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx));
return -1; return TSDB_CODE_INVALID_PARA;
} }
nFree = TDB_PAGE_NFREE(pPage); nFree = TDB_PAGE_NFREE(pPage);
@ -470,7 +476,7 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) {
TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData);
} else { } else {
tdbError("tdb/page-free: invalid cell size: %d", szCell); tdbError("tdb/page-free: invalid cell size: %d", szCell);
return -1; return TSDB_CODE_INVALID_PARA;
} }
} }
@ -502,7 +508,9 @@ static int tdbPageDefragment(SPage *pPage) {
int32_t nCell = TDB_PAGE_NCELLS(pPage); int32_t nCell = TDB_PAGE_NCELLS(pPage);
SCellIdx *aCellIdx = (SCellIdx *)tdbOsMalloc(sizeof(SCellIdx) * nCell); SCellIdx *aCellIdx = (SCellIdx *)tdbOsMalloc(sizeof(SCellIdx) * nCell);
if (aCellIdx == NULL) return -1; if (aCellIdx == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t iCell = 0; iCell < nCell; iCell++) { for (int32_t iCell = 0; iCell < nCell; iCell++) {
aCellIdx[iCell].iCell = iCell; aCellIdx[iCell].iCell = iCell;
aCellIdx[iCell].offset = TDB_PAGE_CELL_OFFSET_AT(pPage, iCell); aCellIdx[iCell].offset = TDB_PAGE_CELL_OFFSET_AT(pPage, iCell);

244
source/libs/tdb/src/db/tdbPager.c
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tdbInt.h"
#include "crypt.h" #include "crypt.h"
#include "tdbInt.h"
#include "tglobal.h" #include "tglobal.h"
/* /*
#pragma pack(push, 1) #pragma pack(push, 1)
@ -41,9 +41,10 @@ struct hashset_st {
static const unsigned int prime = 39; static const unsigned int prime = 39;
static const unsigned int prime2 = 5009; static const unsigned int prime2 = 5009;
hashset_t hashset_create(void) { static hashset_t hashset_create(void) {
hashset_t set = tdbOsCalloc(1, sizeof(struct hashset_st)); hashset_t set = tdbOsCalloc(1, sizeof(struct hashset_st));
if (!set) { if (!set) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
@ -52,6 +53,7 @@ hashset_t hashset_create(void) {
set->items = tdbOsCalloc(set->capacity, sizeof(size_t)); set->items = tdbOsCalloc(set->capacity, sizeof(size_t));
if (!set->items) { if (!set->items) {
tdbOsFree(set); tdbOsFree(set);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
set->mask = set->capacity - 1; set->mask = set->capacity - 1;
@ -69,7 +71,7 @@ void hashset_destroy(hashset_t set) {
} }
} }
int hashset_add_member(hashset_t set, void *item) { static int hashset_add_member(hashset_t set, void *item) {
size_t value = (size_t)item; size_t value = (size_t)item;
size_t h; size_t h;
@ -88,7 +90,7 @@ int hashset_add_member(hashset_t set, void *item) {
return 1; return 1;
} }
int hashset_add(hashset_t set, void *item) { static int hashset_add(hashset_t set, void *item) {
int ret = hashset_add_member(set, item); int ret = hashset_add_member(set, item);
size_t old_capacity = set->capacity; size_t old_capacity = set->capacity;
@ -113,7 +115,7 @@ int hashset_add(hashset_t set, void *item) {
return ret; return ret;
} }
int hashset_remove(hashset_t set, void *item) { static int hashset_remove(hashset_t set, void *item) {
size_t value = (size_t)item; size_t value = (size_t)item;
for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) {
@ -127,7 +129,7 @@ int hashset_remove(hashset_t set, void *item) {
return 0; return 0;
} }
int hashset_contains(hashset_t set, void *item) { static int hashset_contains(hashset_t set, void *item) {
size_t value = (size_t)item; size_t value = (size_t)item;
for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) {
@ -177,7 +179,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
+ fsize + 8 + 1; /* jFileName */ + fsize + 8 + 1; /* jFileName */
pPtr = (uint8_t *)tdbOsCalloc(1, zsize); pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) { if (pPtr == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pPager = (SPager *)pPtr; pPager = (SPager *)pPtr;
@ -198,12 +200,12 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
if (TDB_FD_INVALID(pPager->fd)) { if (TDB_FD_INVALID(pPager->fd)) {
// if (pPager->fd < 0) { // if (pPager->fd < 0) {
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
ret = tdbGnrtFileID(pPager->fd, pPager->fid, false); ret = tdbGnrtFileID(pPager->fd, pPager->fid, false);
if (ret < 0) { if (ret < 0) {
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
// pPager->jfd = -1; // pPager->jfd = -1;
@ -221,11 +223,6 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
int tdbPagerClose(SPager *pPager) { int tdbPagerClose(SPager *pPager) {
if (pPager) { if (pPager) {
/*
if (pPager->inTran) {
tdbOsClose(pPager->jfd);
}
*/
tdbOsClose(pPager->fd); tdbOsClose(pPager->fd);
tdbOsFree(pPager); tdbOsFree(pPager);
} }
@ -254,8 +251,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
!hashset_contains(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { !hashset_contains(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) {
ret = tdbPagerWritePageToJournal(pPager, pPage); ret = tdbPagerWritePageToJournal(pPager, pPage);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write page to journal since %s", tstrerror(terrno)); tdbError("failed to write page to journal since %s", tstrerror(ret));
return -1; return ret;
} }
if (pPager->pActiveTxn->jPageSet) { if (pPager->pActiveTxn->jPageSet) {
@ -278,11 +275,13 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) {
pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
if (TDB_FD_INVALID(pTxn->jfd)) { if (TDB_FD_INVALID(pTxn->jfd)) {
tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
pTxn->jPageSet = hashset_create(); pTxn->jPageSet = hashset_create();
if (pTxn->jPageSet == NULL) {
return terrno;
}
pPager->pActiveTxn = pTxn; pPager->pActiveTxn = pTxn;
@ -319,8 +318,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
ret = tdbOsFSync(pTxn->jfd); ret = tdbOsFSync(pTxn->jfd);
if (ret < 0) { if (ret < 0) {
tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
// loop to write the dirty pages to file // loop to write the dirty pages to file
@ -331,13 +329,13 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
if (pPage->nOverflow != 0) { if (pPage->nOverflow != 0) {
tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow); tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
ret = tdbPagerPWritePageToDB(pPager, pPage); ret = tdbPagerPWritePageToDB(pPager, pPage);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write page to db since %s", tstrerror(terrno)); tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1; return ret;
} }
} }
@ -368,8 +366,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
// sync the db file // sync the db file
if (tdbOsFSync(pPager->fd) < 0) { if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
return 0; return 0;
@ -382,14 +379,12 @@ int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) {
// remove the journal file // remove the journal file
if (tdbOsClose(pTxn->jfd) < 0) { if (tdbOsClose(pTxn->jfd) < 0) {
tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) {
tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
// pPager->inTran = 0; // pPager->inTran = 0;
@ -408,8 +403,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
ret = tdbOsFSync(pTxn->jfd); ret = tdbOsFSync(pTxn->jfd);
if (ret < 0) { if (ret < 0) {
tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
// loop to write the dirty pages to file // loop to write the dirty pages to file
@ -426,7 +420,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
ret = tdbPagerPWritePageToDB(pPager, pPage); ret = tdbPagerPWritePageToDB(pPager, pPage);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write page to db since %s", tstrerror(terrno)); tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1; return ret;
} }
} }
@ -445,36 +439,28 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
tdbPCacheRelease(pPager->pCache, pPage, pTxn); tdbPCacheRelease(pPager->pCache, pPage, pTxn);
} }
/*
tdbTrace("reset dirty tree: %p", &pPager->rbt);
tRBTreeCreate(&pPager->rbt, pageCmpFn);
// sync the db file
if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
*/
return 0; return 0;
} }
static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, const char* function, static char *tdbEncryptPage(SPager *pPager, char *pPageData, int32_t pageSize, const char *function, int64_t offset) {
int64_t offset){
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey; char *encryptKey = pPager->pEnv->encryptKey;
char* buf = pPageData; char *buf = pPageData;
if(encryptAlgorithm == DND_CA_SM4){ if (encryptAlgorithm == DND_CA_SM4) {
//tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); // tdbInfo("CBC_Encrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
//ASSERT(strlen(encryptKey) > 0); // ASSERT(strlen(encryptKey) > 0);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Encrypt", offset, pPage->pData[0]);
buf = taosMemoryMalloc(pageSize); buf = taosMemoryMalloc(pageSize);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
unsigned char packetData[128]; unsigned char packetData[128];
int32_t count = 0; int32_t count = 0;
while (count < pageSize) { while (count < pageSize) {
@ -488,19 +474,19 @@ static char* tdbEncryptPage(SPager *pPager, char* pPageData, int32_t pageSize, c
int32_t newLen = CBC_Encrypt(&opts); int32_t newLen = CBC_Encrypt(&opts);
memcpy(buf + count, packetData, newLen); memcpy(buf + count, packetData, newLen);
count += newLen; count += newLen;
} }
//tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function); // tdbInfo("CBC tdb offset:%" PRId64 ", Encrypt count:%d %s", offset, count, function);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Encrypt", offset, (uint8_t)buf[0]);
} }
return buf; return buf;
} }
void tdbFreeEncryptBuf(SPager *pPager, char* buf){ void tdbFreeEncryptBuf(SPager *pPager, char *buf) {
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
if(encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf); if (encryptAlgorithm == DND_CA_SM4) taosMemoryFreeClear(buf);
} }
// recovery dirty pages // recovery dirty pages
int tdbPagerAbort(SPager *pPager, TXN *pTxn) { int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
@ -518,26 +504,24 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
ret = tdbOsFSync(pTxn->jfd); ret = tdbOsFSync(pTxn->jfd);
if (ret < 0) { if (ret < 0) {
tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
tdb_fd_t jfd = pTxn->jfd; tdb_fd_t jfd = pTxn->jfd;
ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize);
if (ret < 0) { if (ret < 0) {
return -1; return ret;
} }
if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) {
tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize);
if (pageBuf == NULL) { if (pageBuf == NULL) {
return -1; return terrno = TSDB_CODE_OUT_OF_MEMORY;
} }
tdbDebug("pager/abort: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId); tdbDebug("pager/abort: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId);
@ -549,7 +533,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); int ret = tdbOsRead(jfd, &pgno, sizeof(pgno));
if (ret < 0) { if (ret < 0) {
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
tdbTrace("pager/abort: restore pgno:%d,", pgno); tdbTrace("pager/abort: restore pgno:%d,", pgno);
@ -559,27 +543,28 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); ret = tdbOsRead(jfd, pageBuf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
i64 offset = pPager->pageSize * (pgno - 1); i64 offset = pPager->pageSize * (pgno - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
if (buf == NULL) {
return terrno;
}
ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize); ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
pPager->pageSize); pPager->pageSize);
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
@ -587,9 +572,8 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
if (tdbOsFSync(pPager->fd) < 0) { if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
@ -617,8 +601,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
// 4, remove the journal file // 4, remove the journal file
if (tdbOsClose(pTxn->jfd) < 0) { if (tdbOsClose(pTxn->jfd) < 0) {
tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
char jTxnFileName[TDB_FILENAME_LEN]; char jTxnFileName[TDB_FILENAME_LEN];
@ -626,8 +609,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) {
tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
// pPager->inTran = 0; // pPager->inTran = 0;
@ -658,7 +640,7 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) {
ret = tdbPagerPWritePageToDB(pPager, pPage); ret = tdbPagerPWritePageToDB(pPager, pPage);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write page to db since %s", tstrerror(terrno)); tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1; return ret;
} }
tdbTrace("tdb/flush:%p, pgno:%d, %d/%d/%d", pPager, pgno, pPager->dbOrigSize, pPager->dbFileSize, maxPgno); tdbTrace("tdb/flush:%p, pgno:%d, %d/%d/%d", pPager, pgno, pPager->dbOrigSize, pPager->dbFileSize, maxPgno);
@ -717,13 +699,13 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa
ret = tdbPagerAllocPage(pPager, &pgno, pTxn); ret = tdbPagerAllocPage(pPager, &pgno, pTxn);
if (ret < 0) { if (ret < 0) {
tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno);
return -1; return ret;
} }
} }
if (pgno == 0) { if (pgno == 0) {
tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
// fetch a page container // fetch a page container
@ -739,7 +721,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa
ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage); ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage);
if (ret < 0) { if (ret < 0) {
tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage); tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage);
return -1; return ret;
} }
} }
@ -748,11 +730,11 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa
if (!TDB_PAGE_INITIALIZED(pPage)) { if (!TDB_PAGE_INITIALIZED(pPage)) {
tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage); tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
if (pPage->pPager != pPager) { if (pPage->pPager != pPager) {
tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager); tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager);
return -1; return TSDB_CODE_INVALID_DATA_FMT;
} }
*ppgno = pgno; *ppgno = pgno;
@ -771,12 +753,17 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
SPgno pgno = TDB_PAGE_PGNO(pPage); SPgno pgno = TDB_PAGE_PGNO(pPage);
if (pPager->frps) { if (pPager->frps) {
taosArrayPush(pPager->frps, &pgno); if (taosArrayPush(pPager->frps, &pgno) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pPage->pPager = NULL; pPage->pPager = NULL;
return code; return code;
} }
pPager->frps = taosArrayInit(8, sizeof(SPgno)); pPager->frps = taosArrayInit(8, sizeof(SPgno));
if (pPager->frps == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// memset(pPage->pData, 0, pPage->pageSize); // memset(pPage->pData, 0, pPage->pageSize);
tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno); tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno);
// printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno); // printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno);
@ -785,7 +772,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
taosArrayDestroy(pPager->frps); taosArrayDestroy(pPager->frps);
pPager->frps = NULL; pPager->frps = NULL;
return -1; return code;
} }
while (TARRAY_SIZE(pPager->frps) > 0) { while (TARRAY_SIZE(pPager->frps) > 0) {
@ -796,7 +783,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
taosArrayDestroy(pPager->frps); taosArrayDestroy(pPager->frps);
pPager->frps = NULL; pPager->frps = NULL;
return -1; return code;
} }
} }
@ -822,7 +809,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn); code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn);
if (code < 0) { if (code < 0) {
return 0; return code;
} }
code = tdbTbcMoveToFirst(pCur); code = tdbTbcMoveToFirst(pCur);
@ -924,20 +911,19 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
} }
int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm; int32_t encryptAlgorithm = pPager->pEnv->encryptAlgorithm;
char* encryptKey = pPager->pEnv->encryptKey; char *encryptKey = pPager->pEnv->encryptKey;
if(encryptAlgorithm == DND_CA_SM4){ if (encryptAlgorithm == DND_CA_SM4) {
//tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__); // tdbInfo("CBC_Decrypt key:%d %s %s", encryptAlgorithm, encryptKey, __FUNCTION__);
//ASSERT(strlen(encryptKey) > 0); // ASSERT(strlen(encryptKey) > 0);
//uint8_t flags = pPage->pData[0]; // uint8_t flags = pPage->pData[0];
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d before Decrypt", ((i64)pPage->pageSize) * (pgno - 1), flags);
unsigned char packetData[128]; unsigned char packetData[128];
int32_t count = 0; int32_t count = 0;
while(count < pPage->pageSize) while (count < pPage->pageSize) {
{
SCryptOpts opts = {0}; SCryptOpts opts = {0};
opts.len = 128; opts.len = 128;
opts.source = pPage->pData + count; opts.source = pPage->pData + count;
@ -945,20 +931,23 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
opts.unitLen = 128; opts.unitLen = 128;
strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN); strncpy(opts.key, encryptKey, ENCRYPT_KEY_LEN);
int newLen = CBC_Decrypt(&opts); int newLen = CBC_Decrypt(&opts);
memcpy(pPage->pData + count, packetData, newLen); memcpy(pPage->pData + count, packetData, newLen);
count += newLen; count += newLen;
} }
//tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count, __FUNCTION__); // tdbInfo("CBC tdb offset:%" PRId64 ", Decrypt count:%d %s", ((i64)pPage->pageSize) * (pgno - 1), count,
// __FUNCTION__);
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d after Decrypt %s", ((i64)pPage->pageSize) * (pgno - 1),
// pPage->pData[0], __FUNCTION__);
} }
} else { } else {
init = 0; init = 0;
} }
//tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0], __FUNCTION__); // tdbInfo("CBC tdb offset:%" PRId64 ", flag:%d initPage %s", ((i64)pPage->pageSize) * (pgno - 1), pPage->pData[0],
// __FUNCTION__);
ret = (*initPage)(pPage, arg, init); ret = (*initPage)(pPage, arg, init);
if (ret < 0) { if (ret < 0) {
@ -1001,16 +990,14 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
if (ret < 0) { if (ret < 0) {
tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName, tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName,
pgno, pPager->pActiveTxn->txnId); pgno, pPager->pActiveTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize); ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno), tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno),
pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId); pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
return 0; return 0;
@ -1044,15 +1031,14 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) {
offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
char* buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset); char *buf = tdbEncryptPage(pPager, pPage->pData, pPage->pageSize, __FUNCTION__, offset);
ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset); ret = tdbOsPWrite(pPager->fd, buf, pPage->pageSize, offset);
if (ret < 0) { if (ret < 0) {
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName,
pPage->pageSize); pPage->pageSize);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
@ -1072,18 +1058,17 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize);
if (ret < 0) { if (ret < 0) {
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) {
tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
pageBuf = tdbOsCalloc(1, pPager->pageSize); pageBuf = tdbOsCalloc(1, pPager->pageSize);
if (pageBuf == NULL) { if (pageBuf == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
tdbDebug("pager/restore: %p, %d/%d, txnId:%s", pPager, pPager->dbOrigSize, pPager->dbFileSize, jFileName); tdbDebug("pager/restore: %p, %d/%d, txnId:%s", pPager, pPager->dbOrigSize, pPager->dbFileSize, jFileName);
@ -1095,7 +1080,7 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); int ret = tdbOsRead(jfd, &pgno, sizeof(pgno));
if (ret < 0) { if (ret < 0) {
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
tdbTrace("pager/restore: restore pgno:%d,", pgno); tdbTrace("pager/restore: restore pgno:%d,", pgno);
@ -1103,27 +1088,28 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); ret = tdbOsRead(jfd, pageBuf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
i64 offset = pPager->pageSize * (pgno - 1); i64 offset = pPager->pageSize * (pgno - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
char* buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset); char *buf = tdbEncryptPage(pPager, pageBuf, pPager->pageSize, __FUNCTION__, offset);
if (buf == NULL) {
return terrno;
}
ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize); ret = tdbOsWrite(pPager->fd, buf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName,
pPager->pageSize); pPager->pageSize);
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
tdbFreeEncryptBuf(pPager, buf); tdbFreeEncryptBuf(pPager, buf);
@ -1131,9 +1117,8 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
if (tdbOsFSync(pPager->fd) < 0) { if (tdbOsFSync(pPager->fd) < 0) {
tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
@ -1160,21 +1145,27 @@ static int32_t txnIdCompareDesc(const void *pLeft, const void *pRight) {
} }
int tdbPagerRestoreJournals(SPager *pPager) { int tdbPagerRestoreJournals(SPager *pPager) {
int32_t code = 0;
tdbDirEntryPtr pDirEntry; tdbDirEntryPtr pDirEntry;
tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName);
if (pDir == NULL) { if (pDir == NULL) {
tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno));
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
SArray *pTxnList = taosArrayInit(16, sizeof(int64_t)); SArray *pTxnList = taosArrayInit(16, sizeof(int64_t));
if (pTxnList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
while ((pDirEntry = tdbReadDir(pDir)) != NULL) { while ((pDirEntry = tdbReadDir(pDir)) != NULL) {
char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry));
if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) {
int64_t txnId = -1; int64_t txnId = -1;
sscanf(name, TDB_MAINDB_NAME "-journal.%" PRId64, &txnId); sscanf(name, TDB_MAINDB_NAME "-journal.%" PRId64, &txnId);
taosArrayPush(pTxnList, &txnId); if (taosArrayPush(pTxnList, &txnId) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
} }
taosArraySort(pTxnList, txnIdCompareDesc); taosArraySort(pTxnList, txnIdCompareDesc);
@ -1185,12 +1176,12 @@ int tdbPagerRestoreJournals(SPager *pPager) {
memcpy(jname, pPager->pEnv->dbName, dirLen); memcpy(jname, pPager->pEnv->dbName, dirLen);
jname[dirLen] = '/'; jname[dirLen] = '/';
sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId);
if (tdbPagerRestore(pPager, jname) < 0) { code = tdbPagerRestore(pPager, jname);
if (code) {
taosArrayDestroy(pTxnList); taosArrayDestroy(pTxnList);
tdbCloseDir(&pDir); tdbCloseDir(&pDir);
tdbError("failed to restore file due to %s. jFileName:%s", strerror(code), jname);
tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname); return code;
return -1;
} }
} }
@ -1205,7 +1196,7 @@ int tdbPagerRollback(SPager *pPager) {
tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName);
if (pDir == NULL) { if (pDir == NULL) {
tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno));
return -1; return terrno = TAOS_SYSTEM_ERROR(errno);
} }
while ((pDirEntry = tdbReadDir(pDir)) != NULL) { while ((pDirEntry = tdbReadDir(pDir)) != NULL) {
@ -1221,8 +1212,7 @@ int tdbPagerRollback(SPager *pPager) {
tdbCloseDir(&pDir); tdbCloseDir(&pDir);
tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name); tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
} }
} }

10
source/libs/tdb/src/db/tdbTable.c
View File

@ -41,7 +41,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb)); pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb));
if (pTb == NULL) { if (pTb == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
// pTb->pEnv // pTb->pEnv
@ -54,7 +54,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
pPager = tdbEnvGetPager(pEnv, fFullName); pPager = tdbEnvGetPager(pEnv, fFullName);
if (!pPager) { if (!pPager) {
tdbOsFree(pTb); tdbOsFree(pTb);
return -1; return terrno;
} }
ret = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pData, &nData); ret = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pData, &nData);
@ -74,7 +74,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager);
if (ret < 0) { if (ret < 0) {
tdbOsFree(pTb); tdbOsFree(pTb);
return -1; return ret;
} }
tdbEnvAddPager(pEnv, pPager); tdbEnvAddPager(pEnv, pPager);
@ -109,7 +109,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
ret = tdbPagerRestoreJournals(pPager); ret = tdbPagerRestoreJournals(pPager);
if (ret < 0) { if (ret < 0) {
tdbOsFree(pTb); tdbOsFree(pTb);
return -1; return ret;
} }
} else { } else {
tdbPagerRollback(pPager); tdbPagerRollback(pPager);
@ -119,7 +119,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt)); ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt));
if (ret < 0) { if (ret < 0) {
tdbOsFree(pTb); tdbOsFree(pTb);
return -1; return ret;
} }
*ppTb = pTb; *ppTb = pTb;

4
source/libs/tdb/src/db/tdbTxn.c
View File

@ -20,7 +20,7 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void
// not support read-committed version at the moment // not support read-committed version at the moment
if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) { if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) {
tdbError("tdb/txn: invalid txn flags: %" PRId32, flags); tdbError("tdb/txn: invalid txn flags: %" PRId32, flags);
return -1; return TSDB_CODE_INVALID_PARA;
} }
pTxn->flags = flags; pTxn->flags = flags;
@ -39,7 +39,7 @@ int tdbTxnCloseImpl(TXN *pTxn) {
} }
if (pTxn->jfd) { if (pTxn->jfd) {
tdbOsClose(pTxn->jfd); TAOS_UNUSED(tdbOsClose(pTxn->jfd));
ASSERT(pTxn->jfd == NULL); ASSERT(pTxn->jfd == NULL);
} }

7
source/libs/tdb/src/db/tdbUtil.c
View File

@ -38,9 +38,8 @@ void tdbFree(void *p) {
int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique) { int tdbGnrtFileID(tdb_fd_t fd, uint8_t *fileid, bool unique) {
int64_t stDev = 0, stIno = 0; int64_t stDev = 0, stIno = 0;
if (taosDevInoFile(fd, &stDev, &stIno) < 0) { int32_t code = taosDevInoFile(fd, &stDev, &stIno);
return -1; return code;
}
memset(fileid, 0, TDB_FILE_ID_LEN); memset(fileid, 0, TDB_FILE_ID_LEN);
@ -59,7 +58,7 @@ int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size) {
ret = tdbOsFileSize(fd, &szBytes); ret = tdbOsFileSize(fd, &szBytes);
if (ret < 0) { if (ret < 0) {
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
*size = szBytes / szPage; *size = szBytes / szPage;

19
source/libs/tdb/src/inc/tdbInt.h
View File

@ -17,6 +17,7 @@
#define _TD_TDB_INTERNAL_H_ #define _TD_TDB_INTERNAL_H_
#include "tdb.h" #include "tdb.h"
#include "tutil.h"
#include "tdef.h" #include "tdef.h"
#include "tlog.h" #include "tlog.h"
@ -338,15 +339,15 @@ static inline int tdbTryLockPage(tdb_spinlock_t *pLock) {
((*(pPage)->xCellSize)(pPage, pCell, 0, NULL, NULL) + (pPage)->pPageMethods->szOffset) ((*(pPage)->xCellSize)(pPage, pCell, 0, NULL, NULL) + (pPage)->pPageMethods->szOffset)
#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) #define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset)
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg);
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg);
void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt));
void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)); void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt));
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl);
int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt); int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt);
int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt); int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt);
void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int copyOvflCells); int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl);
int tdbPageCapacity(int pageSize, int amHdrSize); int tdbPageCapacity(int pageSize, int amHdrSize);
static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { static inline SCell *tdbPageGetCell(SPage *pPage, int idx) {
SCell *pCell; SCell *pCell;

30
source/os/src/osFile.c
View File

@ -67,7 +67,7 @@ typedef struct TdFile {
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) {
#ifdef WINDOWS #ifdef WINDOWS
char tmpPath[PATH_MAX]; char tmpPath[PATH_MAX];
int32_t len = (int32_t)strlen(inputTmpDir); int32_t len = (int32_t)strlen(inputTmpDir);
memcpy(tmpPath, inputTmpDir, len); memcpy(tmpPath, inputTmpDir, len);
@ -269,13 +269,13 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
#else #else
if (pFile == NULL || pFile->fd < 0) { if (pFile == NULL || pFile->fd < 0) {
return -1; return TSDB_CODE_INVALID_PARA;
} }
struct stat fileStat; struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat); int32_t code = fstat(pFile->fd, &fileStat);
if (code < 0) { if (code < 0) {
printf("taosFStatFile run fstat fail."); printf("taosFStatFile run fstat fail.");
return code; return TAOS_SYSTEM_ERROR(errno);
} }
if (stDev != NULL) { if (stDev != NULL) {
@ -1239,7 +1239,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
#ifdef WINDOWS #ifdef WINDOWS
size_t bufferSize = 512; size_t bufferSize = 512;
*ptrBuf = taosMemoryMalloc(bufferSize); *ptrBuf = taosMemoryMalloc(bufferSize);
if (*ptrBuf == NULL) goto END; if (*ptrBuf == NULL) goto END;
size_t bytesRead = 0; size_t bytesRead = 0;
size_t totalBytesRead = 0; size_t totalBytesRead = 0;
@ -1274,7 +1274,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
ret = getline(ptrBuf, &len, pFile->fp); ret = getline(ptrBuf, &len, pFile->fp);
#endif #endif
END: END:
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockUnlock(&(pFile->rwlock)); taosThreadRwlockUnlock(&(pFile->rwlock));
#endif #endif
@ -1413,34 +1413,30 @@ int32_t taosLinkFile(char *src, char *dst) {
return 0; return 0;
} }
FILE* taosOpenCFile(const char* filename, const char* mode) { FILE *taosOpenCFile(const char *filename, const char *mode) { return fopen(filename, mode); }
return fopen(filename, mode);
}
int taosSeekCFile(FILE* file, int64_t offset, int whence) { int taosSeekCFile(FILE *file, int64_t offset, int whence) {
#ifdef WINDOWS #ifdef WINDOWS
return _fseeki64(file, offset, whence); return _fseeki64(file, offset, whence);
#else #else
return fseeko(file, offset, whence); return fseeko(file, offset, whence);
#endif #endif
} }
size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream ) { size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream) {
return fread(buffer, size, count, stream); return fread(buffer, size, count, stream);
} }
size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream) { size_t taosWriteToCFile(const void *ptr, size_t size, size_t nitems, FILE *stream) {
return fwrite(ptr, size, nitems, stream); return fwrite(ptr, size, nitems, stream);
} }
int taosCloseCFile(FILE *f) { int taosCloseCFile(FILE *f) { return fclose(f); }
return fclose(f);
}
int taosSetAutoDelFile(char* path) { int taosSetAutoDelFile(char *path) {
#ifdef WINDOWS #ifdef WINDOWS
return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY); return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY);
#else #else
return unlink(path); return unlink(path);
#endif #endif
} }