From 5a37fdf28fb9302f0754effb15a87e88271c07c0 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 25 Feb 2022 01:47:20 +0800 Subject: [PATCH] [TD-13062]: cross platform file system. --- include/common/ttszip.h | 6 +- include/libs/wal/wal.h | 8 +- include/os/osFile.h | 78 ++- include/util/tfile.h | 26 +- source/common/src/ttszip.c | 135 ++-- source/dnode/mgmt/impl/inc/dndEnv.h | 2 +- source/dnode/mgmt/impl/src/dndBnode.c | 20 +- source/dnode/mgmt/impl/src/dndEnv.c | 28 +- source/dnode/mgmt/impl/src/dndMgmt.c | 21 +- source/dnode/mgmt/impl/src/dndMnode.c | 20 +- source/dnode/mgmt/impl/src/dndQnode.c | 20 +- source/dnode/mgmt/impl/src/dndSnode.c | 20 +- source/dnode/mgmt/impl/src/dndVnodes.c | 20 +- source/dnode/mnode/impl/src/mndTelem.c | 41 +- source/dnode/mnode/impl/test/trans/trans.cpp | 14 +- source/dnode/mnode/sdb/src/sdbFile.c | 50 +- source/dnode/vnode/src/inc/tqInt.h | 4 +- source/dnode/vnode/src/inc/tsdbFile.h | 28 +- source/dnode/vnode/src/meta/metaBDBImpl.c | 1 + source/dnode/vnode/src/tq/tqMetaStore.c | 58 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 45 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 12 +- source/libs/CMakeLists.txt | 1 + source/libs/function/src/taggfunction.c | 8 +- .../index/inc/index_fst_counting_writer.h | 2 +- .../index/src/index_fst_counting_writer.c | 36 +- source/libs/index/src/index_tfile.c | 8 +- source/libs/tdb/src/db/tdbEnv.c | 12 +- source/libs/tdb/src/db/tdbPgFile.c | 12 +- source/libs/tdb/src/inc/tdbPgFile.h | 2 +- source/libs/tfs/test/tfsTest.cpp | 18 +- source/libs/transport/inc/rpcUdp.h | 2 +- source/libs/transport/test/pushServer.c | 16 +- source/libs/transport/test/rserver.c | 16 +- source/libs/wal/src/walMeta.c | 34 +- source/libs/wal/src/walMgmt.c | 14 +- source/libs/wal/src/walRead.c | 44 +- source/libs/wal/src/walSeek.c | 52 +- source/libs/wal/src/walWrite.c | 60 +- source/os/src/osFile.c | 578 ++++++++++-------- source/os/src/osRand.c | 10 +- source/os/src/osSocket.c | 1 + source/os/src/osSysinfo.c | 101 ++- source/os/src/osTimezone.c | 13 +- source/util/src/tconfig.c | 21 +- source/util/src/tfile.c | 210 +++---- source/util/src/tlog.c | 106 ++-- source/util/src/tpagedbuf.c | 27 +- tools/shell/src/backup/shellCheck.c | 8 +- tools/shell/src/backup/shellImport.c | 12 +- tools/shell/src/shellEngine.c | 74 +-- 51 files changed, 1129 insertions(+), 1026 deletions(-) diff --git a/include/common/ttszip.h b/include/common/ttszip.h index 38699ae791..d83b0066d5 100644 --- a/include/common/ttszip.h +++ b/include/common/ttszip.h @@ -73,9 +73,9 @@ typedef struct STSGroupBlockInfoEx { } STSGroupBlockInfoEx; typedef struct STSBuf { - FILE* f; - char path[PATH_MAX]; - uint32_t fileSize; + TdFilePtr pFile; + char path[PATH_MAX]; + uint32_t fileSize; // todo use array STSGroupBlockInfoEx* pData; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 45f1d88c30..f90dbb97fe 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -120,8 +120,8 @@ typedef struct SWal { int32_t fsyncSeq; // meta SWalVer vers; - int64_t writeLogTfd; - int64_t writeIdxTfd; + TdFilePtr pWriteLogTFile; + TdFilePtr pWriteIdxTFile; int32_t writeCur; SArray *fileInfoSet; // status @@ -138,8 +138,8 @@ typedef struct SWal { typedef struct SWalReadHandle { SWal *pWal; - int64_t readLogTfd; - int64_t readIdxTfd; + TdFilePtr pReadLogTFile; + TdFilePtr pReadIdxTFile; int64_t curFileFirstVer; int64_t curVersion; int64_t capacity; diff --git a/include/os/osFile.h b/include/os/osFile.h index ac399fa3be..59492f6694 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -22,49 +22,63 @@ extern "C" { #include "osSocket.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -typedef int32_t FileFd; -#else -typedef int32_t FileFd; +#ifndef ALLOW_FORBID_FUNC + #define open OPEN_FUNC_TAOS_FORBID + #define fopen FOPEN_FUNC_TAOS_FORBID + // #define close CLOSE_FUNC_TAOS_FORBID + // #define fclose FCLOSE_FUNC_TAOS_FORBID #endif -#define FD_INITIALIZER ((int32_t)-1) - #ifndef PATH_MAX #define PATH_MAX 256 #endif -int32_t taosLockFile(FileFd fd); -int32_t taosUnLockFile(FileFd fd); - -int32_t taosUmaskFile(FileFd fd); - +typedef struct TdFile *TdFilePtr; + +#define TD_FILE_CTEATE 0x0001 +#define TD_FILE_WRITE 0x0002 +#define TD_FILE_READ 0x0004 +#define TD_FILE_TRUNC 0x0008 +#define TD_FILE_APPEND 0x0010 +#define TD_FILE_TEXT 0x0020 +#define TD_FILE_AUTO_DEL 0x0040 +#define TD_FILE_EXCL 0x0080 + +int32_t taosLockFile(TdFilePtr pFile); +int32_t taosUnLockFile(TdFilePtr pFile); + +int32_t taosUmaskFile(int32_t maskVal); + int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); -int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime); - -FileFd taosOpenFileWrite(const char *path); -FileFd taosOpenFileCreateWrite(const char *path); -FileFd taosOpenFileCreateWriteTrunc(const char *path); -FileFd taosOpenFileCreateWriteAppend(const char *path); -FileFd taosOpenFileRead(const char *path); -FileFd taosOpenFileReadWrite(const char *path); - -int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence); -int32_t taosFtruncateFile(FileFd fd, int64_t length); -int32_t taosFsyncFile(FileFd fd); - -int64_t taosReadFile(FileFd fd, void *buf, int64_t count); -int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count); - -void taosCloseFile(FileFd fd); - +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); + +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); + +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length); +int32_t taosFsyncFile(TdFilePtr pFile); + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count); +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset); +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); +void taosFprintfFile(TdFilePtr pFile, const char *format, ...); +size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); +int32_t taosEOFFile(TdFilePtr pFile); + +int64_t taosCloseFile(TdFilePtr *ppFile); + int32_t taosRenameFile(const char *oldName, const char *newName); int64_t taosCopyFile(const char *from, const char *to); - + void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); + +int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size); +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size); -int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); +void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); +bool taosValidFile(TdFilePtr pFile); + +int taosGetErrorFile(TdFilePtr pFile); #ifdef __cplusplus } diff --git a/include/util/tfile.h b/include/util/tfile.h index d3813051a4..59953de861 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -30,20 +30,20 @@ void tfCleanup(); // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused -int64_t tfOpenRead(const char *pathname); -int64_t tfOpenReadWrite(const char *pathname); -int64_t tfOpenCreateWrite(const char *pathname); -int64_t tfOpenCreateWriteAppend(const char *pathname); +// int64_t tfOpenRead(const char *pathname); +// int64_t tfOpenReadWrite(const char *pathname); +// int64_t tfOpenCreateWrite(const char *pathname); +// int64_t tfOpenCreateWriteAppend(const char *pathname); -int64_t tfClose(int64_t tfd); -int64_t tfWrite(int64_t tfd, void *buf, int64_t count); -int64_t tfRead(int64_t tfd, void *buf, int64_t count); -int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); -int32_t tfFsync(int64_t tfd); -bool tfValid(int64_t tfd); -int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); -int32_t tfFtruncate(int64_t tfd, int64_t length); -void * tfMmapReadOnly(int64_t tfd, int64_t length); +// int64_t tfClose(int64_t tfd); +// int64_t tfWrite(int64_t tfd, void *buf, int64_t count); +// int64_t tfRead(int64_t tfd, void *buf, int64_t count); +// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); +// int32_t tfFsync(int64_t tfd); +// bool tfValid(int64_t tfd); +// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); +// int32_t tfFtruncate(int64_t tfd, int64_t length); +// void * tfMmapReadOnly(int64_t tfd, int64_t length); #ifdef __cplusplus } #endif diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 3265ea5547..f69bfd6e88 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -24,8 +24,9 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { pTSBuf->autoDelete = autoDelete; taosGetTmpfilePath(osTempDir(), "join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "wb+"); - if (pTSBuf->f == NULL) { + // pTSBuf->pFile = fopen(pTSBuf->path, "wb+"); + pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } @@ -60,8 +61,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - pTSBuf->f = fopen(pTSBuf->path, "rb+"); - if (pTSBuf->f == NULL) { + // pTSBuf->pFile = fopen(pTSBuf->path, "rb+"); + pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ); + if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } @@ -72,9 +74,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // validate the file magic number STSBufFileHeader header = {0}; - int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET); + int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); UNUSED(ret); - size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f); + size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader)); UNUSED(sz); // invalid file @@ -112,8 +114,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - //int64_t pos = ftell(pTSBuf->f); //pos not used - sz = fread(buf, infoSize, 1, pTSBuf->f); + //int64_t pos = ftell(pTSBuf->pFile); //pos not used + sz = taosReadFile(pTSBuf->pFile, buf, infoSize); UNUSED(sz); // the length value for each vnode is not kept in file, so does not set the length value @@ -123,22 +125,22 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { } free(buf); - ret = fseek(pTSBuf->f, 0, SEEK_END); + ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END); UNUSED(ret); - struct stat fileStat; - if (fstat(fileno(pTSBuf->f), &fileStat) != 0) { + int64_t file_size; + if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) { tsBufDestroy(pTSBuf); return NULL; } - pTSBuf->fileSize = (uint32_t)fileStat.st_size; + pTSBuf->fileSize = (uint32_t)file_size; tsBufResetPos(pTSBuf); // ascending by default pTSBuf->cur.order = TSDB_ORDER_ASC; -// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), +// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->pFile), // pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); return pTSBuf; @@ -156,7 +158,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { tfree(pTSBuf->block.payload); if (!pTSBuf->remainOpen) { - fclose(pTSBuf->f); + taosCloseFile(&pTSBuf->pFile); } if (pTSBuf->autoDelete) { @@ -253,7 +255,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); + int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET); assert(r == 0); /* @@ -264,30 +266,30 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * both side has the compressed length is used to support load data forwards/backwords. */ int32_t metaLen = 0; - metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); int32_t trueLen = pBlock->tag.nLen; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); float tfloat = (float)pBlock->tag.d; - metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.i, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen); } else { trueLen = 0; - metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); } - fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); + taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); + taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); + taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); - metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t) taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); assert(metaLen == getTagAreaLength(&pBlock->tag)); int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; @@ -332,20 +334,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { * the end of each comp data block */ int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); - int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); - size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); - sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR); + size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); UNUSED(sz); pBlock->compLen = pBlock->padding; offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - ret = fseek(pTSBuf->f, -offset, SEEK_CUR); + ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(ret); } - int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); - ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); + ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); // NOTE: mix types tags are not supported size_t sz = 0; @@ -356,23 +358,23 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { memset(tp, 0, pBlock->tag.nLen + 1); pBlock->tag.pz = tp; - sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); UNUSED(sz); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { float tfloat = 0; - sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen); pBlock->tag.d = (double)tfloat; UNUSED(sz); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value - sz = fread(&pBlock->tag.i, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen); UNUSED(sz); } - sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); UNUSED(sz); - sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); UNUSED(sz); - sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); if (decomp) { pTSBuf->tsData.len = @@ -381,11 +383,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { } // read the comp length at the length of comp block - sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); assert(pBlock->padding == pBlock->compLen); int32_t n = 0; - sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen)); if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { assert(n == 0); } else { @@ -396,7 +398,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // for backwards traverse, set the start position at the end of previous block if (order == TSDB_ORDER_DESC) { - int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); + int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(r); } @@ -512,7 +514,7 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num // todo opt performance by cache blocks info static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { return -1; } @@ -531,7 +533,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int STSBlock* pBlock = &pTSBuf->block; int32_t compBlockSize = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); + int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR); UNUSED(ret); } @@ -548,7 +550,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo offset = pBlockInfo->offset + pBlockInfo->compLen; } - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } @@ -618,11 +620,11 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn return -1; } - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } - fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f); + taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo)); return 0; } @@ -636,19 +638,19 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { } int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { - if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { + if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { return -1; } assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC); - int32_t r = fseek(pTSBuf->f, 0, SEEK_SET); + int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); if (r != 0) { // qError("fseek failed, errno:%d", errno); return -1; } - size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); + size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader)); if (ws != 1) { // qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); return -1; @@ -823,12 +825,12 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { pBlockInfoEx->info.id = id; } - int32_t r = fseek(pDestBuf->f, 0, SEEK_END); + int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END); assert(r == 0); int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; - int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); + int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size); if (written == -1 || written != size) { return -1; @@ -839,17 +841,18 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { int32_t oldSize = pDestBuf->fileSize; // file meta data may be cached, close and reopen the file for accurate file size. - fclose(pDestBuf->f); - pDestBuf->f = fopen(pDestBuf->path, "rb+"); - if (pDestBuf->f == NULL) { + taosCloseFile(&pDestBuf->pFile); + // pDestBuf->pFile = fopen(pDestBuf->path, "rb+"); + pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ); + if (pDestBuf->pFile == NULL) { return -1; } - struct stat fileStat; - if (fstat(fileno(pDestBuf->f), &fileStat) != 0) { + int64_t file_size; + if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) { return -1; } - pDestBuf->fileSize = (uint32_t)fileStat.st_size; + pDestBuf->fileSize = (uint32_t)file_size; assert(pDestBuf->fileSize == oldSize + size); @@ -868,13 +871,13 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ // update prev vnode length info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); - int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET); if (ret == -1) { // qError("fseek failed, errno:%d", errno); tsBufDestroy(pTSBuf); return NULL; } - size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); + size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len); if (sz != len) { // qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); tsBufDestroy(pTSBuf); @@ -893,7 +896,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ } // TODO taosFsync?? -// if (taosFsync(fileno(pTSBuf->f)) == -1) { +// if (taosFsync(fileno(pTSBuf->pFile)) == -1) { //// qError("fsync failed, errno:%d", errno); // tsBufDestroy(pTSBuf); // return NULL; @@ -1071,15 +1074,15 @@ int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, in *len = 0; *numOfBlocks = 0; - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); + if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { + int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile)); // qError("%p: fseek failed: %s", pSql, tstrerror(code)); return code; } - size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); + size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen); if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); + int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile)); // tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); return code; } diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 7465dc14f7..cbd5eb5827 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -125,7 +125,7 @@ typedef struct SDnode { EStat stat; SDnodeObjCfg cfg; SDnodeDir dir; - FileFd lockFd; + TdFilePtr pLockFile; SDnodeMgmt dmgmt; SMnodeMgmt mmgmt; SQnodeMgmt qmgmt; diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 610579f6f5..81b020c152 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -62,14 +62,15 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_BNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_BNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) { PRASE_BNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 5bc3b756ea..247f52f958 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -59,31 +59,31 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); } -static FileFd dndCheckRunning(char *dataDir) { +static TdFilePtr dndCheckRunning(char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); - FileFd fd = taosOpenFileCreateWriteTrunc(filepath); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - return -1; + return NULL; } - int32_t ret = taosLockFile(fd); + int32_t ret = taosLockFile(pFile); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - taosCloseFile(fd); - return -1; + taosCloseFile(&pFile); + return NULL; } - return fd; + return pFile; } static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { - pDnode->lockFd = dndCheckRunning(pCfg->dataDir); - if (pDnode->lockFd < 0) { + pDnode->pLockFile = dndCheckRunning(pCfg->dataDir); + if (pDnode->pLockFile == NULL) { return -1; } @@ -147,10 +147,10 @@ static void dndCloseImp(SDnode *pDnode) { tfree(pDnode->dir.snode); tfree(pDnode->dir.bnode); - if (pDnode->lockFd >= 0) { - taosUnLockFile(pDnode->lockFd); - taosCloseFile(pDnode->lockFd); - pDnode->lockFd = 0; + if (pDnode->pLockFile != NULL) { + taosUnLockFile(pDnode->pLockFile); + taosCloseFile(&pDnode->pLockFile); + pDnode->pLockFile = NULL; } } diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 4827065e87..e0d7f299b3 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -185,16 +185,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) { int32_t maxLen = 256 * 1024; char *content = calloc(1, maxLen + 1); cJSON *root = NULL; - FILE *fp = NULL; - fp = fopen(pMgmt->file, "r"); - if (fp == NULL) { + // fp = fopen(pMgmt->file, "r"); + TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", pMgmt->file); code = 0; goto PRASE_DNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", pMgmt->file); goto PRASE_DNODE_OVER; @@ -286,7 +286,7 @@ static int32_t dndReadDnodes(SDnode *pDnode) { PRASE_DNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); @@ -309,8 +309,9 @@ PRASE_DNODE_OVER: static int32_t dndWriteDnodes(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; - FILE *fp = fopen(pMgmt->file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(pMgmt->file, "w"); + TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { dError("failed to write %s since %s", pMgmt->file, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -341,9 +342,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { } len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); terrno = 0; diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index b69516b1b7..e1b16a188f 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -63,14 +63,15 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_MNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_MNODE_OVER; @@ -143,7 +144,7 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { PRASE_MNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -155,8 +156,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -184,9 +186,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { } len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index e5600fcaf0..93e2209610 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -62,14 +62,15 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_QNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_QNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { PRASE_QNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index fb108559b1..4906aef246 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -62,14 +62,15 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_SNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_SNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) { PRASE_SNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index a0c0a02431..f20493aa7f 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -219,14 +219,15 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - fp = fopen(file, "r"); - if (fp == NULL) { + // fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_VNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_VNODE_OVER; @@ -304,7 +305,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ PRASE_VNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); return code; } @@ -315,8 +316,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes); snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write %s since %s", file, terrstr()); return -1; @@ -347,9 +349,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); terrno = 0; diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 2f7d5e10b6..7c1215ea42 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -62,12 +62,13 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { size_t size = 0; int32_t done = 0; - FILE* fp = fopen("/proc/cpuinfo", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/proc/cpuinfo", "r"); + TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ); + if (pFile == NULL) { return; } - while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { + while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { const char* v = strchr(line, ':') + 2; @@ -83,20 +84,21 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; - FILE* fp = fopen("/etc/os-release", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/etc/os-release", "r"); + TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ); + if (pFile == NULL) { return; } - while ((size = tgetline(&line, &size, fp)) != -1) { + while ((size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "PRETTY_NAME", 11) == 0) { const char* p = strchr(line, '=') + 1; @@ -109,20 +111,21 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; - FILE* fp = fopen("/proc/meminfo", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/proc/meminfo", "r"); + TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ); + if (pFile == NULL) { return; } - while ((size = tgetline(&line, &size, fp)) != -1) { + while ((size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "MemTotal", 8) == 0) { const char* p = strchr(line, ':') + 1; @@ -132,8 +135,8 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) { @@ -252,16 +255,16 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { static void mndGetEmail(SMnode* pMnode, char* filepath) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - int32_t fd = taosOpenFileRead(filepath); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) { return; } - if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { + if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } - taosCloseFile(fd); + taosCloseFile(&pFile); } int32_t mndInitTelem(SMnode* pMnode) { diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 88d4fc4f75..cea93017f4 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -27,25 +27,25 @@ class MndTestTrans : public ::testing::Test { static void KillThenRestartServer() { char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; - FileFd fd = taosOpenFileRead(file); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = malloc(size); - int32_t readLen = taosReadFile(fd, buffer, size); + int32_t readLen = taosReadFile(pFile, buffer, size); if (readLen < 0 || readLen == size) { ASSERT(1); } - taosCloseFile(fd); + taosCloseFile(&pFile); test.ServerStop(); - fd = taosOpenFileCreateWriteTrunc(file); - int32_t writeLen = taosWriteFile(fd, buffer, readLen); + pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + int32_t writeLen = taosWriteFile(pFile, buffer, readLen); if (writeLen < 0 || writeLen == readLen) { ASSERT(1); } free(buffer); - taosFsyncFile(fd); - taosCloseFile(fd); + taosFsyncFile(pFile); + taosCloseFile(&pFile); taosMsleep(1000); test.ServerStart(); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6d17423324..a3fe4dfb14 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -37,8 +37,8 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) { return 0; } -static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { - int32_t ret = taosReadFile(fd, &pSdb->curVer, sizeof(int64_t)); +static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { + int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t maxId = -1; - ret = taosReadFile(fd, &maxId, sizeof(int64_t)); + ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t ver = -1; - ret = taosReadFile(fd, &ver, sizeof(int64_t)); + ret = taosReadFile(pFile, &ver, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -81,7 +81,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { } char reserve[SDB_RESERVE_SIZE] = {0}; - ret = taosReadFile(fd, reserve, sizeof(reserve)); + ret = taosReadFile(pFile, reserve, sizeof(reserve)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -94,8 +94,8 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { return 0; } -static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { - if (taosWriteFile(fd, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { +static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { + if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -105,7 +105,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { if (i < SDB_MAX) { maxId = pSdb->maxId[i]; } - if (taosWriteFile(fd, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { + if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -116,14 +116,14 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { if (i < SDB_MAX) { ver = pSdb->tableVer[i]; } - if (taosWriteFile(fd, &ver, sizeof(int64_t)) != sizeof(int64_t)) { + if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } char reserve[SDB_RESERVE_SIZE] = {0}; - if (taosWriteFile(fd, reserve, sizeof(reserve)) != sizeof(reserve)) { + if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -148,25 +148,25 @@ int32_t sdbReadFile(SSdb *pSdb) { snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); mDebug("start to read file:%s", file); - FileFd fd = taosOpenFileRead(file); - if (fd <= 0) { + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { free(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, terrstr()); return 0; } - if (sdbReadFileHead(pSdb, fd) != 0) { + if (sdbReadFileHead(pSdb, pFile) != 0) { mError("failed to read file:%s head since %s", file, terrstr()); pSdb->curVer = -1; free(pRaw); - taosCloseFile(fd); + taosCloseFile(&pFile); return -1; } while (1) { readLen = sizeof(SSdbRaw); - ret = taosReadFile(fd, pRaw, readLen); + ret = taosReadFile(pFile, pRaw, readLen); if (ret == 0) break; if (ret < 0) { @@ -182,7 +182,7 @@ int32_t sdbReadFile(SSdb *pSdb) { } readLen = pRaw->dataLen + sizeof(int32_t); - ret = taosReadFile(fd, pRaw->pData, readLen); + ret = taosReadFile(pFile, pRaw->pData, readLen); if (ret < 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, tstrerror(code)); @@ -214,7 +214,7 @@ int32_t sdbReadFile(SSdb *pSdb) { mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer); PARSE_SDB_DATA_ERROR: - taosCloseFile(fd); + taosCloseFile(&pFile); sdbFreeRaw(pRaw); terrno = code; @@ -232,16 +232,16 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer, pSdb->lastCommitVer); - FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile); - if (fd <= 0) { + TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to open file:%s for write since %s", tmpfile, terrstr()); return -1; } - if (sdbWriteFileHead(pSdb, fd) != 0) { + if (sdbWriteFileHead(pSdb, pFile) != 0) { mError("failed to write file:%s head since %s", tmpfile, terrstr()); - taosCloseFile(fd); + taosCloseFile(&pFile); return -1; } @@ -269,7 +269,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { if (pRaw != NULL) { pRaw->status = pRow->status; int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen; - if (taosWriteFile(fd, pRaw, writeLen) != writeLen) { + if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); @@ -277,7 +277,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen); - if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { + if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); @@ -296,14 +296,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } if (code == 0) { - code = taosFsyncFile(fd); + code = taosFsyncFile(pFile); if (code != 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to write file:%s since %s", tmpfile, tstrerror(code)); } } - taosCloseFile(fd); + taosCloseFile(&pFile); if (code == 0) { code = taosRenameFile(tmpfile, curfile); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 344ad992f0..e87de10912 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -141,9 +141,9 @@ typedef struct { STqMetaList* unconnectTopic; // TODO:temporaral use, to be replaced by unified tfile - int fileFd; + TdFilePtr pFile; // TODO:temporaral use, to be replaced by unified tfile - int idxFd; + TdFilePtr pIdxFile; char* dirPath; int32_t tqConfigFlag; diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 4a79c6358d..9fe753c7d8 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -28,17 +28,17 @@ #define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_F(tf) (&((tf)->f)) -#define TSDB_FILE_FD(tf) ((tf)->fd) +#define TSDB_FILE_PFILE(tf) ((tf)->pFile) #define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) +#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL) #define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) -#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) +#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL) #define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level) #define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id) #define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did) #define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname) #define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf)) +#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf)) #define TSDB_FILE_STATE(tf) ((tf)->state) #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) @@ -178,10 +178,10 @@ typedef struct { } SDFInfo; typedef struct { - SDFInfo info; - STfsFile f; - int fd; - uint8_t state; + SDFInfo info; + STfsFile f; + TdFilePtr pFile; + uint8_t state; } SDFile; void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); @@ -198,8 +198,8 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { ASSERT(!TSDB_FILE_OPENED(pDFile)); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); + if (pDFile->pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -209,7 +209,7 @@ static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { if (TSDB_FILE_OPENED(pDFile)) { - close(pDFile->fd); + taosCloseFile(&pDFile->pFile); TSDB_FILE_SET_CLOSED(pDFile); } } @@ -217,7 +217,7 @@ static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence); + int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); if (loffset < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -229,7 +229,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte); + int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); if (nwrite < nbyte) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -271,7 +271,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(T static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t nread = taosReadFile(pDFile->fd, buf, nbyte); + int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); if (nread < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index e5ccd02e48..3c464391d2 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #include "db.h" #include "metaDef.h" diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index d40cc2294f..6d5085ee74 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -34,11 +34,11 @@ static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) { } } -static inline int tqSeekLastPage(int fd) { - int offset = lseek(fd, 0, SEEK_END); +static inline int64_t tqSeekLastPage(TdFilePtr pFile) { + int offset = taosLSeekFile(pFile, 0, SEEK_END); int pageNo = offset / TQ_PAGE_SIZE; int curPageOffset = pageNo * TQ_PAGE_SIZE; - return lseek(fd, curPageOffset, SEEK_SET); + return taosLSeekFile(pFile, curPageOffset, SEEK_SET); } // TODO: the struct is tightly coupled with index entry @@ -52,10 +52,10 @@ typedef struct STqIdxPageBuf { char buffer[TQ_IDX_PAGE_BODY_SIZE]; } STqIdxPageBuf; -static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { - int offset = tqSeekLastPage(fd); +static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) { + int offset = tqSeekLastPage(pFile); int nBytes; - if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { + if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -65,7 +65,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { } ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset); - return lseek(fd, offset, SEEK_SET); + return taosLSeekFile(pFile, offset, SEEK_SET); } STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, @@ -95,15 +95,15 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F tqError("failed to create dir:%s since %s ", name, terrstr()); } strcat(name, "/" TQ_IDX_NAME); - int idxFd = open(name, O_RDWR | O_CREAT, 0755); - if (idxFd < 0) { + TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pIdxFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s ", name, terrstr()); // free memory return NULL; } - pMeta->idxFd = idxFd; + pMeta->pIdxFile = pIdxFile; pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -113,14 +113,14 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F strcpy(name, path); strcat(name, "/" TQ_META_NAME); - int fileFd = open(name, O_RDWR | O_CREAT, 0755); - if (fileFd < 0) { + TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s", name, terrstr()); return NULL; } - pMeta->fileFd = fileFd; + pMeta->pFile = pFile; pMeta->pSerializer = serializer; pMeta->pDeserializer = deserializer; @@ -136,7 +136,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F int idxRead; int allocated = TQ_PAGE_SIZE; bool readEnd = false; - while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { + while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) { if (idxRead == -1) { // TODO: handle error terrno = TAOS_SYSTEM_ERROR(errno); @@ -152,7 +152,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); - lseek(fileFd, pNode->handle.offset, SEEK_SET); + taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET); if (allocated < pNode->handle.serializedSize) { void* ptr = realloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { @@ -163,7 +163,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F allocated = pNode->handle.serializedSize; } serializedObj->ssize = pNode->handle.serializedSize; - if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { + if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { // TODO: read error } if (serializedObj->action == TQ_ACTION_INUSE) { @@ -237,8 +237,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { // commit data and idx tqStorePersist(pMeta); ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL); - close(pMeta->fileFd); - close(pMeta->idxFd); + taosCloseFile(&pMeta->pFile); + taosCloseFile(&pMeta->pIdxFile); // free memory for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; @@ -263,8 +263,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { } int32_t tqStoreDelete(STqMetaStore* pMeta) { - close(pMeta->fileFd); - close(pMeta->idxFd); + taosCloseFile(&pMeta->pFile); + taosCloseFile(&pMeta->pIdxFile); // free memory for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; @@ -302,12 +302,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { pSHead->checksum = 0; pSHead->ssize = sizeof(STqSerializedHead); /*int allocatedSize = sizeof(STqSerializedHead);*/ - int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); + int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR); - tqReadLastPage(pMeta->idxFd, &idxBuf); + tqReadLastPage(pMeta->pIdxFile, &idxBuf); if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) { - lseek(pMeta->idxFd, 0, SEEK_END); + taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END); memset(&idxBuf, 0, TQ_PAGE_SIZE); idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; } else { @@ -329,7 +329,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } else { pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); } - nBytes = write(pMeta->fileFd, pSHead, pSHead->ssize); + nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); ASSERT(nBytes == pSHead->ssize); } @@ -340,7 +340,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } else { pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); } - int nBytesTxn = write(pMeta->fileFd, pSHead, pSHead->ssize); + int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); ASSERT(nBytesTxn == pSHead->ssize); nBytes += nBytesTxn; } @@ -355,7 +355,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { idxBuf.head.writeOffset += TQ_IDX_SIZE; if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { - nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE); + nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE); // TODO: handle error with tfile ASSERT(nBytes == TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE); @@ -391,13 +391,13 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { free(pSHead); // TODO: write new version in tfile if ((char*)bufPtr != idxBuf.buffer) { - int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset); + int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset); // TODO: handle error in tfile ASSERT(nBytes == idxBuf.head.writeOffset); } // TODO: using fsync in tfile - fsync(pMeta->idxFd); - fsync(pMeta->fileFd); + taosFsyncFile(pMeta->pIdxFile); + taosFsyncFile(pMeta->pFile); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index e5d5ef513f..9a444a1c17 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -416,8 +416,8 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname); tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname); - int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -436,9 +436,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE); - if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); remove(tfname); return -1; } @@ -446,7 +446,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { // Encode file status and write to file if (fsheader.len > 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { - close(fd); + taosCloseFile(&pFile); remove(tfname); return -1; } @@ -455,9 +455,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { tsdbEncodeFSStatus(&ptr, pStatus); taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len); - if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) { + if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); (void)remove(tfname); taosTZfree(pBuf); return -1; @@ -465,15 +465,15 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { } // fsync, close and rename - if (taosFsyncFile(fd) < 0) { + if (taosFsyncFile(pFile) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); remove(tfname); taosTZfree(pBuf); return -1; } - (void)close(fd); + (void)taosCloseFile(&pFile); (void)taosRenameFile(tfname, cfname); taosTZfree(pBuf); @@ -652,7 +652,7 @@ static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { static int tsdbOpenFSFromCurrent(STsdb *pRepo) { STsdbFS * pfs = REPO_FS(pRepo); - int fd = -1; + TdFilePtr pFile = NULL; void * buffer = NULL; SFSHeader fsheader; char current[TSDB_FILENAME_LEN] = "\0"; @@ -661,8 +661,8 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); // current file exists, try to recover - fd = open(current, O_RDONLY | O_BINARY); - if (fd < 0) { + pFile = taosOpenFile(current, TD_FILE_READ); + if (pFile == NULL) { tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -672,7 +672,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { goto _err; } - int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE); + int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE); if (nread < 0) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current, strerror(errno)); @@ -706,7 +706,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { goto _err; } - nread = (int)taosReadFile(fd, buffer, fsheader.len); + nread = (int)taosReadFile(pFile, buffer, fsheader.len); if (nread < 0) { tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -732,13 +732,13 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { } taosTZfree(buffer); - close(fd); + taosCloseFile(&pFile); return 0; _err: - if (fd >= 0) { - close(fd); + if (pFile != NULL) { + taosCloseFile(&pFile); } taosTZfree(buffer); return -1; @@ -1244,18 +1244,17 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { } if (tsdbForceKeepFile) { - struct stat tfstat; - + int64_t file_size; // Get real file size - if (fstat(pDFile->fd, &tfstat) < 0) { + if (taosFStatFile(pDFile->pFile, &file_size, NULL) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosArrayDestroy(fArray); return -1; } - if (pDFile->info.size != tfstat.st_size) { + if (pDFile->info.size != file_size) { int64_t tfsize = pDFile->info.size; - pDFile->info.size = tfstat.st_size; + pDFile->info.size = file_size; tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 579160bcb2..ff79d91081 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -355,8 +355,8 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pDFile->pFile < 0) { if (errno == ENOENT) { // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); @@ -366,8 +366,8 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { } tfree(s); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pDFile->pFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -460,7 +460,7 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { return -1; } - if (taosFtruncateFile(df.fd, df.info.size) < 0) { + if (taosFtruncateFile(df.pFile, df.info.size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); tsdbCloseDFile(&df); return -1; @@ -541,7 +541,7 @@ static int tsdbRollBackDFile(SDFile *pDFile) { return -1; } - if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) { + if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); tsdbCloseDFile(&df); return -1; diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index aa6d204310..e07e46948f 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,3 +1,4 @@ +add_definitions("-D ALLOW_FORBID_FUNC") add_subdirectory(transport) add_subdirectory(sync) add_subdirectory(tdb) diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 1aa5871468..2ff2edf84e 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -3960,14 +3960,14 @@ static void ts_comp_finalize(SqlFunctionCtx *pCtx) { // qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); // TODO refactor transfer ownership of current file - *(FILE **)pCtx->pOutput = pTSbuf->f; + *(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile; pResInfo->complete = true; // get the file size - struct stat fStat; - if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) { - pResInfo->numOfRes = fStat.st_size; + int64_t file_size; + if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) { + pResInfo->numOfRes = (uint32_t )file_size; } pTSbuf->remainOpen = true; diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index 1e0a88e17f..86e829aa95 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -38,7 +38,7 @@ typedef struct WriterCtx { WriterType type; union { struct { - int fd; + TdFilePtr pFile; bool readOnly; char buf[256]; int size; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index b57f639726..86aa257b48 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -19,7 +19,7 @@ static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { if (ctx->type == TFile) { - assert(len == tfWrite(ctx->file.fd, buf, len)); + assert(len == taosWriteFile(ctx->file.pFile, buf, len)); } else { memcpy(ctx->mem.buf + ctx->offset, buf, len); } @@ -33,7 +33,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { nRead = len < ctx->file.size ? len : ctx->file.size; memcpy(buf, ctx->file.ptr, nRead); #else - nRead = tfRead(ctx->file.fd, buf, len); + nRead = taosReadFile(ctx->file.pFile, buf, len); #endif } else { memcpy(buf, ctx->mem.buf + ctx->offset, len); @@ -45,13 +45,13 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { int nRead = 0; if (ctx->type == TFile) { - // tfLseek(ctx->file.fd, offset, 0); + // tfLseek(ctx->file.pFile, offset, 0); #ifdef USE_MMAP int32_t last = ctx->file.size - offset; nRead = last >= len ? len : last; memcpy(buf, ctx->file.ptr + offset, nRead); #else - nRead = tfPread(ctx->file.fd, buf, len, offset); + nRead = taosPReadFile(ctx->file.pFile, buf, len, offset); #endif } else { // refactor later @@ -69,9 +69,9 @@ static int writeCtxGetSize(WriterCtx* ctx) { } static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { - // taosFsyncFile(ctx->file.fd); - tfFsync(ctx->file.fd); - // tfFlush(ctx->file.fd); + // taosFsyncFile(ctx->file.pFile); + taosFsyncFile(ctx->file.pFile); + // tfFlush(ctx->file.pFile); } else { // do nothing } @@ -87,25 +87,25 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ugly code, refactor later ctx->file.readOnly = readOnly; if (readOnly == false) { - // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); - ctx->file.fd = tfOpenCreateWriteAppend(path); - tfFtruncate(ctx->file.fd, 0); - struct stat fstat; - stat(path, &fstat); - ctx->file.size = fstat.st_size; + // ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + taosFtruncateFile(ctx->file.pFile, 0); + int64_t file_size; + taosStatFile(path, &file_size, NULL); + ctx->file.size = (int)file_size; } else { - // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); - ctx->file.fd = tfOpenRead(path); + // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); struct stat fstat; stat(path, &fstat); ctx->file.size = fstat.st_size; #ifdef USE_MMAP - ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size); + ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); #endif } memcpy(ctx->file.buf, path, strlen(path)); - if (ctx->file.fd < 0) { + if (ctx->file.pFile < 0) { indexError("failed to open file, error %d", errno); goto END; } @@ -133,7 +133,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { free(ctx->mem.buf); } else { ctx->flush(ctx); - tfClose(ctx->file.fd); + taosCloseFile(&ctx->file.pFile); if (ctx->file.readOnly) { #ifdef USE_MMAP munmap(ctx->file.ptr, ctx->file.size); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 282059df29..e44f8fc1c3 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -586,11 +586,11 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); if (nread == -1) { - indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), - errno, reader->ctx->file.fd, reader->ctx->file.buf); + indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), + errno, reader->ctx->file.buf); } else { - indexInfo("actual Read: %d, to read: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), - reader->ctx->file.fd, reader->ctx->file.buf); + indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), + reader->ctx->file.buf); } // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 3670c770ab..2afce30828 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -18,7 +18,7 @@ struct STDbEnv { char * rootDir; // root directory of the environment char * jname; // journal file name - int jfd; // journal file fd + TdFilePtr jpFile; // journal file fd pgsz_t pgSize; // page size cachesz_t cacheSize; // total cache size STDbList dbList; // TDB List @@ -55,7 +55,7 @@ int tdbEnvCreate(TENV **ppEnv, const char *rootDir) { pEnv->rootDir = (char *)(&pEnv[1]); pEnv->jname = pEnv->rootDir + slen + 1; - pEnv->jfd = -1; + pEnv->jpFile = NULL; pEnv->pgSize = TDB_DEFAULT_PGSIZE; pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE; @@ -139,8 +139,8 @@ static int tdbEnvDestroy(TENV *pEnv) { } int tdbEnvBeginTxn(TENV *pEnv) { - pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755); - if (pEnv->jfd < 0) { + pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pEnv->jpFile < 0) { return -1; } @@ -149,8 +149,8 @@ int tdbEnvBeginTxn(TENV *pEnv) { int tdbEnvCommit(TENV *pEnv) { /* TODO */ - close(pEnv->jfd); - pEnv->jfd = -1; + taosCloseFile(&pEnv->jpFile); + pEnv->jpFile = NULL; return 0; } diff --git a/source/libs/tdb/src/db/tdbPgFile.c b/source/libs/tdb/src/db/tdbPgFile.c index ee5b486f7b..48f8ed1792 100644 --- a/source/libs/tdb/src/db/tdbPgFile.c +++ b/source/libs/tdb/src/db/tdbPgFile.c @@ -51,10 +51,10 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { pPgFile->fname = (char *)(&(pPgFile[1])); memcpy(pPgFile->fname, fname, fnameLen); pPgFile->fname[fnameLen] = '\0'; - pPgFile->fd = -1; + pPgFile->pFile = NULL; - pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755); - if (pPgFile->fd < 0) { + pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pPgFile->pFile == NULL) { // TODO: handle error return -1; } @@ -95,8 +95,8 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { int pgFileClose(SPgFile *pPgFile) { if (pPgFile) { - if (pPgFile->fd >= 0) { - close(pPgFile->fd); + if (pPgFile->pFile >= 0) { + taosCloseFile(&pPgFile->pFile); } tfree(pPgFile->fname); @@ -201,7 +201,7 @@ static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) { pTData = pData; szToRead = pgSize; for (; szToRead > 0;) { - rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize); + rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize); if (rsize < 0) { if (errno == EINTR) { continue; diff --git a/source/libs/tdb/src/inc/tdbPgFile.h b/source/libs/tdb/src/inc/tdbPgFile.h index 2a7116a0dd..eaeebf6b9d 100644 --- a/source/libs/tdb/src/inc/tdbPgFile.h +++ b/source/libs/tdb/src/inc/tdbPgFile.h @@ -38,7 +38,7 @@ struct SPgFile { uint8_t fileid[TDB_FILE_ID_LEN]; // file id pgno_t lsize; // page file logical size (for count) pgno_t fsize; // real file size on disk (for rollback) - int fd; + TdFilePtr pFile; SPgFileListNode envHash; SPgFileListNode envPgfList; }; diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 178d115c59..af66304f84 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -230,10 +230,11 @@ TEST_F(TfsTest, 04_File) { EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); - FILE *fp = fopen(f1.aname, "w"); - ASSERT_NE(fp, nullptr); - fwrite("12345678", 1, 5, fp); - fclose(fp); + // FILE *fp = fopen(f1.aname, "w"); + TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, "12345678", 5); + taosCloseFile(&pFile); char base[128] = {0}; tfsBasename(&f1, base); @@ -638,10 +639,11 @@ TEST_F(TfsTest, 05_MultiDisk) { EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); - FILE *fp = fopen(f1.aname, "w"); - ASSERT_NE(fp, nullptr); - fwrite("12345678", 1, 5, fp); - fclose(fp); + // FILE *fp = fopen(f1.aname, "w"); + TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, "12345678", 5); + taosCloseFile(&pFile); char base[128] = {0}; tfsBasename(&f1, base); diff --git a/source/libs/transport/inc/rpcUdp.h b/source/libs/transport/inc/rpcUdp.h index c1da6a9240..0c651d07ed 100644 --- a/source/libs/transport/inc/rpcUdp.h +++ b/source/libs/transport/inc/rpcUdp.h @@ -30,7 +30,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t void taosFreeMsgHdr(void *hdr); int taosMsgHdrSize(void *hdr); -void taosSendMsgHdr(void *hdr, int fd); +void taosSendMsgHdr(void *hdr, TdFilePtr pFile); void taosInitMsgHdr(void **hdr, void *dest, int maxPkts); void taosSetMsgHdrData(void *hdr, char *data, int dataLen); diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 0ccfde16ce..a1c181ac98 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -22,7 +22,7 @@ int msgSize = 128; int commit = 0; -int dataFd = -1; +TdFilePtr pDataFile = NULL; STaosQueue *qhandle = NULL; STaosQset * qset = NULL; @@ -43,8 +43,8 @@ void processShellMsg() { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - if (dataFd >= 0) { - if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + if (pDataFile != NULL) { + if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { tInfo("failed to write data file, reason:%s", strerror(errno)); } } @@ -52,7 +52,7 @@ void processShellMsg() { if (commit >= 2) { num += numOfMsgs; - // if (taosFsync(dataFd) < 0) { + // if (taosFsync(pDataFile) < 0) { // tInfo("failed to flush data to file, reason:%s", strerror(errno)); //} @@ -181,8 +181,8 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + pDataFile = taosOpenFile(dataName, TD_FILE_APPEND | TD_FILE_CTEATE | TD_FILE_WRITE); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); } qhandle = taosOpenQueue(); qset = taosOpenQset(); @@ -190,8 +190,8 @@ int main(int argc, char *argv[]) { processShellMsg(); - if (dataFd >= 0) { - close(dataFd); + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); remove(dataName); } diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 4337cd242c..794497500c 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -22,7 +22,7 @@ int msgSize = 128; int commit = 0; -int dataFd = -1; +TdFilePtr pDataFile = NULL; STaosQueue *qhandle = NULL; STaosQset * qset = NULL; @@ -43,8 +43,8 @@ void processShellMsg() { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - if (dataFd >= 0) { - if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + if (pDataFile >= 0) { + if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { tInfo("failed to write data file, reason:%s", strerror(errno)); } } @@ -52,7 +52,7 @@ void processShellMsg() { if (commit >= 2) { num += numOfMsgs; - // if (taosFsync(dataFd) < 0) { + // if (taosFsync(pDataFile) < 0) { // tInfo("failed to flush data to file, reason:%s", strerror(errno)); //} @@ -170,8 +170,8 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + pDataFile = taosOpenFile(dataName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); } qhandle = taosOpenQueue(); qset = taosOpenQset(); @@ -179,8 +179,8 @@ int main(int argc, char *argv[]) { processShellMsg(); - if (dataFd >= 0) { - close(dataFd); + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); remove(dataName); } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 2d6fb8fc76..ae0b0bd849 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -69,8 +69,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size); pLastFileInfo->fileSize = statbuf.st_size; - FileFd fd = taosOpenFileRead(fnameStr); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -79,15 +79,15 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { char* buf = malloc(readSize + 5); if (buf == NULL) { - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; } - taosLSeekFile(fd, -readSize, SEEK_END); - if (readSize != taosReadFile(fd, buf, readSize)) { + taosLSeekFile(pFile, -readSize, SEEK_END); + if (readSize != taosReadFile(pFile, buf, readSize)) { free(buf); - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -108,12 +108,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { // file has to be deleted free(buf); - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } } - taosCloseFile(fd); + taosCloseFile(&pFile); SWalHead *lastEntry = (SWalHead*)found; return lastEntry->head.version; @@ -364,18 +364,18 @@ int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer + 1, fnameStr); - FileFd metaFd = taosOpenFileCreateWrite(fnameStr); - if (metaFd < 0) { + TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE); + if (pMataFile == NULL) { return -1; } char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); - if (len != taosWriteFile(metaFd, serialized, len)) { + if (len != taosWriteFile(pMataFile, serialized, len)) { // TODO:clean file return -1; } - taosCloseFile(metaFd); + taosCloseFile(&pMataFile); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -404,20 +404,20 @@ int walLoadMeta(SWal* pWal) { return -1; } memset(buf, 0, size + 5); - FileFd fd = taosOpenFileRead(fnameStr); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pFile == NULL) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - if (taosReadFile(fd, buf, size) != size) { + if (taosReadFile(pFile, buf, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(fd); + taosCloseFile(&pFile); free(buf); return -1; } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); - taosCloseFile(fd); + taosCloseFile(&pFile); free(buf); return code; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d5c28d9d9b..d3cd23f284 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -89,8 +89,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // open meta walResetVer(&pWal->vers); - pWal->writeLogTfd = -1; - pWal->writeIdxTfd = -1; + pWal->pWriteLogTFile = NULL; + pWal->pWriteIdxTFile = NULL; pWal->writeCur = -1; pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { @@ -164,10 +164,10 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { void walClose(SWal *pWal) { pthread_mutex_lock(&pWal->mutex); - tfClose(pWal->writeLogTfd); - pWal->writeLogTfd = -1; - tfClose(pWal->writeIdxTfd); - pWal->writeIdxTfd = -1; + taosCloseFile(&pWal->pWriteLogTFile); + pWal->pWriteLogTFile = NULL; + taosCloseFile(&pWal->pWriteIdxTFile); + pWal->pWriteIdxTFile = NULL; walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; @@ -207,7 +207,7 @@ static void walFsyncAll() { if (walNeedFsync(pWal)) { wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); - int32_t code = tfFsync(pWal->writeLogTfd); + int32_t code = taosFsyncFile(pWal->pWriteLogTFile); if (code != 0) { wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1d9201f69d..159d281759 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -25,8 +25,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { } pRead->pWal = pWal; - pRead->readIdxTfd = -1; - pRead->readLogTfd = -1; + pRead->pReadIdxTFile = NULL; + pRead->pReadLogTFile = NULL; pRead->curVersion = -1; pRead->curFileFirstVer = -1; pRead->capacity = 0; @@ -41,8 +41,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { } void walCloseReadHandle(SWalReadHandle *pRead) { - tfClose(pRead->readIdxTfd); - tfClose(pRead->readLogTfd); + taosCloseFile(&pRead->pReadIdxTFile); + taosCloseFile(&pRead->pReadLogTFile); tfree(pRead->pHead); free(pRead); } @@ -52,24 +52,24 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; - int64_t idxTfd = pRead->readIdxTfd; - int64_t logTfd = pRead->readLogTfd; + TdFilePtr pIdxTFile = pRead->pReadIdxTFile; + TdFilePtr pLogTFile = pRead->pReadLogTFile; // seek position int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); - code = tfLseek(idxTfd, offset, SEEK_SET); + code = taosLSeekFile(pIdxTFile, offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } // TODO:deserialize ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -80,24 +80,24 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { char fnameStr[WAL_FILE_LEN]; - tfClose(pRead->readIdxTfd); - tfClose(pRead->readLogTfd); + taosCloseFile(&pRead->pReadIdxTFile); + taosCloseFile(&pRead->pReadLogTFile); walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); - int64_t logTfd = tfOpenRead(fnameStr); - if (logTfd < 0) { + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); - int64_t idxTfd = tfOpenRead(fnameStr); - if (idxTfd < 0) { + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pIdxTFile == NULL) { return -1; } - pRead->readLogTfd = logTfd; - pRead->readIdxTfd = idxTfd; + pRead->pReadLogTFile = pLogTFile; + pRead->pReadIdxTFile = pIdxTFile; return 0; } @@ -145,9 +145,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } } - if (!tfValid(pRead->readLogTfd)) return -1; + if (!taosValidFile(pRead->pReadLogTFile)) return -1; - code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); + code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { return -1; } @@ -165,7 +165,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { pRead->pHead = ptr; pRead->capacity = pRead->pHead->head.len; } - if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { + if (pRead->pHead->head.len != taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.len)) { return -1; } @@ -202,7 +202,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { } *ppHead = ptr; } - if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { return -1; } // TODO: endian compatibility processing after read @@ -215,7 +215,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { *ppHead = NULL; return -1; } - if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { + if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } // TODO: endian compatibility processing after read diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 91b172444b..7aae9138c1 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -23,25 +23,25 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd = pWal->writeIdxTfd; - int64_t logTfd = pWal->writeLogTfd; + TdFilePtr pIdxTFile = pWal->pWriteIdxTFile; + TdFilePtr pLogTFile = pWal->pWriteLogTFile; // seek position int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = tfLseek(idxTfd, idxOff, SEEK_SET); + code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; // TODO:deserialize - code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)); + code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -50,43 +50,43 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) { } int walSetWrite(SWal* pWal) { - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } // switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; + pWal->pWriteIdxTFile = pIdxTFile; + pWal->pWriteLogTFile = pLogTFile; return 0; } int walChangeWrite(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; - if (pWal->writeLogTfd != -1) { - code = tfClose(pWal->writeLogTfd); + if (pWal->pWriteLogTFile != NULL) { + code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - if (pWal->writeIdxTfd != -1) { - code = tfClose(pWal->writeIdxTfd); + if (pWal->pWriteIdxTFile != NULL) { + code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -103,23 +103,23 @@ int walChangeWrite(SWal* pWal, int64_t ver) { int64_t fileFirstVer = pFileInfo->firstVer; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - pWal->writeIdxTfd = -1; + pWal->pWriteIdxTFile = NULL; return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { - tfClose(idxTfd); + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile < 0) { + taosCloseFile(&pIdxTFile); terrno = TAOS_SYSTEM_ERROR(errno); - pWal->writeLogTfd = -1; + pWal->pWriteLogTFile = NULL; return -1; } - pWal->writeLogTfd = logTfd; - pWal->writeIdxTfd = idxTfd; + pWal->pWriteLogTFile = pLogTFile; + pWal->pWriteIdxTFile = pIdxTFile; pWal->writeCur = idx; return fileFirstVer; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index a4b34dee37..f9bb168234 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -65,15 +65,15 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - int64_t idxTfd = tfOpenReadWrite(fnameStr); + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); // TODO:change to deserialize function - if (idxTfd < 0) { + if (pIdxTFile == NULL) { pthread_mutex_unlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = tfLseek(idxTfd, idxOff, SEEK_SET); + code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code < 0) { pthread_mutex_unlock(&pWal->mutex); return -1; @@ -81,20 +81,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // read idx file and get log file pos // TODO:change to deserialize function SWalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { pthread_mutex_unlock(&pWal->mutex); return -1; } ASSERT(entry.ver == ver); walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - int64_t logTfd = tfOpenReadWrite(fnameStr); - if (logTfd < 0) { + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); + if (pLogTFile < 0) { // TODO pthread_mutex_unlock(&pWal->mutex); return -1; } - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { // TODO pthread_mutex_unlock(&pWal->mutex); @@ -102,8 +102,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } // validate offset SWalHead head; - ASSERT(tfValid(logTfd)); - int size = tfRead(logTfd, &head, sizeof(SWalHead)); + ASSERT(taosValidFile(pLogTFile)); + int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead)); if (size != sizeof(SWalHead)) { return -1; } @@ -118,11 +118,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) { return -1; } // truncate old files - code = tfFtruncate(logTfd, entry.offset); + code = taosFtruncateFile(pLogTFile, entry.offset); if (code < 0) { return -1; } - code = tfFtruncate(idxTfd, idxOff); + code = taosFtruncateFile(pIdxTFile, idxOff); if (code < 0) { return -1; } @@ -203,31 +203,31 @@ int32_t walEndSnapshot(SWal *pWal) { int walRoll(SWal *pWal) { int code = 0; - if (pWal->writeIdxTfd != -1) { - code = tfClose(pWal->writeIdxTfd); + if (pWal->pWriteIdxTFile != NULL) { + code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { return -1; } } - if (pWal->writeLogTfd != -1) { - code = tfClose(pWal->writeLogTfd); + if (pWal->pWriteLogTFile != NULL) { + code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { return -1; } } - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; // create new file int64_t newFileFirstVersion = pWal->vers.lastVer + 1; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, newFileFirstVersion, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -237,8 +237,8 @@ int walRoll(SWal *pWal) { } // switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; + pWal->pWriteIdxTFile = pIdxTFile; + pWal->pWriteLogTFile = pLogTFile; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; ASSERT(pWal->writeCur >= 0); @@ -248,7 +248,7 @@ int walRoll(SWal *pWal) { static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; - int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry)); + int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { terrno = TAOS_SYSTEM_ERROR(errno); // TODO truncate @@ -282,16 +282,16 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in // must truncate explicitly first return -1; } - /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ + /*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/ ASSERT(pWal->writeCur >= 0); pthread_mutex_lock(&pWal->mutex); - if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) { + if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) { walSetWrite(pWal); - tfLseek(pWal->writeLogTfd, 0, SEEK_END); - tfLseek(pWal->writeIdxTfd, 0, SEEK_END); + taosLSeekFile(pWal->pWriteLogTFile, 0, SEEK_END); + taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END); } pWal->writeHead.head.version = index; @@ -302,14 +302,14 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { // ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) { + if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) { // ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), @@ -336,7 +336,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); - if (tfFsync(pWal->writeLogTfd) < 0) { + if (taosFsyncFile(pWal->pWriteLogTFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 5a67ab73eb..4bfe618f2c 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -12,38 +12,45 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#define ALLOW_FORBID_FUNC #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -#include + #include -#if defined(_MSDOS) -#define open _open -#endif + #if defined(_MSDOS) + #define open _open + #endif -#if defined(_WIN32) -extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */ -extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ -#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) -#define open openU -#else /* _ANSI_SOURCE */ -#define open openA -#endif /* defined(_UTF8_SOURCE) */ -#endif /* defined(_WIN32) */ + #if defined(_WIN32) + extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */ + extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ + #if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) + #define open openU + #else /* _ANSI_SOURCE */ + #define open openA + #endif /* defined(_UTF8_SOURCE) */ + #endif /* defined(_WIN32) */ #else -#include -#include -#include -#include -#include + #include + #include + #include + #include + #include + #define LINUX_FILE_NO_TEXT_OPTION 0 + #define O_TEXT LINUX_FILE_NO_TEXT_OPTION #endif -void taosCloseFile(FileFd fd) { - close(fd); - fd = FD_INITIALIZER; -} +typedef int32_t FileFd; + +typedef struct TdFile { + int refId; + FileFd fd; + FILE *fp; +}*TdFilePtr,TdFile; + + void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, char *dstPath) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -96,13 +103,166 @@ void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, ch #endif } -int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { +int64_t taosCopyFile(const char *from, const char *to) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + char buffer[4096]; + int64_t size = 0; + int64_t bytes; + + // fidfrom = open(from, O_RDONLY); + TdFilePtr pFileFrom = taosOpenFile(from,TD_FILE_READ); + if (pFileFrom == NULL) goto _err; + + // fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); + TdFilePtr pFileTo = taosOpenFile(to,TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_EXCL); + if (pFileTo == NULL) goto _err; + + while (true) { + bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer)); + if (bytes < 0) goto _err; + if (bytes == 0) break; + + size += bytes; + + if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) goto _err; + if (bytes < sizeof(buffer)) break; + } + + taosFsyncFile(pFileTo); + + taosCloseFile(&pFileFrom); + taosCloseFile(&pFileTo); + return size; + +_err: + if (pFileFrom != NULL) taosCloseFile(&pFileFrom); + if (pFileTo != NULL) taosCloseFile(&pFileTo); + remove(to); + return -1; +#endif +} + +int32_t taosRenameFile(const char *oldName, const char *newName) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); + if (code < 0) { + //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } + + return code; +#else + int32_t code = rename(oldName, newName); + if (code < 0) { + //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } + + return code; +#endif +} + +int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); + if (code < 0) { + return code; + } + + if (size != NULL) { + *size = fileStat.st_size; + } + + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } + + return 0; +#endif +} + +void autoDelFileListAdd(const char *path) { + return; +} + +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return NULL; +#else + int access = O_BINARY; + char *mode = NULL; + access |= (tdFileOptions & TD_FILE_CTEATE) ? O_CREAT : 0; + if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ)) + { + access |= O_RDWR; + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; + }else if(tdFileOptions & TD_FILE_WRITE) { + access |= O_WRONLY; + mode = (tdFileOptions & TD_FILE_TEXT) ? "wt" : "wb"; + }else if(tdFileOptions & TD_FILE_READ) { + access |= O_RDONLY; + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb"; + } + access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0; + access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; + access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; + access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; + if(tdFileOptions & TD_FILE_AUTO_DEL) { + autoDelFileListAdd(path); + } + int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); + if(fd == -1) { + return NULL; + } + FILE* fp = fdopen(fd, mode); + if (fp == NULL) { + close(fd); + return NULL; + } + TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile)); + if (pFile == NULL) { + close(fd); + fclose(fp); + return NULL; + } + pFile->fd = fd; + pFile->fp = fp; + pFile->refId = 0; + return pFile; +#endif +} + +int64_t taosCloseFile(TdFilePtr *ppFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + if(ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) { + return 0; + } + fsync((*ppFile)->fd); + close((*ppFile)->fd); + (*ppFile)->fd = -1; + (*ppFile)->fp = NULL; + (*ppFile)->refId = 0; + free(*ppFile); + *ppFile = NULL; + return 0; +#endif +} + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { + if(pFile == NULL) { + return 0; + } int64_t leftbytes = count; int64_t readbytes; char * tbuf = (char *)buf; while (leftbytes > 0) { - readbytes = read(fd, (void *)tbuf, (uint32_t)leftbytes); + readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); if (readbytes < 0) { if (errno == EINTR) { continue; @@ -120,13 +280,20 @@ int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { return count; } -int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { - int64_t nleft = n; +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { + if(pFile == NULL) { + return 0; + } + return pread(pFile->fd, buf, count, offset); +} + +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { + int64_t nleft = count; int64_t nwritten = 0; char * tbuf = (char *)buf; while (nleft > 0) { - nwritten = write(fd, (void *)tbuf, (uint32_t)nleft); + nwritten = write(pFile->fd, (void *)tbuf, (uint32_t)nleft); if (nwritten < 0) { if (errno == EINTR) { continue; @@ -137,48 +304,119 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { tbuf += nwritten; } - return n; + return count; } -int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { + return (int64_t)lseek(pFile->fd, (long)offset, whence); +} -int64_t taosCopyFile(const char *from, const char *to) { +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else - char buffer[4096]; - int fidto = -1, fidfrom = -1; - int64_t size = 0; - int64_t bytes; - - fidfrom = open(from, O_RDONLY); - if (fidfrom < 0) goto _err; - - fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); - if (fidto < 0) goto _err; - - while (true) { - bytes = taosReadFile(fidfrom, buffer, sizeof(buffer)); - if (bytes < 0) goto _err; - if (bytes == 0) break; - - size += bytes; - - if (taosWriteFile(fidto, (void *)buffer, bytes) < bytes) goto _err; - if (bytes < sizeof(buffer)) break; + struct stat fileStat; + int32_t code = fstat(pFile->fd, &fileStat); + if (code < 0) { + return code; } - taosFsyncFile(fidto); + if (size != NULL) { + *size = fileStat.st_size; + } - taosCloseFile(fidfrom); - taosCloseFile(fidto); - return size; + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } -_err: - if (fidfrom >= 0) taosCloseFile(fidfrom); - if (fidto >= 0) taosCloseFile(fidto); - remove(to); - return -1; + return 0; +#endif +} + +int32_t taosLockFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); +#endif +} + +int32_t taosUnLockFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); +#endif +} + +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + if (pFile->fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + + LARGE_INTEGER li_0; + li_0.QuadPart = (int64_t)0; + BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); + if (!cur) { + uError("SetFilePointerEx Error getting current position in file.\n"); + return -1; + } + + LARGE_INTEGER li_size; + li_size.QuadPart = l_size; + BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); + if (cur2 == 0) { + int error = GetLastError(); + uError("SetFilePointerEx GetLastError is: %d\n", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + if (!SetEndOfFile(h)) { + int error = GetLastError(); + uError("SetEndOfFile GetLastError is:%d", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + return 0; +#else + return ftruncate(pFile->fd, l_size); +#endif +} + +int32_t taosFsyncFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + if (pFile->fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + + return FlushFileBuffers(h); +#else + return fsync(pFile->fd); #endif } @@ -301,12 +539,12 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { #else -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; while (leftbytes > 0) { - sentbytes = sendfile(dfd, sfd, offset, leftbytes); + sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes); if (sentbytes == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { continue; @@ -323,215 +561,49 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) { return size; } -int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) { - return taosSendFile(fileno(outfile), fileno(infile), offset, size); +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { + return taosSendFile(pFileOut->fd, pFileIn, offset, size); } #endif -int32_t taosFtruncateFile(FileFd fd, int64_t l_size) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd < 0) { - errno = EBADF; - uError("%s\n", "fd arg was negative"); - return -1; - } - HANDLE h = (HANDLE)_get_osfhandle(fd); - - LARGE_INTEGER li_0; - li_0.QuadPart = (int64_t)0; - BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); - if (!cur) { - uError("SetFilePointerEx Error getting current position in file.\n"); - return -1; - } - - LARGE_INTEGER li_size; - li_size.QuadPart = l_size; - BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); - if (cur2 == 0) { - int error = GetLastError(); - uError("SetFilePointerEx GetLastError is: %d\n", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - if (!SetEndOfFile(h)) { - int error = GetLastError(); - uError("SetEndOfFile GetLastError is:%d", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - return 0; -#else - return ftruncate(fd, l_size); +#ifdef __GNUC__ + __attribute__((format(printf, 2, 3))) #endif +void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { + va_list ap; + va_start(ap, format); + fprintf(pFile->fp, format, ap); + va_end(ap); + fflush(pFile->fp); } -int32_t taosFsyncFile(FileFd fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd < 0) { - errno = EBADF; - uError("%s\n", "fd arg was negative"); - return -1; - } +void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { + if (pFile == NULL) return NULL; - HANDLE h = (HANDLE)_get_osfhandle(fd); - - return FlushFileBuffers(h); -#else - return fsync(fd); -#endif + void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); + return ptr; } -int32_t taosRenameFile(const char *oldName, const char *newName) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); - if (code < 0) { - //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); - } - - return code; -#else - int32_t code = rename(oldName, newName); - if (code < 0) { - //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); - } - - return code; -#endif +bool taosValidFile(TdFilePtr pFile) { + return pFile != NULL; } -int32_t taosLockFile(int32_t fd) { +int32_t taosUmaskFile(int32_t maskVal) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else - return (int32_t)flock(fd, LOCK_EX | LOCK_NB); + return umask(maskVal); #endif } -int32_t taosUnLockFile(int32_t fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return (int32_t)flock(fd, LOCK_UN | LOCK_NB); -#endif +int taosGetErrorFile(TdFilePtr pFile) { + return errno; } - -int32_t taosUmaskFile(int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return umask(val); -#endif +size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf) { + return getline(ptrBuf, NULL, pFile->fp); } - -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - struct stat fileStat; - int32_t code = stat(path, &fileStat); - if (code < 0) { - return code; - } - - if (size != NULL) { - *size = fileStat.st_size; - } - - if (mtime != NULL) { - *mtime = fileStat.st_mtime; - } - - return 0; -#endif -} - -int32_t taosFStatFile(int32_t fd, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - struct stat fileStat; - int32_t code = fstat(fd, &fileStat); - if (code < 0) { - return code; - } - - if (size != NULL) { - *size = fileStat.st_size; - } - - if (mtime != NULL) { - *mtime = fileStat.st_mtime; - } - - return 0; -#endif -} - -int32_t taosOpenFileWrite(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWrite(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWriteTrunc(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWriteAppend(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -FileFd taosOpenFileRead(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -FileFd taosOpenFileReadWrite(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDWR, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - - +int32_t taosEOFFile(TdFilePtr pFile) { + return feof(pFile->fp); +} \ No newline at end of file diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 973323a346..b81e41b3cf 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -24,18 +24,18 @@ uint32_t taosRand(void) { return rand(); } uint32_t taosSafeRand(void) { - int fd; + TdFilePtr pFile; int seed; - fd = open("/dev/urandom", 0); - if (fd < 0) { + pFile = taosOpenFile("/dev/urandom", TD_FILE_READ); + if (pFile == NULL) { seed = (int)time(0); } else { - int len = read(fd, &seed, sizeof(seed)); + int len = taosReadFile(pFile, &seed, sizeof(seed)); if (len < 0) { seed = (int)time(0); } - close(fd); + taosCloseFile(&pFile); } return (uint32_t)seed; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 8afb4ae8b7..07d30276b7 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#define ALLOW_FORBID_FUNC #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 5bbd7e59a5..ed289c6934 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -408,19 +408,17 @@ bool taosGetSysMemory(float *memoryUsedMB) { } bool taosGetProcMemory(float *memoryUsedMB) { - FILE *fp = fopen(tsProcMemFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcMemFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcMemFile); return false; } ssize_t _bytes = 0; - size_t len; char * line = NULL; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { break; } @@ -431,7 +429,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { if (line == NULL) { //printf("read file:%s failed", tsProcMemFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -440,24 +438,24 @@ bool taosGetProcMemory(float *memoryUsedMB) { sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { - FILE *fp = fopen(tsSysCpuFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsSysCpuFile, "r"); + TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsSysCpuFile); return false; } - size_t len; char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); + ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { //printf("read file:%s failed", tsSysCpuFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -465,24 +463,24 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { - FILE *fp = fopen(tsProcCpuFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcCpuFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcCpuFile); return false; } - size_t len = 0; char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); + ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { //printf("read file:%s failed", tsProcCpuFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -495,8 +493,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } @@ -556,19 +554,17 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes = 0; - FILE *fp = fopen(tsSysNetFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsSysNetFile, "r"); + TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsSysNetFile); return false; } ssize_t _bytes = 0; - size_t len = 2048; - char * line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); + char * line = NULL; + while (!taosEOFFile(pFile)) { int64_t o_rbytes = 0; int64_t rpackts = 0; int64_t o_tbytes = 0; @@ -581,12 +577,12 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { int64_t nouse6 = 0; char nouse0[200] = {0}; - _bytes = getline(&line, &len, fp); + _bytes = taosGetLineFile(pFile, &line); if (_bytes < 0) { break; } - line[len - 1] = 0; + line[_bytes - 1] = 0; if (strstr(line, "lo:") != NULL) { continue; @@ -601,8 +597,8 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes += (o_rbytes + o_tbytes); } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } @@ -644,22 +640,20 @@ bool taosGetBandSpeed(float *bandSpeedKb) { } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { - FILE *fp = fopen(tsProcIOFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcIOFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcIOFile); return false; } ssize_t _bytes = 0; - size_t len; char * line = NULL; char tmp[10]; int readIndex = 0; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { break; } @@ -675,8 +669,8 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { if (readIndex >= 2) break; } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); if (readIndex < 2) { //printf("read file:%s failed", tsProcIOFile); @@ -839,15 +833,15 @@ void taosSetCoreDump(bool enable) { } int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { - int fd; int len = 0; - fd = open("/proc/sys/kernel/random/uuid", 0); - if (fd < 0) { + // fd = open("/proc/sys/kernel/random/uuid", 0); + TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ); + if (pFile == NULL) { return -1; } else { - len = read(fd, uid, uidlen); - close(fd); + len = taosReadFile(pFile, uid, uidlen); + taosCloseFile(&pFile); } if (len >= 36) { @@ -862,16 +856,17 @@ char *taosGetCmdlineByPID(int pid) { static char cmdline[1024]; sprintf(cmdline, "/proc/%d/cmdline", pid); - int fd = open(cmdline, O_RDONLY); - if (fd >= 0) { - int n = read(fd, cmdline, sizeof(cmdline) - 1); + // int fd = open(cmdline, O_RDONLY); + TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ); + if (pFile != NULL) { + int n = taosReadFile(pFile, cmdline, sizeof(cmdline) - 1); if (n < 0) n = 0; if (n > 0 && cmdline[n - 1] == '\n') --n; cmdline[n] = 0; - close(fd); + taosCloseFile(&pFile); } else { cmdline[0] = 0; } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index a0ea01596f..07846781ad 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -150,17 +150,18 @@ void taosGetSystemTimezone(char *outTimezone) { localtime_r(&tx1, &tm1); /* load time zone string from /etc/timezone */ - FILE *f = fopen("/etc/timezone", "r"); + // FILE *f = fopen("/etc/timezone", "r"); + TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); char buf[68] = {0}; - if (f != NULL) { - int len = fread(buf, 64, 1, f); - if (len < 64 && ferror(f)) { - fclose(f); + if (pFile != NULL) { + int len = taosReadFile(pFile, buf, 64); + if (len < 64 && taosGetErrorFile(pFile)) { + taosCloseFile(&pFile); // printf("read /etc/timezone error, reason:%s", strerror(errno)); return; } - fclose(f); + taosCloseFile(&pFile); buf[sizeof(buf) - 1] = 0; char *lineEnd = strstr(buf, "\n"); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 0f4ba419f3..7ad08e44d5 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -571,31 +571,28 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { } int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { - char *line, *name, *value, *value2, *value3; + char *line = NULL, *name, *value, *value2, *value3; int olen, vlen, vlen2, vlen3; ssize_t _bytes = 0; - size_t len = 1024; - FILE *fp = fopen(filepath, "r"); - if (fp == NULL) { + // FILE *fp = fopen(filepath, "r"); + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - line = malloc(len); - - while (!feof(fp)) { - memset(line, 0, len); + while (!taosEOFFile(pFile)) { name = value = value2 = value3 = NULL; olen = vlen = vlen2 = vlen3 = 0; - _bytes = tgetline(&line, &len, fp); + _bytes = taosGetLineFile(pFile, &line); if (_bytes < 0) { break; } - line[len - 1] = 0; + line[_bytes - 1] = 0; paGetToken(line, &name, &olen); if (olen == 0) continue; @@ -616,8 +613,8 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { // taosReadConfigOption(name, value, value2, value3); } - fclose(fp); - tfree(line); + taosCloseFile(&pFile); + if(line != NULL) tfree(line); uInfo("load from cfg file %s success", filepath); return 0; diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 0f68e9204d..d10ea6a934 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -24,147 +24,147 @@ static int32_t tsFileRsetId = -1; static int8_t tfInited = 0; -static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); } +// static void tfCloseFile(void *p) { taosCloseFile((TdFilePtr)(uintptr_t)p); } int32_t tfInit() { - int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); - if (old == 1) return 0; - tsFileRsetId = taosOpenRef(2000, tfCloseFile); - if (tsFileRsetId > 0) { - return 0; - } else { - atomic_store_8(&tfInited, 0); - return -1; - } + // int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); + // if (old == 1) return 0; + // tsFileRsetId = taosOpenRef(2000, tfCloseFile); + // if (tsFileRsetId > 0) { + // return 0; + // } else { + // atomic_store_8(&tfInited, 0); + // return -1; + // } } void tfCleanup() { - atomic_store_8(&tfInited, 0); - if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); - tsFileRsetId = -1; + // atomic_store_8(&tfInited, 0); + // if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); + // tsFileRsetId = -1; } -static int64_t tfOpenImp(int32_t fd) { - if (fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } +// static int64_t tfOpenImp(TdFilePtr pFile) { +// if (pFile == NULL) { +// terrno = TAOS_SYSTEM_ERROR(errno); +// return -1; +// } - void * p = (void *)(int64_t)fd; - int64_t rid = taosAddRef(tsFileRsetId, p); - if (rid < 0) taosCloseFile(fd); +// void * p = (void *)(int64_t)pFile; +// int64_t rid = taosAddRef(tsFileRsetId, p); +// if (rid < 0) taosCloseFile(&pFile); - return rid; -} +// return rid; +// } -int64_t tfOpenRead(const char *pathname, int32_t flags) { - int32_t fd = taosOpenFileRead(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenRead(const char *pathname, int32_t flags) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ); +// return tfOpenImp(fd); +// } -int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { - int32_t fd = taosOpenFileReadWrite(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ | TD_FILE_WRITE); +// return tfOpenImp(fd); +// } -int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = taosOpenFileCreateWrite(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE); +// return tfOpenImp(fd); +// } -int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = taosOpenFileCreateWriteAppend(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); +// return tfOpenImp(fd); +// } -int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } +// int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } -int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = taosWriteFile(fd, buf, count); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = taosWriteFile(pFile, buf, count); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int64_t tfRead(int64_t tfd, void *buf, int64_t count) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfRead(int64_t tfd, void *buf, int64_t count) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = taosReadFile(fd, buf, count); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = taosReadFile(pFile, buf, count); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = pread(fd, buf, count, offset); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = pread(fd, buf, count, offset); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int32_t tfFsync(int64_t tfd) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int32_t tfFsync(int64_t tfd) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFsyncFile(fd); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int32_t code = taosFsyncFile(pFile); - taosReleaseRef(tsFileRsetId, tfd); - return code; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return code; +// } -bool tfValid(int64_t tfd) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return false; +// bool tfValid(int64_t tfd) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return false; - taosReleaseRef(tsFileRsetId, tfd); - return true; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return true; +// } -int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int64_t ret = taosLSeekFile(fd, offset, whence); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int64_t ret = taosLSeekFile(fd, offset, whence); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int32_t tfFtruncate(int64_t tfd, int64_t length) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int32_t tfFtruncate(int64_t tfd, int64_t length) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFtruncateFile(fd, length); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int32_t code = taosFtruncateFile(fd, length); - taosReleaseRef(tsFileRsetId, tfd); - return code; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return code; +// } -void *tfMmapReadOnly(int64_t tfd, int64_t length) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return NULL; - int32_t fd = (int32_t)(uintptr_t)p; +// void *tfMmapReadOnly(int64_t tfd, int64_t length) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return NULL; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); - taosReleaseRef(tsFileRsetId, tfd); - return ptr; -} +// void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); +// taosReleaseRef(tsFileRsetId, tfd); +// return ptr; +// } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index ed55ef91b0..a69cf215d9 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -47,7 +47,7 @@ typedef struct { int32_t buffEnd; int32_t buffSize; int32_t minBuffSize; - int32_t fd; + TdFilePtr pFile; int32_t stop; pthread_t asyncThread; pthread_mutex_t buffMutex; @@ -101,7 +101,7 @@ static SLogObj tsLogObj = {.fileNum = 1}; static void *taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); -static void taosCloseLogByFd(int32_t oldFd); +static void taosCloseLogByFd(TdFilePtr pFile); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); extern void taosPrintCfg(); static int32_t taosCompressFile(char *srcFileName, char *destFileName); @@ -150,11 +150,11 @@ void taosCloseLog() { // taosCloseLog(); } -static bool taosLockLogFile(int32_t fd) { - if (fd < 0) return false; +static bool taosLockLogFile(TdFilePtr pFile) { + if (pFile == NULL) return false; if (tsLogObj.fileNum > 1) { - int32_t ret = taosLockFile(fd); + int32_t ret = taosLockFile(pFile); if (ret == 0) { return true; } @@ -163,11 +163,11 @@ static bool taosLockLogFile(int32_t fd) { return false; } -static void taosUnLockLogFile(int32_t fd) { - if (fd < 0) return; +static void taosUnLockLogFile(TdFilePtr pFile) { + if (pFile == NULL) return; if (tsLogObj.fileNum > 1) { - taosUnLockFile(fd); + taosUnLockFile(pFile); } } @@ -201,22 +201,22 @@ static void *taosThreadToOpenNewFile(void *param) { taosUmaskFile(0); - int32_t fd = taosOpenFileCreateWriteTrunc(name); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { tsLogObj.openInProgress = 0; tsLogObj.lines = tsLogObj.maxLines - 1000; - uError("open new log file fail! fd:%d reason:%s, reuse lastlog", fd, strerror(errno)); + uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); return NULL; } - taosLockLogFile(fd); - (void)taosLSeekFile(fd, 0, SEEK_SET); + taosLockLogFile(pFile); + (void)taosLSeekFile(pFile, 0, SEEK_SET); - int32_t oldFd = tsLogObj.logHandle->fd; - tsLogObj.logHandle->fd = fd; + TdFilePtr pOldFile = tsLogObj.logHandle->pFile; + tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - taosCloseLogByFd(oldFd); + taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); @@ -262,8 +262,8 @@ void taosResetLog() { } static bool taosCheckFileIsOpen(char *logFileName) { - int32_t fd = taosOpenFileWrite(logFileName); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(logFileName, TD_FILE_WRITE); + if (pFile == NULL) { if (errno == ENOENT) { return false; } else { @@ -272,12 +272,12 @@ static bool taosCheckFileIsOpen(char *logFileName) { } } - if (taosLockLogFile(fd)) { - taosUnLockLogFile(fd); - taosCloseFile(fd); + if (taosLockLogFile(pFile)) { + taosUnLockLogFile(pFile); + taosCloseFile(&pFile); return false; } else { - taosCloseFile(fd); + taosCloseFile(&pFile); return true; } } @@ -350,31 +350,31 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { pthread_mutex_init(&tsLogObj.logMutex, NULL); taosUmaskFile(0); - tsLogObj.logHandle->fd = taosOpenFileCreateWrite(fileName); + tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); - if (tsLogObj.logHandle->fd < 0) { + if (tsLogObj.logHandle->pFile == NULL) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - taosLockLogFile(tsLogObj.logHandle->fd); + taosLockLogFile(tsLogObj.logHandle->pFile); // only an estimate for number of lines int64_t filesize = 0; - if (taosFStatFile(tsLogObj.logHandle->fd, &filesize, NULL) < 0) { + if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) { printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } size = (int32_t)filesize; tsLogObj.lines = size / 60; - taosLSeekFile(tsLogObj.logHandle->fd, 0, SEEK_END); + taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END); sprintf(name, "==================================================\n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); sprintf(name, " new log file \n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); sprintf(name, "==================================================\n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); return 0; } @@ -416,11 +416,11 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { buffer[len++] = '\n'; buffer[len] = 0; - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWriteFile(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -430,7 +430,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); } void taosDumpData(unsigned char *msg, int32_t len) { @@ -445,7 +445,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { pos += 3; if (c >= 16) { temp[pos++] = '\n'; - taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); c = 0; pos = 0; } @@ -453,7 +453,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { temp[pos++] = '\n'; - taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { @@ -483,11 +483,11 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . buffer[len++] = '\n'; buffer[len] = 0; - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWriteFile(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -497,19 +497,19 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } } - if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); } #if 0 void taosCloseLog() { - taosCloseLogByFd(tsLogObj.logHandle->fd); + taosCloseLogByFd(tsLogObj.logHandle->pFile); } #endif -static void taosCloseLogByFd(int32_t fd) { - if (fd >= 0) { - taosUnLockLogFile(fd); - taosCloseFile(fd); +static void taosCloseLogByFd(TdFilePtr pFile) { + if (pFile != NULL) { + taosUnLockLogFile(pFile); + taosCloseFile(&pFile); } } @@ -645,12 +645,12 @@ static void taosWriteLog(SLogBuff *tLogBuff) { } if (start < end) { - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff), end); } dbgWN++; @@ -707,17 +707,17 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t ret = 0; int32_t len = 0; char *data = malloc(compressSize); - FILE *srcFp = NULL; // gzFile dstFp = NULL; - srcFp = fopen(srcFileName, "r"); - if (srcFp == NULL) { + // srcFp = fopen(srcFileName, "r"); + TdFilePtr pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ); + if (pSrcFile == NULL) { ret = -1; goto cmp_end; } - int32_t fd = taosOpenFileCreateWriteTrunc(destFileName); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(destFileName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { ret = -2; goto cmp_end; } @@ -735,8 +735,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { // } cmp_end: - if (srcFp) { - fclose(srcFp); + if (pSrcFile) { + taosCloseFile(&pSrcFile); } // if (dstFp) { // gzclose(dstFp); diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 0e8d85492c..ca260f0df5 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -32,7 +32,7 @@ typedef struct SDiskbasedBuf { int32_t numOfPages; int64_t totalBufSize; uint64_t fileSize; // disk file size - FILE* file; + TdFilePtr pFile; int32_t allocateId; // allocated page id char* path; // file path int32_t pageSize; // current used page size @@ -67,7 +67,7 @@ static void printStatisData(const SDiskbasedBuf* pBuf); pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. pResBuf->allocateId = -1; pResBuf->comp = true; - pResBuf->file = NULL; + pResBuf->pFile = NULL; pResBuf->qId = qId; pResBuf->fileSize = 0; @@ -94,8 +94,9 @@ static void printStatisData(const SDiskbasedBuf* pBuf); } static int32_t createDiskFile(SDiskbasedBuf* pBuf) { - pBuf->file = fopen(pBuf->path, "wb+"); - if (pBuf->file == NULL) { + // pBuf->file = fopen(pBuf->path, "wb+"); + pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (pBuf->pFile == NULL) { // qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } @@ -168,13 +169,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { pg->offset = allocatePositionInFile(pBuf, size); pBuf->nextPos += size; - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - ret = (int32_t) fwrite(t, 1, size, pBuf->file); + ret = (int32_t) taosWriteFile(pBuf->pFile, t, size); if (ret != size) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -199,13 +200,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { } // 3. write to disk. - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - ret = (int32_t)fwrite(t, 1, size, pBuf->file); + ret = (int32_t) taosWriteFile(pBuf->pFile, t, size); if (ret != size) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -233,7 +234,7 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { int32_t ret = TSDB_CODE_SUCCESS; assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); - if (pBuf->file == NULL) { + if (pBuf->pFile == NULL) { if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) { terrno = ret; return NULL; @@ -245,14 +246,14 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { // load file block data in disk static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { ret = TAOS_SYSTEM_ERROR(errno); return ret; } SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg); - ret = (int32_t)fread(pPage->data, 1, pg->length, pBuf->file); + ret = (int32_t)taosReadFile(pBuf->pFile, pPage->data, pg->length); if (ret != pg->length) { ret = TAOS_SYSTEM_ERROR(errno); return ret; @@ -495,12 +496,12 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) { printStatisData(pBuf); - if (pBuf->file != NULL) { + if (pBuf->pFile != NULL) { uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n", pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId); - fclose(pBuf->file); + taosCloseFile(&pBuf->pFile); } else { uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId); } diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c index 7fc8b1409a..33d25b6746 100644 --- a/tools/shell/src/backup/shellCheck.c +++ b/tools/shell/src/backup/shellCheck.c @@ -116,7 +116,7 @@ static void *shellCheckThreadFp(void *arg) { char file[32] = {0}; snprintf(file, 32, "tb%d.txt", pThread->threadIndex); - FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (!fp) { fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno)); return NULL; @@ -133,7 +133,7 @@ static void *shellCheckThreadFp(void *arg) { int32_t code = taos_errno(pSql); if (code != 0) { int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname); - fwrite(sql, 1, len, fp); + taosWriteFile(pFile, sql, len); atomic_add_fetch_32(&errorNum, 1); } @@ -145,8 +145,8 @@ static void *shellCheckThreadFp(void *arg) { taos_free_result(pSql); } - taosFsync(fileno(fp)); - fclose(fp); + taosFsync(pFile); + taosCloseFile(&pFile); return NULL; } diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index 222d69e854..ce15212f86 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -138,7 +138,6 @@ static void shellSourceFile(TAOS *con, char *fptr) { char * cmd = malloc(tsMaxSQLStringLen); size_t cmd_len = 0; char * line = NULL; - size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); @@ -171,8 +170,9 @@ static void shellSourceFile(TAOS *con, char *fptr) { } */ - FILE *f = fopen(fname, "r"); - if (f == NULL) { + // FILE *f = fopen(fname, "r"); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); free(cmd); @@ -182,7 +182,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stdout, "begin import file:%s\n", fname); int lineNo = 0; - while ((read_len = getline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetLineFile(pFile, &line)) != -1) { ++lineNo; if (read_len >= tsMaxSQLStringLen) continue; line[--read_len] = '\0'; @@ -215,9 +215,9 @@ static void shellSourceFile(TAOS *con, char *fptr) { } free(cmd); - if (line) free(line); + if(line != NULL) free(line); wordfree(&full_path); - fclose(f); + taosCloseFile(&pFile); } void* shellImportThreadFp(void *arg) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 33774bdd05..906a9ce5f0 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -459,44 +459,44 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { return buf; } -static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) { +static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int precision) { if (val == NULL) { - fprintf(fp, "%s", TSDB_DATA_NULL_STR); + taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR); return; } char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: - fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); + taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); break; case TSDB_DATA_TYPE_TINYINT: - fprintf(fp, "%d", *((int8_t *)val)); + taosFprintfFile(pFile, "%d", *((int8_t *)val)); break; case TSDB_DATA_TYPE_SMALLINT: - fprintf(fp, "%d", *((int16_t *)val)); + taosFprintfFile(pFile, "%d", *((int16_t *)val)); break; case TSDB_DATA_TYPE_INT: - fprintf(fp, "%d", *((int32_t *)val)); + taosFprintfFile(pFile, "%d", *((int32_t *)val)); break; case TSDB_DATA_TYPE_BIGINT: - fprintf(fp, "%" PRId64, *((int64_t *)val)); + taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - fprintf(fp, "%.5f", GET_FLOAT_VAL(val)); + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); break; case TSDB_DATA_TYPE_DOUBLE: - fprintf(fp, "%.9f", GET_DOUBLE_VAL(val)); + taosFprintfFile(pFile, "%.9f", GET_DOUBLE_VAL(val)); break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memcpy(buf, val, length); buf[length] = 0; - fprintf(fp, "\'%s\'", buf); + taosFprintfFile(pFile, "\'%s\'", buf); break; case TSDB_DATA_TYPE_TIMESTAMP: formatTimestamp(buf, *(int64_t *)val, precision); - fprintf(fp, "'%s'", buf); + taosFprintfFile(pFile, "'%s'", buf); break; default: break; @@ -516,8 +516,9 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { return -1; } - FILE *fp = fopen(full_path.we_wordv[0], "w"); - if (fp == NULL) { + // FILE *fp = fopen(full_path.we_wordv[0], "w"); + TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); wordfree(&full_path); return -1; @@ -531,29 +532,29 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { for (int col = 0; col < num_fields; col++) { if (col > 0) { - fprintf(fp, ","); + taosFprintfFile(pFile, ","); } - fprintf(fp, "%s", fields[col].name); + taosFprintfFile(pFile, "%s", fields[col].name); } - fputc('\n', fp); + taosFprintfFile(pFile, "\n"); int numOfRows = 0; do { int32_t *length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { if (i > 0) { - fputc(',', fp); + taosFprintfFile(pFile, "\n"); } - dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision); + dumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision); } - fputc('\n', fp); + taosFprintfFile(pFile, "\n"); numOfRows++; row = taos_fetch_row(tres); } while (row != NULL); result = 0; - fclose(fp); + taosCloseFile(&pFile); return numOfRows; } @@ -897,14 +898,14 @@ void read_history() { history.hstart = 0; history.hend = 0; char *line = NULL; - size_t line_size = 0; int read_size = 0; char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); - FILE *f = fopen(f_history, "r"); - if (f == NULL) { + // FILE *f = fopen(f_history, "r"); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ); + if (pFile == NULL) { #ifndef WINDOWS if (errno != ENOENT) { fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno)); @@ -913,7 +914,7 @@ void read_history() { return; } - while ((read_size = tgetline(&line, &line_size, f)) != -1) { + while ((read_size = taosGetLineFile(pFile, &line)) != -1) { line[read_size - 1] = '\0'; history.hist[history.hend] = strdup(line); @@ -924,16 +925,17 @@ void read_history() { } } - free(line); - fclose(f); + if(line != NULL) free(line); + taosCloseFile(&pFile); } void write_history() { char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); - FILE *f = fopen(f_history, "w"); - if (f == NULL) { + // FILE *f = fopen(f_history, "w"); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { #ifndef WINDOWS fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); #endif @@ -942,12 +944,12 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { - fprintf(f, "%s\n", history.hist[i]); + taosFprintfFile(pFile, "%s\n", history.hist[i]); tfree(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } - fclose(f); + taosCloseFile(&pFile); } void taos_error(TAOS_RES *tres, int64_t st) { @@ -969,7 +971,6 @@ void source_file(TAOS *con, char *fptr) { char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; - size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); @@ -989,15 +990,16 @@ void source_file(TAOS *con, char *fptr) { } */ - FILE *f = fopen(fname, "r"); - if (f == NULL) { + // FILE *f = fopen(fname, "r"); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); free(cmd); return; } - while ((read_len = tgetline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetLineFile(pFile, &line)) != -1) { if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; @@ -1020,9 +1022,9 @@ void source_file(TAOS *con, char *fptr) { } free(cmd); - if (line) free(line); + if(line != NULL) free(line); wordfree(&full_path); - fclose(f); + taosCloseFile(&pFile); } void shellGetGrantInfo(void *con) {