xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-13062]<fix>: cross platform file system.

This commit is contained in:
afwerar 2022-02-25 01:47:20 +08:00
parent 68514f3dfd
commit 5a37fdf28f
51 changed files with 1129 additions and 1026 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/common/ttszip.h
View File

@ -73,9 +73,9 @@ typedef struct STSGroupBlockInfoEx {
} STSGroupBlockInfoEx;
typedef struct STSBuf {
FILE* f;
char path[PATH_MAX];
uint32_t fileSize;
TdFilePtr pFile;
char path[PATH_MAX];
uint32_t fileSize;
// todo use array
STSGroupBlockInfoEx* pData;

8
include/libs/wal/wal.h
View File

@ -120,8 +120,8 @@ typedef struct SWal {
int32_t fsyncSeq;
// meta
SWalVer vers;
int64_t writeLogTfd;
int64_t writeIdxTfd;
TdFilePtr pWriteLogTFile;
TdFilePtr pWriteIdxTFile;
int32_t writeCur;
SArray *fileInfoSet;
// status
@ -138,8 +138,8 @@ typedef struct SWal {
typedef struct SWalReadHandle {
SWal *pWal;
int64_t readLogTfd;
int64_t readIdxTfd;
TdFilePtr pReadLogTFile;
TdFilePtr pReadIdxTFile;
int64_t curFileFirstVer;
int64_t curVersion;
int64_t capacity;

78
include/os/osFile.h
View File

@ -22,49 +22,63 @@ extern "C" {
#include "osSocket.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
typedef int32_t FileFd;
#else
typedef int32_t FileFd;
#ifndef ALLOW_FORBID_FUNC
#define open OPEN_FUNC_TAOS_FORBID
#define fopen FOPEN_FUNC_TAOS_FORBID
// #define close CLOSE_FUNC_TAOS_FORBID
// #define fclose FCLOSE_FUNC_TAOS_FORBID
#endif
#define FD_INITIALIZER ((int32_t)-1)
#ifndef PATH_MAX
#define PATH_MAX 256
#endif
int32_t taosLockFile(FileFd fd);
int32_t taosUnLockFile(FileFd fd);
int32_t taosUmaskFile(FileFd fd);
typedef struct TdFile *TdFilePtr;
#define TD_FILE_CTEATE 0x0001
#define TD_FILE_WRITE 0x0002
#define TD_FILE_READ 0x0004
#define TD_FILE_TRUNC 0x0008
#define TD_FILE_APPEND 0x0010
#define TD_FILE_TEXT 0x0020
#define TD_FILE_AUTO_DEL 0x0040
#define TD_FILE_EXCL 0x0080
int32_t taosLockFile(TdFilePtr pFile);
int32_t taosUnLockFile(TdFilePtr pFile);
int32_t taosUmaskFile(int32_t maskVal);
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime);
FileFd taosOpenFileWrite(const char *path);
FileFd taosOpenFileCreateWrite(const char *path);
FileFd taosOpenFileCreateWriteTrunc(const char *path);
FileFd taosOpenFileCreateWriteAppend(const char *path);
FileFd taosOpenFileRead(const char *path);
FileFd taosOpenFileReadWrite(const char *path);
int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence);
int32_t taosFtruncateFile(FileFd fd, int64_t length);
int32_t taosFsyncFile(FileFd fd);
int64_t taosReadFile(FileFd fd, void *buf, int64_t count);
int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count);
void taosCloseFile(FileFd fd);
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length);
int32_t taosFsyncFile(TdFilePtr pFile);
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
int32_t taosEOFFile(TdFilePtr pFile);
int64_t taosCloseFile(TdFilePtr *ppFile);
int32_t taosRenameFile(const char *oldName, const char *newName);
int64_t taosCopyFile(const char *from, const char *to);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size);
int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
bool taosValidFile(TdFilePtr pFile);
int taosGetErrorFile(TdFilePtr pFile);
#ifdef __cplusplus
}

26
include/util/tfile.h
View File

@ -30,20 +30,20 @@ void tfCleanup();
// the same syntax as UNIX standard open/close/read/write
// but FD is int64_t and will never be reused
int64_t tfOpenRead(const char *pathname);
int64_t tfOpenReadWrite(const char *pathname);
int64_t tfOpenCreateWrite(const char *pathname);
int64_t tfOpenCreateWriteAppend(const char *pathname);
// int64_t tfOpenRead(const char *pathname);
// int64_t tfOpenReadWrite(const char *pathname);
// int64_t tfOpenCreateWrite(const char *pathname);
// int64_t tfOpenCreateWriteAppend(const char *pathname);
int64_t tfClose(int64_t tfd);
int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
int64_t tfRead(int64_t tfd, void *buf, int64_t count);
int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset);
int32_t tfFsync(int64_t tfd);
bool tfValid(int64_t tfd);
int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
int32_t tfFtruncate(int64_t tfd, int64_t length);
void * tfMmapReadOnly(int64_t tfd, int64_t length);
// int64_t tfClose(int64_t tfd);
// int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
// int64_t tfRead(int64_t tfd, void *buf, int64_t count);
// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset);
// int32_t tfFsync(int64_t tfd);
// bool tfValid(int64_t tfd);
// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
// int32_t tfFtruncate(int64_t tfd, int64_t length);
// void * tfMmapReadOnly(int64_t tfd, int64_t length);
#ifdef __cplusplus
}
#endif

135
source/common/src/ttszip.c
View File

@ -24,8 +24,9 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
pTSBuf->autoDelete = autoDelete;
taosGetTmpfilePath(osTempDir(), "join", pTSBuf->path);
pTSBuf->f = fopen(pTSBuf->path, "wb+");
if (pTSBuf->f == NULL) {
// pTSBuf->pFile = fopen(pTSBuf->path, "wb+");
pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (pTSBuf->pFile == NULL) {
free(pTSBuf);
return NULL;
}
@ -60,8 +61,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path));
pTSBuf->f = fopen(pTSBuf->path, "rb+");
if (pTSBuf->f == NULL) {
// pTSBuf->pFile = fopen(pTSBuf->path, "rb+");
pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ);
if (pTSBuf->pFile == NULL) {
free(pTSBuf);
return NULL;
}
@ -72,9 +74,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
// validate the file magic number
STSBufFileHeader header = {0};
int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET);
int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET);
UNUSED(ret);
size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f);
size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader));
UNUSED(sz);
// invalid file
@ -112,8 +114,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return NULL;
}
//int64_t pos = ftell(pTSBuf->f); //pos not used
sz = fread(buf, infoSize, 1, pTSBuf->f);
//int64_t pos = ftell(pTSBuf->pFile); //pos not used
sz = taosReadFile(pTSBuf->pFile, buf, infoSize);
UNUSED(sz);
// the length value for each vnode is not kept in file, so does not set the length value
@ -123,22 +125,22 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
}
free(buf);
ret = fseek(pTSBuf->f, 0, SEEK_END);
ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END);
UNUSED(ret);
struct stat fileStat;
if (fstat(fileno(pTSBuf->f), &fileStat) != 0) {
int64_t file_size;
if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) {
tsBufDestroy(pTSBuf);
return NULL;
}
pTSBuf->fileSize = (uint32_t)fileStat.st_size;
pTSBuf->fileSize = (uint32_t)file_size;
tsBufResetPos(pTSBuf);
// ascending by default
pTSBuf->cur.order = TSDB_ORDER_ASC;
// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f),
// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->pFile),
// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete);
return pTSBuf;
@ -156,7 +158,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
tfree(pTSBuf->block.payload);
if (!pTSBuf->remainOpen) {
fclose(pTSBuf->f);
taosCloseFile(&pTSBuf->pFile);
}
if (pTSBuf->autoDelete) {
@ -253,7 +255,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize,
TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize);
int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET);
int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET);
assert(r == 0);
/*
@ -264,30 +266,30 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
* both side has the compressed length is used to support load data forwards/backwords.
*/
int32_t metaLen = 0;
metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
int32_t trueLen = pBlock->tag.nLen;
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
float tfloat = (float)pBlock->tag.d;
metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)fwrite(&pBlock->tag.i, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen);
} else {
trueLen = 0;
metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen));
}
fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem));
taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen);
taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t) taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen));
assert(metaLen == getTagAreaLength(&pBlock->tag));
int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen;
@ -332,20 +334,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
* the end of each comp data block
*/
int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen));
int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR);
size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f);
sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR);
size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding));
sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
UNUSED(sz);
pBlock->compLen = pBlock->padding;
offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
ret = fseek(pTSBuf->f, -offset, SEEK_CUR);
ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR);
UNUSED(ret);
}
int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f);
ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
// NOTE: mix types tags are not supported
size_t sz = 0;
@ -356,23 +358,23 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
memset(tp, 0, pBlock->tag.nLen + 1);
pBlock->tag.pz = tp;
sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
UNUSED(sz);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
float tfloat = 0;
sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen);
pBlock->tag.d = (double)tfloat;
UNUSED(sz);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value
sz = fread(&pBlock->tag.i, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen);
UNUSED(sz);
}
sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem));
UNUSED(sz);
sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
UNUSED(sz);
sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen);
if (decomp) {
pTSBuf->tsData.len =
@ -381,11 +383,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
}
// read the comp length at the length of comp block
sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding));
assert(pBlock->padding == pBlock->compLen);
int32_t n = 0;
sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen));
if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) {
assert(n == 0);
} else {
@ -396,7 +398,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
// for backwards traverse, set the start position at the end of previous block
if (order == TSDB_ORDER_DESC) {
int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR);
UNUSED(r);
}
@ -512,7 +514,7 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num
// todo opt performance by cache blocks info
static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) {
if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) {
return -1;
}
@ -531,7 +533,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int
STSBlock* pBlock = &pTSBuf->block;
int32_t compBlockSize =
pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR);
UNUSED(ret);
}
@ -548,7 +550,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo
offset = pBlockInfo->offset + pBlockInfo->compLen;
}
if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) {
return -1;
}
@ -618,11 +620,11 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn
return -1;
}
if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) {
return -1;
}
fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f);
taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo));
return 0;
}
@ -636,19 +638,19 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) {
}
int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
return -1;
}
assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC);
int32_t r = fseek(pTSBuf->f, 0, SEEK_SET);
int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET);
if (r != 0) {
// qError("fseek failed, errno:%d", errno);
return -1;
}
size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f);
size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader));
if (ws != 1) {
// qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader));
return -1;
@ -823,12 +825,12 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
pBlockInfoEx->info.id = id;
}
int32_t r = fseek(pDestBuf->f, 0, SEEK_END);
int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END);
assert(r == 0);
int64_t offset = getDataStartOffset();
int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size);
if (written == -1 || written != size) {
return -1;
@ -839,17 +841,18 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
int32_t oldSize = pDestBuf->fileSize;
// file meta data may be cached, close and reopen the file for accurate file size.
fclose(pDestBuf->f);
pDestBuf->f = fopen(pDestBuf->path, "rb+");
if (pDestBuf->f == NULL) {
taosCloseFile(&pDestBuf->pFile);
// pDestBuf->pFile = fopen(pDestBuf->path, "rb+");
pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ);
if (pDestBuf->pFile == NULL) {
return -1;
}
struct stat fileStat;
if (fstat(fileno(pDestBuf->f), &fileStat) != 0) {
int64_t file_size;
if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) {
return -1;
}
pDestBuf->fileSize = (uint32_t)fileStat.st_size;
pDestBuf->fileSize = (uint32_t)file_size;
assert(pDestBuf->fileSize == oldSize + size);
@ -868,13 +871,13 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
// update prev vnode length info in file
TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo);
int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET);
if (ret == -1) {
// qError("fseek failed, errno:%d", errno);
tsBufDestroy(pTSBuf);
return NULL;
}
size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f);
size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len);
if (sz != len) {
// qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len);
tsBufDestroy(pTSBuf);
@ -893,7 +896,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
}
// TODO taosFsync??
// if (taosFsync(fileno(pTSBuf->f)) == -1) {
// if (taosFsync(fileno(pTSBuf->pFile)) == -1) {
//// qError("fsync failed, errno:%d", errno);
// tsBufDestroy(pTSBuf);
// return NULL;
@ -1071,15 +1074,15 @@ int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, in
*len = 0;
*numOfBlocks = 0;
if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) {
int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile));
// qError("%p: fseek failed: %s", pSql, tstrerror(code));
return code;
}
size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f);
size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen);
if (s != pBlockInfo->compLen) {
int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile));
// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
return code;
}

2
source/dnode/mgmt/impl/inc/dndEnv.h
View File

@ -125,7 +125,7 @@ typedef struct SDnode {
EStat stat;
SDnodeObjCfg cfg;
SDnodeDir dir;
FileFd lockFd;
TdFilePtr pLockFile;
SDnodeMgmt dmgmt;
SMnodeMgmt mmgmt;
SQnodeMgmt qmgmt;

20
source/dnode/mgmt/impl/src/dndBnode.c
View File

@ -62,14 +62,15 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_BNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_BNODE_OVER;
@ -102,7 +103,7 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) {
PRASE_BNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
@ -114,8 +115,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
@ -130,9 +132,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];

28
source/dnode/mgmt/impl/src/dndEnv.c
View File

@ -59,31 +59,31 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
}
static FileFd dndCheckRunning(char *dataDir) {
static TdFilePtr dndCheckRunning(char *dataDir) {
char filepath[PATH_MAX] = {0};
snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
FileFd fd = taosOpenFileCreateWriteTrunc(filepath);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s, quit", filepath, terrstr());
return -1;
return NULL;
}
int32_t ret = taosLockFile(fd);
int32_t ret = taosLockFile(pFile);
if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s, quit", filepath, terrstr());
taosCloseFile(fd);
return -1;
taosCloseFile(&pFile);
return NULL;
}
return fd;
return pFile;
}
static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) {
pDnode->lockFd = dndCheckRunning(pCfg->dataDir);
if (pDnode->lockFd < 0) {
pDnode->pLockFile = dndCheckRunning(pCfg->dataDir);
if (pDnode->pLockFile == NULL) {
return -1;
}
@ -147,10 +147,10 @@ static void dndCloseImp(SDnode *pDnode) {
tfree(pDnode->dir.snode);
tfree(pDnode->dir.bnode);
if (pDnode->lockFd >= 0) {
taosUnLockFile(pDnode->lockFd);
taosCloseFile(pDnode->lockFd);
pDnode->lockFd = 0;
if (pDnode->pLockFile != NULL) {
taosUnLockFile(pDnode->pLockFile);
taosCloseFile(&pDnode->pLockFile);
pDnode->pLockFile = NULL;
}
}

21
source/dnode/mgmt/impl/src/dndMgmt.c
View File

@ -185,16 +185,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
int32_t maxLen = 256 * 1024;
char *content = calloc(1, maxLen + 1);
cJSON *root = NULL;
FILE *fp = NULL;
fp = fopen(pMgmt->file, "r");
if (fp == NULL) {
// fp = fopen(pMgmt->file, "r");
TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", pMgmt->file);
code = 0;
goto PRASE_DNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", pMgmt->file);
goto PRASE_DNODE_OVER;
@ -286,7 +286,7 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
PRASE_DNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) {
dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file);
@ -309,8 +309,9 @@ PRASE_DNODE_OVER:
static int32_t dndWriteDnodes(SDnode *pDnode) {
SDnodeMgmt *pMgmt = &pDnode->dmgmt;
FILE *fp = fopen(pMgmt->file, "w");
if (fp == NULL) {
// FILE *fp = fopen(pMgmt->file, "w");
TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
dError("failed to write %s since %s", pMgmt->file, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -341,9 +342,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) {
}
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
terrno = 0;

20
source/dnode/mgmt/impl/src/dndMnode.c
View File

@ -63,14 +63,15 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_MNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_MNODE_OVER;
@ -143,7 +144,7 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
PRASE_MNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
@ -155,8 +156,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
@ -184,9 +186,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
}
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];

20
source/dnode/mgmt/impl/src/dndQnode.c
View File

@ -62,14 +62,15 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_QNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_QNODE_OVER;
@ -102,7 +103,7 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
PRASE_QNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
@ -114,8 +115,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
@ -130,9 +132,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];

20
source/dnode/mgmt/impl/src/dndSnode.c
View File

@ -62,14 +62,15 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_SNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_SNODE_OVER;
@ -102,7 +103,7 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) {
PRASE_SNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
@ -114,8 +115,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
@ -130,9 +132,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];

20
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -219,14 +219,15 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
fp = fopen(file, "r");
if (fp == NULL) {
// fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_VNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_VNODE_OVER;
@ -304,7 +305,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
PRASE_VNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
return code;
}
@ -315,8 +316,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes);
snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write %s since %s", file, terrstr());
return -1;
@ -347,9 +349,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " ]\n");
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
terrno = 0;

41
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -62,12 +62,13 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
size_t size = 0;
int32_t done = 0;
FILE* fp = fopen("/proc/cpuinfo", "r");
if (fp == NULL) {
// FILE* fp = fopen("/proc/cpuinfo", "r");
TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ);
if (pFile == NULL) {
return;
}
while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) {
while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
const char* v = strchr(line, ':') + 2;
@ -83,20 +84,21 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
}
}
free(line);
fclose(fp);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
FILE* fp = fopen("/etc/os-release", "r");
if (fp == NULL) {
// FILE* fp = fopen("/etc/os-release", "r");
TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ);
if (pFile == NULL) {
return;
}
while ((size = tgetline(&line, &size, fp)) != -1) {
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "PRETTY_NAME", 11) == 0) {
const char* p = strchr(line, '=') + 1;
@ -109,20 +111,21 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
}
}
free(line);
fclose(fp);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
FILE* fp = fopen("/proc/meminfo", "r");
if (fp == NULL) {
// FILE* fp = fopen("/proc/meminfo", "r");
TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ);
if (pFile == NULL) {
return;
}
while ((size = tgetline(&line, &size, fp)) != -1) {
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "MemTotal", 8) == 0) {
const char* p = strchr(line, ':') + 1;
@ -132,8 +135,8 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
}
}
free(line);
fclose(fp);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) {
@ -252,16 +255,16 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
static void mndGetEmail(SMnode* pMnode, char* filepath) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
int32_t fd = taosOpenFileRead(filepath);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
if (pFile == NULL) {
return;
}
if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
}
taosCloseFile(fd);
taosCloseFile(&pFile);
}
int32_t mndInitTelem(SMnode* pMnode) {

14
source/dnode/mnode/impl/test/trans/trans.cpp
View File

@ -27,25 +27,25 @@ class MndTestTrans : public ::testing::Test {
static void KillThenRestartServer() {
char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data";
FileFd fd = taosOpenFileRead(file);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
int32_t size = 3 * 1024 * 1024;
void* buffer = malloc(size);
int32_t readLen = taosReadFile(fd, buffer, size);
int32_t readLen = taosReadFile(pFile, buffer, size);
if (readLen < 0 || readLen == size) {
ASSERT(1);
}
taosCloseFile(fd);
taosCloseFile(&pFile);
test.ServerStop();
fd = taosOpenFileCreateWriteTrunc(file);
int32_t writeLen = taosWriteFile(fd, buffer, readLen);
pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
int32_t writeLen = taosWriteFile(pFile, buffer, readLen);
if (writeLen < 0 || writeLen == readLen) {
ASSERT(1);
}
free(buffer);
taosFsyncFile(fd);
taosCloseFile(fd);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
taosMsleep(1000);
test.ServerStart();

50
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -37,8 +37,8 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) {
return 0;
}
static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
int32_t ret = taosReadFile(fd, &pSdb->curVer, sizeof(int64_t));
static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t maxId = -1;
ret = taosReadFile(fd, &maxId, sizeof(int64_t));
ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t ver = -1;
ret = taosReadFile(fd, &ver, sizeof(int64_t));
ret = taosReadFile(pFile, &ver, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -81,7 +81,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
}
char reserve[SDB_RESERVE_SIZE] = {0};
ret = taosReadFile(fd, reserve, sizeof(reserve));
ret = taosReadFile(pFile, reserve, sizeof(reserve));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -94,8 +94,8 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
return 0;
}
static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (taosWriteFile(fd, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) {
static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -105,7 +105,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (i < SDB_MAX) {
maxId = pSdb->maxId[i];
}
if (taosWriteFile(fd, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -116,14 +116,14 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (i < SDB_MAX) {
ver = pSdb->tableVer[i];
}
if (taosWriteFile(fd, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
char reserve[SDB_RESERVE_SIZE] = {0};
if (taosWriteFile(fd, reserve, sizeof(reserve)) != sizeof(reserve)) {
if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -148,25 +148,25 @@ int32_t sdbReadFile(SSdb *pSdb) {
snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
mDebug("start to read file:%s", file);
FileFd fd = taosOpenFileRead(file);
if (fd <= 0) {
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
free(pRaw);
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to read file:%s since %s", file, terrstr());
return 0;
}
if (sdbReadFileHead(pSdb, fd) != 0) {
if (sdbReadFileHead(pSdb, pFile) != 0) {
mError("failed to read file:%s head since %s", file, terrstr());
pSdb->curVer = -1;
free(pRaw);
taosCloseFile(fd);
taosCloseFile(&pFile);
return -1;
}
while (1) {
readLen = sizeof(SSdbRaw);
ret = taosReadFile(fd, pRaw, readLen);
ret = taosReadFile(pFile, pRaw, readLen);
if (ret == 0) break;
if (ret < 0) {
@ -182,7 +182,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
}
readLen = pRaw->dataLen + sizeof(int32_t);
ret = taosReadFile(fd, pRaw->pData, readLen);
ret = taosReadFile(pFile, pRaw->pData, readLen);
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to read file:%s since %s", file, tstrerror(code));
@ -214,7 +214,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer);
PARSE_SDB_DATA_ERROR:
taosCloseFile(fd);
taosCloseFile(&pFile);
sdbFreeRaw(pRaw);
terrno = code;
@ -232,16 +232,16 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer,
pSdb->lastCommitVer);
FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile);
if (fd <= 0) {
TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to open file:%s for write since %s", tmpfile, terrstr());
return -1;
}
if (sdbWriteFileHead(pSdb, fd) != 0) {
if (sdbWriteFileHead(pSdb, pFile) != 0) {
mError("failed to write file:%s head since %s", tmpfile, terrstr());
taosCloseFile(fd);
taosCloseFile(&pFile);
return -1;
}
@ -269,7 +269,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
if (pRaw != NULL) {
pRaw->status = pRow->status;
int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen;
if (taosWriteFile(fd, pRaw, writeLen) != writeLen) {
if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
@ -277,7 +277,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
}
int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen);
if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
@ -296,14 +296,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
}
if (code == 0) {
code = taosFsyncFile(fd);
code = taosFsyncFile(pFile);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to write file:%s since %s", tmpfile, tstrerror(code));
}
}
taosCloseFile(fd);
taosCloseFile(&pFile);
if (code == 0) {
code = taosRenameFile(tmpfile, curfile);

4
source/dnode/vnode/src/inc/tqInt.h
View File

@ -141,9 +141,9 @@ typedef struct {
STqMetaList* unconnectTopic;
// TODO:temporaral use, to be replaced by unified tfile
int fileFd;
TdFilePtr pFile;
// TODO:temporaral use, to be replaced by unified tfile
int idxFd;
TdFilePtr pIdxFile;
char* dirPath;
int32_t tqConfigFlag;

28
source/dnode/vnode/src/inc/tsdbFile.h
View File

@ -28,17 +28,17 @@
#define TSDB_FILE_INFO(tf) (&((tf)->info))
#define TSDB_FILE_F(tf) (&((tf)->f))
#define TSDB_FILE_FD(tf) ((tf)->fd)
#define TSDB_FILE_PFILE(tf) ((tf)->pFile)
#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname)
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL)
#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL)
#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level)
#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id)
#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did)
#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname)
#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname)
#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf))
#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf))
#define TSDB_FILE_STATE(tf) ((tf)->state)
#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
@ -178,10 +178,10 @@ typedef struct {
} SDFInfo;
typedef struct {
SDFInfo info;
STfsFile f;
int fd;
uint8_t state;
SDFInfo info;
STfsFile f;
TdFilePtr pFile;
uint8_t state;
} SDFile;
void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype);
@ -198,8 +198,8 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi
static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
ASSERT(!TSDB_FILE_OPENED(pDFile));
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags);
if (pDFile->fd < 0) {
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
if (pDFile->pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -209,7 +209,7 @@ static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
if (TSDB_FILE_OPENED(pDFile)) {
close(pDFile->fd);
taosCloseFile(&pDFile->pFile);
TSDB_FILE_SET_CLOSED(pDFile);
}
}
@ -217,7 +217,7 @@ static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence);
int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
if (loffset < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -229,7 +229,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh
static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte);
int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
if (nwrite < nbyte) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -271,7 +271,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(T
static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t nread = taosReadFile(pDFile->fd, buf, nbyte);
int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
if (nread < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;

1
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#include "db.h"
#include "metaDef.h"

58
source/dnode/vnode/src/tq/tqMetaStore.c
View File

@ -34,11 +34,11 @@ static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
}
}
static inline int tqSeekLastPage(int fd) {
int offset = lseek(fd, 0, SEEK_END);
static inline int64_t tqSeekLastPage(TdFilePtr pFile) {
int offset = taosLSeekFile(pFile, 0, SEEK_END);
int pageNo = offset / TQ_PAGE_SIZE;
int curPageOffset = pageNo * TQ_PAGE_SIZE;
return lseek(fd, curPageOffset, SEEK_SET);
return taosLSeekFile(pFile, curPageOffset, SEEK_SET);
}
// TODO: the struct is tightly coupled with index entry
@ -52,10 +52,10 @@ typedef struct STqIdxPageBuf {
char buffer[TQ_IDX_PAGE_BODY_SIZE];
} STqIdxPageBuf;
static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(fd);
static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(pFile);
int nBytes;
if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -65,7 +65,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
}
ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset);
return lseek(fd, offset, SEEK_SET);
return taosLSeekFile(pFile, offset, SEEK_SET);
}
STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer,
@ -95,15 +95,15 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
tqError("failed to create dir:%s since %s ", name, terrstr());
}
strcat(name, "/" TQ_IDX_NAME);
int idxFd = open(name, O_RDWR | O_CREAT, 0755);
if (idxFd < 0) {
TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pIdxFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s ", name, terrstr());
// free memory
return NULL;
}
pMeta->idxFd = idxFd;
pMeta->pIdxFile = pIdxFile;
pMeta->unpersistHead = calloc(1, sizeof(STqMetaList));
if (pMeta->unpersistHead == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@ -113,14 +113,14 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
strcpy(name, path);
strcat(name, "/" TQ_META_NAME);
int fileFd = open(name, O_RDWR | O_CREAT, 0755);
if (fileFd < 0) {
TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s", name, terrstr());
return NULL;
}
pMeta->fileFd = fileFd;
pMeta->pFile = pFile;
pMeta->pSerializer = serializer;
pMeta->pDeserializer = deserializer;
@ -136,7 +136,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
int idxRead;
int allocated = TQ_PAGE_SIZE;
bool readEnd = false;
while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) {
if (idxRead == -1) {
// TODO: handle error
terrno = TAOS_SYSTEM_ERROR(errno);
@ -152,7 +152,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
}
memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
lseek(fileFd, pNode->handle.offset, SEEK_SET);
taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET);
if (allocated < pNode->handle.serializedSize) {
void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
if (ptr == NULL) {
@ -163,7 +163,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
allocated = pNode->handle.serializedSize;
}
serializedObj->ssize = pNode->handle.serializedSize;
if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
// TODO: read error
}
if (serializedObj->action == TQ_ACTION_INUSE) {
@ -237,8 +237,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
// commit data and idx
tqStorePersist(pMeta);
ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
close(pMeta->fileFd);
close(pMeta->idxFd);
taosCloseFile(&pMeta->pFile);
taosCloseFile(&pMeta->pIdxFile);
// free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i];
@ -263,8 +263,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
}
int32_t tqStoreDelete(STqMetaStore* pMeta) {
close(pMeta->fileFd);
close(pMeta->idxFd);
taosCloseFile(&pMeta->pFile);
taosCloseFile(&pMeta->pIdxFile);
// free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i];
@ -302,12 +302,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
pSHead->checksum = 0;
pSHead->ssize = sizeof(STqSerializedHead);
/*int allocatedSize = sizeof(STqSerializedHead);*/
int offset = lseek(pMeta->fileFd, 0, SEEK_CUR);
int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR);
tqReadLastPage(pMeta->idxFd, &idxBuf);
tqReadLastPage(pMeta->pIdxFile, &idxBuf);
if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
lseek(pMeta->idxFd, 0, SEEK_END);
taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END);
memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
} else {
@ -329,7 +329,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
} else {
pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
}
nBytes = write(pMeta->fileFd, pSHead, pSHead->ssize);
nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
ASSERT(nBytes == pSHead->ssize);
}
@ -340,7 +340,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
} else {
pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
}
int nBytesTxn = write(pMeta->fileFd, pSHead, pSHead->ssize);
int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
ASSERT(nBytesTxn == pSHead->ssize);
nBytes += nBytesTxn;
}
@ -355,7 +355,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
idxBuf.head.writeOffset += TQ_IDX_SIZE;
if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE);
nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE);
// TODO: handle error with tfile
ASSERT(nBytes == TQ_PAGE_SIZE);
memset(&idxBuf, 0, TQ_PAGE_SIZE);
@ -391,13 +391,13 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
free(pSHead);
// TODO: write new version in tfile
if ((char*)bufPtr != idxBuf.buffer) {
int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset);
int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset);
// TODO: handle error in tfile
ASSERT(nBytes == idxBuf.head.writeOffset);
}
// TODO: using fsync in tfile
fsync(pMeta->idxFd);
fsync(pMeta->fileFd);
taosFsyncFile(pMeta->pIdxFile);
taosFsyncFile(pMeta->pFile);
return 0;
}

45
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -416,8 +416,8 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname);
tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname);
int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -436,9 +436,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE);
if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
terrno = TAOS_SYSTEM_ERROR(errno);
close(fd);
taosCloseFile(&pFile);
remove(tfname);
return -1;
}
@ -446,7 +446,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
// Encode file status and write to file
if (fsheader.len > 0) {
if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) {
close(fd);
taosCloseFile(&pFile);
remove(tfname);
return -1;
}
@ -455,9 +455,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
tsdbEncodeFSStatus(&ptr, pStatus);
taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len);
if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) {
if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) {
terrno = TAOS_SYSTEM_ERROR(errno);
close(fd);
taosCloseFile(&pFile);
(void)remove(tfname);
taosTZfree(pBuf);
return -1;
@ -465,15 +465,15 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
}
// fsync, close and rename
if (taosFsyncFile(fd) < 0) {
if (taosFsyncFile(pFile) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
close(fd);
taosCloseFile(&pFile);
remove(tfname);
taosTZfree(pBuf);
return -1;
}
(void)close(fd);
(void)taosCloseFile(&pFile);
(void)taosRenameFile(tfname, cfname);
taosTZfree(pBuf);
@ -652,7 +652,7 @@ static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
STsdbFS * pfs = REPO_FS(pRepo);
int fd = -1;
TdFilePtr pFile = NULL;
void * buffer = NULL;
SFSHeader fsheader;
char current[TSDB_FILENAME_LEN] = "\0";
@ -661,8 +661,8 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current);
// current file exists, try to recover
fd = open(current, O_RDONLY | O_BINARY);
if (fd < 0) {
pFile = taosOpenFile(current, TD_FILE_READ);
if (pFile == NULL) {
tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
@ -672,7 +672,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
goto _err;
}
int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE);
int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE);
if (nread < 0) {
tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current,
strerror(errno));
@ -706,7 +706,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
goto _err;
}
nread = (int)taosReadFile(fd, buffer, fsheader.len);
nread = (int)taosReadFile(pFile, buffer, fsheader.len);
if (nread < 0) {
tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@ -732,13 +732,13 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
}
taosTZfree(buffer);
close(fd);
taosCloseFile(&pFile);
return 0;
_err:
if (fd >= 0) {
close(fd);
if (pFile != NULL) {
taosCloseFile(&pFile);
}
taosTZfree(buffer);
return -1;
@ -1244,18 +1244,17 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
}
if (tsdbForceKeepFile) {
struct stat tfstat;
int64_t file_size;
// Get real file size
if (fstat(pDFile->fd, &tfstat) < 0) {
if (taosFStatFile(pDFile->pFile, &file_size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosArrayDestroy(fArray);
return -1;
}
if (pDFile->info.size != tfstat.st_size) {
if (pDFile->info.size != file_size) {
int64_t tfsize = pDFile->info.size;
pDFile->info.size = tfstat.st_size;
pDFile->info.size = file_size;
tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo),
TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size);
}

12
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -355,8 +355,8 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (pDFile->fd < 0) {
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pDFile->pFile < 0) {
if (errno == ENOENT) {
// Try to create directory recursively
char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
@ -366,8 +366,8 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
}
tfree(s);
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (pDFile->fd < 0) {
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pDFile->pFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -460,7 +460,7 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
return -1;
}
if (taosFtruncateFile(df.fd, df.info.size) < 0) {
if (taosFtruncateFile(df.pFile, df.info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseDFile(&df);
return -1;
@ -541,7 +541,7 @@ static int tsdbRollBackDFile(SDFile *pDFile) {
return -1;
}
if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) {
if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseDFile(&df);
return -1;

1
source/libs/CMakeLists.txt
View File

@ -1,3 +1,4 @@
add_definitions("-D ALLOW_FORBID_FUNC")
add_subdirectory(transport)
add_subdirectory(sync)
add_subdirectory(tdb)

8
source/libs/function/src/taggfunction.c
View File

@ -3960,14 +3960,14 @@ static void ts_comp_finalize(SqlFunctionCtx *pCtx) {
// qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal);
// TODO refactor transfer ownership of current file
*(FILE **)pCtx->pOutput = pTSbuf->f;
*(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile;
pResInfo->complete = true;
// get the file size
struct stat fStat;
if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) {
pResInfo->numOfRes = fStat.st_size;
int64_t file_size;
if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) {
pResInfo->numOfRes = (uint32_t )file_size;
}
pTSbuf->remainOpen = true;

2
source/libs/index/inc/index_fst_counting_writer.h
View File

@ -38,7 +38,7 @@ typedef struct WriterCtx {
WriterType type;
union {
struct {
int fd;
TdFilePtr pFile;
bool readOnly;
char buf[256];
int size;

36
source/libs/index/src/index_fst_counting_writer.c
View File

@ -19,7 +19,7 @@
static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
if (ctx->type == TFile) {
assert(len == tfWrite(ctx->file.fd, buf, len));
assert(len == taosWriteFile(ctx->file.pFile, buf, len));
} else {
memcpy(ctx->mem.buf + ctx->offset, buf, len);
}
@ -33,7 +33,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
nRead = len < ctx->file.size ? len : ctx->file.size;
memcpy(buf, ctx->file.ptr, nRead);
#else
nRead = tfRead(ctx->file.fd, buf, len);
nRead = taosReadFile(ctx->file.pFile, buf, len);
#endif
} else {
memcpy(buf, ctx->mem.buf + ctx->offset, len);
@ -45,13 +45,13 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) {
int nRead = 0;
if (ctx->type == TFile) {
// tfLseek(ctx->file.fd, offset, 0);
// tfLseek(ctx->file.pFile, offset, 0);
#ifdef USE_MMAP
int32_t last = ctx->file.size - offset;
nRead = last >= len ? len : last;
memcpy(buf, ctx->file.ptr + offset, nRead);
#else
nRead = tfPread(ctx->file.fd, buf, len, offset);
nRead = taosPReadFile(ctx->file.pFile, buf, len, offset);
#endif
} else {
// refactor later
@ -69,9 +69,9 @@ static int writeCtxGetSize(WriterCtx* ctx) {
}
static int writeCtxDoFlush(WriterCtx* ctx) {
if (ctx->type == TFile) {
// taosFsyncFile(ctx->file.fd);
tfFsync(ctx->file.fd);
// tfFlush(ctx->file.fd);
// taosFsyncFile(ctx->file.pFile);
taosFsyncFile(ctx->file.pFile);
// tfFlush(ctx->file.pFile);
} else {
// do nothing
}
@ -87,25 +87,25 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
// ugly code, refactor later
ctx->file.readOnly = readOnly;
if (readOnly == false) {
// ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.fd = tfOpenCreateWriteAppend(path);
tfFtruncate(ctx->file.fd, 0);
struct stat fstat;
stat(path, &fstat);
ctx->file.size = fstat.st_size;
// ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
taosFtruncateFile(ctx->file.pFile, 0);
int64_t file_size;
taosStatFile(path, &file_size, NULL);
ctx->file.size = (int)file_size;
} else {
// ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.fd = tfOpenRead(path);
// ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
struct stat fstat;
stat(path, &fstat);
ctx->file.size = fstat.st_size;
#ifdef USE_MMAP
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size);
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
#endif
}
memcpy(ctx->file.buf, path, strlen(path));
if (ctx->file.fd < 0) {
if (ctx->file.pFile < 0) {
indexError("failed to open file, error %d", errno);
goto END;
}
@ -133,7 +133,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
free(ctx->mem.buf);
} else {
ctx->flush(ctx);
tfClose(ctx->file.fd);
taosCloseFile(&ctx->file.pFile);
if (ctx->file.readOnly) {
#ifdef USE_MMAP
munmap(ctx->file.ptr, ctx->file.size);

8
source/libs/index/src/index_tfile.c
View File

@ -586,11 +586,11 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0);
if (nread == -1) {
indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
errno, reader->ctx->file.fd, reader->ctx->file.buf);
indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf),
errno, reader->ctx->file.buf);
} else {
indexInfo("actual Read: %d, to read: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
reader->ctx->file.fd, reader->ctx->file.buf);
indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf),
reader->ctx->file.buf);
}
// assert(nread == sizeof(buf));
memcpy(&reader->header, buf, sizeof(buf));

12
source/libs/tdb/src/db/tdbEnv.c
View File

@ -18,7 +18,7 @@
struct STDbEnv {
char * rootDir; // root directory of the environment
char * jname; // journal file name
int jfd; // journal file fd
TdFilePtr jpFile; // journal file fd
pgsz_t pgSize; // page size
cachesz_t cacheSize; // total cache size
STDbList dbList; // TDB List
@ -55,7 +55,7 @@ int tdbEnvCreate(TENV **ppEnv, const char *rootDir) {
pEnv->rootDir = (char *)(&pEnv[1]);
pEnv->jname = pEnv->rootDir + slen + 1;
pEnv->jfd = -1;
pEnv->jpFile = NULL;
pEnv->pgSize = TDB_DEFAULT_PGSIZE;
pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE;
@ -139,8 +139,8 @@ static int tdbEnvDestroy(TENV *pEnv) {
}
int tdbEnvBeginTxn(TENV *pEnv) {
pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755);
if (pEnv->jfd < 0) {
pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pEnv->jpFile < 0) {
return -1;
}
@ -149,8 +149,8 @@ int tdbEnvBeginTxn(TENV *pEnv) {
int tdbEnvCommit(TENV *pEnv) {
/* TODO */
close(pEnv->jfd);
pEnv->jfd = -1;
taosCloseFile(&pEnv->jpFile);
pEnv->jpFile = NULL;
return 0;
}

12
source/libs/tdb/src/db/tdbPgFile.c
View File

@ -51,10 +51,10 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
pPgFile->fname = (char *)(&(pPgFile[1]));
memcpy(pPgFile->fname, fname, fnameLen);
pPgFile->fname[fnameLen] = '\0';
pPgFile->fd = -1;
pPgFile->pFile = NULL;
pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755);
if (pPgFile->fd < 0) {
pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pPgFile->pFile == NULL) {
// TODO: handle error
return -1;
}
@ -95,8 +95,8 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
int pgFileClose(SPgFile *pPgFile) {
if (pPgFile) {
if (pPgFile->fd >= 0) {
close(pPgFile->fd);
if (pPgFile->pFile >= 0) {
taosCloseFile(&pPgFile->pFile);
}
tfree(pPgFile->fname);
@ -201,7 +201,7 @@ static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) {
pTData = pData;
szToRead = pgSize;
for (; szToRead > 0;) {
rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize);
rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize);
if (rsize < 0) {
if (errno == EINTR) {
continue;

2
source/libs/tdb/src/inc/tdbPgFile.h
View File

@ -38,7 +38,7 @@ struct SPgFile {
uint8_t fileid[TDB_FILE_ID_LEN]; // file id
pgno_t lsize; // page file logical size (for count)
pgno_t fsize; // real file size on disk (for rollback)
int fd;
TdFilePtr pFile;
SPgFileListNode envHash;
SPgFileListNode envPgfList;
};

18
source/libs/tfs/test/tfsTest.cpp
View File

@ -230,10 +230,11 @@ TEST_F(TfsTest, 04_File) {
EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0);
FILE *fp = fopen(f1.aname, "w");
ASSERT_NE(fp, nullptr);
fwrite("12345678", 1, 5, fp);
fclose(fp);
// FILE *fp = fopen(f1.aname, "w");
TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
ASSERT_NE(pFile, nullptr);
taosWriteFile(pFile, "12345678", 5);
taosCloseFile(&pFile);
char base[128] = {0};
tfsBasename(&f1, base);
@ -638,10 +639,11 @@ TEST_F(TfsTest, 05_MultiDisk) {
EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0);
FILE *fp = fopen(f1.aname, "w");
ASSERT_NE(fp, nullptr);
fwrite("12345678", 1, 5, fp);
fclose(fp);
// FILE *fp = fopen(f1.aname, "w");
TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
ASSERT_NE(pFile, nullptr);
taosWriteFile(pFile, "12345678", 5);
taosCloseFile(&pFile);
char base[128] = {0};
tfsBasename(&f1, base);

2
source/libs/transport/inc/rpcUdp.h
View File

@ -30,7 +30,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
void taosFreeMsgHdr(void *hdr);
int taosMsgHdrSize(void *hdr);
void taosSendMsgHdr(void *hdr, int fd);
void taosSendMsgHdr(void *hdr, TdFilePtr pFile);
void taosInitMsgHdr(void **hdr, void *dest, int maxPkts);
void taosSetMsgHdrData(void *hdr, char *data, int dataLen);

16
source/libs/transport/test/pushServer.c
View File

@ -22,7 +22,7 @@
int msgSize = 128;
int commit = 0;
int dataFd = -1;
TdFilePtr pDataFile = NULL;
STaosQueue *qhandle = NULL;
STaosQset * qset = NULL;
@ -43,8 +43,8 @@ void processShellMsg() {
for (int i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg);
if (dataFd >= 0) {
if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
if (pDataFile != NULL) {
if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
tInfo("failed to write data file, reason:%s", strerror(errno));
}
}
@ -52,7 +52,7 @@ void processShellMsg() {
if (commit >= 2) {
num += numOfMsgs;
// if (taosFsync(dataFd) < 0) {
// if (taosFsync(pDataFile) < 0) {
// tInfo("failed to flush data to file, reason:%s", strerror(errno));
//}
@ -181,8 +181,8 @@ int main(int argc, char *argv[]) {
tInfo("RPC server is running, ctrl-c to exit");
if (commit) {
dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno));
pDataFile = taosOpenFile(dataName, TD_FILE_APPEND | TD_FILE_CTEATE | TD_FILE_WRITE);
if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno));
}
qhandle = taosOpenQueue();
qset = taosOpenQset();
@ -190,8 +190,8 @@ int main(int argc, char *argv[]) {
processShellMsg();
if (dataFd >= 0) {
close(dataFd);
if (pDataFile != NULL) {
taosCloseFile(&pDataFile);
remove(dataName);
}

16
source/libs/transport/test/rserver.c
View File

@ -22,7 +22,7 @@
int msgSize = 128;
int commit = 0;
int dataFd = -1;
TdFilePtr pDataFile = NULL;
STaosQueue *qhandle = NULL;
STaosQset * qset = NULL;
@ -43,8 +43,8 @@ void processShellMsg() {
for (int i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg);
if (dataFd >= 0) {
if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
if (pDataFile >= 0) {
if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
tInfo("failed to write data file, reason:%s", strerror(errno));
}
}
@ -52,7 +52,7 @@ void processShellMsg() {
if (commit >= 2) {
num += numOfMsgs;
// if (taosFsync(dataFd) < 0) {
// if (taosFsync(pDataFile) < 0) {
// tInfo("failed to flush data to file, reason:%s", strerror(errno));
//}
@ -170,8 +170,8 @@ int main(int argc, char *argv[]) {
tInfo("RPC server is running, ctrl-c to exit");
if (commit) {
dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno));
pDataFile = taosOpenFile(dataName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno));
}
qhandle = taosOpenQueue();
qset = taosOpenQset();
@ -179,8 +179,8 @@ int main(int argc, char *argv[]) {
processShellMsg();
if (dataFd >= 0) {
close(dataFd);
if (pDataFile != NULL) {
taosCloseFile(&pDataFile);
remove(dataName);
}

34
source/libs/wal/src/walMeta.c
View File

@ -69,8 +69,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size);
pLastFileInfo->fileSize = statbuf.st_size;
FileFd fd = taosOpenFileRead(fnameStr);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -79,15 +79,15 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
char* buf = malloc(readSize + 5);
if (buf == NULL) {
taosCloseFile(fd);
taosCloseFile(&pFile);
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
return -1;
}
taosLSeekFile(fd, -readSize, SEEK_END);
if (readSize != taosReadFile(fd, buf, readSize)) {
taosLSeekFile(pFile, -readSize, SEEK_END);
if (readSize != taosReadFile(pFile, buf, readSize)) {
free(buf);
taosCloseFile(fd);
taosCloseFile(&pFile);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -108,12 +108,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) {
// file has to be deleted
free(buf);
taosCloseFile(fd);
taosCloseFile(&pFile);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
}
taosCloseFile(fd);
taosCloseFile(&pFile);
SWalHead *lastEntry = (SWalHead*)found;
return lastEntry->head.version;
@ -364,18 +364,18 @@ int walSaveMeta(SWal* pWal) {
int metaVer = walFindCurMetaVer(pWal);
char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer + 1, fnameStr);
FileFd metaFd = taosOpenFileCreateWrite(fnameStr);
if (metaFd < 0) {
TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE);
if (pMataFile == NULL) {
return -1;
}
char* serialized = walMetaSerialize(pWal);
int len = strlen(serialized);
if (len != taosWriteFile(metaFd, serialized, len)) {
if (len != taosWriteFile(pMataFile, serialized, len)) {
// TODO:clean file
return -1;
}
taosCloseFile(metaFd);
taosCloseFile(&pMataFile);
// delete old file
if (metaVer > -1) {
walBuildMetaName(pWal, metaVer, fnameStr);
@ -404,20 +404,20 @@ int walLoadMeta(SWal* pWal) {
return -1;
}
memset(buf, 0, size + 5);
FileFd fd = taosOpenFileRead(fnameStr);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pFile == NULL) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
if (taosReadFile(fd, buf, size) != size) {
if (taosReadFile(pFile, buf, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(fd);
taosCloseFile(&pFile);
free(buf);
return -1;
}
// load into fileInfoSet
int code = walMetaDeserialize(pWal, buf);
taosCloseFile(fd);
taosCloseFile(&pFile);
free(buf);
return code;
}

14
source/libs/wal/src/walMgmt.c
View File

@ -89,8 +89,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
// open meta
walResetVer(&pWal->vers);
pWal->writeLogTfd = -1;
pWal->writeIdxTfd = -1;
pWal->pWriteLogTFile = NULL;
pWal->pWriteIdxTFile = NULL;
pWal->writeCur = -1;
pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo));
if (pWal->fileInfoSet == NULL) {
@ -164,10 +164,10 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
void walClose(SWal *pWal) {
pthread_mutex_lock(&pWal->mutex);
tfClose(pWal->writeLogTfd);
pWal->writeLogTfd = -1;
tfClose(pWal->writeIdxTfd);
pWal->writeIdxTfd = -1;
taosCloseFile(&pWal->pWriteLogTFile);
pWal->pWriteLogTFile = NULL;
taosCloseFile(&pWal->pWriteIdxTFile);
pWal->pWriteIdxTFile = NULL;
walSaveMeta(pWal);
taosArrayDestroy(pWal->fileInfoSet);
pWal->fileInfoSet = NULL;
@ -207,7 +207,7 @@ static void walFsyncAll() {
if (walNeedFsync(pWal)) {
wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq,
atomic_load_32(&tsWal.seq));
int32_t code = tfFsync(pWal->writeLogTfd);
int32_t code = taosFsyncFile(pWal->pWriteLogTFile);
if (code != 0) {
wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(code));

44
source/libs/wal/src/walRead.c
View File

@ -25,8 +25,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
}
pRead->pWal = pWal;
pRead->readIdxTfd = -1;
pRead->readLogTfd = -1;
pRead->pReadIdxTFile = NULL;
pRead->pReadLogTFile = NULL;
pRead->curVersion = -1;
pRead->curFileFirstVer = -1;
pRead->capacity = 0;
@ -41,8 +41,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
}
void walCloseReadHandle(SWalReadHandle *pRead) {
tfClose(pRead->readIdxTfd);
tfClose(pRead->readLogTfd);
taosCloseFile(&pRead->pReadIdxTFile);
taosCloseFile(&pRead->pReadLogTFile);
tfree(pRead->pHead);
free(pRead);
}
@ -52,24 +52,24 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; }
static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
int code = 0;
int64_t idxTfd = pRead->readIdxTfd;
int64_t logTfd = pRead->readLogTfd;
TdFilePtr pIdxTFile = pRead->pReadIdxTFile;
TdFilePtr pLogTFile = pRead->pReadLogTFile;
// seek position
int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
code = tfLseek(idxTfd, offset, SEEK_SET);
code = taosLSeekFile(pIdxTFile, offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
// TODO:deserialize
ASSERT(entry.ver == ver);
code = tfLseek(logTfd, entry.offset, SEEK_SET);
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -80,24 +80,24 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
char fnameStr[WAL_FILE_LEN];
tfClose(pRead->readIdxTfd);
tfClose(pRead->readLogTfd);
taosCloseFile(&pRead->pReadIdxTFile);
taosCloseFile(&pRead->pReadLogTFile);
walBuildLogName(pRead->pWal, fileFirstVer, fnameStr);
int64_t logTfd = tfOpenRead(fnameStr);
if (logTfd < 0) {
TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pLogTFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
int64_t idxTfd = tfOpenRead(fnameStr);
if (idxTfd < 0) {
TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pIdxTFile == NULL) {
return -1;
}
pRead->readLogTfd = logTfd;
pRead->readIdxTfd = idxTfd;
pRead->pReadLogTFile = pLogTFile;
pRead->pReadIdxTFile = pIdxTFile;
return 0;
}
@ -145,9 +145,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
}
}
if (!tfValid(pRead->readLogTfd)) return -1;
if (!taosValidFile(pRead->pReadLogTFile)) return -1;
code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead));
code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead));
if (code != sizeof(SWalHead)) {
return -1;
}
@ -165,7 +165,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
pRead->pHead = ptr;
pRead->capacity = pRead->pHead->head.len;
}
if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) {
if (pRead->pHead->head.len != taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.len)) {
return -1;
}
@ -202,7 +202,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
}
*ppHead = ptr;
}
if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
return -1;
}
// TODO: endian compatibility processing after read
@ -215,7 +215,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
*ppHead = NULL;
return -1;
}
if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
return -1;
}
// TODO: endian compatibility processing after read

52
source/libs/wal/src/walSeek.c
View File

@ -23,25 +23,25 @@
static int walSeekWritePos(SWal* pWal, int64_t ver) {
int code = 0;
int64_t idxTfd = pWal->writeIdxTfd;
int64_t logTfd = pWal->writeLogTfd;
TdFilePtr pIdxTFile = pWal->pWriteIdxTFile;
TdFilePtr pLogTFile = pWal->pWriteLogTFile;
// seek position
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET);
code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
SWalIdxEntry entry;
// TODO:deserialize
code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry));
code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
ASSERT(entry.ver == ver);
code = tfLseek(logTfd, entry.offset, SEEK_SET);
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -50,43 +50,43 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) {
}
int walSetWrite(SWal* pWal) {
int64_t idxTfd, logTfd;
TdFilePtr pIdxTFile, pLogTFile;
SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
ASSERT(pRet != NULL);
int64_t fileFirstVer = pRet->firstVer;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr);
if (idxTfd < 0) {
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr);
if (logTfd < 0) {
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
// switch file
pWal->writeIdxTfd = idxTfd;
pWal->writeLogTfd = logTfd;
pWal->pWriteIdxTFile = pIdxTFile;
pWal->pWriteLogTFile = pLogTFile;
return 0;
}
int walChangeWrite(SWal* pWal, int64_t ver) {
int code = 0;
int64_t idxTfd, logTfd;
TdFilePtr pIdxTFile, pLogTFile;
char fnameStr[WAL_FILE_LEN];
if (pWal->writeLogTfd != -1) {
code = tfClose(pWal->writeLogTfd);
if (pWal->pWriteLogTFile != NULL) {
code = taosCloseFile(&pWal->pWriteLogTFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
if (pWal->writeIdxTfd != -1) {
code = tfClose(pWal->writeIdxTfd);
if (pWal->pWriteIdxTFile != NULL) {
code = taosCloseFile(&pWal->pWriteIdxTFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -103,23 +103,23 @@ int walChangeWrite(SWal* pWal, int64_t ver) {
int64_t fileFirstVer = pFileInfo->firstVer;
walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr);
if (idxTfd < 0) {
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
pWal->writeIdxTfd = -1;
pWal->pWriteIdxTFile = NULL;
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr);
if (logTfd < 0) {
tfClose(idxTfd);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile < 0) {
taosCloseFile(&pIdxTFile);
terrno = TAOS_SYSTEM_ERROR(errno);
pWal->writeLogTfd = -1;
pWal->pWriteLogTFile = NULL;
return -1;
}
pWal->writeLogTfd = logTfd;
pWal->writeIdxTfd = idxTfd;
pWal->pWriteLogTFile = pLogTFile;
pWal->pWriteIdxTFile = pIdxTFile;
pWal->writeCur = idx;
return fileFirstVer;
}

60
source/libs/wal/src/walWrite.c
View File

@ -65,15 +65,15 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
int64_t idxTfd = tfOpenReadWrite(fnameStr);
TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
// TODO:change to deserialize function
if (idxTfd < 0) {
if (pIdxTFile == NULL) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET);
code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET);
if (code < 0) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
@ -81,20 +81,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
// read idx file and get log file pos
// TODO:change to deserialize function
SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
ASSERT(entry.ver == ver);
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
int64_t logTfd = tfOpenReadWrite(fnameStr);
if (logTfd < 0) {
TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
if (pLogTFile < 0) {
// TODO
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
code = tfLseek(logTfd, entry.offset, SEEK_SET);
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
// TODO
pthread_mutex_unlock(&pWal->mutex);
@ -102,8 +102,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
// validate offset
SWalHead head;
ASSERT(tfValid(logTfd));
int size = tfRead(logTfd, &head, sizeof(SWalHead));
ASSERT(taosValidFile(pLogTFile));
int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead));
if (size != sizeof(SWalHead)) {
return -1;
}
@ -118,11 +118,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
return -1;
}
// truncate old files
code = tfFtruncate(logTfd, entry.offset);
code = taosFtruncateFile(pLogTFile, entry.offset);
if (code < 0) {
return -1;
}
code = tfFtruncate(idxTfd, idxOff);
code = taosFtruncateFile(pIdxTFile, idxOff);
if (code < 0) {
return -1;
}
@ -203,31 +203,31 @@ int32_t walEndSnapshot(SWal *pWal) {
int walRoll(SWal *pWal) {
int code = 0;
if (pWal->writeIdxTfd != -1) {
code = tfClose(pWal->writeIdxTfd);
if (pWal->pWriteIdxTFile != NULL) {
code = taosCloseFile(&pWal->pWriteIdxTFile);
if (code != 0) {
return -1;
}
}
if (pWal->writeLogTfd != -1) {
code = tfClose(pWal->writeLogTfd);
if (pWal->pWriteLogTFile != NULL) {
code = taosCloseFile(&pWal->pWriteLogTFile);
if (code != 0) {
return -1;
}
}
int64_t idxTfd, logTfd;
TdFilePtr pIdxTFile, pLogTFile;
// create new file
int64_t newFileFirstVersion = pWal->vers.lastVer + 1;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, newFileFirstVersion, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr);
if (idxTfd < 0) {
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildLogName(pWal, newFileFirstVersion, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr);
if (logTfd < 0) {
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -237,8 +237,8 @@ int walRoll(SWal *pWal) {
}
// switch file
pWal->writeIdxTfd = idxTfd;
pWal->writeLogTfd = logTfd;
pWal->pWriteIdxTFile = pIdxTFile;
pWal->pWriteLogTFile = pLogTFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
ASSERT(pWal->writeCur >= 0);
@ -248,7 +248,7 @@ int walRoll(SWal *pWal) {
static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
SWalIdxEntry entry = {.ver = ver, .offset = offset};
int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry));
int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry));
if (size != sizeof(SWalIdxEntry)) {
terrno = TAOS_SYSTEM_ERROR(errno);
// TODO truncate
@ -282,16 +282,16 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
// must truncate explicitly first
return -1;
}
/*if (!tfValid(pWal->writeLogTfd)) return -1;*/
/*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/
ASSERT(pWal->writeCur >= 0);
pthread_mutex_lock(&pWal->mutex);
if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) {
if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) {
walSetWrite(pWal);
tfLseek(pWal->writeLogTfd, 0, SEEK_END);
tfLseek(pWal->writeIdxTfd, 0, SEEK_END);
taosLSeekFile(pWal->pWriteLogTFile, 0, SEEK_END);
taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END);
}
pWal->writeHead.head.version = index;
@ -302,14 +302,14 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead);
pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen);
if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
// ftruncate
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
}
if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) {
if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) {
// ftruncate
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
@ -336,7 +336,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
void walFsync(SWal *pWal, bool forceFsync) {
if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
if (tfFsync(pWal->writeLogTfd) < 0) {
if (taosFsyncFile(pWal->pWriteLogTFile) < 0) {
wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
strerror(errno));
}

578
source/os/src/osFile.c
View File

@ -12,38 +12,45 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#include "os.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#include <io.h>
#include <io.h>
#if defined(_MSDOS)
#define open _open
#endif
#if defined(_MSDOS)
#define open _open
#endif
#if defined(_WIN32)
extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */
extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */
#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
#define open openU
#else /* _ANSI_SOURCE */
#define open openA
#endif /* defined(_UTF8_SOURCE) */
#endif /* defined(_WIN32) */
#if defined(_WIN32)
extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */
extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */
#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
#define open openU
#else /* _ANSI_SOURCE */
#define open openA
#endif /* defined(_UTF8_SOURCE) */
#endif /* defined(_WIN32) */
#else
#include <fcntl.h>
#include <sys/file.h>
#include <sys/sendfile.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/file.h>
#include <sys/sendfile.h>
#include <sys/stat.h>
#include <unistd.h>
#define LINUX_FILE_NO_TEXT_OPTION 0
#define O_TEXT LINUX_FILE_NO_TEXT_OPTION
#endif
void taosCloseFile(FileFd fd) {
close(fd);
fd = FD_INITIALIZER;
}
typedef int32_t FileFd;
typedef struct TdFile {
int refId;
FileFd fd;
FILE *fp;
}*TdFilePtr,TdFile;
void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, char *dstPath) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
@ -96,13 +103,166 @@ void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, ch
#endif
}
int64_t taosReadFile(FileFd fd, void *buf, int64_t count) {
int64_t taosCopyFile(const char *from, const char *to) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
char buffer[4096];
int64_t size = 0;
int64_t bytes;
// fidfrom = open(from, O_RDONLY);
TdFilePtr pFileFrom = taosOpenFile(from,TD_FILE_READ);
if (pFileFrom == NULL) goto _err;
// fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755);
TdFilePtr pFileTo = taosOpenFile(to,TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_EXCL);
if (pFileTo == NULL) goto _err;
while (true) {
bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer));
if (bytes < 0) goto _err;
if (bytes == 0) break;
size += bytes;
if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) goto _err;
if (bytes < sizeof(buffer)) break;
}
taosFsyncFile(pFileTo);
taosCloseFile(&pFileFrom);
taosCloseFile(&pFileTo);
return size;
_err:
if (pFileFrom != NULL) taosCloseFile(&pFileFrom);
if (pFileTo != NULL) taosCloseFile(&pFileTo);
remove(to);
return -1;
#endif
}
int32_t taosRenameFile(const char *oldName, const char *newName) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
if (code < 0) {
//printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
}
return code;
#else
int32_t code = rename(oldName, newName);
if (code < 0) {
//printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
}
return code;
#endif
}
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
struct stat fileStat;
int32_t code = stat(path, &fileStat);
if (code < 0) {
return code;
}
if (size != NULL) {
*size = fileStat.st_size;
}
if (mtime != NULL) {
*mtime = fileStat.st_mtime;
}
return 0;
#endif
}
void autoDelFileListAdd(const char *path) {
return;
}
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return NULL;
#else
int access = O_BINARY;
char *mode = NULL;
access |= (tdFileOptions & TD_FILE_CTEATE) ? O_CREAT : 0;
if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ))
{
access |= O_RDWR;
mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+";
}else if(tdFileOptions & TD_FILE_WRITE) {
access |= O_WRONLY;
mode = (tdFileOptions & TD_FILE_TEXT) ? "wt" : "wb";
}else if(tdFileOptions & TD_FILE_READ) {
access |= O_RDONLY;
mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb";
}
access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0;
access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0;
access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0;
access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0;
if(tdFileOptions & TD_FILE_AUTO_DEL) {
autoDelFileListAdd(path);
}
int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO);
if(fd == -1) {
return NULL;
}
FILE* fp = fdopen(fd, mode);
if (fp == NULL) {
close(fd);
return NULL;
}
TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile));
if (pFile == NULL) {
close(fd);
fclose(fp);
return NULL;
}
pFile->fd = fd;
pFile->fp = fp;
pFile->refId = 0;
return pFile;
#endif
}
int64_t taosCloseFile(TdFilePtr *ppFile) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
if(ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) {
return 0;
}
fsync((*ppFile)->fd);
close((*ppFile)->fd);
(*ppFile)->fd = -1;
(*ppFile)->fp = NULL;
(*ppFile)->refId = 0;
free(*ppFile);
*ppFile = NULL;
return 0;
#endif
}
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
if(pFile == NULL) {
return 0;
}
int64_t leftbytes = count;
int64_t readbytes;
char * tbuf = (char *)buf;
while (leftbytes > 0) {
readbytes = read(fd, (void *)tbuf, (uint32_t)leftbytes);
readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes);
if (readbytes < 0) {
if (errno == EINTR) {
continue;
@ -120,13 +280,20 @@ int64_t taosReadFile(FileFd fd, void *buf, int64_t count) {
return count;
}
int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) {
int64_t nleft = n;
int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) {
if(pFile == NULL) {
return 0;
}
return pread(pFile->fd, buf, count, offset);
}
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
int64_t nleft = count;
int64_t nwritten = 0;
char * tbuf = (char *)buf;
while (nleft > 0) {
nwritten = write(fd, (void *)tbuf, (uint32_t)nleft);
nwritten = write(pFile->fd, (void *)tbuf, (uint32_t)nleft);
if (nwritten < 0) {
if (errno == EINTR) {
continue;
@ -137,48 +304,119 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) {
tbuf += nwritten;
}
return n;
return count;
}
int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); }
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
return (int64_t)lseek(pFile->fd, (long)offset, whence);
}
int64_t taosCopyFile(const char *from, const char *to) {
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
char buffer[4096];
int fidto = -1, fidfrom = -1;
int64_t size = 0;
int64_t bytes;
fidfrom = open(from, O_RDONLY);
if (fidfrom < 0) goto _err;
fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755);
if (fidto < 0) goto _err;
while (true) {
bytes = taosReadFile(fidfrom, buffer, sizeof(buffer));
if (bytes < 0) goto _err;
if (bytes == 0) break;
size += bytes;
if (taosWriteFile(fidto, (void *)buffer, bytes) < bytes) goto _err;
if (bytes < sizeof(buffer)) break;
struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat);
if (code < 0) {
return code;
}
taosFsyncFile(fidto);
if (size != NULL) {
*size = fileStat.st_size;
}
taosCloseFile(fidfrom);
taosCloseFile(fidto);
return size;
if (mtime != NULL) {
*mtime = fileStat.st_mtime;
}
_err:
if (fidfrom >= 0) taosCloseFile(fidfrom);
if (fidto >= 0) taosCloseFile(fidto);
remove(to);
return -1;
return 0;
#endif
}
int32_t taosLockFile(TdFilePtr pFile) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB);
#endif
}
int32_t taosUnLockFile(TdFilePtr pFile) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB);
#endif
}
int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
if (pFile->fd < 0) {
errno = EBADF;
uError("%s\n", "fd arg was negative");
return -1;
}
HANDLE h = (HANDLE)_get_osfhandle(pFile->fd);
LARGE_INTEGER li_0;
li_0.QuadPart = (int64_t)0;
BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT);
if (!cur) {
uError("SetFilePointerEx Error getting current position in file.\n");
return -1;
}
LARGE_INTEGER li_size;
li_size.QuadPart = l_size;
BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN);
if (cur2 == 0) {
int error = GetLastError();
uError("SetFilePointerEx GetLastError is: %d\n", error);
switch (error) {
case ERROR_INVALID_HANDLE:
errno = EBADF;
break;
default:
errno = EIO;
break;
}
return -1;
}
if (!SetEndOfFile(h)) {
int error = GetLastError();
uError("SetEndOfFile GetLastError is:%d", error);
switch (error) {
case ERROR_INVALID_HANDLE:
errno = EBADF;
break;
default:
errno = EIO;
break;
}
return -1;
}
return 0;
#else
return ftruncate(pFile->fd, l_size);
#endif
}
int32_t taosFsyncFile(TdFilePtr pFile) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
if (pFile->fd < 0) {
errno = EBADF;
uError("%s\n", "fd arg was negative");
return -1;
}
HANDLE h = (HANDLE)_get_osfhandle(pFile->fd);
return FlushFileBuffers(h);
#else
return fsync(pFile->fd);
#endif
}
@ -301,12 +539,12 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) {
#else
int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) {
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) {
int64_t leftbytes = size;
int64_t sentbytes;
while (leftbytes > 0) {
sentbytes = sendfile(dfd, sfd, offset, leftbytes);
sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes);
if (sentbytes == -1) {
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
continue;
@ -323,215 +561,49 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) {
return size;
}
int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) {
return taosSendFile(fileno(outfile), fileno(infile), offset, size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) {
return taosSendFile(pFileOut->fd, pFileIn, offset, size);
}
#endif
int32_t taosFtruncateFile(FileFd fd, int64_t l_size) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
if (fd < 0) {
errno = EBADF;
uError("%s\n", "fd arg was negative");
return -1;
}
HANDLE h = (HANDLE)_get_osfhandle(fd);
LARGE_INTEGER li_0;
li_0.QuadPart = (int64_t)0;
BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT);
if (!cur) {
uError("SetFilePointerEx Error getting current position in file.\n");
return -1;
}
LARGE_INTEGER li_size;
li_size.QuadPart = l_size;
BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN);
if (cur2 == 0) {
int error = GetLastError();
uError("SetFilePointerEx GetLastError is: %d\n", error);
switch (error) {
case ERROR_INVALID_HANDLE:
errno = EBADF;
break;
default:
errno = EIO;
break;
}
return -1;
}
if (!SetEndOfFile(h)) {
int error = GetLastError();
uError("SetEndOfFile GetLastError is:%d", error);
switch (error) {
case ERROR_INVALID_HANDLE:
errno = EBADF;
break;
default:
errno = EIO;
break;
}
return -1;
}
return 0;
#else
return ftruncate(fd, l_size);
#ifdef __GNUC__
__attribute__((format(printf, 2, 3)))
#endif
void taosFprintfFile(TdFilePtr pFile, const char *format, ...) {
va_list ap;
va_start(ap, format);
fprintf(pFile->fp, format, ap);
va_end(ap);
fflush(pFile->fp);
}
int32_t taosFsyncFile(FileFd fd) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
if (fd < 0) {
errno = EBADF;
uError("%s\n", "fd arg was negative");
return -1;
}
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) {
if (pFile == NULL) return NULL;
HANDLE h = (HANDLE)_get_osfhandle(fd);
return FlushFileBuffers(h);
#else
return fsync(fd);
#endif
void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0);
return ptr;
}
int32_t taosRenameFile(const char *oldName, const char *newName) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
if (code < 0) {
//printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
}
return code;
#else
int32_t code = rename(oldName, newName);
if (code < 0) {
//printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
}
return code;
#endif
bool taosValidFile(TdFilePtr pFile) {
return pFile != NULL;
}
int32_t taosLockFile(int32_t fd) {
int32_t taosUmaskFile(int32_t maskVal) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return (int32_t)flock(fd, LOCK_EX | LOCK_NB);
return umask(maskVal);
#endif
}
int32_t taosUnLockFile(int32_t fd) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return (int32_t)flock(fd, LOCK_UN | LOCK_NB);
#endif
int taosGetErrorFile(TdFilePtr pFile) {
return errno;
}
int32_t taosUmaskFile(int32_t val) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return umask(val);
#endif
size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf) {
return getline(ptrBuf, NULL, pFile->fp);
}
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
struct stat fileStat;
int32_t code = stat(path, &fileStat);
if (code < 0) {
return code;
}
if (size != NULL) {
*size = fileStat.st_size;
}
if (mtime != NULL) {
*mtime = fileStat.st_mtime;
}
return 0;
#endif
}
int32_t taosFStatFile(int32_t fd, int64_t *size, int32_t *mtime) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
struct stat fileStat;
int32_t code = fstat(fd, &fileStat);
if (code < 0) {
return code;
}
if (size != NULL) {
*size = fileStat.st_size;
}
if (mtime != NULL) {
*mtime = fileStat.st_mtime;
}
return 0;
#endif
}
int32_t taosOpenFileWrite(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
int32_t taosOpenFileCreateWrite(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
int32_t taosOpenFileCreateWriteTrunc(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
int32_t taosOpenFileCreateWriteAppend(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
FileFd taosOpenFileRead(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
FileFd taosOpenFileReadWrite(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_RDWR, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
int32_t taosEOFFile(TdFilePtr pFile) {
return feof(pFile->fp);
}

10
source/os/src/osRand.c
View File

@ -24,18 +24,18 @@
uint32_t taosRand(void) { return rand(); }
uint32_t taosSafeRand(void) {
int fd;
TdFilePtr pFile;
int seed;
fd = open("/dev/urandom", 0);
if (fd < 0) {
pFile = taosOpenFile("/dev/urandom", TD_FILE_READ);
if (pFile == NULL) {
seed = (int)time(0);
} else {
int len = read(fd, &seed, sizeof(seed));
int len = taosReadFile(pFile, &seed, sizeof(seed));
if (len < 0) {
seed = (int)time(0);
}
close(fd);
taosCloseFile(&pFile);
}
return (uint32_t)seed;

1
source/os/src/osSocket.c
View File

@ -14,6 +14,7 @@
*/
#define _DEFAULT_SOURCE
#define ALLOW_FORBID_FUNC
#include "os.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)

101
source/os/src/osSysinfo.c
View File

@ -408,19 +408,17 @@ bool taosGetSysMemory(float *memoryUsedMB) {
}
bool taosGetProcMemory(float *memoryUsedMB) {
FILE *fp = fopen(tsProcMemFile, "r");
if (fp == NULL) {
// FILE *fp = fopen(tsProcMemFile, "r");
TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ);
if (pFile == NULL) {
//printf("open file:%s failed", tsProcMemFile);
return false;
}
ssize_t _bytes = 0;
size_t len;
char * line = NULL;
while (!feof(fp)) {
tfree(line);
len = 0;
_bytes = getline(&line, &len, fp);
while (!taosEOFFile(pFile)) {
_bytes = taosGetLineFile(pFile, &line);
if ((_bytes < 0) || (line == NULL)) {
break;
}
@ -431,7 +429,7 @@ bool taosGetProcMemory(float *memoryUsedMB) {
if (line == NULL) {
//printf("read file:%s failed", tsProcMemFile);
fclose(fp);
taosCloseFile(&pFile);
return false;
}
@ -440,24 +438,24 @@ bool taosGetProcMemory(float *memoryUsedMB) {
sscanf(line, "%s %" PRId64, tmp, &memKB);
*memoryUsedMB = (float)((double)memKB / 1024);
tfree(line);
fclose(fp);
if(line != NULL) tfree(line);
taosCloseFile(&pFile);
return true;
}
static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
FILE *fp = fopen(tsSysCpuFile, "r");
if (fp == NULL) {
// FILE *fp = fopen(tsSysCpuFile, "r");
TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ);
if (pFile == NULL) {
//printf("open file:%s failed", tsSysCpuFile);
return false;
}
size_t len;
char * line = NULL;
ssize_t _bytes = getline(&line, &len, fp);
ssize_t _bytes = taosGetLineFile(pFile, &line);
if ((_bytes < 0) || (line == NULL)) {
//printf("read file:%s failed", tsSysCpuFile);
fclose(fp);
taosCloseFile(&pFile);
return false;
}
@ -465,24 +463,24 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system,
&cpuInfo->idle);
tfree(line);
fclose(fp);
if(line != NULL) tfree(line);
taosCloseFile(&pFile);
return true;
}
static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
FILE *fp = fopen(tsProcCpuFile, "r");
if (fp == NULL) {
// FILE *fp = fopen(tsProcCpuFile, "r");
TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ);
if (pFile == NULL) {
//printf("open file:%s failed", tsProcCpuFile);
return false;
}
size_t len = 0;
char * line = NULL;
ssize_t _bytes = getline(&line, &len, fp);
ssize_t _bytes = taosGetLineFile(pFile, &line);
if ((_bytes < 0) || (line == NULL)) {
//printf("read file:%s failed", tsProcCpuFile);
fclose(fp);
taosCloseFile(&pFile);
return false;
}
@ -495,8 +493,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
}
}
tfree(line);
fclose(fp);
if(line != NULL) tfree(line);
taosCloseFile(&pFile);
return true;
}
@ -556,19 +554,17 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
*bytes = 0;
FILE *fp = fopen(tsSysNetFile, "r");
if (fp == NULL) {
// FILE *fp = fopen(tsSysNetFile, "r");
TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ);
if (pFile == NULL) {
//printf("open file:%s failed", tsSysNetFile);
return false;
}
ssize_t _bytes = 0;
size_t len = 2048;
char * line = calloc(1, len);
while (!feof(fp)) {
memset(line, 0, len);
char * line = NULL;
while (!taosEOFFile(pFile)) {
int64_t o_rbytes = 0;
int64_t rpackts = 0;
int64_t o_tbytes = 0;
@ -581,12 +577,12 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
int64_t nouse6 = 0;
char nouse0[200] = {0};
_bytes = getline(&line, &len, fp);
_bytes = taosGetLineFile(pFile, &line);
if (_bytes < 0) {
break;
}
line[len - 1] = 0;
line[_bytes - 1] = 0;
if (strstr(line, "lo:") != NULL) {
continue;
@ -601,8 +597,8 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
*bytes += (o_rbytes + o_tbytes);
}
tfree(line);
fclose(fp);
if(line != NULL) tfree(line);
taosCloseFile(&pFile);
return true;
}
@ -644,22 +640,20 @@ bool taosGetBandSpeed(float *bandSpeedKb) {
}
bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
FILE *fp = fopen(tsProcIOFile, "r");
if (fp == NULL) {
// FILE *fp = fopen(tsProcIOFile, "r");
TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ);
if (pFile == NULL) {
//printf("open file:%s failed", tsProcIOFile);
return false;
}
ssize_t _bytes = 0;
size_t len;
char * line = NULL;
char tmp[10];
int readIndex = 0;
while (!feof(fp)) {
tfree(line);
len = 0;
_bytes = getline(&line, &len, fp);
while (!taosEOFFile(pFile)) {
_bytes = taosGetLineFile(pFile, &line);
if ((_bytes < 0) || (line == NULL)) {
break;
}
@ -675,8 +669,8 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
if (readIndex >= 2) break;
}
tfree(line);
fclose(fp);
if(line != NULL) tfree(line);
taosCloseFile(&pFile);
if (readIndex < 2) {
//printf("read file:%s failed", tsProcIOFile);
@ -839,15 +833,15 @@ void taosSetCoreDump(bool enable) {
}
int32_t taosGetSystemUUID(char *uid, int32_t uidlen) {
int fd;
int len = 0;
fd = open("/proc/sys/kernel/random/uuid", 0);
if (fd < 0) {
// fd = open("/proc/sys/kernel/random/uuid", 0);
TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ);
if (pFile == NULL) {
return -1;
} else {
len = read(fd, uid, uidlen);
close(fd);
len = taosReadFile(pFile, uid, uidlen);
taosCloseFile(&pFile);
}
if (len >= 36) {
@ -862,16 +856,17 @@ char *taosGetCmdlineByPID(int pid) {
static char cmdline[1024];
sprintf(cmdline, "/proc/%d/cmdline", pid);
int fd = open(cmdline, O_RDONLY);
if (fd >= 0) {
int n = read(fd, cmdline, sizeof(cmdline) - 1);
// int fd = open(cmdline, O_RDONLY);
TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ);
if (pFile != NULL) {
int n = taosReadFile(pFile, cmdline, sizeof(cmdline) - 1);
if (n < 0) n = 0;
if (n > 0 && cmdline[n - 1] == '\n') --n;
cmdline[n] = 0;
close(fd);
taosCloseFile(&pFile);
} else {
cmdline[0] = 0;
}

13
source/os/src/osTimezone.c
View File

@ -150,17 +150,18 @@ void taosGetSystemTimezone(char *outTimezone) {
localtime_r(&tx1, &tm1);
/* load time zone string from /etc/timezone */
FILE *f = fopen("/etc/timezone", "r");
// FILE *f = fopen("/etc/timezone", "r");
TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ);
char buf[68] = {0};
if (f != NULL) {
int len = fread(buf, 64, 1, f);
if (len < 64 && ferror(f)) {
fclose(f);
if (pFile != NULL) {
int len = taosReadFile(pFile, buf, 64);
if (len < 64 && taosGetErrorFile(pFile)) {
taosCloseFile(&pFile);
// printf("read /etc/timezone error, reason:%s", strerror(errno));
return;
}
fclose(f);
taosCloseFile(&pFile);
buf[sizeof(buf) - 1] = 0;
char *lineEnd = strstr(buf, "\n");

21
source/util/src/tconfig.c
View File

@ -571,31 +571,28 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) {
}
int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
char *line, *name, *value, *value2, *value3;
char *line = NULL, *name, *value, *value2, *value3;
int olen, vlen, vlen2, vlen3;
ssize_t _bytes = 0;
size_t len = 1024;
FILE *fp = fopen(filepath, "r");
if (fp == NULL) {
// FILE *fp = fopen(filepath, "r");
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
line = malloc(len);
while (!feof(fp)) {
memset(line, 0, len);
while (!taosEOFFile(pFile)) {
name = value = value2 = value3 = NULL;
olen = vlen = vlen2 = vlen3 = 0;
_bytes = tgetline(&line, &len, fp);
_bytes = taosGetLineFile(pFile, &line);
if (_bytes < 0) {
break;
}
line[len - 1] = 0;
line[_bytes - 1] = 0;
paGetToken(line, &name, &olen);
if (olen == 0) continue;
@ -616,8 +613,8 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
// taosReadConfigOption(name, value, value2, value3);
}
fclose(fp);
tfree(line);
taosCloseFile(&pFile);
if(line != NULL) tfree(line);
uInfo("load from cfg file %s success", filepath);
return 0;

210
source/util/src/tfile.c
View File

@ -24,147 +24,147 @@ static int32_t tsFileRsetId = -1;
static int8_t tfInited = 0;
static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); }
// static void tfCloseFile(void *p) { taosCloseFile((TdFilePtr)(uintptr_t)p); }
int32_t tfInit() {
int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1);
if (old == 1) return 0;
tsFileRsetId = taosOpenRef(2000, tfCloseFile);
if (tsFileRsetId > 0) {
return 0;
} else {
atomic_store_8(&tfInited, 0);
return -1;
}
// int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1);
// if (old == 1) return 0;
// tsFileRsetId = taosOpenRef(2000, tfCloseFile);
// if (tsFileRsetId > 0) {
// return 0;
// } else {
// atomic_store_8(&tfInited, 0);
// return -1;
// }
}
void tfCleanup() {
atomic_store_8(&tfInited, 0);
if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
tsFileRsetId = -1;
// atomic_store_8(&tfInited, 0);
// if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
// tsFileRsetId = -1;
}
static int64_t tfOpenImp(int32_t fd) {
if (fd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
// static int64_t tfOpenImp(TdFilePtr pFile) {
// if (pFile == NULL) {
// terrno = TAOS_SYSTEM_ERROR(errno);
// return -1;
// }
void * p = (void *)(int64_t)fd;
int64_t rid = taosAddRef(tsFileRsetId, p);
if (rid < 0) taosCloseFile(fd);
// void * p = (void *)(int64_t)pFile;
// int64_t rid = taosAddRef(tsFileRsetId, p);
// if (rid < 0) taosCloseFile(&pFile);
return rid;
}
// return rid;
// }
int64_t tfOpenRead(const char *pathname, int32_t flags) {
int32_t fd = taosOpenFileRead(pathname);
return tfOpenImp(fd);
}
// int64_t tfOpenRead(const char *pathname, int32_t flags) {
// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ);
// return tfOpenImp(fd);
// }
int64_t tfOpenReadWrite(const char *pathname, int32_t flags) {
int32_t fd = taosOpenFileReadWrite(pathname);
return tfOpenImp(fd);
}
// int64_t tfOpenReadWrite(const char *pathname, int32_t flags) {
// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ | TD_FILE_WRITE);
// return tfOpenImp(fd);
// }
int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) {
int32_t fd = taosOpenFileCreateWrite(pathname);
return tfOpenImp(fd);
}
// int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) {
// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE);
// return tfOpenImp(fd);
// }
int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) {
int32_t fd = taosOpenFileCreateWriteAppend(pathname);
return tfOpenImp(fd);
}
// int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) {
// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
// return tfOpenImp(fd);
// }
int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); }
// int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); }
int64_t tfWrite(int64_t tfd, void *buf, int64_t count) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int64_t tfWrite(int64_t tfd, void *buf, int64_t count) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
int64_t ret = taosWriteFile(fd, buf, count);
if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
// int64_t ret = taosWriteFile(pFile, buf, count);
// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return ret;
// }
int64_t tfRead(int64_t tfd, void *buf, int64_t count) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int64_t tfRead(int64_t tfd, void *buf, int64_t count) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
int64_t ret = taosReadFile(fd, buf, count);
if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
// int64_t ret = taosReadFile(pFile, buf, count);
// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return ret;
// }
int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
int64_t ret = pread(fd, buf, count, offset);
if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
// int64_t ret = pread(fd, buf, count, offset);
// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return ret;
// }
int32_t tfFsync(int64_t tfd) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int32_t tfFsync(int64_t tfd) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int32_t code = taosFsyncFile(fd);
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
// int32_t code = taosFsyncFile(pFile);
taosReleaseRef(tsFileRsetId, tfd);
return code;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return code;
// }
bool tfValid(int64_t tfd) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return false;
// bool tfValid(int64_t tfd) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return false;
taosReleaseRef(tsFileRsetId, tfd);
return true;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return true;
// }
int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int64_t ret = taosLSeekFile(fd, offset, whence);
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
// int64_t ret = taosLSeekFile(fd, offset, whence);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return ret;
// }
int32_t tfFtruncate(int64_t tfd, int64_t length) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int32_t tfFtruncate(int64_t tfd, int64_t length) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int32_t code = taosFtruncateFile(fd, length);
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
// int32_t code = taosFtruncateFile(fd, length);
taosReleaseRef(tsFileRsetId, tfd);
return code;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return code;
// }
void *tfMmapReadOnly(int64_t tfd, int64_t length) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return NULL;
int32_t fd = (int32_t)(uintptr_t)p;
// void *tfMmapReadOnly(int64_t tfd, int64_t length) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return NULL;
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0);
taosReleaseRef(tsFileRsetId, tfd);
return ptr;
}
// void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0);
// taosReleaseRef(tsFileRsetId, tfd);
// return ptr;
// }

106
source/util/src/tlog.c
View File

@ -47,7 +47,7 @@ typedef struct {
int32_t buffEnd;
int32_t buffSize;
int32_t minBuffSize;
int32_t fd;
TdFilePtr pFile;
int32_t stop;
pthread_t asyncThread;
pthread_mutex_t buffMutex;
@ -101,7 +101,7 @@ static SLogObj tsLogObj = {.fileNum = 1};
static void *taosAsyncOutputLog(void *param);
static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen);
static SLogBuff *taosLogBuffNew(int32_t bufSize);
static void taosCloseLogByFd(int32_t oldFd);
static void taosCloseLogByFd(TdFilePtr pFile);
static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum);
extern void taosPrintCfg();
static int32_t taosCompressFile(char *srcFileName, char *destFileName);
@ -150,11 +150,11 @@ void taosCloseLog() {
// taosCloseLog();
}
static bool taosLockLogFile(int32_t fd) {
if (fd < 0) return false;
static bool taosLockLogFile(TdFilePtr pFile) {
if (pFile == NULL) return false;
if (tsLogObj.fileNum > 1) {
int32_t ret = taosLockFile(fd);
int32_t ret = taosLockFile(pFile);
if (ret == 0) {
return true;
}
@ -163,11 +163,11 @@ static bool taosLockLogFile(int32_t fd) {
return false;
}
static void taosUnLockLogFile(int32_t fd) {
if (fd < 0) return;
static void taosUnLockLogFile(TdFilePtr pFile) {
if (pFile == NULL) return;
if (tsLogObj.fileNum > 1) {
taosUnLockFile(fd);
taosUnLockFile(pFile);
}
}
@ -201,22 +201,22 @@ static void *taosThreadToOpenNewFile(void *param) {
taosUmaskFile(0);
int32_t fd = taosOpenFileCreateWriteTrunc(name);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
tsLogObj.openInProgress = 0;
tsLogObj.lines = tsLogObj.maxLines - 1000;
uError("open new log file fail! fd:%d reason:%s, reuse lastlog", fd, strerror(errno));
uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno));
return NULL;
}
taosLockLogFile(fd);
(void)taosLSeekFile(fd, 0, SEEK_SET);
taosLockLogFile(pFile);
(void)taosLSeekFile(pFile, 0, SEEK_SET);
int32_t oldFd = tsLogObj.logHandle->fd;
tsLogObj.logHandle->fd = fd;
TdFilePtr pOldFile = tsLogObj.logHandle->pFile;
tsLogObj.logHandle->pFile = pFile;
tsLogObj.lines = 0;
tsLogObj.openInProgress = 0;
taosCloseLogByFd(oldFd);
taosCloseLogByFd(pOldFile);
uInfo(" new log file:%d is opened", tsLogObj.flag);
uInfo("==================================");
@ -262,8 +262,8 @@ void taosResetLog() {
}
static bool taosCheckFileIsOpen(char *logFileName) {
int32_t fd = taosOpenFileWrite(logFileName);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(logFileName, TD_FILE_WRITE);
if (pFile == NULL) {
if (errno == ENOENT) {
return false;
} else {
@ -272,12 +272,12 @@ static bool taosCheckFileIsOpen(char *logFileName) {
}
}
if (taosLockLogFile(fd)) {
taosUnLockLogFile(fd);
taosCloseFile(fd);
if (taosLockLogFile(pFile)) {
taosUnLockLogFile(pFile);
taosCloseFile(&pFile);
return false;
} else {
taosCloseFile(fd);
taosCloseFile(&pFile);
return true;
}
}
@ -350,31 +350,31 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
pthread_mutex_init(&tsLogObj.logMutex, NULL);
taosUmaskFile(0);
tsLogObj.logHandle->fd = taosOpenFileCreateWrite(fileName);
tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE);
if (tsLogObj.logHandle->fd < 0) {
if (tsLogObj.logHandle->pFile == NULL) {
printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
return -1;
}
taosLockLogFile(tsLogObj.logHandle->fd);
taosLockLogFile(tsLogObj.logHandle->pFile);
// only an estimate for number of lines
int64_t filesize = 0;
if (taosFStatFile(tsLogObj.logHandle->fd, &filesize, NULL) < 0) {
if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) {
printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno));
return -1;
}
size = (int32_t)filesize;
tsLogObj.lines = size / 60;
taosLSeekFile(tsLogObj.logHandle->fd, 0, SEEK_END);
taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END);
sprintf(name, "==================================================\n");
taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
sprintf(name, " new log file \n");
taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
sprintf(name, "==================================================\n");
taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
return 0;
}
@ -416,11 +416,11 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
buffer[len++] = '\n';
buffer[len] = 0;
if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) {
if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) {
if (tsAsyncLog) {
taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
} else {
taosWriteFile(tsLogObj.logHandle->fd, buffer, len);
taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
}
if (tsLogObj.maxLines > 0) {
@ -430,7 +430,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
}
}
if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len);
if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len);
}
void taosDumpData(unsigned char *msg, int32_t len) {
@ -445,7 +445,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
pos += 3;
if (c >= 16) {
temp[pos++] = '\n';
taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos);
c = 0;
pos = 0;
}
@ -453,7 +453,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
temp[pos++] = '\n';
taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos);
}
void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) {
@ -483,11 +483,11 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
buffer[len++] = '\n';
buffer[len] = 0;
if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) {
if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) {
if (tsAsyncLog) {
taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
} else {
taosWriteFile(tsLogObj.logHandle->fd, buffer, len);
taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
}
if (tsLogObj.maxLines > 0) {
@ -497,19 +497,19 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
}
}
if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len);
if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len);
}
#if 0
void taosCloseLog() {
taosCloseLogByFd(tsLogObj.logHandle->fd);
taosCloseLogByFd(tsLogObj.logHandle->pFile);
}
#endif
static void taosCloseLogByFd(int32_t fd) {
if (fd >= 0) {
taosUnLockLogFile(fd);
taosCloseFile(fd);
static void taosCloseLogByFd(TdFilePtr pFile) {
if (pFile != NULL) {
taosUnLockLogFile(pFile);
taosCloseFile(&pFile);
}
}
@ -645,12 +645,12 @@ static void taosWriteLog(SLogBuff *tLogBuff) {
}
if (start < end) {
taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize);
taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, pollSize);
} else {
int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start;
taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize);
taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, tsize);
taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end);
taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff), end);
}
dbgWN++;
@ -707,17 +707,17 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
int32_t ret = 0;
int32_t len = 0;
char *data = malloc(compressSize);
FILE *srcFp = NULL;
// gzFile dstFp = NULL;
srcFp = fopen(srcFileName, "r");
if (srcFp == NULL) {
// srcFp = fopen(srcFileName, "r");
TdFilePtr pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ);
if (pSrcFile == NULL) {
ret = -1;
goto cmp_end;
}
int32_t fd = taosOpenFileCreateWriteTrunc(destFileName);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(destFileName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
ret = -2;
goto cmp_end;
}
@ -735,8 +735,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
// }
cmp_end:
if (srcFp) {
fclose(srcFp);
if (pSrcFile) {
taosCloseFile(&pSrcFile);
}
// if (dstFp) {
// gzclose(dstFp);

27
source/util/src/tpagedbuf.c
View File

@ -32,7 +32,7 @@ typedef struct SDiskbasedBuf {
int32_t numOfPages;
int64_t totalBufSize;
uint64_t fileSize; // disk file size
FILE* file;
TdFilePtr pFile;
int32_t allocateId; // allocated page id
char* path; // file path
int32_t pageSize; // current used page size
@ -67,7 +67,7 @@ static void printStatisData(const SDiskbasedBuf* pBuf);
pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit.
pResBuf->allocateId = -1;
pResBuf->comp = true;
pResBuf->file = NULL;
pResBuf->pFile = NULL;
pResBuf->qId = qId;
pResBuf->fileSize = 0;
@ -94,8 +94,9 @@ static void printStatisData(const SDiskbasedBuf* pBuf);
}
static int32_t createDiskFile(SDiskbasedBuf* pBuf) {
pBuf->file = fopen(pBuf->path, "wb+");
if (pBuf->file == NULL) {
// pBuf->file = fopen(pBuf->path, "wb+");
pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (pBuf->pFile == NULL) {
// qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
@ -168,13 +169,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
pg->offset = allocatePositionInFile(pBuf, size);
pBuf->nextPos += size;
int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
ret = (int32_t) fwrite(t, 1, size, pBuf->file);
ret = (int32_t) taosWriteFile(pBuf->pFile, t, size);
if (ret != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
@ -199,13 +200,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
}
// 3. write to disk.
int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
ret = (int32_t)fwrite(t, 1, size, pBuf->file);
ret = (int32_t) taosWriteFile(pBuf->pFile, t, size);
if (ret != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
@ -233,7 +234,7 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
int32_t ret = TSDB_CODE_SUCCESS;
assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages);
if (pBuf->file == NULL) {
if (pBuf->pFile == NULL) {
if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) {
terrno = ret;
return NULL;
@ -245,14 +246,14 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
// load file block data in disk
static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
if (ret != 0) {
ret = TAOS_SYSTEM_ERROR(errno);
return ret;
}
SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg);
ret = (int32_t)fread(pPage->data, 1, pg->length, pBuf->file);
ret = (int32_t)taosReadFile(pBuf->pFile, pPage->data, pg->length);
if (ret != pg->length) {
ret = TAOS_SYSTEM_ERROR(errno);
return ret;
@ -495,12 +496,12 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) {
printStatisData(pBuf);
if (pBuf->file != NULL) {
if (pBuf->pFile != NULL) {
uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n",
pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId);
fclose(pBuf->file);
taosCloseFile(&pBuf->pFile);
} else {
uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId);
}

8
tools/shell/src/backup/shellCheck.c
View File

@ -116,7 +116,7 @@ static void *shellCheckThreadFp(void *arg) {
char file[32] = {0};
snprintf(file, 32, "tb%d.txt", pThread->threadIndex);
FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (!fp) {
fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno));
return NULL;
@ -133,7 +133,7 @@ static void *shellCheckThreadFp(void *arg) {
int32_t code = taos_errno(pSql);
if (code != 0) {
int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname);
fwrite(sql, 1, len, fp);
taosWriteFile(pFile, sql, len);
atomic_add_fetch_32(&errorNum, 1);
}
@ -145,8 +145,8 @@ static void *shellCheckThreadFp(void *arg) {
taos_free_result(pSql);
}
taosFsync(fileno(fp));
fclose(fp);
taosFsync(pFile);
taosCloseFile(&pFile);
return NULL;
}

12
tools/shell/src/backup/shellImport.c
View File

@ -138,7 +138,6 @@ static void shellSourceFile(TAOS *con, char *fptr) {
char * cmd = malloc(tsMaxSQLStringLen);
size_t cmd_len = 0;
char * line = NULL;
size_t line_len = 0;
if (wordexp(fptr, &full_path, 0) != 0) {
fprintf(stderr, "ERROR: illegal file name\n");
@ -171,8 +170,9 @@ static void shellSourceFile(TAOS *con, char *fptr) {
}
*/
FILE *f = fopen(fname, "r");
if (f == NULL) {
// FILE *f = fopen(fname, "r");
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) {
fprintf(stderr, "ERROR: failed to open file %s\n", fname);
wordfree(&full_path);
free(cmd);
@ -182,7 +182,7 @@ static void shellSourceFile(TAOS *con, char *fptr) {
fprintf(stdout, "begin import file:%s\n", fname);
int lineNo = 0;
while ((read_len = getline(&line, &line_len, f)) != -1) {
while ((read_len = taosGetLineFile(pFile, &line)) != -1) {
++lineNo;
if (read_len >= tsMaxSQLStringLen) continue;
line[--read_len] = '\0';
@ -215,9 +215,9 @@ static void shellSourceFile(TAOS *con, char *fptr) {
}
free(cmd);
if (line) free(line);
if(line != NULL) free(line);
wordfree(&full_path);
fclose(f);
taosCloseFile(&pFile);
}
void* shellImportThreadFp(void *arg)

74
tools/shell/src/shellEngine.c
View File

@ -459,44 +459,44 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) {
return buf;
}
static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) {
static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int precision) {
if (val == NULL) {
fprintf(fp, "%s", TSDB_DATA_NULL_STR);
taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR);
return;
}
char buf[TSDB_MAX_BYTES_PER_ROW];
switch (field->type) {
case TSDB_DATA_TYPE_BOOL:
fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
break;
case TSDB_DATA_TYPE_TINYINT:
fprintf(fp, "%d", *((int8_t *)val));
taosFprintfFile(pFile, "%d", *((int8_t *)val));
break;
case TSDB_DATA_TYPE_SMALLINT:
fprintf(fp, "%d", *((int16_t *)val));
taosFprintfFile(pFile, "%d", *((int16_t *)val));
break;
case TSDB_DATA_TYPE_INT:
fprintf(fp, "%d", *((int32_t *)val));
taosFprintfFile(pFile, "%d", *((int32_t *)val));
break;
case TSDB_DATA_TYPE_BIGINT:
fprintf(fp, "%" PRId64, *((int64_t *)val));
taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val));
break;
case TSDB_DATA_TYPE_FLOAT:
fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val));
break;
case TSDB_DATA_TYPE_DOUBLE:
fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
taosFprintfFile(pFile, "%.9f", GET_DOUBLE_VAL(val));
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
memcpy(buf, val, length);
buf[length] = 0;
fprintf(fp, "\'%s\'", buf);
taosFprintfFile(pFile, "\'%s\'", buf);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
formatTimestamp(buf, *(int64_t *)val, precision);
fprintf(fp, "'%s'", buf);
taosFprintfFile(pFile, "'%s'", buf);
break;
default:
break;
@ -516,8 +516,9 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) {
return -1;
}
FILE *fp = fopen(full_path.we_wordv[0], "w");
if (fp == NULL) {
// FILE *fp = fopen(full_path.we_wordv[0], "w");
TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]);
wordfree(&full_path);
return -1;
@ -531,29 +532,29 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) {
for (int col = 0; col < num_fields; col++) {
if (col > 0) {
fprintf(fp, ",");
taosFprintfFile(pFile, ",");
}
fprintf(fp, "%s", fields[col].name);
taosFprintfFile(pFile, "%s", fields[col].name);
}
fputc('\n', fp);
taosFprintfFile(pFile, "\n");
int numOfRows = 0;
do {
int32_t *length = taos_fetch_lengths(tres);
for (int i = 0; i < num_fields; i++) {
if (i > 0) {
fputc(',', fp);
taosFprintfFile(pFile, "\n");
}
dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision);
dumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision);
}
fputc('\n', fp);
taosFprintfFile(pFile, "\n");
numOfRows++;
row = taos_fetch_row(tres);
} while (row != NULL);
result = 0;
fclose(fp);
taosCloseFile(&pFile);
return numOfRows;
}
@ -897,14 +898,14 @@ void read_history() {
history.hstart = 0;
history.hend = 0;
char *line = NULL;
size_t line_size = 0;
int read_size = 0;
char f_history[TSDB_FILENAME_LEN];
get_history_path(f_history);
FILE *f = fopen(f_history, "r");
if (f == NULL) {
// FILE *f = fopen(f_history, "r");
TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ);
if (pFile == NULL) {
#ifndef WINDOWS
if (errno != ENOENT) {
fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno));
@ -913,7 +914,7 @@ void read_history() {
return;
}
while ((read_size = tgetline(&line, &line_size, f)) != -1) {
while ((read_size = taosGetLineFile(pFile, &line)) != -1) {
line[read_size - 1] = '\0';
history.hist[history.hend] = strdup(line);
@ -924,16 +925,17 @@ void read_history() {
}
}
free(line);
fclose(f);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
void write_history() {
char f_history[TSDB_FILENAME_LEN];
get_history_path(f_history);
FILE *f = fopen(f_history, "w");
if (f == NULL) {
// FILE *f = fopen(f_history, "w");
TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
#ifndef WINDOWS
fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno));
#endif
@ -942,12 +944,12 @@ void write_history() {
for (int i = history.hstart; i != history.hend;) {
if (history.hist[i] != NULL) {
fprintf(f, "%s\n", history.hist[i]);
taosFprintfFile(pFile, "%s\n", history.hist[i]);
tfree(history.hist[i]);
}
i = (i + 1) % MAX_HISTORY_SIZE;
}
fclose(f);
taosCloseFile(&pFile);
}
void taos_error(TAOS_RES *tres, int64_t st) {
@ -969,7 +971,6 @@ void source_file(TAOS *con, char *fptr) {
char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1);
size_t cmd_len = 0;
char *line = NULL;
size_t line_len = 0;
if (wordexp(fptr, &full_path, 0) != 0) {
fprintf(stderr, "ERROR: illegal file name\n");
@ -989,15 +990,16 @@ void source_file(TAOS *con, char *fptr) {
}
*/
FILE *f = fopen(fname, "r");
if (f == NULL) {
// FILE *f = fopen(fname, "r");
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) {
fprintf(stderr, "ERROR: failed to open file %s\n", fname);
wordfree(&full_path);
free(cmd);
return;
}
while ((read_len = tgetline(&line, &line_len, f)) != -1) {
while ((read_len = taosGetLineFile(pFile, &line)) != -1) {
if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue;
line[--read_len] = '\0';
@ -1020,9 +1022,9 @@ void source_file(TAOS *con, char *fptr) {
}
free(cmd);
if (line) free(line);
if(line != NULL) free(line);
wordfree(&full_path);
fclose(f);
taosCloseFile(&pFile);
}
void shellGetGrantInfo(void *con) {