xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[td-13039] merge 3.0

This commit is contained in:
Haojun Liao 2022-02-25 10:30:44 +08:00
parent 5d07ef4e43 c9c75bdf79
commit 40da10fabd
70 changed files with 1854 additions and 1239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/common/ttszip.h
View File

@ -73,9 +73,9 @@ typedef struct STSGroupBlockInfoEx {
} STSGroupBlockInfoEx;
typedef struct STSBuf {
FILE* f;
char path[PATH_MAX];
uint32_t fileSize;
TdFilePtr pFile;
char path[PATH_MAX];
uint32_t fileSize;
// todo use array
STSGroupBlockInfoEx* pData;

5
include/libs/nodes/nodes.h
View File

@ -62,7 +62,6 @@ typedef enum ENodeType {
QUERY_NODE_NODE_LIST,
QUERY_NODE_FILL,
QUERY_NODE_RAW_EXPR, // Only be used in parser module.
QUERY_NODE_COLUMN_REF,
QUERY_NODE_TARGET,
QUERY_NODE_TUPLE_DESC,
QUERY_NODE_SLOT_DESC,
@ -81,7 +80,9 @@ typedef enum ENodeType {
// physical plan node
QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_PROJECT
QUERY_NODE_PHYSICAL_PLAN_PROJECT,
QUERY_NODE_PHYSICAL_PLAN_JOIN,
QUERY_NODE_PHYSICAL_PLAN_AGG
} ENodeType;
/**

16
include/libs/nodes/plannodes.h
View File

@ -69,8 +69,6 @@ typedef struct SSlotDescNode {
ENodeType type;
int16_t slotId;
SDataType dataType;
int16_t srcTupleId;
int16_t srcSlotId;
bool reserve;
bool output;
} SSlotDescNode;
@ -115,6 +113,20 @@ typedef struct SProjectPhysiNode {
SNodeList* pProjections;
} SProjectPhysiNode;
typedef struct SJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
SNode* pOnConditions; // in or out tuple ?
SNodeList* pTargets;
} SJoinPhysiNode;
typedef struct SAggPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function
SNodeList* pGroupKeys; // SColumnRefNode list
SNodeList* pAggFuncs;
} SAggPhysiNode;
#ifdef __cplusplus
}
#endif

16
include/libs/nodes/querynodes.h
View File

@ -58,15 +58,17 @@ typedef struct SColumnNode {
char tableAlias[TSDB_TABLE_NAME_LEN];
char colName[TSDB_COL_NAME_LEN];
SNode* pProjectRef;
} SColumnNode;
typedef struct SColumnRefNode {
ENodeType type;
SDataType dataType;
int16_t tupleId;
int16_t slotId;
int16_t columnId;
} SColumnRefNode;
} SColumnNode;
// typedef struct SColumnRefNode {
// ENodeType type;
// SDataType dataType;
// int16_t tupleId;
// int16_t slotId;
// int16_t columnId;
// } SColumnRefNode;
typedef struct STargetNode {
ENodeType type;

8
include/libs/wal/wal.h
View File

@ -120,8 +120,8 @@ typedef struct SWal {
int32_t fsyncSeq;
// meta
SWalVer vers;
int64_t writeLogTfd;
int64_t writeIdxTfd;
TdFilePtr pWriteLogTFile;
TdFilePtr pWriteIdxTFile;
int32_t writeCur;
SArray *fileInfoSet;
// status
@ -138,8 +138,8 @@ typedef struct SWal {
typedef struct SWalReadHandle {
SWal *pWal;
int64_t readLogTfd;
int64_t readIdxTfd;
TdFilePtr pReadLogTFile;
TdFilePtr pReadIdxTFile;
int64_t curFileFirstVer;
int64_t curVersion;
int64_t capacity;

78
include/os/osFile.h
View File

@ -22,49 +22,63 @@ extern "C" {
#include "osSocket.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
typedef int32_t FileFd;
#else
typedef int32_t FileFd;
#ifndef ALLOW_FORBID_FUNC
#define open OPEN_FUNC_TAOS_FORBID
#define fopen FOPEN_FUNC_TAOS_FORBID
// #define close CLOSE_FUNC_TAOS_FORBID
// #define fclose FCLOSE_FUNC_TAOS_FORBID
#endif
#define FD_INITIALIZER ((int32_t)-1)
#ifndef PATH_MAX
#define PATH_MAX 256
#endif
int32_t taosLockFile(FileFd fd);
int32_t taosUnLockFile(FileFd fd);
int32_t taosUmaskFile(FileFd fd);
typedef struct TdFile *TdFilePtr;
#define TD_FILE_CTEATE 0x0001
#define TD_FILE_WRITE 0x0002
#define TD_FILE_READ 0x0004
#define TD_FILE_TRUNC 0x0008
#define TD_FILE_APPEND 0x0010
#define TD_FILE_TEXT 0x0020
#define TD_FILE_AUTO_DEL 0x0040
#define TD_FILE_EXCL 0x0080
int32_t taosLockFile(TdFilePtr pFile);
int32_t taosUnLockFile(TdFilePtr pFile);
int32_t taosUmaskFile(int32_t maskVal);
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime);
FileFd taosOpenFileWrite(const char *path);
FileFd taosOpenFileCreateWrite(const char *path);
FileFd taosOpenFileCreateWriteTrunc(const char *path);
FileFd taosOpenFileCreateWriteAppend(const char *path);
FileFd taosOpenFileRead(const char *path);
FileFd taosOpenFileReadWrite(const char *path);
int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence);
int32_t taosFtruncateFile(FileFd fd, int64_t length);
int32_t taosFsyncFile(FileFd fd);
int64_t taosReadFile(FileFd fd, void *buf, int64_t count);
int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count);
void taosCloseFile(FileFd fd);
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length);
int32_t taosFsyncFile(TdFilePtr pFile);
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
int32_t taosEOFFile(TdFilePtr pFile);
int64_t taosCloseFile(TdFilePtr *ppFile);
int32_t taosRenameFile(const char *oldName, const char *newName);
int64_t taosCopyFile(const char *from, const char *to);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size);
int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
bool taosValidFile(TdFilePtr pFile);
int taosGetErrorFile(TdFilePtr pFile);
#ifdef __cplusplus
}

26
include/util/tfile.h
View File

@ -30,20 +30,20 @@ void tfCleanup();
// the same syntax as UNIX standard open/close/read/write
// but FD is int64_t and will never be reused
int64_t tfOpenRead(const char *pathname);
int64_t tfOpenReadWrite(const char *pathname);
int64_t tfOpenCreateWrite(const char *pathname);
int64_t tfOpenCreateWriteAppend(const char *pathname);
// int64_t tfOpenRead(const char *pathname);
// int64_t tfOpenReadWrite(const char *pathname);
// int64_t tfOpenCreateWrite(const char *pathname);
// int64_t tfOpenCreateWriteAppend(const char *pathname);
int64_t tfClose(int64_t tfd);
int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
int64_t tfRead(int64_t tfd, void *buf, int64_t count);
int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset);
int32_t tfFsync(int64_t tfd);
bool tfValid(int64_t tfd);
int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
int32_t tfFtruncate(int64_t tfd, int64_t length);
void * tfMmapReadOnly(int64_t tfd, int64_t length);
// int64_t tfClose(int64_t tfd);
// int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
// int64_t tfRead(int64_t tfd, void *buf, int64_t count);
// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset);
// int32_t tfFsync(int64_t tfd);
// bool tfValid(int64_t tfd);
// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
// int32_t tfFtruncate(int64_t tfd, int64_t length);
// void * tfMmapReadOnly(int64_t tfd, int64_t length);
#ifdef __cplusplus
}
#endif

4
source/client/test/clientTests.cpp
View File

@ -53,8 +53,10 @@ TEST(testCase, driverInit_Test) {
// taos_init();
}
#if 0
#if 1
TEST(testCase, connect_Test) {
// taos_options(TSDB_OPTION_CONFIGDIR, "/home/ubuntu/first/cfg");
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
if (pConn == NULL) {
printf("failed to connect to server, reason:%s\n", taos_errstr(NULL));

135
source/common/src/ttszip.c
View File

@ -24,8 +24,9 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
pTSBuf->autoDelete = autoDelete;
taosGetTmpfilePath(osTempDir(), "join", pTSBuf->path);
pTSBuf->f = fopen(pTSBuf->path, "wb+");
if (pTSBuf->f == NULL) {
// pTSBuf->pFile = fopen(pTSBuf->path, "wb+");
pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (pTSBuf->pFile == NULL) {
free(pTSBuf);
return NULL;
}
@ -60,8 +61,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path));
pTSBuf->f = fopen(pTSBuf->path, "rb+");
if (pTSBuf->f == NULL) {
// pTSBuf->pFile = fopen(pTSBuf->path, "rb+");
pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ);
if (pTSBuf->pFile == NULL) {
free(pTSBuf);
return NULL;
}
@ -72,9 +74,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
// validate the file magic number
STSBufFileHeader header = {0};
int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET);
int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET);
UNUSED(ret);
size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f);
size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader));
UNUSED(sz);
// invalid file
@ -112,8 +114,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return NULL;
}
//int64_t pos = ftell(pTSBuf->f); //pos not used
sz = fread(buf, infoSize, 1, pTSBuf->f);
//int64_t pos = ftell(pTSBuf->pFile); //pos not used
sz = taosReadFile(pTSBuf->pFile, buf, infoSize);
UNUSED(sz);
// the length value for each vnode is not kept in file, so does not set the length value
@ -123,22 +125,22 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
}
free(buf);
ret = fseek(pTSBuf->f, 0, SEEK_END);
ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END);
UNUSED(ret);
struct stat fileStat;
if (fstat(fileno(pTSBuf->f), &fileStat) != 0) {
int64_t file_size;
if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) {
tsBufDestroy(pTSBuf);
return NULL;
}
pTSBuf->fileSize = (uint32_t)fileStat.st_size;
pTSBuf->fileSize = (uint32_t)file_size;
tsBufResetPos(pTSBuf);
// ascending by default
pTSBuf->cur.order = TSDB_ORDER_ASC;
// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f),
// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->pFile),
// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete);
return pTSBuf;
@ -156,7 +158,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
tfree(pTSBuf->block.payload);
if (!pTSBuf->remainOpen) {
fclose(pTSBuf->f);
taosCloseFile(&pTSBuf->pFile);
}
if (pTSBuf->autoDelete) {
@ -253,7 +255,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize,
TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize);
int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET);
int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET);
assert(r == 0);
/*
@ -264,30 +266,30 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
* both side has the compressed length is used to support load data forwards/backwords.
*/
int32_t metaLen = 0;
metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
int32_t trueLen = pBlock->tag.nLen;
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
float tfloat = (float)pBlock->tag.d;
metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)fwrite(&pBlock->tag.i, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen);
} else {
trueLen = 0;
metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen));
}
fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem));
taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen);
taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t) taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen));
assert(metaLen == getTagAreaLength(&pBlock->tag));
int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen;
@ -332,20 +334,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
* the end of each comp data block
*/
int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen));
int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR);
size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f);
sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR);
size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding));
sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
UNUSED(sz);
pBlock->compLen = pBlock->padding;
offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
ret = fseek(pTSBuf->f, -offset, SEEK_CUR);
ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR);
UNUSED(ret);
}
int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f);
ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
// NOTE: mix types tags are not supported
size_t sz = 0;
@ -356,23 +358,23 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
memset(tp, 0, pBlock->tag.nLen + 1);
pBlock->tag.pz = tp;
sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
UNUSED(sz);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
float tfloat = 0;
sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen);
pBlock->tag.d = (double)tfloat;
UNUSED(sz);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value
sz = fread(&pBlock->tag.i, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen);
UNUSED(sz);
}
sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem));
UNUSED(sz);
sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
UNUSED(sz);
sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen);
if (decomp) {
pTSBuf->tsData.len =
@ -381,11 +383,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
}
// read the comp length at the length of comp block
sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding));
assert(pBlock->padding == pBlock->compLen);
int32_t n = 0;
sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen));
if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) {
assert(n == 0);
} else {
@ -396,7 +398,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
// for backwards traverse, set the start position at the end of previous block
if (order == TSDB_ORDER_DESC) {
int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR);
UNUSED(r);
}
@ -512,7 +514,7 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num
// todo opt performance by cache blocks info
static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) {
if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) {
return -1;
}
@ -531,7 +533,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int
STSBlock* pBlock = &pTSBuf->block;
int32_t compBlockSize =
pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR);
UNUSED(ret);
}
@ -548,7 +550,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo
offset = pBlockInfo->offset + pBlockInfo->compLen;
}
if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) {
return -1;
}
@ -618,11 +620,11 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn
return -1;
}
if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) {
return -1;
}
fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f);
taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo));
return 0;
}
@ -636,19 +638,19 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) {
}
int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
return -1;
}
assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC);
int32_t r = fseek(pTSBuf->f, 0, SEEK_SET);
int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET);
if (r != 0) {
// qError("fseek failed, errno:%d", errno);
return -1;
}
size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f);
size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader));
if (ws != 1) {
// qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader));
return -1;
@ -823,12 +825,12 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
pBlockInfoEx->info.id = id;
}
int32_t r = fseek(pDestBuf->f, 0, SEEK_END);
int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END);
assert(r == 0);
int64_t offset = getDataStartOffset();
int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size);
if (written == -1 || written != size) {
return -1;
@ -839,17 +841,18 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
int32_t oldSize = pDestBuf->fileSize;
// file meta data may be cached, close and reopen the file for accurate file size.
fclose(pDestBuf->f);
pDestBuf->f = fopen(pDestBuf->path, "rb+");
if (pDestBuf->f == NULL) {
taosCloseFile(&pDestBuf->pFile);
// pDestBuf->pFile = fopen(pDestBuf->path, "rb+");
pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ);
if (pDestBuf->pFile == NULL) {
return -1;
}
struct stat fileStat;
if (fstat(fileno(pDestBuf->f), &fileStat) != 0) {
int64_t file_size;
if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) {
return -1;
}
pDestBuf->fileSize = (uint32_t)fileStat.st_size;
pDestBuf->fileSize = (uint32_t)file_size;
assert(pDestBuf->fileSize == oldSize + size);
@ -868,13 +871,13 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
// update prev vnode length info in file
TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo);
int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET);
if (ret == -1) {
// qError("fseek failed, errno:%d", errno);
tsBufDestroy(pTSBuf);
return NULL;
}
size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f);
size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len);
if (sz != len) {
// qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len);
tsBufDestroy(pTSBuf);
@ -893,7 +896,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
}
// TODO taosFsync??
// if (taosFsync(fileno(pTSBuf->f)) == -1) {
// if (taosFsync(fileno(pTSBuf->pFile)) == -1) {
//// qError("fsync failed, errno:%d", errno);
// tsBufDestroy(pTSBuf);
// return NULL;
@ -1071,15 +1074,15 @@ int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, in
*len = 0;
*numOfBlocks = 0;
if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) {
int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile));
// qError("%p: fseek failed: %s", pSql, tstrerror(code));
return code;
}
size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f);
size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen);
if (s != pBlockInfo->compLen) {
int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile));
// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
return code;
}

2
source/dnode/mgmt/impl/inc/dndEnv.h
View File

@ -125,7 +125,7 @@ typedef struct SDnode {
EStat stat;
SDnodeObjCfg cfg;
SDnodeDir dir;
FileFd lockFd;
TdFilePtr pLockFile;
SDnodeMgmt dmgmt;
SMnodeMgmt mmgmt;
SQnodeMgmt qmgmt;

20
source/dnode/mgmt/impl/src/dndBnode.c
View File

@ -62,14 +62,15 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_BNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_BNODE_OVER;
@ -102,7 +103,7 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) {
PRASE_BNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
@ -114,8 +115,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
@ -130,9 +132,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];

28
source/dnode/mgmt/impl/src/dndEnv.c
View File

@ -59,31 +59,31 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
}
static FileFd dndCheckRunning(char *dataDir) {
static TdFilePtr dndCheckRunning(char *dataDir) {
char filepath[PATH_MAX] = {0};
snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
FileFd fd = taosOpenFileCreateWriteTrunc(filepath);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s, quit", filepath, terrstr());
return -1;
return NULL;
}
int32_t ret = taosLockFile(fd);
int32_t ret = taosLockFile(pFile);
if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s, quit", filepath, terrstr());
taosCloseFile(fd);
return -1;
taosCloseFile(&pFile);
return NULL;
}
return fd;
return pFile;
}
static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) {
pDnode->lockFd = dndCheckRunning(pCfg->dataDir);
if (pDnode->lockFd < 0) {
pDnode->pLockFile = dndCheckRunning(pCfg->dataDir);
if (pDnode->pLockFile == NULL) {
return -1;
}
@ -147,10 +147,10 @@ static void dndCloseImp(SDnode *pDnode) {
tfree(pDnode->dir.snode);
tfree(pDnode->dir.bnode);
if (pDnode->lockFd >= 0) {
taosUnLockFile(pDnode->lockFd);
taosCloseFile(pDnode->lockFd);
pDnode->lockFd = 0;
if (pDnode->pLockFile != NULL) {
taosUnLockFile(pDnode->pLockFile);
taosCloseFile(&pDnode->pLockFile);
pDnode->pLockFile = NULL;
}
}

21
source/dnode/mgmt/impl/src/dndMgmt.c
View File

@ -185,16 +185,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
int32_t maxLen = 256 * 1024;
char *content = calloc(1, maxLen + 1);
cJSON *root = NULL;
FILE *fp = NULL;
fp = fopen(pMgmt->file, "r");
if (fp == NULL) {
// fp = fopen(pMgmt->file, "r");
TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", pMgmt->file);
code = 0;
goto PRASE_DNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", pMgmt->file);
goto PRASE_DNODE_OVER;
@ -286,7 +286,7 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
PRASE_DNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) {
dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file);
@ -309,8 +309,9 @@ PRASE_DNODE_OVER:
static int32_t dndWriteDnodes(SDnode *pDnode) {
SDnodeMgmt *pMgmt = &pDnode->dmgmt;
FILE *fp = fopen(pMgmt->file, "w");
if (fp == NULL) {
// FILE *fp = fopen(pMgmt->file, "w");
TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
dError("failed to write %s since %s", pMgmt->file, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -341,9 +342,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) {
}
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
terrno = 0;

20
source/dnode/mgmt/impl/src/dndMnode.c
View File

@ -63,14 +63,15 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_MNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_MNODE_OVER;
@ -143,7 +144,7 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
PRASE_MNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
@ -155,8 +156,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
@ -184,9 +186,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
}
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];

20
source/dnode/mgmt/impl/src/dndQnode.c
View File

@ -62,14 +62,15 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_QNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_QNODE_OVER;
@ -102,7 +103,7 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
PRASE_QNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
@ -114,8 +115,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
@ -130,9 +132,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];

20
source/dnode/mgmt/impl/src/dndSnode.c
View File

@ -62,14 +62,15 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_SNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_SNODE_OVER;
@ -102,7 +103,7 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) {
PRASE_SNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
@ -114,8 +115,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
@ -130,9 +132,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];

20
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -219,14 +219,15 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
fp = fopen(file, "r");
if (fp == NULL) {
// fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_VNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_VNODE_OVER;
@ -304,7 +305,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
PRASE_VNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
return code;
}
@ -315,8 +316,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes);
snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write %s since %s", file, terrstr());
return -1;
@ -347,9 +349,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " ]\n");
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
terrno = 0;

41
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -62,12 +62,13 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
size_t size = 0;
int32_t done = 0;
FILE* fp = fopen("/proc/cpuinfo", "r");
if (fp == NULL) {
// FILE* fp = fopen("/proc/cpuinfo", "r");
TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ);
if (pFile == NULL) {
return;
}
while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) {
while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
const char* v = strchr(line, ':') + 2;
@ -83,20 +84,21 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
}
}
free(line);
fclose(fp);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
FILE* fp = fopen("/etc/os-release", "r");
if (fp == NULL) {
// FILE* fp = fopen("/etc/os-release", "r");
TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ);
if (pFile == NULL) {
return;
}
while ((size = tgetline(&line, &size, fp)) != -1) {
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "PRETTY_NAME", 11) == 0) {
const char* p = strchr(line, '=') + 1;
@ -109,20 +111,21 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
}
}
free(line);
fclose(fp);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
FILE* fp = fopen("/proc/meminfo", "r");
if (fp == NULL) {
// FILE* fp = fopen("/proc/meminfo", "r");
TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ);
if (pFile == NULL) {
return;
}
while ((size = tgetline(&line, &size, fp)) != -1) {
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "MemTotal", 8) == 0) {
const char* p = strchr(line, ':') + 1;
@ -132,8 +135,8 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
}
}
free(line);
fclose(fp);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) {
@ -252,16 +255,16 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
static void mndGetEmail(SMnode* pMnode, char* filepath) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
int32_t fd = taosOpenFileRead(filepath);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
if (pFile == NULL) {
return;
}
if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
}
taosCloseFile(fd);
taosCloseFile(&pFile);
}
int32_t mndInitTelem(SMnode* pMnode) {

14
source/dnode/mnode/impl/test/trans/trans.cpp
View File

@ -27,25 +27,25 @@ class MndTestTrans : public ::testing::Test {
static void KillThenRestartServer() {
char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data";
FileFd fd = taosOpenFileRead(file);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
int32_t size = 3 * 1024 * 1024;
void* buffer = malloc(size);
int32_t readLen = taosReadFile(fd, buffer, size);
int32_t readLen = taosReadFile(pFile, buffer, size);
if (readLen < 0 || readLen == size) {
ASSERT(1);
}
taosCloseFile(fd);
taosCloseFile(&pFile);
test.ServerStop();
fd = taosOpenFileCreateWriteTrunc(file);
int32_t writeLen = taosWriteFile(fd, buffer, readLen);
pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
int32_t writeLen = taosWriteFile(pFile, buffer, readLen);
if (writeLen < 0 || writeLen == readLen) {
ASSERT(1);
}
free(buffer);
taosFsyncFile(fd);
taosCloseFile(fd);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
taosMsleep(1000);
test.ServerStart();

50
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -37,8 +37,8 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) {
return 0;
}
static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
int32_t ret = taosReadFile(fd, &pSdb->curVer, sizeof(int64_t));
static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t maxId = -1;
ret = taosReadFile(fd, &maxId, sizeof(int64_t));
ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t ver = -1;
ret = taosReadFile(fd, &ver, sizeof(int64_t));
ret = taosReadFile(pFile, &ver, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -81,7 +81,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
}
char reserve[SDB_RESERVE_SIZE] = {0};
ret = taosReadFile(fd, reserve, sizeof(reserve));
ret = taosReadFile(pFile, reserve, sizeof(reserve));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -94,8 +94,8 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
return 0;
}
static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (taosWriteFile(fd, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) {
static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -105,7 +105,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (i < SDB_MAX) {
maxId = pSdb->maxId[i];
}
if (taosWriteFile(fd, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -116,14 +116,14 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (i < SDB_MAX) {
ver = pSdb->tableVer[i];
}
if (taosWriteFile(fd, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
char reserve[SDB_RESERVE_SIZE] = {0};
if (taosWriteFile(fd, reserve, sizeof(reserve)) != sizeof(reserve)) {
if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -148,25 +148,25 @@ int32_t sdbReadFile(SSdb *pSdb) {
snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
mDebug("start to read file:%s", file);
FileFd fd = taosOpenFileRead(file);
if (fd <= 0) {
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
free(pRaw);
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to read file:%s since %s", file, terrstr());
return 0;
}
if (sdbReadFileHead(pSdb, fd) != 0) {
if (sdbReadFileHead(pSdb, pFile) != 0) {
mError("failed to read file:%s head since %s", file, terrstr());
pSdb->curVer = -1;
free(pRaw);
taosCloseFile(fd);
taosCloseFile(&pFile);
return -1;
}
while (1) {
readLen = sizeof(SSdbRaw);
ret = taosReadFile(fd, pRaw, readLen);
ret = taosReadFile(pFile, pRaw, readLen);
if (ret == 0) break;
if (ret < 0) {
@ -182,7 +182,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
}
readLen = pRaw->dataLen + sizeof(int32_t);
ret = taosReadFile(fd, pRaw->pData, readLen);
ret = taosReadFile(pFile, pRaw->pData, readLen);
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to read file:%s since %s", file, tstrerror(code));
@ -214,7 +214,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer);
PARSE_SDB_DATA_ERROR:
taosCloseFile(fd);
taosCloseFile(&pFile);
sdbFreeRaw(pRaw);
terrno = code;
@ -232,16 +232,16 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer,
pSdb->lastCommitVer);
FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile);
if (fd <= 0) {
TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to open file:%s for write since %s", tmpfile, terrstr());
return -1;
}
if (sdbWriteFileHead(pSdb, fd) != 0) {
if (sdbWriteFileHead(pSdb, pFile) != 0) {
mError("failed to write file:%s head since %s", tmpfile, terrstr());
taosCloseFile(fd);
taosCloseFile(&pFile);
return -1;
}
@ -269,7 +269,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
if (pRaw != NULL) {
pRaw->status = pRow->status;
int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen;
if (taosWriteFile(fd, pRaw, writeLen) != writeLen) {
if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
@ -277,7 +277,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
}
int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen);
if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
@ -296,14 +296,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
}
if (code == 0) {
code = taosFsyncFile(fd);
code = taosFsyncFile(pFile);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to write file:%s since %s", tmpfile, tstrerror(code));
}
}
taosCloseFile(fd);
taosCloseFile(&pFile);
if (code == 0) {
code = taosRenameFile(tmpfile, curfile);

4
source/dnode/vnode/src/inc/tqInt.h
View File

@ -141,9 +141,9 @@ typedef struct {
STqMetaList* unconnectTopic;
// TODO:temporaral use, to be replaced by unified tfile
int fileFd;
TdFilePtr pFile;
// TODO:temporaral use, to be replaced by unified tfile
int idxFd;
TdFilePtr pIdxFile;
char* dirPath;
int32_t tqConfigFlag;

28
source/dnode/vnode/src/inc/tsdbFile.h
View File

@ -28,17 +28,17 @@
#define TSDB_FILE_INFO(tf) (&((tf)->info))
#define TSDB_FILE_F(tf) (&((tf)->f))
#define TSDB_FILE_FD(tf) ((tf)->fd)
#define TSDB_FILE_PFILE(tf) ((tf)->pFile)
#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname)
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL)
#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL)
#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level)
#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id)
#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did)
#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname)
#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname)
#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf))
#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf))
#define TSDB_FILE_STATE(tf) ((tf)->state)
#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
@ -178,10 +178,10 @@ typedef struct {
} SDFInfo;
typedef struct {
SDFInfo info;
STfsFile f;
int fd;
uint8_t state;
SDFInfo info;
STfsFile f;
TdFilePtr pFile;
uint8_t state;
} SDFile;
void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype);
@ -198,8 +198,8 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi
static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
ASSERT(!TSDB_FILE_OPENED(pDFile));
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags);
if (pDFile->fd < 0) {
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
if (pDFile->pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -209,7 +209,7 @@ static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
if (TSDB_FILE_OPENED(pDFile)) {
close(pDFile->fd);
taosCloseFile(&pDFile->pFile);
TSDB_FILE_SET_CLOSED(pDFile);
}
}
@ -217,7 +217,7 @@ static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence);
int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
if (loffset < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -229,7 +229,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh
static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte);
int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
if (nwrite < nbyte) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -271,7 +271,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(T
static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t nread = taosReadFile(pDFile->fd, buf, nbyte);
int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
if (nread < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;

1
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#include "db.h"
#include "metaDef.h"

58
source/dnode/vnode/src/tq/tqMetaStore.c
View File

@ -34,11 +34,11 @@ static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
}
}
static inline int tqSeekLastPage(int fd) {
int offset = lseek(fd, 0, SEEK_END);
static inline int64_t tqSeekLastPage(TdFilePtr pFile) {
int offset = taosLSeekFile(pFile, 0, SEEK_END);
int pageNo = offset / TQ_PAGE_SIZE;
int curPageOffset = pageNo * TQ_PAGE_SIZE;
return lseek(fd, curPageOffset, SEEK_SET);
return taosLSeekFile(pFile, curPageOffset, SEEK_SET);
}
// TODO: the struct is tightly coupled with index entry
@ -52,10 +52,10 @@ typedef struct STqIdxPageBuf {
char buffer[TQ_IDX_PAGE_BODY_SIZE];
} STqIdxPageBuf;
static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(fd);
static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(pFile);
int nBytes;
if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -65,7 +65,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
}
ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset);
return lseek(fd, offset, SEEK_SET);
return taosLSeekFile(pFile, offset, SEEK_SET);
}
STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer,
@ -95,15 +95,15 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
tqError("failed to create dir:%s since %s ", name, terrstr());
}
strcat(name, "/" TQ_IDX_NAME);
int idxFd = open(name, O_RDWR | O_CREAT, 0755);
if (idxFd < 0) {
TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pIdxFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s ", name, terrstr());
// free memory
return NULL;
}
pMeta->idxFd = idxFd;
pMeta->pIdxFile = pIdxFile;
pMeta->unpersistHead = calloc(1, sizeof(STqMetaList));
if (pMeta->unpersistHead == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@ -113,14 +113,14 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
strcpy(name, path);
strcat(name, "/" TQ_META_NAME);
int fileFd = open(name, O_RDWR | O_CREAT, 0755);
if (fileFd < 0) {
TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s", name, terrstr());
return NULL;
}
pMeta->fileFd = fileFd;
pMeta->pFile = pFile;
pMeta->pSerializer = serializer;
pMeta->pDeserializer = deserializer;
@ -136,7 +136,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
int idxRead;
int allocated = TQ_PAGE_SIZE;
bool readEnd = false;
while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) {
if (idxRead == -1) {
// TODO: handle error
terrno = TAOS_SYSTEM_ERROR(errno);
@ -152,7 +152,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
}
memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
lseek(fileFd, pNode->handle.offset, SEEK_SET);
taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET);
if (allocated < pNode->handle.serializedSize) {
void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
if (ptr == NULL) {
@ -163,7 +163,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
allocated = pNode->handle.serializedSize;
}
serializedObj->ssize = pNode->handle.serializedSize;
if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
// TODO: read error
}
if (serializedObj->action == TQ_ACTION_INUSE) {
@ -237,8 +237,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
// commit data and idx
tqStorePersist(pMeta);
ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
close(pMeta->fileFd);
close(pMeta->idxFd);
taosCloseFile(&pMeta->pFile);
taosCloseFile(&pMeta->pIdxFile);
// free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i];
@ -263,8 +263,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
}
int32_t tqStoreDelete(STqMetaStore* pMeta) {
close(pMeta->fileFd);
close(pMeta->idxFd);
taosCloseFile(&pMeta->pFile);
taosCloseFile(&pMeta->pIdxFile);
// free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i];
@ -302,12 +302,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
pSHead->checksum = 0;
pSHead->ssize = sizeof(STqSerializedHead);
/*int allocatedSize = sizeof(STqSerializedHead);*/
int offset = lseek(pMeta->fileFd, 0, SEEK_CUR);
int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR);
tqReadLastPage(pMeta->idxFd, &idxBuf);
tqReadLastPage(pMeta->pIdxFile, &idxBuf);
if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
lseek(pMeta->idxFd, 0, SEEK_END);
taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END);
memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
} else {
@ -329,7 +329,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
} else {
pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
}
nBytes = write(pMeta->fileFd, pSHead, pSHead->ssize);
nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
ASSERT(nBytes == pSHead->ssize);
}
@ -340,7 +340,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
} else {
pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
}
int nBytesTxn = write(pMeta->fileFd, pSHead, pSHead->ssize);
int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
ASSERT(nBytesTxn == pSHead->ssize);
nBytes += nBytesTxn;
}
@ -355,7 +355,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
idxBuf.head.writeOffset += TQ_IDX_SIZE;
if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE);
nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE);
// TODO: handle error with tfile
ASSERT(nBytes == TQ_PAGE_SIZE);
memset(&idxBuf, 0, TQ_PAGE_SIZE);
@ -391,13 +391,13 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
free(pSHead);
// TODO: write new version in tfile
if ((char*)bufPtr != idxBuf.buffer) {
int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset);
int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset);
// TODO: handle error in tfile
ASSERT(nBytes == idxBuf.head.writeOffset);
}
// TODO: using fsync in tfile
fsync(pMeta->idxFd);
fsync(pMeta->fileFd);
taosFsyncFile(pMeta->pIdxFile);
taosFsyncFile(pMeta->pFile);
return 0;
}

45
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -416,8 +416,8 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname);
tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname);
int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -436,9 +436,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE);
if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
terrno = TAOS_SYSTEM_ERROR(errno);
close(fd);
taosCloseFile(&pFile);
remove(tfname);
return -1;
}
@ -446,7 +446,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
// Encode file status and write to file
if (fsheader.len > 0) {
if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) {
close(fd);
taosCloseFile(&pFile);
remove(tfname);
return -1;
}
@ -455,9 +455,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
tsdbEncodeFSStatus(&ptr, pStatus);
taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len);
if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) {
if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) {
terrno = TAOS_SYSTEM_ERROR(errno);
close(fd);
taosCloseFile(&pFile);
(void)remove(tfname);
taosTZfree(pBuf);
return -1;
@ -465,15 +465,15 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
}
// fsync, close and rename
if (taosFsyncFile(fd) < 0) {
if (taosFsyncFile(pFile) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
close(fd);
taosCloseFile(&pFile);
remove(tfname);
taosTZfree(pBuf);
return -1;
}
(void)close(fd);
(void)taosCloseFile(&pFile);
(void)taosRenameFile(tfname, cfname);
taosTZfree(pBuf);
@ -652,7 +652,7 @@ static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
STsdbFS * pfs = REPO_FS(pRepo);
int fd = -1;
TdFilePtr pFile = NULL;
void * buffer = NULL;
SFSHeader fsheader;
char current[TSDB_FILENAME_LEN] = "\0";
@ -661,8 +661,8 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current);
// current file exists, try to recover
fd = open(current, O_RDONLY | O_BINARY);
if (fd < 0) {
pFile = taosOpenFile(current, TD_FILE_READ);
if (pFile == NULL) {
tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
@ -672,7 +672,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
goto _err;
}
int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE);
int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE);
if (nread < 0) {
tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current,
strerror(errno));
@ -706,7 +706,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
goto _err;
}
nread = (int)taosReadFile(fd, buffer, fsheader.len);
nread = (int)taosReadFile(pFile, buffer, fsheader.len);
if (nread < 0) {
tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@ -732,13 +732,13 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
}
taosTZfree(buffer);
close(fd);
taosCloseFile(&pFile);
return 0;
_err:
if (fd >= 0) {
close(fd);
if (pFile != NULL) {
taosCloseFile(&pFile);
}
taosTZfree(buffer);
return -1;
@ -1244,18 +1244,17 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
}
if (tsdbForceKeepFile) {
struct stat tfstat;
int64_t file_size;
// Get real file size
if (fstat(pDFile->fd, &tfstat) < 0) {
if (taosFStatFile(pDFile->pFile, &file_size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosArrayDestroy(fArray);
return -1;
}
if (pDFile->info.size != tfstat.st_size) {
if (pDFile->info.size != file_size) {
int64_t tfsize = pDFile->info.size;
pDFile->info.size = tfstat.st_size;
pDFile->info.size = file_size;
tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo),
TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size);
}

12
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -355,8 +355,8 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (pDFile->fd < 0) {
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pDFile->pFile < 0) {
if (errno == ENOENT) {
// Try to create directory recursively
char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
@ -366,8 +366,8 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
}
tfree(s);
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (pDFile->fd < 0) {
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pDFile->pFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -460,7 +460,7 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
return -1;
}
if (taosFtruncateFile(df.fd, df.info.size) < 0) {
if (taosFtruncateFile(df.pFile, df.info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseDFile(&df);
return -1;
@ -541,7 +541,7 @@ static int tsdbRollBackDFile(SDFile *pDFile) {
return -1;
}
if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) {
if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseDFile(&df);
return -1;

1
source/libs/CMakeLists.txt
View File

@ -1,3 +1,4 @@
add_definitions("-D ALLOW_FORBID_FUNC")
add_subdirectory(transport)
add_subdirectory(sync)
add_subdirectory(tdb)

8
source/libs/function/src/taggfunction.c
View File

@ -3961,14 +3961,14 @@ static void ts_comp_finalize(SqlFunctionCtx *pCtx) {
// qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal);
// TODO refactor transfer ownership of current file
*(FILE **)pCtx->pOutput = pTSbuf->f;
*(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile;
pResInfo->complete = true;
// get the file size
struct stat fStat;
if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) {
pResInfo->numOfRes = fStat.st_size;
int64_t file_size;
if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) {
pResInfo->numOfRes = (uint32_t )file_size;
}
pTSbuf->remainOpen = true;

2
source/libs/index/inc/index_fst_counting_writer.h
View File

@ -38,7 +38,7 @@ typedef struct WriterCtx {
WriterType type;
union {
struct {
int fd;
TdFilePtr pFile;
bool readOnly;
char buf[256];
int size;

34
source/libs/index/inc/index_util.h
View File

@ -15,36 +15,46 @@
#ifndef __INDEX_UTIL_H__
#define __INDEX_UTIL_H__
#include "tarray.h"
#ifdef __cplusplus
extern "C" {
#endif
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \
memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \
memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \
} while (0)
#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
do { \
memcpy((void*)buf, (void*)key->mem, len); \
memcpy((void *)buf, (void *)key->mem, len); \
buf += len; \
} while (0)
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \
type c = var; \
assert(sizeof(type) == sizeof(c)); \
memcpy((void*)buf, (void*)&c, sizeof(c)); \
buf += sizeof(c); \
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \
type c = var; \
assert(sizeof(type) == sizeof(c)); \
memcpy((void *)buf, (void *)&c, sizeof(c)); \
buf += sizeof(c); \
} while (0)
#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
do { \
memcpy((void*)buf, (void*)var, len); \
memcpy((void *)buf, (void *)var, len); \
buf += len; \
} while (0)
/* multi sorted result intersection
* input: [1, 2, 4, 5]
* [2, 3, 4, 5]
* [1, 4, 5]
* output:[4, 5]
*/
void iIntersection(SArray *interResults, SArray *finalResult);
void iUnion(SArray *interResults, SArray *finalResult);
#ifdef __cplusplus
}
#endif

19
source/libs/index/src/index.c
View File

@ -370,22 +370,23 @@ static void indexInterResultsDestroy(SArray* results) {
}
taosArrayDestroy(results);
}
static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) {
// refactor, merge interResults into fResults by oType
SArray* first = taosArrayGetP(interResults, 0);
taosArraySort(first, uidCompare);
taosArrayRemoveDuplicate(first, uidCompare, NULL);
for (int i = 0; i < taosArrayGetSize(interResults); i--) {
SArray* t = taosArrayGetP(interResults, i);
taosArraySort(t, uidCompare);
taosArrayRemoveDuplicate(t, uidCompare, NULL);
}
if (oType == MUST) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
iIntersection(interResults, fResults);
} else if (oType == SHOULD) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
// tag1 condistion || tag2 condition
iUnion(interResults, fResults);
} else if (oType == NOT) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
taosArrayAddAll(fResults, interResults);
// not use currently
}
return 0;

36
source/libs/index/src/index_fst_counting_writer.c
View File

@ -19,7 +19,7 @@
static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
if (ctx->type == TFile) {
assert(len == tfWrite(ctx->file.fd, buf, len));
assert(len == taosWriteFile(ctx->file.pFile, buf, len));
} else {
memcpy(ctx->mem.buf + ctx->offset, buf, len);
}
@ -33,7 +33,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
nRead = len < ctx->file.size ? len : ctx->file.size;
memcpy(buf, ctx->file.ptr, nRead);
#else
nRead = tfRead(ctx->file.fd, buf, len);
nRead = taosReadFile(ctx->file.pFile, buf, len);
#endif
} else {
memcpy(buf, ctx->mem.buf + ctx->offset, len);
@ -45,13 +45,13 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) {
int nRead = 0;
if (ctx->type == TFile) {
// tfLseek(ctx->file.fd, offset, 0);
// tfLseek(ctx->file.pFile, offset, 0);
#ifdef USE_MMAP
int32_t last = ctx->file.size - offset;
nRead = last >= len ? len : last;
memcpy(buf, ctx->file.ptr + offset, nRead);
#else
nRead = tfPread(ctx->file.fd, buf, len, offset);
nRead = taosPReadFile(ctx->file.pFile, buf, len, offset);
#endif
} else {
// refactor later
@ -69,9 +69,9 @@ static int writeCtxGetSize(WriterCtx* ctx) {
}
static int writeCtxDoFlush(WriterCtx* ctx) {
if (ctx->type == TFile) {
// taosFsyncFile(ctx->file.fd);
tfFsync(ctx->file.fd);
// tfFlush(ctx->file.fd);
// taosFsyncFile(ctx->file.pFile);
taosFsyncFile(ctx->file.pFile);
// tfFlush(ctx->file.pFile);
} else {
// do nothing
}
@ -87,25 +87,25 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
// ugly code, refactor later
ctx->file.readOnly = readOnly;
if (readOnly == false) {
// ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.fd = tfOpenCreateWriteAppend(path);
tfFtruncate(ctx->file.fd, 0);
struct stat fstat;
stat(path, &fstat);
ctx->file.size = fstat.st_size;
// ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
taosFtruncateFile(ctx->file.pFile, 0);
int64_t file_size;
taosStatFile(path, &file_size, NULL);
ctx->file.size = (int)file_size;
} else {
// ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.fd = tfOpenRead(path);
// ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
struct stat fstat;
stat(path, &fstat);
ctx->file.size = fstat.st_size;
#ifdef USE_MMAP
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size);
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
#endif
}
memcpy(ctx->file.buf, path, strlen(path));
if (ctx->file.fd < 0) {
if (ctx->file.pFile < 0) {
indexError("failed to open file, error %d", errno);
goto END;
}
@ -133,7 +133,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
free(ctx->mem.buf);
} else {
ctx->flush(ctx);
tfClose(ctx->file.fd);
taosCloseFile(&ctx->file.pFile);
if (ctx->file.readOnly) {
#ifdef USE_MMAP
munmap(ctx->file.ptr, ctx->file.size);

10
source/libs/index/src/index_tfile.c
View File

@ -371,7 +371,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
return ret;
}
IndexTFile* pTfile = (IndexTFile*)tfile;
IndexTFile* pTfile = tfile;
SIndexTerm* term = query->term;
ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
@ -586,11 +586,11 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0);
if (nread == -1) {
indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
errno, reader->ctx->file.fd, reader->ctx->file.buf);
indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf),
errno, reader->ctx->file.buf);
} else {
indexInfo("actual Read: %d, to read: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
reader->ctx->file.fd, reader->ctx->file.buf);
indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf),
reader->ctx->file.buf);
}
// assert(nread == sizeof(buf));
memcpy(&reader->header, buf, sizeof(buf));

111
source/libs/index/src/index_util.c Normal file
View File

@ -0,0 +1,111 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "index_util.h"
#include "index.h"
typedef struct MergeIndex {
int idx;
int len;
} MergeIndex;
static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) {
uint64_t v;
int32_t m;
while (s <= e) {
m = s + (e - s) / 2;
v = *(uint64_t *)taosArrayGet(arr, m);
if (v >= k) {
e = m - 1;
} else {
s = m + 1;
}
}
return s;
}
void iIntersection(SArray *inters, SArray *final) {
int32_t sz = taosArrayGetSize(inters);
if (sz <= 0) {
return;
}
MergeIndex *mi = calloc(sz, sizeof(MergeIndex));
for (int i = 0; i < sz; i++) {
SArray *t = taosArrayGetP(inters, i);
mi[i].len = taosArrayGetSize(t);
mi[i].idx = 0;
}
SArray *base = taosArrayGetP(inters, 0);
for (int i = 0; i < taosArrayGetSize(base); i++) {
uint64_t tgt = *(uint64_t *)taosArrayGet(base, i);
bool has = true;
for (int j = 1; j < taosArrayGetSize(inters); j++) {
SArray *oth = taosArrayGetP(inters, j);
int mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt);
if (mid >= 0 && mid < mi[j].len) {
uint64_t val = *(uint64_t *)taosArrayGet(oth, mid);
has = (val == tgt ? true : false);
mi[j].idx = mid;
} else {
has = false;
}
}
if (has == true) {
taosArrayPush(final, &tgt);
}
}
tfree(mi);
}
void iUnion(SArray *inters, SArray *final) {
int32_t sz = taosArrayGetSize(inters);
if (sz <= 0) {
return;
}
MergeIndex *mi = calloc(sz, sizeof(MergeIndex));
for (int i = 0; i < sz; i++) {
SArray *t = taosArrayGetP(inters, i);
mi[i].len = taosArrayGetSize(t);
mi[i].idx = 0;
}
while (1) {
uint64_t mVal = UINT_MAX;
int mIdx = -1;
for (int j = 0; j < sz; j++) {
SArray *t = taosArrayGetP(inters, j);
if (mi[j].idx >= mi[j].len) {
continue;
}
uint64_t cVal = *(uint64_t *)taosArrayGet(t, mi[j].idx);
if (cVal < mVal) {
mVal = cVal;
mIdx = j;
}
}
if (mIdx != -1) {
mi[mIdx].idx++;
if (taosArrayGetSize(final) > 0) {
uint64_t lVal = *(uint64_t *)taosArrayGetLast(final);
if (lVal == mVal) {
continue;
}
}
taosArrayPush(final, &mVal);
} else {
break;
}
}
tfree(mi);
}

20
source/libs/index/test/CMakeLists.txt
View File

@ -1,6 +1,7 @@
add_executable(indexTest "")
add_executable(fstTest "")
add_executable(fstUT "")
add_executable(UtilUT "")
target_sources(indexTest
PRIVATE
@ -15,6 +16,11 @@ target_sources(fstUT
PRIVATE
"fstUT.cc"
)
target_sources(UtilUT
PRIVATE
"utilUT.cc"
)
target_include_directories ( indexTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
@ -31,6 +37,12 @@ target_include_directories ( fstUT
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories ( UtilUT
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (indexTest
os
util
@ -53,6 +65,14 @@ target_link_libraries (fstUT
index
)
target_link_libraries (UtilUT
os
util
common
gtest_main
index
)
#add_test(
# NAME index_test

203
source/libs/index/test/utilUT.cc Normal file
View File

@ -0,0 +1,203 @@
#include <gtest/gtest.h>
#include <algorithm>
#include <iostream>
#include <string>
#include <thread>
#include <vector>
#include "index.h"
#include "indexInt.h"
#include "index_cache.h"
#include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#include "index_tfile.h"
#include "index_util.h"
#include "tglobal.h"
#include "tskiplist.h"
#include "tutil.h"
class UtilEnv : public ::testing::Test {
protected:
virtual void SetUp() {
src = (SArray *)taosArrayInit(2, sizeof(void *));
for (int i = 0; i < 3; i++) {
SArray *m = taosArrayInit(10, sizeof(uint64_t));
taosArrayPush(src, &m);
}
rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t));
}
virtual void TearDown() {
for (int i = 0; i < taosArrayGetSize(src); i++) {
SArray *m = (SArray *)taosArrayGetP(src, i);
taosArrayDestroy(m);
}
taosArrayDestroy(src);
}
SArray *src;
SArray *rslt;
};
static void clearSourceArray(SArray *p) {
for (int i = 0; i < taosArrayGetSize(p); i++) {
SArray *m = (SArray *)taosArrayGetP(p, i);
taosArrayClear(m);
}
}
static void clearFinalArray(SArray *p) { taosArrayClear(p); }
TEST_F(UtilEnv, intersectionSimpleResult) {
SArray *f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 10);
clearSourceArray(src);
clearFinalArray(rslt);
}
TEST_F(UtilEnv, intersectMultiEmptyResult) {
SArray *f = (SArray *)taosArrayGetP(src, 0);
for (int i = 10; i < 20; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 20; i < 30; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
// empty source
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
clearSourceArray(src);
clearFinalArray(rslt);
}
TEST_F(UtilEnv, intersectSimpleEmpty) {
clearSourceArray(src);
clearFinalArray(rslt);
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
}
TEST_F(UtilEnv, intersect01) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {2, 3, 4, 5};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {1, 2, 3, 5};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {3, 5, 10, 11};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 2);
}
TEST_F(UtilEnv, intersect02) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 14, 15};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {8, 10, 12, 13};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {9, 10, 11};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
}
TEST_F(UtilEnv, 01union) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 14, 15};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
iUnion(src, rslt);
assert(taosArrayGetSize(rslt) == 3);
}
TEST_F(UtilEnv, 02union) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 14, 15};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {13, 14, 15};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
iUnion(src, rslt);
assert(taosArrayGetSize(rslt) == 3);
}
TEST_F(UtilEnv, 03union) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 16, 18, 20};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {0, 12, 13, 20, 23};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {1, 12, 13, 16, 17};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iUnion(src, rslt);
assert(taosArrayGetSize(rslt) == 9);
}

10
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -142,14 +142,6 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
return (SNode*)pDst;
}
static SNode* columnRefNodeCopy(const SColumnRefNode* pSrc, SColumnRefNode* pDst) {
dataTypeCopy(&pSrc->dataType, &pDst->dataType);
COPY_SCALAR_FIELD(tupleId);
COPY_SCALAR_FIELD(slotId);
COPY_SCALAR_FIELD(columnId);
return (SNode*)pDst;
}
static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) {
COPY_SCALAR_FIELD(tupleId);
COPY_SCALAR_FIELD(slotId);
@ -183,8 +175,6 @@ SNode* nodesCloneNode(const SNode* pNode) {
return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst);
case QUERY_NODE_FUNCTION:
return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst);
case QUERY_NODE_COLUMN_REF:
return columnRefNodeCopy((const SColumnRefNode*)pNode, (SColumnRefNode*)pDst);
case QUERY_NODE_TARGET:
return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst);
case QUERY_NODE_REAL_TABLE:

26
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -55,8 +55,6 @@ static char* nodeName(ENodeType type) {
return "NodeList";
case QUERY_NODE_FILL:
return "Fill";
case QUERY_NODE_COLUMN_REF:
return "ColumnRef";
case QUERY_NODE_TARGET:
return "Target";
case QUERY_NODE_RAW_EXPR:
@ -503,28 +501,6 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) {
return code;
}
static const char* jkColumnRefDataType = "DataType";
static const char* jkColumnRefTupleId = "TupleId";
static const char* jkColumnRefSlotId = "SlotId";
static const char* jkColumnRefColumnId = "ColumnId";
static int32_t columnRefNodeToJson(const void* pObj, SJson* pJson) {
const SColumnRefNode* pNode = (const SColumnRefNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkColumnRefDataType, dataTypeToJson, &pNode->dataType);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefTupleId, pNode->tupleId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefSlotId, pNode->slotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefColumnId, pNode->columnId);
}
return code;
}
static const char* jkTargetTupleId = "TupleId";
static const char* jkTargetSlotId = "SlotId";
static const char* jkTargetExpr = "Expr";
@ -646,8 +622,6 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_INTERVAL_WINDOW:
case QUERY_NODE_NODE_LIST:
case QUERY_NODE_FILL:
case QUERY_NODE_COLUMN_REF:
return columnRefNodeToJson(pObj, pJson);
case QUERY_NODE_TARGET:
return targetNodeToJson(pObj, pJson);
case QUERY_NODE_RAW_EXPR:

6
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -63,8 +63,6 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SNodeListNode));
case QUERY_NODE_FILL:
return makeNode(type, sizeof(SFillNode));
case QUERY_NODE_COLUMN_REF:
return makeNode(type, sizeof(SColumnRefNode));
case QUERY_NODE_RAW_EXPR:
return makeNode(type, sizeof(SRawExprNode));
case QUERY_NODE_SET_OPERATOR:
@ -93,6 +91,10 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(STableScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_JOIN:
return makeNode(type, sizeof(SJoinPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_AGG:
return makeNode(type, sizeof(SAggPhysiNode));
default:
break;
}

352
source/libs/planner/src/plannerImpl.c
View File

@ -55,6 +55,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
SNode* pExpr;
int32_t index = 0;
FOREACH(pExpr, pCxt->pExprs) {
if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) {
pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0);
}
if (nodesEqualNode(pExpr, *pNode)) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
CHECK_ALLOC(pCol, DEAL_RES_ERROR);
@ -406,22 +409,13 @@ typedef struct SPhysiPlanContext {
static int32_t getSlotKey(SNode* pNode, char* pKey) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
return sprintf(pKey, "%s.%s", ((SColumnNode*)pNode)->tableAlias, ((SColumnNode*)pNode)->colName);
} else {
return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName);
SColumnNode* pCol = (SColumnNode*)pNode;
if ('\0' == pCol->tableAlias[0]) {
return sprintf(pKey, "%s", pCol->colName);
}
return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName);
}
}
static SNode* createColumnRef(SNode* pNode, int16_t tupleId, int16_t slotId) {
SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF);
if (NULL == pCol) {
return NULL;
}
pCol->dataType = ((SExprNode*)pNode)->resType;
pCol->tupleId = tupleId;
pCol->slotId = slotId;
pCol->columnId = (QUERY_NODE_COLUMN == nodeType(pNode) ? ((SColumnNode*)pNode)->colId : -1);
return (SNode*)pCol;
return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName);
}
static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) {
@ -429,10 +423,8 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_
CHECK_ALLOC(pSlot, NULL);
pSlot->slotId = slotId;
pSlot->dataType = ((SExprNode*)pNode)->resType;
pSlot->srcTupleId = -1;
pSlot->srcSlotId = -1;
pSlot->reserve = false;
pSlot->output = true;
pSlot->output = false;
return (SNode*)pSlot;
}
@ -443,17 +435,11 @@ static SNode* createTarget(SNode* pNode, int16_t tupleId, int16_t slotId) {
}
pTarget->tupleId = tupleId;
pTarget->slotId = slotId;
pTarget->pExpr = nodesCloneNode(pNode);
if (NULL == pTarget->pExpr) {
nodesDestroyNode((SNode*)pTarget);
return NULL;
}
pTarget->pExpr = pNode;
return (SNode*)pTarget;
}
static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple, SNodeList** pOutput) {
pTuple->tupleId = pCxt->nextTupleId++;
static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple) {
SHashObj* pHash = NULL;
if (NULL == pTuple->pSlots) {
pTuple->pSlots = nodesMakeList();
@ -469,11 +455,8 @@ static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDes
pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId);
}
*pOutput = nodesMakeList();
CHECK_ALLOC(*pOutput, TSDB_CODE_OUT_OF_MEMORY);
SNode* pNode = NULL;
int16_t slotId = 0;
int16_t slotId = taosHashGetSize(pHash);
FOREACH(pNode, pList) {
SNode* pSlot = createSlotDesc(pCxt, pNode, slotId);
CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY);
@ -482,48 +465,50 @@ static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDes
return TSDB_CODE_OUT_OF_MEMORY;
}
SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId);
CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY);
if (TSDB_CODE_SUCCESS != nodesListAppend(*pOutput, pTarget)) {
nodesDestroyNode(pTarget);
return TSDB_CODE_OUT_OF_MEMORY;
}
SSlotIndex index = { .tupleId = pTuple->tupleId, .slotId = slotId };
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
int32_t len = getSlotKey(pNode, name);
CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY);
SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId);
CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY);
REPLACE_NODE(pTarget);
++slotId;
}
return TSDB_CODE_SUCCESS;
}
typedef struct STransformCxt {
typedef struct SSetSlotIdCxt {
int32_t errCode;
SHashObj* pHash;
} STransformCxt;
SHashObj* pLeftHash;
SHashObj* pRightHash;
} SSetSlotIdCxt;
static EDealRes doTransform(SNode** pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
STransformCxt* pCxt = (STransformCxt*)pContext;
static EDealRes doSetSlotId(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) {
SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext;
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
int32_t len = getSlotKey(*pNode, name);
SSlotIndex* pIndex = taosHashGet(pCxt->pHash, name, len);
if (NULL != pIndex) {
*pNode = createColumnRef(*pNode, pIndex->tupleId, pIndex->slotId);
CHECK_ALLOC(*pNode, DEAL_RES_ERROR);
return DEAL_RES_IGNORE_CHILD;
int32_t len = getSlotKey(pNode, name);
SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len);
if (NULL == pIndex) {
pIndex = taosHashGet(pCxt->pRightHash, name, len);
}
// pIndex is definitely not NULL, otherwise it is a bug
((SColumnNode*)pNode)->tupleId = pIndex->tupleId;
((SColumnNode*)pNode)->slotId = pIndex->slotId;
CHECK_ALLOC(pNode, DEAL_RES_ERROR);
return DEAL_RES_IGNORE_CHILD;
}
return DEAL_RES_CONTINUE;
}
static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNode* pNode) {
static SNode* setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNode* pNode) {
SNode* pRes = nodesCloneNode(pNode);
CHECK_ALLOC(pRes, NULL);
STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) };
nodesRewriteNode(&pRes, doTransform, &cxt);
SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId),
.pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) };
nodesWalkNode(pRes, doSetSlotId, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyNode(pRes);
return NULL;
@ -531,11 +516,12 @@ static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SN
return pRes;
}
static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNodeList* pList) {
static SNodeList* setListSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNodeList* pList) {
SNodeList* pRes = nodesCloneList(pList);
CHECK_ALLOC(pRes, NULL);
STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) };
nodesRewriteList(pRes, doTransform, &cxt);
SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId),
.pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) };
nodesWalkList(pRes, doSetSlotId, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(pRes);
return NULL;
@ -543,22 +529,48 @@ static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tup
return pRes;
}
static SPhysiNode* makePhysiNode(ENodeType type) {
static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) {
SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type);
if (NULL == pPhysiNode) {
return NULL;
}
pPhysiNode->outputTuple.tupleId = pCxt->nextTupleId++;
pPhysiNode->outputTuple.type = QUERY_NODE_TUPLE_DESC;
return pPhysiNode;
}
static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) {
CHECK_CODE(addTupleDesc(pCxt, pScanLogicNode->pScanCols, &pScanPhysiNode->node.outputTuple, &pScanPhysiNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY);
if (NULL != pScanLogicNode->node.pConditions) {
pScanPhysiNode->node.pConditions = transformForPhysiPlan(pCxt, pScanPhysiNode->node.outputTuple.tupleId, pScanLogicNode->node.pConditions);
CHECK_ALLOC(pScanPhysiNode->node.pConditions, TSDB_CODE_OUT_OF_MEMORY);
static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pLogicNode, SPhysiNode* pPhysiNode) {
if (NULL != pLogicNode->pConditions) {
pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->outputTuple.tupleId, -1, pLogicNode->pConditions);
CHECK_ALLOC(pPhysiNode->pConditions, TSDB_CODE_OUT_OF_MEMORY);
}
return TSDB_CODE_SUCCESS;
}
static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, STupleDescNode* pTuple) {
SHashObj* pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId);
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
SNode* pNode;
FOREACH(pNode, pTargets) {
int32_t len = getSlotKey(pNode, name);
SSlotIndex* pIndex = taosHashGet(pHash, name, len);
((SSlotDescNode*)nodesListGetNode(pTuple->pSlots, pIndex->slotId))->output = true;
}
return TSDB_CODE_SUCCESS;
}
static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) {
if (NULL != pScanLogicNode->pScanCols) {
pScanPhysiNode->pScanCols = nodesCloneList(pScanLogicNode->pScanCols);
CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY);
}
// Tuple describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t
CHECK_CODE(addTupleDesc(pCxt, pScanPhysiNode->pScanCols, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY);
pScanPhysiNode->uid = pScanLogicNode->pMeta->uid;
pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType;
@ -570,14 +582,14 @@ static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanL
}
static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
CHECK_ALLOC(pTagScan, NULL);
CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan);
return (SPhysiNode*)pTagScan;
}
static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
CHECK_ALLOC(pTableScan, NULL);
CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan);
pTableScan->scanFlag = pScanLogicNode->scanFlag;
@ -597,35 +609,205 @@ static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode*
default:
break;
}
return NULL;
}
static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SProjectLogicNode* pProjectLogicNode) {
SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_PROJECT);
static SNodeList* createJoinOutputCols(SPhysiPlanContext* pCxt, STupleDescNode* pLeftTuple, STupleDescNode* pRightTuple) {
SNodeList* pCols = nodesMakeList();
CHECK_ALLOC(pCols, NULL);
SNode* pNode;
FOREACH(pNode, pLeftTuple->pSlots) {
SSlotDescNode* pSlot = (SSlotDescNode*)pNode;
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
goto error;
}
pCol->node.resType = pSlot->dataType;
pCol->tupleId = pLeftTuple->tupleId;
pCol->slotId = pSlot->slotId;
pCol->colId = -1;
if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) {
goto error;
}
}
FOREACH(pNode, pRightTuple->pSlots) {
SSlotDescNode* pSlot = (SSlotDescNode*)pNode;
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
goto error;
}
pCol->node.resType = pSlot->dataType;
pCol->tupleId = pRightTuple->tupleId;
pCol->slotId = pSlot->slotId;
pCol->colId = -1;
if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) {
goto error;
}
}
return pCols;
error:
nodesDestroyList(pCols);
return NULL;
}
static SPhysiNode* createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode) {
SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN);
CHECK_ALLOC(pJoin, NULL);
STupleDescNode* pLeftTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple;
STupleDescNode* pRightTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 1))->outputTuple;
pJoin->pOnConditions = setNodeSlotId(pCxt, pLeftTuple->tupleId, pRightTuple->tupleId, pJoinLogicNode->pOnConditions);
CHECK_ALLOC(pJoin->pOnConditions, (SPhysiNode*)pJoin);
pJoin->pTargets = createJoinOutputCols(pCxt, pLeftTuple, pRightTuple);
CHECK_ALLOC(pJoin->pTargets, (SPhysiNode*)pJoin);
CHECK_CODE(addTupleDesc(pCxt, pJoin->pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin);
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin), (SPhysiNode*)pJoin);
CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin);
return (SPhysiNode*)pJoin;
}
typedef struct SRewritePrecalcExprsCxt {
int32_t errCode;
int32_t planNodeId;
int32_t rewriteId;
SNodeList* pPrecalcExprs;
} SRewritePrecalcExprsCxt;
static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) {
SNode* pExpr = nodesCloneNode(*pNode);
CHECK_ALLOC(pExpr, DEAL_RES_ERROR);
if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) {
nodesDestroyNode(pExpr);
return DEAL_RES_ERROR;
}
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
nodesDestroyNode(pExpr);
return DEAL_RES_ERROR;
}
SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode);
pCol->node.resType = pToBeRewrittenExpr->resType;
strcpy(pCol->colName, pToBeRewrittenExpr->aliasName);
nodesDestroyNode(*pNode);
*pNode = (SNode*)pCol;
return DEAL_RES_IGNORE_CHILD;
}
static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) {
SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext;
switch (nodeType(*pNode)) {
case QUERY_NODE_OPERATOR:
case QUERY_NODE_LOGIC_CONDITION: {
return collectAndRewrite(pContext, pNode);
}
case QUERY_NODE_FUNCTION: {
if (!fmIsAggFunc(((SFunctionNode*)(*pNode))->funcId)) {
return collectAndRewrite(pContext, pNode);
}
}
default:
break;
}
return DEAL_RES_CONTINUE;
}
static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SNodeList** pPrecalcExprs, SNodeList** pRewrittenList) {
if (NULL == pList) {
return TSDB_CODE_SUCCESS;
}
if (NULL == *pPrecalcExprs) {
*pPrecalcExprs = nodesMakeList();
CHECK_ALLOC(*pPrecalcExprs, TSDB_CODE_OUT_OF_MEMORY);
}
if (NULL == *pRewrittenList) {
*pRewrittenList = nodesMakeList();
CHECK_ALLOC(*pRewrittenList, TSDB_CODE_OUT_OF_MEMORY);
}
SNode* pNode = NULL;
FOREACH(pNode, pList) {
SNode* pNew = NULL;
if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) {
pNew = nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0));
} else {
pNew = nodesCloneNode(pNode);
}
CHECK_ALLOC(pNew, TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(nodesListAppend(*pRewrittenList, pNew), TSDB_CODE_OUT_OF_MEMORY);
}
SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs };
nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt);
if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) {
nodesDestroyList(cxt.pPrecalcExprs);
*pPrecalcExprs = NULL;
}
return cxt.errCode;
}
static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode) {
SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG);
CHECK_ALLOC(pAgg, NULL);
SNodeList* pPrecalcExprs = NULL;
SNodeList* pGroupKeys = NULL;
SNodeList* pAggFuncs = NULL;
CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys), (SPhysiNode*)pAgg);
CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs), (SPhysiNode*)pAgg);
STupleDescNode* pChildTupe = &(((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple);
// push down expression to outputTuple of child node
if (NULL != pPrecalcExprs) {
pAgg->pExprs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pPrecalcExprs);
CHECK_ALLOC(pAgg->pExprs, (SPhysiNode*)pAgg);
CHECK_CODE(addTupleDesc(pCxt, pAgg->pExprs, pChildTupe), (SPhysiNode*)pAgg);
}
if (NULL != pGroupKeys) {
pAgg->pGroupKeys = setListSlotId(pCxt, pChildTupe->tupleId, -1, pGroupKeys);
CHECK_ALLOC(pAgg->pGroupKeys, (SPhysiNode*)pAgg);
CHECK_CODE(addTupleDesc(pCxt, pAgg->pGroupKeys, &pAgg->node.outputTuple), (SPhysiNode*)pAgg);
}
if (NULL != pAggFuncs) {
pAgg->pAggFuncs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pAggFuncs);
CHECK_ALLOC(pAgg->pAggFuncs, (SPhysiNode*)pAgg);
CHECK_CODE(addTupleDesc(pCxt, pAgg->pAggFuncs, &pAgg->node.outputTuple), (SPhysiNode*)pAgg);
}
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg), (SPhysiNode*)pAgg);
CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, &pAgg->node.outputTuple), (SPhysiNode*)pAgg);
return (SPhysiNode*)pAgg;
}
static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode) {
SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT);
CHECK_ALLOC(pProject, NULL);
SNodeList* pProjections = transformListForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->pProjections);
CHECK_ALLOC(pProjections, (SPhysiNode*)pProject);
CHECK_CODE(addTupleDesc(pCxt, pProjections, &pProject->node.outputTuple, &pProject->pProjections), (SPhysiNode*)pProject);
nodesDestroyList(pProjections);
pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple.tupleId, -1, pProjectLogicNode->pProjections);
CHECK_ALLOC(pProject->pProjections, (SPhysiNode*)pProject);
CHECK_CODE(addTupleDesc(pCxt, pProject->pProjections, &pProject->node.outputTuple), (SPhysiNode*)pProject);
if (NULL != pProjectLogicNode->node.pConditions) {
pProject->node.pConditions = transformForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->node.pConditions);
CHECK_ALLOC(pProject->node.pConditions, (SPhysiNode*)pProject);
}
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject), (SPhysiNode*)pProject);
return (SPhysiNode*)pProject;
}
static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) {
SNodeList* pChildern = nodesMakeList();
CHECK_ALLOC(pChildern, NULL);
SNodeList* pChildren = nodesMakeList();
CHECK_ALLOC(pChildren, NULL);
SNode* pLogicChild;
FOREACH(pLogicChild, pLogicPlan->pChildren) {
SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild);
if (TSDB_CODE_SUCCESS != nodesListAppend(pChildern, pChildPhyNode)) {
if (TSDB_CODE_SUCCESS != nodesListAppend(pChildren, pChildPhyNode)) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
nodesDestroyList(pChildern);
nodesDestroyList(pChildren);
return NULL;
}
}
@ -636,22 +818,22 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPl
pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan);
break;
case QUERY_NODE_LOGIC_PLAN_JOIN:
pPhyNode = createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicPlan);
break;
case QUERY_NODE_LOGIC_PLAN_AGG:
pPhyNode = createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicPlan);
break;
case QUERY_NODE_LOGIC_PLAN_PROJECT:
pPhyNode = createProjectPhysiNode(pCxt, (SProjectLogicNode*)pLogicPlan);
pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan);
break;
default:
break;
}
if (NULL != pPhyNode) {
pPhyNode->pChildren = pChildern;
SNode* pChild;
FOREACH(pChild, pPhyNode->pChildren) {
((SPhysiNode*)pChild)->pParent = pPhyNode;
}
pPhyNode->pChildren = pChildren;
SNode* pChild;
FOREACH(pChild, pPhyNode->pChildren) {
((SPhysiNode*)pChild)->pParent = pPhyNode;
}
return pPhyNode;

4
source/libs/planner/test/newPlannerTest.cpp
View File

@ -123,8 +123,8 @@ TEST_F(NewPlannerTest, simple) {
TEST_F(NewPlannerTest, groupBy) {
setDatabase("root", "test");
bind("SELECT count(*) FROM t1");
ASSERT_TRUE(run());
// bind("SELECT count(*) FROM t1");
// ASSERT_TRUE(run());
bind("SELECT c1, count(*) FROM t1 GROUP BY c1");
ASSERT_TRUE(run());

10
source/libs/scalar/inc/filterInt.h
View File

@ -301,11 +301,11 @@ typedef struct SFilterInfo {
#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type)
#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes)
#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId)
#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId)
#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc))
#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnNode *)((fi)->desc))->node.resType.type)
#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnNode *)((fi)->desc))->node.resType.bytes)
#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId)
#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId)
#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnNode *)((fi)->desc))
#define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri)))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)

24
source/libs/scalar/src/filter.c
View File

@ -915,14 +915,14 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) {
if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) {
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
int32_t type;
void *v;
if (nodeType(node) == QUERY_NODE_COLUMN_REF) {
if (nodeType(node) == QUERY_NODE_COLUMN) {
type = FLD_TYPE_COLUMN;
v = node;
} else {
@ -1455,7 +1455,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i];
SColumnRefNode *refNode = (SColumnRefNode *)field->desc;
SColumnNode *refNode = (SColumnNode *)field->desc;
qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId);
}
@ -1484,7 +1484,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
char str[512] = {0};
SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
SColumnRefNode *refNode = (SColumnRefNode *)left->desc;
SColumnNode *refNode = (SColumnNode *)left->desc;
if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){
len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str);
}
@ -3488,7 +3488,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN_REF == nodeType(*pNode)) {
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
return DEAL_RES_CONTINUE;
}
@ -3511,7 +3511,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_ERROR;
}
if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) {
if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
@ -3522,12 +3522,12 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
} else {
if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) {
if ((QUERY_NODE_COLUMN != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) {
if ((QUERY_NODE_COLUMN != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
@ -3537,7 +3537,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) {
if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
SNode *t = node->pLeft;
node->pLeft = node->pRight;
node->pRight = t;
@ -3550,10 +3550,10 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
}
if (OP_TYPE_IN != node->opType) {
SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft;
SColumnNode *refNode = (SColumnNode *)node->pLeft;
SValueNode *valueNode = (SValueNode *)node->pRight;
int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type);
if (0 != type && type != refNode->dataType.type) {
int32_t type = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type);
if (0 != type && type != refNode->node.resType.type) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}

12
source/libs/scalar/src/scalar.c
View File

@ -134,13 +134,13 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
break;
}
case QUERY_NODE_COLUMN_REF: {
case QUERY_NODE_COLUMN: {
if (NULL == ctx) {
sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx);
SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
SColumnRefNode *ref = (SColumnRefNode *)node;
SColumnNode *ref = (SColumnNode *)node;
if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) {
sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock));
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
@ -281,7 +281,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu
SScalarFuncExecFuncs ffpSet = {0};
int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
if (code) {
sclError( "fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
sclError(
"fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_RET(code);
}
@ -299,7 +300,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu
for (int32_t i = 0; i < rowNum; ++i) {
code = (*ffpSet.process)(params, node->pParameterList->length, output);
if (code) {
sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
sclError(
"scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_JRET(code);
}
@ -599,7 +601,7 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) {
EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN_REF == nodeType(pNode)) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
return DEAL_RES_CONTINUE;
}

18
source/libs/scalar/test/filter/filterTests.cpp
View File

@ -76,15 +76,15 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
}
void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF);
SColumnRefNode *rnode = (SColumnRefNode *)node;
rnode->dataType.type = dataType;
rnode->dataType.bytes = dataBytes;
SNode *node = nodesMakeNode(QUERY_NODE_COLUMN);
SColumnNode *rnode = (SColumnNode *)node;
rnode->node.resType.type = dataType;
rnode->node.resType.bytes = dataBytes;
rnode->tupleId = 0;
if (NULL == block) {
rnode->slotId = 2;
rnode->columnId = 55;
rnode->colId = 55;
*pNode = (SNode *)rnode;
return;
@ -120,7 +120,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
taosArrayPush(res->pDataBlock, &idata);
rnode->slotId = 2;
rnode->columnId = 55;
rnode->colId = 55;
*block = res;
} else {
@ -137,7 +137,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
res->info.numOfCols++;
rnode->slotId = idx;
rnode->columnId = 55 + idx;
rnode->colId = 55 + idx;
}
*pNode = (SNode *)rnode;
@ -259,7 +259,7 @@ TEST(columnTest, smallint_column_greater_double_value) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
stat.colId = ((SColumnRefNode *)pLeft)->columnId;
stat.colId = ((SColumnNode *)pLeft)->colId;
stat.max = 10;
stat.min = 5;
stat.numOfNull = 0;
@ -310,7 +310,7 @@ TEST(columnTest, int_column_greater_smallint_value) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
stat.colId = ((SColumnRefNode *)pLeft)->columnId;
stat.colId = ((SColumnNode *)pLeft)->colId;
stat.max = 10;
stat.min = 5;
stat.numOfNull = 0;

12
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -75,10 +75,10 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
}
void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF);
SColumnRefNode *rnode = (SColumnRefNode *)node;
rnode->dataType.type = dataType;
rnode->dataType.bytes = dataBytes;
SNode *node = nodesMakeNode(QUERY_NODE_COLUMN);
SColumnNode *rnode = (SColumnNode *)node;
rnode->node.resType.type = dataType;
rnode->node.resType.bytes = dataBytes;
rnode->tupleId = 0;
if (NULL == *block) {
@ -111,7 +111,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
taosArrayPush(res->pDataBlock, &idata);
rnode->slotId = 2;
rnode->columnId = 55;
rnode->colId = 55;
*block = res;
} else {
@ -126,7 +126,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
taosArrayPush(res->pDataBlock, &idata);
rnode->slotId = idx;
rnode->columnId = 55 + idx;
rnode->colId = 55 + idx;
}
*pNode = (SNode *)rnode;

12
source/libs/tdb/src/db/tdbEnv.c
View File

@ -18,7 +18,7 @@
struct STDbEnv {
char * rootDir; // root directory of the environment
char * jname; // journal file name
int jfd; // journal file fd
TdFilePtr jpFile; // journal file fd
pgsz_t pgSize; // page size
cachesz_t cacheSize; // total cache size
STDbList dbList; // TDB List
@ -55,7 +55,7 @@ int tdbEnvCreate(TENV **ppEnv, const char *rootDir) {
pEnv->rootDir = (char *)(&pEnv[1]);
pEnv->jname = pEnv->rootDir + slen + 1;
pEnv->jfd = -1;
pEnv->jpFile = NULL;
pEnv->pgSize = TDB_DEFAULT_PGSIZE;
pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE;
@ -139,8 +139,8 @@ static int tdbEnvDestroy(TENV *pEnv) {
}
int tdbEnvBeginTxn(TENV *pEnv) {
pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755);
if (pEnv->jfd < 0) {
pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pEnv->jpFile < 0) {
return -1;
}
@ -149,8 +149,8 @@ int tdbEnvBeginTxn(TENV *pEnv) {
int tdbEnvCommit(TENV *pEnv) {
/* TODO */
close(pEnv->jfd);
pEnv->jfd = -1;
taosCloseFile(&pEnv->jpFile);
pEnv->jpFile = NULL;
return 0;
}

12
source/libs/tdb/src/db/tdbPgFile.c
View File

@ -51,10 +51,10 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
pPgFile->fname = (char *)(&(pPgFile[1]));
memcpy(pPgFile->fname, fname, fnameLen);
pPgFile->fname[fnameLen] = '\0';
pPgFile->fd = -1;
pPgFile->pFile = NULL;
pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755);
if (pPgFile->fd < 0) {
pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pPgFile->pFile == NULL) {
// TODO: handle error
return -1;
}
@ -95,8 +95,8 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
int pgFileClose(SPgFile *pPgFile) {
if (pPgFile) {
if (pPgFile->fd >= 0) {
close(pPgFile->fd);
if (pPgFile->pFile >= 0) {
taosCloseFile(&pPgFile->pFile);
}
tfree(pPgFile->fname);
@ -201,7 +201,7 @@ static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) {
pTData = pData;
szToRead = pgSize;
for (; szToRead > 0;) {
rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize);
rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize);
if (rsize < 0) {
if (errno == EINTR) {
continue;

2
source/libs/tdb/src/inc/tdbPgFile.h
View File

@ -38,7 +38,7 @@ struct SPgFile {
uint8_t fileid[TDB_FILE_ID_LEN]; // file id
pgno_t lsize; // page file logical size (for count)
pgno_t fsize; // real file size on disk (for rollback)
int fd;
TdFilePtr pFile;
SPgFileListNode envHash;
SPgFileListNode envPgfList;
};

18
source/libs/tfs/test/tfsTest.cpp
View File

@ -230,10 +230,11 @@ TEST_F(TfsTest, 04_File) {
EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0);
FILE *fp = fopen(f1.aname, "w");
ASSERT_NE(fp, nullptr);
fwrite("12345678", 1, 5, fp);
fclose(fp);
// FILE *fp = fopen(f1.aname, "w");
TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
ASSERT_NE(pFile, nullptr);
taosWriteFile(pFile, "12345678", 5);
taosCloseFile(&pFile);
char base[128] = {0};
tfsBasename(&f1, base);
@ -638,10 +639,11 @@ TEST_F(TfsTest, 05_MultiDisk) {
EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0);
FILE *fp = fopen(f1.aname, "w");
ASSERT_NE(fp, nullptr);
fwrite("12345678", 1, 5, fp);
fclose(fp);
// FILE *fp = fopen(f1.aname, "w");
TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
ASSERT_NE(pFile, nullptr);
taosWriteFile(pFile, "12345678", 5);
taosCloseFile(&pFile);
char base[128] = {0};
tfsBasename(&f1, base);

2
source/libs/transport/inc/rpcUdp.h
View File

@ -30,7 +30,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
void taosFreeMsgHdr(void *hdr);
int taosMsgHdrSize(void *hdr);
void taosSendMsgHdr(void *hdr, int fd);
void taosSendMsgHdr(void *hdr, TdFilePtr pFile);
void taosInitMsgHdr(void **hdr, void *dest, int maxPkts);
void taosSetMsgHdrData(void *hdr, char *data, int dataLen);

16
source/libs/transport/test/pushServer.c
View File

@ -22,7 +22,7 @@
int msgSize = 128;
int commit = 0;
int dataFd = -1;
TdFilePtr pDataFile = NULL;
STaosQueue *qhandle = NULL;
STaosQset * qset = NULL;
@ -43,8 +43,8 @@ void processShellMsg() {
for (int i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg);
if (dataFd >= 0) {
if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
if (pDataFile != NULL) {
if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
tInfo("failed to write data file, reason:%s", strerror(errno));
}
}
@ -52,7 +52,7 @@ void processShellMsg() {
if (commit >= 2) {
num += numOfMsgs;
// if (taosFsync(dataFd) < 0) {
// if (taosFsync(pDataFile) < 0) {
// tInfo("failed to flush data to file, reason:%s", strerror(errno));
//}
@ -181,8 +181,8 @@ int main(int argc, char *argv[]) {
tInfo("RPC server is running, ctrl-c to exit");
if (commit) {
dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno));
pDataFile = taosOpenFile(dataName, TD_FILE_APPEND | TD_FILE_CTEATE | TD_FILE_WRITE);
if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno));
}
qhandle = taosOpenQueue();
qset = taosOpenQset();
@ -190,8 +190,8 @@ int main(int argc, char *argv[]) {
processShellMsg();
if (dataFd >= 0) {
close(dataFd);
if (pDataFile != NULL) {
taosCloseFile(&pDataFile);
remove(dataName);
}

16
source/libs/transport/test/rserver.c
View File

@ -22,7 +22,7 @@
int msgSize = 128;
int commit = 0;
int dataFd = -1;
TdFilePtr pDataFile = NULL;
STaosQueue *qhandle = NULL;
STaosQset * qset = NULL;
@ -43,8 +43,8 @@ void processShellMsg() {
for (int i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg);
if (dataFd >= 0) {
if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
if (pDataFile >= 0) {
if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
tInfo("failed to write data file, reason:%s", strerror(errno));
}
}
@ -52,7 +52,7 @@ void processShellMsg() {
if (commit >= 2) {
num += numOfMsgs;
// if (taosFsync(dataFd) < 0) {
// if (taosFsync(pDataFile) < 0) {
// tInfo("failed to flush data to file, reason:%s", strerror(errno));
//}
@ -170,8 +170,8 @@ int main(int argc, char *argv[]) {
tInfo("RPC server is running, ctrl-c to exit");
if (commit) {
dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno));
pDataFile = taosOpenFile(dataName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno));
}
qhandle = taosOpenQueue();
qset = taosOpenQset();
@ -179,8 +179,8 @@ int main(int argc, char *argv[]) {
processShellMsg();
if (dataFd >= 0) {
close(dataFd);
if (pDataFile != NULL) {
taosCloseFile(&pDataFile);
remove(dataName);
}

34
source/libs/wal/src/walMeta.c
View File

@ -69,8 +69,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size);
pLastFileInfo->fileSize = statbuf.st_size;
FileFd fd = taosOpenFileRead(fnameStr);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -79,15 +79,15 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
char* buf = malloc(readSize + 5);
if (buf == NULL) {
taosCloseFile(fd);
taosCloseFile(&pFile);
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
return -1;
}
taosLSeekFile(fd, -readSize, SEEK_END);
if (readSize != taosReadFile(fd, buf, readSize)) {
taosLSeekFile(pFile, -readSize, SEEK_END);
if (readSize != taosReadFile(pFile, buf, readSize)) {
free(buf);
taosCloseFile(fd);
taosCloseFile(&pFile);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -108,12 +108,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) {
// file has to be deleted
free(buf);
taosCloseFile(fd);
taosCloseFile(&pFile);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
}
taosCloseFile(fd);
taosCloseFile(&pFile);
SWalHead *lastEntry = (SWalHead*)found;
return lastEntry->head.version;
@ -364,18 +364,18 @@ int walSaveMeta(SWal* pWal) {
int metaVer = walFindCurMetaVer(pWal);
char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer + 1, fnameStr);
FileFd metaFd = taosOpenFileCreateWrite(fnameStr);
if (metaFd < 0) {
TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE);
if (pMataFile == NULL) {
return -1;
}
char* serialized = walMetaSerialize(pWal);
int len = strlen(serialized);
if (len != taosWriteFile(metaFd, serialized, len)) {
if (len != taosWriteFile(pMataFile, serialized, len)) {
// TODO:clean file
return -1;
}
taosCloseFile(metaFd);
taosCloseFile(&pMataFile);
// delete old file
if (metaVer > -1) {
walBuildMetaName(pWal, metaVer, fnameStr);
@ -404,20 +404,20 @@ int walLoadMeta(SWal* pWal) {
return -1;
}
memset(buf, 0, size + 5);
FileFd fd = taosOpenFileRead(fnameStr);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pFile == NULL) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
if (taosReadFile(fd, buf, size) != size) {
if (taosReadFile(pFile, buf, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(fd);
taosCloseFile(&pFile);
free(buf);
return -1;
}
// load into fileInfoSet
int code = walMetaDeserialize(pWal, buf);
taosCloseFile(fd);
taosCloseFile(&pFile);
free(buf);
return code;
}

14
source/libs/wal/src/walMgmt.c
View File

@ -89,8 +89,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
// open meta
walResetVer(&pWal->vers);
pWal->writeLogTfd = -1;
pWal->writeIdxTfd = -1;
pWal->pWriteLogTFile = NULL;
pWal->pWriteIdxTFile = NULL;
pWal->writeCur = -1;
pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo));
if (pWal->fileInfoSet == NULL) {
@ -164,10 +164,10 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
void walClose(SWal *pWal) {
pthread_mutex_lock(&pWal->mutex);
tfClose(pWal->writeLogTfd);
pWal->writeLogTfd = -1;
tfClose(pWal->writeIdxTfd);
pWal->writeIdxTfd = -1;
taosCloseFile(&pWal->pWriteLogTFile);
pWal->pWriteLogTFile = NULL;
taosCloseFile(&pWal->pWriteIdxTFile);
pWal->pWriteIdxTFile = NULL;
walSaveMeta(pWal);
taosArrayDestroy(pWal->fileInfoSet);
pWal->fileInfoSet = NULL;
@ -207,7 +207,7 @@ static void walFsyncAll() {
if (walNeedFsync(pWal)) {
wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq,
atomic_load_32(&tsWal.seq));
int32_t code = tfFsync(pWal->writeLogTfd);
int32_t code = taosFsyncFile(pWal->pWriteLogTFile);
if (code != 0) {
wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(code));

44
source/libs/wal/src/walRead.c
View File

@ -25,8 +25,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
}
pRead->pWal = pWal;
pRead->readIdxTfd = -1;
pRead->readLogTfd = -1;
pRead->pReadIdxTFile = NULL;
pRead->pReadLogTFile = NULL;
pRead->curVersion = -1;
pRead->curFileFirstVer = -1;
pRead->capacity = 0;
@ -41,8 +41,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
}
void walCloseReadHandle(SWalReadHandle *pRead) {
tfClose(pRead->readIdxTfd);
tfClose(pRead->readLogTfd);
taosCloseFile(&pRead->pReadIdxTFile);
taosCloseFile(&pRead->pReadLogTFile);
tfree(pRead->pHead);
free(pRead);
}
@ -52,24 +52,24 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; }
static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
int code = 0;
int64_t idxTfd = pRead->readIdxTfd;
int64_t logTfd = pRead->readLogTfd;
TdFilePtr pIdxTFile = pRead->pReadIdxTFile;
TdFilePtr pLogTFile = pRead->pReadLogTFile;
// seek position
int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
code = tfLseek(idxTfd, offset, SEEK_SET);
code = taosLSeekFile(pIdxTFile, offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
// TODO:deserialize
ASSERT(entry.ver == ver);
code = tfLseek(logTfd, entry.offset, SEEK_SET);
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -80,24 +80,24 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
char fnameStr[WAL_FILE_LEN];
tfClose(pRead->readIdxTfd);
tfClose(pRead->readLogTfd);
taosCloseFile(&pRead->pReadIdxTFile);
taosCloseFile(&pRead->pReadLogTFile);
walBuildLogName(pRead->pWal, fileFirstVer, fnameStr);
int64_t logTfd = tfOpenRead(fnameStr);
if (logTfd < 0) {
TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pLogTFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
int64_t idxTfd = tfOpenRead(fnameStr);
if (idxTfd < 0) {
TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pIdxTFile == NULL) {
return -1;
}
pRead->readLogTfd = logTfd;
pRead->readIdxTfd = idxTfd;
pRead->pReadLogTFile = pLogTFile;
pRead->pReadIdxTFile = pIdxTFile;
return 0;
}
@ -145,9 +145,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
}
}
if (!tfValid(pRead->readLogTfd)) return -1;
if (!taosValidFile(pRead->pReadLogTFile)) return -1;
code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead));
code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead));
if (code != sizeof(SWalHead)) {
return -1;
}
@ -165,7 +165,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
pRead->pHead = ptr;
pRead->capacity = pRead->pHead->head.len;
}
if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) {
if (pRead->pHead->head.len != taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.len)) {
return -1;
}
@ -202,7 +202,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
}
*ppHead = ptr;
}
if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
return -1;
}
// TODO: endian compatibility processing after read
@ -215,7 +215,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
*ppHead = NULL;
return -1;
}
if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
return -1;
}
// TODO: endian compatibility processing after read

52
source/libs/wal/src/walSeek.c
View File

@ -23,25 +23,25 @@
static int walSeekWritePos(SWal* pWal, int64_t ver) {
int code = 0;
int64_t idxTfd = pWal->writeIdxTfd;
int64_t logTfd = pWal->writeLogTfd;
TdFilePtr pIdxTFile = pWal->pWriteIdxTFile;
TdFilePtr pLogTFile = pWal->pWriteLogTFile;
// seek position
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET);
code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
SWalIdxEntry entry;
// TODO:deserialize
code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry));
code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
ASSERT(entry.ver == ver);
code = tfLseek(logTfd, entry.offset, SEEK_SET);
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -50,43 +50,43 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) {
}
int walSetWrite(SWal* pWal) {
int64_t idxTfd, logTfd;
TdFilePtr pIdxTFile, pLogTFile;
SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
ASSERT(pRet != NULL);
int64_t fileFirstVer = pRet->firstVer;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr);
if (idxTfd < 0) {
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr);
if (logTfd < 0) {
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
// switch file
pWal->writeIdxTfd = idxTfd;
pWal->writeLogTfd = logTfd;
pWal->pWriteIdxTFile = pIdxTFile;
pWal->pWriteLogTFile = pLogTFile;
return 0;
}
int walChangeWrite(SWal* pWal, int64_t ver) {
int code = 0;
int64_t idxTfd, logTfd;
TdFilePtr pIdxTFile, pLogTFile;
char fnameStr[WAL_FILE_LEN];
if (pWal->writeLogTfd != -1) {
code = tfClose(pWal->writeLogTfd);
if (pWal->pWriteLogTFile != NULL) {
code = taosCloseFile(&pWal->pWriteLogTFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
if (pWal->writeIdxTfd != -1) {
code = tfClose(pWal->writeIdxTfd);
if (pWal->pWriteIdxTFile != NULL) {
code = taosCloseFile(&pWal->pWriteIdxTFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@ -103,23 +103,23 @@ int walChangeWrite(SWal* pWal, int64_t ver) {
int64_t fileFirstVer = pFileInfo->firstVer;
walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr);
if (idxTfd < 0) {
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
pWal->writeIdxTfd = -1;
pWal->pWriteIdxTFile = NULL;
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr);
if (logTfd < 0) {
tfClose(idxTfd);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile < 0) {
taosCloseFile(&pIdxTFile);
terrno = TAOS_SYSTEM_ERROR(errno);
pWal->writeLogTfd = -1;
pWal->pWriteLogTFile = NULL;
return -1;
}
pWal->writeLogTfd = logTfd;
pWal->writeIdxTfd = idxTfd;
pWal->pWriteLogTFile = pLogTFile;
pWal->pWriteIdxTFile = pIdxTFile;
pWal->writeCur = idx;
return fileFirstVer;
}

60
source/libs/wal/src/walWrite.c
View File

@ -65,15 +65,15 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
int64_t idxTfd = tfOpenReadWrite(fnameStr);
TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
// TODO:change to deserialize function
if (idxTfd < 0) {
if (pIdxTFile == NULL) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET);
code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET);
if (code < 0) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
@ -81,20 +81,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
// read idx file and get log file pos
// TODO:change to deserialize function
SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
ASSERT(entry.ver == ver);
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
int64_t logTfd = tfOpenReadWrite(fnameStr);
if (logTfd < 0) {
TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
if (pLogTFile < 0) {
// TODO
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
code = tfLseek(logTfd, entry.offset, SEEK_SET);
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
// TODO
pthread_mutex_unlock(&pWal->mutex);
@ -102,8 +102,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
// validate offset
SWalHead head;
ASSERT(tfValid(logTfd));
int size = tfRead(logTfd, &head, sizeof(SWalHead));
ASSERT(taosValidFile(pLogTFile));
int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead));
if (size != sizeof(SWalHead)) {
return -1;
}
@ -118,11 +118,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
return -1;
}
// truncate old files
code = tfFtruncate(logTfd, entry.offset);
code = taosFtruncateFile(pLogTFile, entry.offset);
if (code < 0) {
return -1;
}
code = tfFtruncate(idxTfd, idxOff);
code = taosFtruncateFile(pIdxTFile, idxOff);
if (code < 0) {
return -1;
}
@ -203,31 +203,31 @@ int32_t walEndSnapshot(SWal *pWal) {
int walRoll(SWal *pWal) {
int code = 0;
if (pWal->writeIdxTfd != -1) {
code = tfClose(pWal->writeIdxTfd);
if (pWal->pWriteIdxTFile != NULL) {
code = taosCloseFile(&pWal->pWriteIdxTFile);
if (code != 0) {
return -1;
}
}
if (pWal->writeLogTfd != -1) {
code = tfClose(pWal->writeLogTfd);
if (pWal->pWriteLogTFile != NULL) {
code = taosCloseFile(&pWal->pWriteLogTFile);
if (code != 0) {
return -1;
}
}
int64_t idxTfd, logTfd;
TdFilePtr pIdxTFile, pLogTFile;
// create new file
int64_t newFileFirstVersion = pWal->vers.lastVer + 1;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, newFileFirstVersion, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr);
if (idxTfd < 0) {
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildLogName(pWal, newFileFirstVersion, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr);
if (logTfd < 0) {
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -237,8 +237,8 @@ int walRoll(SWal *pWal) {
}
// switch file
pWal->writeIdxTfd = idxTfd;
pWal->writeLogTfd = logTfd;
pWal->pWriteIdxTFile = pIdxTFile;
pWal->pWriteLogTFile = pLogTFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
ASSERT(pWal->writeCur >= 0);
@ -248,7 +248,7 @@ int walRoll(SWal *pWal) {
static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
SWalIdxEntry entry = {.ver = ver, .offset = offset};
int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry));
int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry));
if (size != sizeof(SWalIdxEntry)) {
terrno = TAOS_SYSTEM_ERROR(errno);
// TODO truncate
@ -282,16 +282,16 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
// must truncate explicitly first
return -1;
}
/*if (!tfValid(pWal->writeLogTfd)) return -1;*/
/*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/
ASSERT(pWal->writeCur >= 0);
pthread_mutex_lock(&pWal->mutex);
if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) {
if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) {
walSetWrite(pWal);
tfLseek(pWal->writeLogTfd, 0, SEEK_END);
tfLseek(pWal->writeIdxTfd, 0, SEEK_END);
taosLSeekFile(pWal->pWriteLogTFile, 0, SEEK_END);
taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END);
}
pWal->writeHead.head.version = index;
@ -302,14 +302,14 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead);
pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen);
if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
// ftruncate
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
}
if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) {
if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) {
// ftruncate
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
@ -336,7 +336,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
void walFsync(SWal *pWal, bool forceFsync) {
if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
if (tfFsync(pWal->writeLogTfd) < 0) {
if (taosFsyncFile(pWal->pWriteLogTFile) < 0) {
wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
strerror(errno));
}

578
source/os/src/osFile.c
View File

@ -12,38 +12,45 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#include "os.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#include <io.h>
#include <io.h>
#if defined(_MSDOS)
#define open _open
#endif
#if defined(_MSDOS)
#define open _open
#endif
#if defined(_WIN32)
extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */
extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */
#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
#define open openU
#else /* _ANSI_SOURCE */
#define open openA
#endif /* defined(_UTF8_SOURCE) */
#endif /* defined(_WIN32) */
#if defined(_WIN32)
extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */
extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */
#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
#define open openU
#else /* _ANSI_SOURCE */
#define open openA
#endif /* defined(_UTF8_SOURCE) */
#endif /* defined(_WIN32) */
#else
#include <fcntl.h>
#include <sys/file.h>
#include <sys/sendfile.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/file.h>
#include <sys/sendfile.h>
#include <sys/stat.h>
#include <unistd.h>
#define LINUX_FILE_NO_TEXT_OPTION 0
#define O_TEXT LINUX_FILE_NO_TEXT_OPTION
#endif
void taosCloseFile(FileFd fd) {
close(fd);
fd = FD_INITIALIZER;
}
typedef int32_t FileFd;
typedef struct TdFile {
int refId;
FileFd fd;
FILE *fp;
}*TdFilePtr,TdFile;
void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, char *dstPath) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
@ -96,13 +103,166 @@ void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, ch
#endif
}
int64_t taosReadFile(FileFd fd, void *buf, int64_t count) {
int64_t taosCopyFile(const char *from, const char *to) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
char buffer[4096];
int64_t size = 0;
int64_t bytes;
// fidfrom = open(from, O_RDONLY);
TdFilePtr pFileFrom = taosOpenFile(from,TD_FILE_READ);
if (pFileFrom == NULL) goto _err;
// fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755);
TdFilePtr pFileTo = taosOpenFile(to,TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_EXCL);
if (pFileTo == NULL) goto _err;
while (true) {
bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer));
if (bytes < 0) goto _err;
if (bytes == 0) break;
size += bytes;
if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) goto _err;
if (bytes < sizeof(buffer)) break;
}
taosFsyncFile(pFileTo);
taosCloseFile(&pFileFrom);
taosCloseFile(&pFileTo);
return size;
_err:
if (pFileFrom != NULL) taosCloseFile(&pFileFrom);
if (pFileTo != NULL) taosCloseFile(&pFileTo);
remove(to);
return -1;
#endif
}
int32_t taosRenameFile(const char *oldName, const char *newName) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
if (code < 0) {
//printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
}
return code;
#else
int32_t code = rename(oldName, newName);
if (code < 0) {
//printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
}
return code;
#endif
}
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
struct stat fileStat;
int32_t code = stat(path, &fileStat);
if (code < 0) {
return code;
}
if (size != NULL) {
*size = fileStat.st_size;
}
if (mtime != NULL) {
*mtime = fileStat.st_mtime;
}
return 0;
#endif
}
void autoDelFileListAdd(const char *path) {
return;
}
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return NULL;
#else
int access = O_BINARY;
char *mode = NULL;
access |= (tdFileOptions & TD_FILE_CTEATE) ? O_CREAT : 0;
if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ))
{
access |= O_RDWR;
mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+";
}else if(tdFileOptions & TD_FILE_WRITE) {
access |= O_WRONLY;
mode = (tdFileOptions & TD_FILE_TEXT) ? "wt" : "wb";
}else if(tdFileOptions & TD_FILE_READ) {
access |= O_RDONLY;
mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb";
}
access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0;
access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0;
access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0;
access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0;
if(tdFileOptions & TD_FILE_AUTO_DEL) {
autoDelFileListAdd(path);
}
int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO);
if(fd == -1) {
return NULL;
}
FILE* fp = fdopen(fd, mode);
if (fp == NULL) {
close(fd);
return NULL;
}
TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile));
if (pFile == NULL) {
close(fd);
fclose(fp);
return NULL;
}
pFile->fd = fd;
pFile->fp = fp;
pFile->refId = 0;
return pFile;
#endif
}
int64_t taosCloseFile(TdFilePtr *ppFile) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
if(ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) {
return 0;
}
fsync((*ppFile)->fd);
close((*ppFile)->fd);
(*ppFile)->fd = -1;
(*ppFile)->fp = NULL;
(*ppFile)->refId = 0;
free(*ppFile);
*ppFile = NULL;
return 0;
#endif
}
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
if(pFile == NULL) {
return 0;
}
int64_t leftbytes = count;
int64_t readbytes;
char * tbuf = (char *)buf;
while (leftbytes > 0) {
readbytes = read(fd, (void *)tbuf, (uint32_t)leftbytes);
readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes);
if (readbytes < 0) {
if (errno == EINTR) {
continue;
@ -120,13 +280,20 @@ int64_t taosReadFile(FileFd fd, void *buf, int64_t count) {
return count;
}
int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) {
int64_t nleft = n;
int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) {
if(pFile == NULL) {
return 0;
}
return pread(pFile->fd, buf, count, offset);
}
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
int64_t nleft = count;
int64_t nwritten = 0;
char * tbuf = (char *)buf;
while (nleft > 0) {
nwritten = write(fd, (void *)tbuf, (uint32_t)nleft);
nwritten = write(pFile->fd, (void *)tbuf, (uint32_t)nleft);
if (nwritten < 0) {
if (errno == EINTR) {
continue;
@ -137,48 +304,119 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) {
tbuf += nwritten;
}
return n;
return count;
}
int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); }
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
return (int64_t)lseek(pFile->fd, (long)offset, whence);
}
int64_t taosCopyFile(const char *from, const char *to) {
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
char buffer[4096];
int fidto = -1, fidfrom = -1;
int64_t size = 0;
int64_t bytes;
fidfrom = open(from, O_RDONLY);
if (fidfrom < 0) goto _err;
fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755);
if (fidto < 0) goto _err;
while (true) {
bytes = taosReadFile(fidfrom, buffer, sizeof(buffer));
if (bytes < 0) goto _err;
if (bytes == 0) break;
size += bytes;
if (taosWriteFile(fidto, (void *)buffer, bytes) < bytes) goto _err;
if (bytes < sizeof(buffer)) break;
struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat);
if (code < 0) {
return code;
}
taosFsyncFile(fidto);
if (size != NULL) {
*size = fileStat.st_size;
}
taosCloseFile(fidfrom);
taosCloseFile(fidto);
return size;
if (mtime != NULL) {
*mtime = fileStat.st_mtime;
}
_err:
if (fidfrom >= 0) taosCloseFile(fidfrom);
if (fidto >= 0) taosCloseFile(fidto);
remove(to);
return -1;
return 0;
#endif
}
int32_t taosLockFile(TdFilePtr pFile) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB);
#endif
}
int32_t taosUnLockFile(TdFilePtr pFile) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB);
#endif
}
int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
if (pFile->fd < 0) {
errno = EBADF;
uError("%s\n", "fd arg was negative");
return -1;
}
HANDLE h = (HANDLE)_get_osfhandle(pFile->fd);
LARGE_INTEGER li_0;
li_0.QuadPart = (int64_t)0;
BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT);
if (!cur) {
uError("SetFilePointerEx Error getting current position in file.\n");
return -1;
}
LARGE_INTEGER li_size;
li_size.QuadPart = l_size;
BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN);
if (cur2 == 0) {
int error = GetLastError();
uError("SetFilePointerEx GetLastError is: %d\n", error);
switch (error) {
case ERROR_INVALID_HANDLE:
errno = EBADF;
break;
default:
errno = EIO;
break;
}
return -1;
}
if (!SetEndOfFile(h)) {
int error = GetLastError();
uError("SetEndOfFile GetLastError is:%d", error);
switch (error) {
case ERROR_INVALID_HANDLE:
errno = EBADF;
break;
default:
errno = EIO;
break;
}
return -1;
}
return 0;
#else
return ftruncate(pFile->fd, l_size);
#endif
}
int32_t taosFsyncFile(TdFilePtr pFile) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
if (pFile->fd < 0) {
errno = EBADF;
uError("%s\n", "fd arg was negative");
return -1;
}
HANDLE h = (HANDLE)_get_osfhandle(pFile->fd);
return FlushFileBuffers(h);
#else
return fsync(pFile->fd);
#endif
}
@ -301,12 +539,12 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) {
#else
int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) {
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) {
int64_t leftbytes = size;
int64_t sentbytes;
while (leftbytes > 0) {
sentbytes = sendfile(dfd, sfd, offset, leftbytes);
sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes);
if (sentbytes == -1) {
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
continue;
@ -323,215 +561,49 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) {
return size;
}
int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) {
return taosSendFile(fileno(outfile), fileno(infile), offset, size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) {
return taosSendFile(pFileOut->fd, pFileIn, offset, size);
}
#endif
int32_t taosFtruncateFile(FileFd fd, int64_t l_size) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
if (fd < 0) {
errno = EBADF;
uError("%s\n", "fd arg was negative");
return -1;
}
HANDLE h = (HANDLE)_get_osfhandle(fd);
LARGE_INTEGER li_0;
li_0.QuadPart = (int64_t)0;
BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT);
if (!cur) {
uError("SetFilePointerEx Error getting current position in file.\n");
return -1;
}
LARGE_INTEGER li_size;
li_size.QuadPart = l_size;
BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN);
if (cur2 == 0) {
int error = GetLastError();
uError("SetFilePointerEx GetLastError is: %d\n", error);
switch (error) {
case ERROR_INVALID_HANDLE:
errno = EBADF;
break;
default:
errno = EIO;
break;
}
return -1;
}
if (!SetEndOfFile(h)) {
int error = GetLastError();
uError("SetEndOfFile GetLastError is:%d", error);
switch (error) {
case ERROR_INVALID_HANDLE:
errno = EBADF;
break;
default:
errno = EIO;
break;
}
return -1;
}
return 0;
#else
return ftruncate(fd, l_size);
#ifdef __GNUC__
__attribute__((format(printf, 2, 3)))
#endif
void taosFprintfFile(TdFilePtr pFile, const char *format, ...) {
va_list ap;
va_start(ap, format);
fprintf(pFile->fp, format, ap);
va_end(ap);
fflush(pFile->fp);
}
int32_t taosFsyncFile(FileFd fd) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
if (fd < 0) {
errno = EBADF;
uError("%s\n", "fd arg was negative");
return -1;
}
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) {
if (pFile == NULL) return NULL;
HANDLE h = (HANDLE)_get_osfhandle(fd);
return FlushFileBuffers(h);
#else
return fsync(fd);
#endif
void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0);
return ptr;
}
int32_t taosRenameFile(const char *oldName, const char *newName) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
if (code < 0) {
//printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
}
return code;
#else
int32_t code = rename(oldName, newName);
if (code < 0) {
//printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
}
return code;
#endif
bool taosValidFile(TdFilePtr pFile) {
return pFile != NULL;
}
int32_t taosLockFile(int32_t fd) {
int32_t taosUmaskFile(int32_t maskVal) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return (int32_t)flock(fd, LOCK_EX | LOCK_NB);
return umask(maskVal);
#endif
}
int32_t taosUnLockFile(int32_t fd) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return (int32_t)flock(fd, LOCK_UN | LOCK_NB);
#endif
int taosGetErrorFile(TdFilePtr pFile) {
return errno;
}
int32_t taosUmaskFile(int32_t val) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return umask(val);
#endif
size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf) {
return getline(ptrBuf, NULL, pFile->fp);
}
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
struct stat fileStat;
int32_t code = stat(path, &fileStat);
if (code < 0) {
return code;
}
if (size != NULL) {
*size = fileStat.st_size;
}
if (mtime != NULL) {
*mtime = fileStat.st_mtime;
}
return 0;
#endif
}
int32_t taosFStatFile(int32_t fd, int64_t *size, int32_t *mtime) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
struct stat fileStat;
int32_t code = fstat(fd, &fileStat);
if (code < 0) {
return code;
}
if (size != NULL) {
*size = fileStat.st_size;
}
if (mtime != NULL) {
*mtime = fileStat.st_mtime;
}
return 0;
#endif
}
int32_t taosOpenFileWrite(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
int32_t taosOpenFileCreateWrite(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
int32_t taosOpenFileCreateWriteTrunc(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
int32_t taosOpenFileCreateWriteAppend(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
FileFd taosOpenFileRead(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
FileFd taosOpenFileReadWrite(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return 0;
#else
return open(path, O_RDWR, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}
int32_t taosEOFFile(TdFilePtr pFile) {
return feof(pFile->fp);
}

10
source/os/src/osRand.c
View File

@ -24,18 +24,18 @@
uint32_t taosRand(void) { return rand(); }
uint32_t taosSafeRand(void) {
int fd;
TdFilePtr pFile;
int seed;
fd = open("/dev/urandom", 0);
if (fd < 0) {
pFile = taosOpenFile("/dev/urandom", TD_FILE_READ);
if (pFile == NULL) {
seed = (int)time(0);
} else {
int len = read(fd, &seed, sizeof(seed));
int len = taosReadFile(pFile, &seed, sizeof(seed));
if (len < 0) {
seed = (int)time(0);
}
close(fd);
taosCloseFile(&pFile);
}
return (uint32_t)seed;

1
source/os/src/osSocket.c
View File

@ -14,6 +14,7 @@
*/
#define _DEFAULT_SOURCE
#define ALLOW_FORBID_FUNC
#include "os.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)

101
source/os/src/osSysinfo.c
View File

@ -408,19 +408,17 @@ bool taosGetSysMemory(float *memoryUsedMB) {
}
bool taosGetProcMemory(float *memoryUsedMB) {
FILE *fp = fopen(tsProcMemFile, "r");
if (fp == NULL) {
// FILE *fp = fopen(tsProcMemFile, "r");
TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ);
if (pFile == NULL) {
//printf("open file:%s failed", tsProcMemFile);
return false;
}
ssize_t _bytes = 0;
size_t len;
char * line = NULL;
while (!feof(fp)) {
tfree(line);
len = 0;
_bytes = getline(&line, &len, fp);
while (!taosEOFFile(pFile)) {
_bytes = taosGetLineFile(pFile, &line);
if ((_bytes < 0) || (line == NULL)) {
break;
}
@ -431,7 +429,7 @@ bool taosGetProcMemory(float *memoryUsedMB) {
if (line == NULL) {
//printf("read file:%s failed", tsProcMemFile);
fclose(fp);
taosCloseFile(&pFile);
return false;
}
@ -440,24 +438,24 @@ bool taosGetProcMemory(float *memoryUsedMB) {
sscanf(line, "%s %" PRId64, tmp, &memKB);
*memoryUsedMB = (float)((double)memKB / 1024);
tfree(line);
fclose(fp);
if(line != NULL) tfree(line);
taosCloseFile(&pFile);
return true;
}
static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
FILE *fp = fopen(tsSysCpuFile, "r");
if (fp == NULL) {
// FILE *fp = fopen(tsSysCpuFile, "r");
TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ);
if (pFile == NULL) {
//printf("open file:%s failed", tsSysCpuFile);
return false;
}
size_t len;
char * line = NULL;
ssize_t _bytes = getline(&line, &len, fp);
ssize_t _bytes = taosGetLineFile(pFile, &line);
if ((_bytes < 0) || (line == NULL)) {
//printf("read file:%s failed", tsSysCpuFile);
fclose(fp);
taosCloseFile(&pFile);
return false;
}
@ -465,24 +463,24 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system,
&cpuInfo->idle);
tfree(line);
fclose(fp);
if(line != NULL) tfree(line);
taosCloseFile(&pFile);
return true;
}
static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
FILE *fp = fopen(tsProcCpuFile, "r");
if (fp == NULL) {
// FILE *fp = fopen(tsProcCpuFile, "r");
TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ);
if (pFile == NULL) {
//printf("open file:%s failed", tsProcCpuFile);
return false;
}
size_t len = 0;
char * line = NULL;
ssize_t _bytes = getline(&line, &len, fp);
ssize_t _bytes = taosGetLineFile(pFile, &line);
if ((_bytes < 0) || (line == NULL)) {
//printf("read file:%s failed", tsProcCpuFile);
fclose(fp);
taosCloseFile(&pFile);
return false;
}
@ -495,8 +493,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
}
}
tfree(line);
fclose(fp);
if(line != NULL) tfree(line);
taosCloseFile(&pFile);
return true;
}
@ -556,19 +554,17 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
*bytes = 0;
FILE *fp = fopen(tsSysNetFile, "r");
if (fp == NULL) {
// FILE *fp = fopen(tsSysNetFile, "r");
TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ);
if (pFile == NULL) {
//printf("open file:%s failed", tsSysNetFile);
return false;
}
ssize_t _bytes = 0;
size_t len = 2048;
char * line = calloc(1, len);
while (!feof(fp)) {
memset(line, 0, len);
char * line = NULL;
while (!taosEOFFile(pFile)) {
int64_t o_rbytes = 0;
int64_t rpackts = 0;
int64_t o_tbytes = 0;
@ -581,12 +577,12 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
int64_t nouse6 = 0;
char nouse0[200] = {0};
_bytes = getline(&line, &len, fp);
_bytes = taosGetLineFile(pFile, &line);
if (_bytes < 0) {
break;
}
line[len - 1] = 0;
line[_bytes - 1] = 0;
if (strstr(line, "lo:") != NULL) {
continue;
@ -601,8 +597,8 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
*bytes += (o_rbytes + o_tbytes);
}
tfree(line);
fclose(fp);
if(line != NULL) tfree(line);
taosCloseFile(&pFile);
return true;
}
@ -644,22 +640,20 @@ bool taosGetBandSpeed(float *bandSpeedKb) {
}
bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
FILE *fp = fopen(tsProcIOFile, "r");
if (fp == NULL) {
// FILE *fp = fopen(tsProcIOFile, "r");
TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ);
if (pFile == NULL) {
//printf("open file:%s failed", tsProcIOFile);
return false;
}
ssize_t _bytes = 0;
size_t len;
char * line = NULL;
char tmp[10];
int readIndex = 0;
while (!feof(fp)) {
tfree(line);
len = 0;
_bytes = getline(&line, &len, fp);
while (!taosEOFFile(pFile)) {
_bytes = taosGetLineFile(pFile, &line);
if ((_bytes < 0) || (line == NULL)) {
break;
}
@ -675,8 +669,8 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
if (readIndex >= 2) break;
}
tfree(line);
fclose(fp);
if(line != NULL) tfree(line);
taosCloseFile(&pFile);
if (readIndex < 2) {
//printf("read file:%s failed", tsProcIOFile);
@ -839,15 +833,15 @@ void taosSetCoreDump(bool enable) {
}
int32_t taosGetSystemUUID(char *uid, int32_t uidlen) {
int fd;
int len = 0;
fd = open("/proc/sys/kernel/random/uuid", 0);
if (fd < 0) {
// fd = open("/proc/sys/kernel/random/uuid", 0);
TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ);
if (pFile == NULL) {
return -1;
} else {
len = read(fd, uid, uidlen);
close(fd);
len = taosReadFile(pFile, uid, uidlen);
taosCloseFile(&pFile);
}
if (len >= 36) {
@ -862,16 +856,17 @@ char *taosGetCmdlineByPID(int pid) {
static char cmdline[1024];
sprintf(cmdline, "/proc/%d/cmdline", pid);
int fd = open(cmdline, O_RDONLY);
if (fd >= 0) {
int n = read(fd, cmdline, sizeof(cmdline) - 1);
// int fd = open(cmdline, O_RDONLY);
TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ);
if (pFile != NULL) {
int n = taosReadFile(pFile, cmdline, sizeof(cmdline) - 1);
if (n < 0) n = 0;
if (n > 0 && cmdline[n - 1] == '\n') --n;
cmdline[n] = 0;
close(fd);
taosCloseFile(&pFile);
} else {
cmdline[0] = 0;
}

13
source/os/src/osTimezone.c
View File

@ -150,17 +150,18 @@ void taosGetSystemTimezone(char *outTimezone) {
localtime_r(&tx1, &tm1);
/* load time zone string from /etc/timezone */
FILE *f = fopen("/etc/timezone", "r");
// FILE *f = fopen("/etc/timezone", "r");
TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ);
char buf[68] = {0};
if (f != NULL) {
int len = fread(buf, 64, 1, f);
if (len < 64 && ferror(f)) {
fclose(f);
if (pFile != NULL) {
int len = taosReadFile(pFile, buf, 64);
if (len < 64 && taosGetErrorFile(pFile)) {
taosCloseFile(&pFile);
// printf("read /etc/timezone error, reason:%s", strerror(errno));
return;
}
fclose(f);
taosCloseFile(&pFile);
buf[sizeof(buf) - 1] = 0;
char *lineEnd = strstr(buf, "\n");

21
source/util/src/tconfig.c
View File

@ -571,31 +571,28 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) {
}
int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
char *line, *name, *value, *value2, *value3;
char *line = NULL, *name, *value, *value2, *value3;
int olen, vlen, vlen2, vlen3;
ssize_t _bytes = 0;
size_t len = 1024;
FILE *fp = fopen(filepath, "r");
if (fp == NULL) {
// FILE *fp = fopen(filepath, "r");
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
line = malloc(len);
while (!feof(fp)) {
memset(line, 0, len);
while (!taosEOFFile(pFile)) {
name = value = value2 = value3 = NULL;
olen = vlen = vlen2 = vlen3 = 0;
_bytes = tgetline(&line, &len, fp);
_bytes = taosGetLineFile(pFile, &line);
if (_bytes < 0) {
break;
}
line[len - 1] = 0;
line[_bytes - 1] = 0;
paGetToken(line, &name, &olen);
if (olen == 0) continue;
@ -616,8 +613,8 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
// taosReadConfigOption(name, value, value2, value3);
}
fclose(fp);
tfree(line);
taosCloseFile(&pFile);
if(line != NULL) tfree(line);
uInfo("load from cfg file %s success", filepath);
return 0;

210
source/util/src/tfile.c
View File

@ -24,147 +24,147 @@ static int32_t tsFileRsetId = -1;
static int8_t tfInited = 0;
static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); }
// static void tfCloseFile(void *p) { taosCloseFile((TdFilePtr)(uintptr_t)p); }
int32_t tfInit() {
int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1);
if (old == 1) return 0;
tsFileRsetId = taosOpenRef(2000, tfCloseFile);
if (tsFileRsetId > 0) {
return 0;
} else {
atomic_store_8(&tfInited, 0);
return -1;
}
// int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1);
// if (old == 1) return 0;
// tsFileRsetId = taosOpenRef(2000, tfCloseFile);
// if (tsFileRsetId > 0) {
// return 0;
// } else {
// atomic_store_8(&tfInited, 0);
// return -1;
// }
}
void tfCleanup() {
atomic_store_8(&tfInited, 0);
if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
tsFileRsetId = -1;
// atomic_store_8(&tfInited, 0);
// if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
// tsFileRsetId = -1;
}
static int64_t tfOpenImp(int32_t fd) {
if (fd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
// static int64_t tfOpenImp(TdFilePtr pFile) {
// if (pFile == NULL) {
// terrno = TAOS_SYSTEM_ERROR(errno);
// return -1;
// }
void * p = (void *)(int64_t)fd;
int64_t rid = taosAddRef(tsFileRsetId, p);
if (rid < 0) taosCloseFile(fd);
// void * p = (void *)(int64_t)pFile;
// int64_t rid = taosAddRef(tsFileRsetId, p);
// if (rid < 0) taosCloseFile(&pFile);
return rid;
}
// return rid;
// }
int64_t tfOpenRead(const char *pathname, int32_t flags) {
int32_t fd = taosOpenFileRead(pathname);
return tfOpenImp(fd);
}
// int64_t tfOpenRead(const char *pathname, int32_t flags) {
// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ);
// return tfOpenImp(fd);
// }
int64_t tfOpenReadWrite(const char *pathname, int32_t flags) {
int32_t fd = taosOpenFileReadWrite(pathname);
return tfOpenImp(fd);
}
// int64_t tfOpenReadWrite(const char *pathname, int32_t flags) {
// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ | TD_FILE_WRITE);
// return tfOpenImp(fd);
// }
int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) {
int32_t fd = taosOpenFileCreateWrite(pathname);
return tfOpenImp(fd);
}
// int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) {
// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE);
// return tfOpenImp(fd);
// }
int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) {
int32_t fd = taosOpenFileCreateWriteAppend(pathname);
return tfOpenImp(fd);
}
// int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) {
// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
// return tfOpenImp(fd);
// }
int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); }
// int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); }
int64_t tfWrite(int64_t tfd, void *buf, int64_t count) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int64_t tfWrite(int64_t tfd, void *buf, int64_t count) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
int64_t ret = taosWriteFile(fd, buf, count);
if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
// int64_t ret = taosWriteFile(pFile, buf, count);
// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return ret;
// }
int64_t tfRead(int64_t tfd, void *buf, int64_t count) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int64_t tfRead(int64_t tfd, void *buf, int64_t count) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
int64_t ret = taosReadFile(fd, buf, count);
if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
// int64_t ret = taosReadFile(pFile, buf, count);
// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return ret;
// }
int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
int64_t ret = pread(fd, buf, count, offset);
if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
// int64_t ret = pread(fd, buf, count, offset);
// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return ret;
// }
int32_t tfFsync(int64_t tfd) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int32_t tfFsync(int64_t tfd) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int32_t code = taosFsyncFile(fd);
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
// int32_t code = taosFsyncFile(pFile);
taosReleaseRef(tsFileRsetId, tfd);
return code;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return code;
// }
bool tfValid(int64_t tfd) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return false;
// bool tfValid(int64_t tfd) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return false;
taosReleaseRef(tsFileRsetId, tfd);
return true;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return true;
// }
int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int64_t ret = taosLSeekFile(fd, offset, whence);
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
// int64_t ret = taosLSeekFile(fd, offset, whence);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return ret;
// }
int32_t tfFtruncate(int64_t tfd, int64_t length) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
// int32_t tfFtruncate(int64_t tfd, int64_t length) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int32_t code = taosFtruncateFile(fd, length);
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
// int32_t code = taosFtruncateFile(fd, length);
taosReleaseRef(tsFileRsetId, tfd);
return code;
}
// taosReleaseRef(tsFileRsetId, tfd);
// return code;
// }
void *tfMmapReadOnly(int64_t tfd, int64_t length) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return NULL;
int32_t fd = (int32_t)(uintptr_t)p;
// void *tfMmapReadOnly(int64_t tfd, int64_t length) {
// void *p = taosAcquireRef(tsFileRsetId, tfd);
// if (p == NULL) return NULL;
// int32_t pFile = (TdFilePtr)(uintptr_t)p;
void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0);
taosReleaseRef(tsFileRsetId, tfd);
return ptr;
}
// void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0);
// taosReleaseRef(tsFileRsetId, tfd);
// return ptr;
// }

106
source/util/src/tlog.c
View File

@ -47,7 +47,7 @@ typedef struct {
int32_t buffEnd;
int32_t buffSize;
int32_t minBuffSize;
int32_t fd;
TdFilePtr pFile;
int32_t stop;
pthread_t asyncThread;
pthread_mutex_t buffMutex;
@ -101,7 +101,7 @@ static SLogObj tsLogObj = {.fileNum = 1};
static void *taosAsyncOutputLog(void *param);
static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen);
static SLogBuff *taosLogBuffNew(int32_t bufSize);
static void taosCloseLogByFd(int32_t oldFd);
static void taosCloseLogByFd(TdFilePtr pFile);
static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum);
extern void taosPrintCfg();
static int32_t taosCompressFile(char *srcFileName, char *destFileName);
@ -150,11 +150,11 @@ void taosCloseLog() {
// taosCloseLog();
}
static bool taosLockLogFile(int32_t fd) {
if (fd < 0) return false;
static bool taosLockLogFile(TdFilePtr pFile) {
if (pFile == NULL) return false;
if (tsLogObj.fileNum > 1) {
int32_t ret = taosLockFile(fd);
int32_t ret = taosLockFile(pFile);
if (ret == 0) {
return true;
}
@ -163,11 +163,11 @@ static bool taosLockLogFile(int32_t fd) {
return false;
}
static void taosUnLockLogFile(int32_t fd) {
if (fd < 0) return;
static void taosUnLockLogFile(TdFilePtr pFile) {
if (pFile == NULL) return;
if (tsLogObj.fileNum > 1) {
taosUnLockFile(fd);
taosUnLockFile(pFile);
}
}
@ -201,22 +201,22 @@ static void *taosThreadToOpenNewFile(void *param) {
taosUmaskFile(0);
int32_t fd = taosOpenFileCreateWriteTrunc(name);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
tsLogObj.openInProgress = 0;
tsLogObj.lines = tsLogObj.maxLines - 1000;
uError("open new log file fail! fd:%d reason:%s, reuse lastlog", fd, strerror(errno));
uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno));
return NULL;
}
taosLockLogFile(fd);
(void)taosLSeekFile(fd, 0, SEEK_SET);
taosLockLogFile(pFile);
(void)taosLSeekFile(pFile, 0, SEEK_SET);
int32_t oldFd = tsLogObj.logHandle->fd;
tsLogObj.logHandle->fd = fd;
TdFilePtr pOldFile = tsLogObj.logHandle->pFile;
tsLogObj.logHandle->pFile = pFile;
tsLogObj.lines = 0;
tsLogObj.openInProgress = 0;
taosCloseLogByFd(oldFd);
taosCloseLogByFd(pOldFile);
uInfo(" new log file:%d is opened", tsLogObj.flag);
uInfo("==================================");
@ -262,8 +262,8 @@ void taosResetLog() {
}
static bool taosCheckFileIsOpen(char *logFileName) {
int32_t fd = taosOpenFileWrite(logFileName);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(logFileName, TD_FILE_WRITE);
if (pFile == NULL) {
if (errno == ENOENT) {
return false;
} else {
@ -272,12 +272,12 @@ static bool taosCheckFileIsOpen(char *logFileName) {
}
}
if (taosLockLogFile(fd)) {
taosUnLockLogFile(fd);
taosCloseFile(fd);
if (taosLockLogFile(pFile)) {
taosUnLockLogFile(pFile);
taosCloseFile(&pFile);
return false;
} else {
taosCloseFile(fd);
taosCloseFile(&pFile);
return true;
}
}
@ -350,31 +350,31 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
pthread_mutex_init(&tsLogObj.logMutex, NULL);
taosUmaskFile(0);
tsLogObj.logHandle->fd = taosOpenFileCreateWrite(fileName);
tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE);
if (tsLogObj.logHandle->fd < 0) {
if (tsLogObj.logHandle->pFile == NULL) {
printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
return -1;
}
taosLockLogFile(tsLogObj.logHandle->fd);
taosLockLogFile(tsLogObj.logHandle->pFile);
// only an estimate for number of lines
int64_t filesize = 0;
if (taosFStatFile(tsLogObj.logHandle->fd, &filesize, NULL) < 0) {
if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) {
printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno));
return -1;
}
size = (int32_t)filesize;
tsLogObj.lines = size / 60;
taosLSeekFile(tsLogObj.logHandle->fd, 0, SEEK_END);
taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END);
sprintf(name, "==================================================\n");
taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
sprintf(name, " new log file \n");
taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
sprintf(name, "==================================================\n");
taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
return 0;
}
@ -416,11 +416,11 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
buffer[len++] = '\n';
buffer[len] = 0;
if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) {
if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) {
if (tsAsyncLog) {
taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
} else {
taosWriteFile(tsLogObj.logHandle->fd, buffer, len);
taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
}
if (tsLogObj.maxLines > 0) {
@ -430,7 +430,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
}
}
if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len);
if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len);
}
void taosDumpData(unsigned char *msg, int32_t len) {
@ -445,7 +445,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
pos += 3;
if (c >= 16) {
temp[pos++] = '\n';
taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos);
c = 0;
pos = 0;
}
@ -453,7 +453,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
temp[pos++] = '\n';
taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos);
}
void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) {
@ -483,11 +483,11 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
buffer[len++] = '\n';
buffer[len] = 0;
if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) {
if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) {
if (tsAsyncLog) {
taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
} else {
taosWriteFile(tsLogObj.logHandle->fd, buffer, len);
taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
}
if (tsLogObj.maxLines > 0) {
@ -497,19 +497,19 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
}
}
if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len);
if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len);
}
#if 0
void taosCloseLog() {
taosCloseLogByFd(tsLogObj.logHandle->fd);
taosCloseLogByFd(tsLogObj.logHandle->pFile);
}
#endif
static void taosCloseLogByFd(int32_t fd) {
if (fd >= 0) {
taosUnLockLogFile(fd);
taosCloseFile(fd);
static void taosCloseLogByFd(TdFilePtr pFile) {
if (pFile != NULL) {
taosUnLockLogFile(pFile);
taosCloseFile(&pFile);
}
}
@ -645,12 +645,12 @@ static void taosWriteLog(SLogBuff *tLogBuff) {
}
if (start < end) {
taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize);
taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, pollSize);
} else {
int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start;
taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize);
taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, tsize);
taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end);
taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff), end);
}
dbgWN++;
@ -707,17 +707,17 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
int32_t ret = 0;
int32_t len = 0;
char *data = malloc(compressSize);
FILE *srcFp = NULL;
// gzFile dstFp = NULL;
srcFp = fopen(srcFileName, "r");
if (srcFp == NULL) {
// srcFp = fopen(srcFileName, "r");
TdFilePtr pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ);
if (pSrcFile == NULL) {
ret = -1;
goto cmp_end;
}
int32_t fd = taosOpenFileCreateWriteTrunc(destFileName);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(destFileName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
ret = -2;
goto cmp_end;
}
@ -735,8 +735,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
// }
cmp_end:
if (srcFp) {
fclose(srcFp);
if (pSrcFile) {
taosCloseFile(&pSrcFile);
}
// if (dstFp) {
// gzclose(dstFp);

61
source/util/src/tpagedbuf.c
View File

@ -27,7 +27,7 @@ struct SDiskbasedBuf {
int32_t numOfPages;
int64_t totalBufSize;
uint64_t fileSize; // disk file size
FILE* file;
TdFilePtr pFile;
int32_t allocateId; // allocated page id
char* path; // file path
int32_t pageSize; // current used page size
@ -48,8 +48,9 @@ struct SDiskbasedBuf {
};
static int32_t createDiskFile(SDiskbasedBuf* pBuf) {
pBuf->file = fopen(pBuf->path, "wb+");
if (pBuf->file == NULL) {
// pBuf->file = fopen(pBuf->path, "wb+");
pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (pBuf->pFile == NULL) {
// qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
@ -140,17 +141,17 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
pg->offset = allocatePositionInFile(pBuf, size);
pBuf->nextPos += size;
int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
ret = (int32_t)fwrite(t, 1, size, pBuf->file);
if (ret != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
ret = (int32_t) taosWriteFile(pBuf->pFile, t, size);
if (ret != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
if (pBuf->fileSize < pg->offset + size) {
pBuf->fileSize = pg->offset + size;
@ -170,18 +171,18 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
pBuf->nextPos += size;
}
// 3. write to disk.
int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
// 3. write to disk.
int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
ret = (int32_t)fwrite(t, 1, size, pBuf->file);
if (ret != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
ret = (int32_t) taosWriteFile(pBuf->pFile, t, size);
if (ret != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
if (pBuf->fileSize < pg->offset + size) {
pBuf->fileSize = pg->offset + size;
@ -210,7 +211,7 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
int32_t ret = TSDB_CODE_SUCCESS;
assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages);
if (pBuf->file == NULL) {
if (pBuf->pFile == NULL) {
if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) {
terrno = ret;
return NULL;
@ -226,14 +227,14 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
// load file block data in disk
static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
if (ret != 0) {
ret = TAOS_SYSTEM_ERROR(errno);
return ret;
}
void* pPage = (void*) GET_DATA_PAYLOAD(pg);
ret = (int32_t)fread(pPage, 1, pg->length, pBuf->file);
ret = (int32_t)taosReadFile(pBuf->pFile, pPage, pg->length);
if (ret != pg->length) {
ret = TAOS_SYSTEM_ERROR(errno);
return ret;
@ -373,7 +374,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem
pPBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit.
pPBuf->allocateId = -1;
pPBuf->comp = true;
pPBuf->file = NULL;
pPBuf->pFile = NULL;
pPBuf->qId = qId;
pPBuf->fileSize = 0;
pPBuf->pFree = taosArrayInit(4, sizeof(SFreeListItem));
@ -538,12 +539,12 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
dBufPrintStatis(pBuf);
if (pBuf->file != NULL) {
if (pBuf->pFile != NULL) {
uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n",
pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId);
fclose(pBuf->file);
taosCloseFile(&pBuf->pFile);
} else {
uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId);
}

8
tools/shell/src/backup/shellCheck.c
View File

@ -116,7 +116,7 @@ static void *shellCheckThreadFp(void *arg) {
char file[32] = {0};
snprintf(file, 32, "tb%d.txt", pThread->threadIndex);
FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (!fp) {
fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno));
return NULL;
@ -133,7 +133,7 @@ static void *shellCheckThreadFp(void *arg) {
int32_t code = taos_errno(pSql);
if (code != 0) {
int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname);
fwrite(sql, 1, len, fp);
taosWriteFile(pFile, sql, len);
atomic_add_fetch_32(&errorNum, 1);
}
@ -145,8 +145,8 @@ static void *shellCheckThreadFp(void *arg) {
taos_free_result(pSql);
}
taosFsync(fileno(fp));
fclose(fp);
taosFsync(pFile);
taosCloseFile(&pFile);
return NULL;
}

12
tools/shell/src/backup/shellImport.c
View File

@ -138,7 +138,6 @@ static void shellSourceFile(TAOS *con, char *fptr) {
char * cmd = malloc(tsMaxSQLStringLen);
size_t cmd_len = 0;
char * line = NULL;
size_t line_len = 0;
if (wordexp(fptr, &full_path, 0) != 0) {
fprintf(stderr, "ERROR: illegal file name\n");
@ -171,8 +170,9 @@ static void shellSourceFile(TAOS *con, char *fptr) {
}
*/
FILE *f = fopen(fname, "r");
if (f == NULL) {
// FILE *f = fopen(fname, "r");
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) {
fprintf(stderr, "ERROR: failed to open file %s\n", fname);
wordfree(&full_path);
free(cmd);
@ -182,7 +182,7 @@ static void shellSourceFile(TAOS *con, char *fptr) {
fprintf(stdout, "begin import file:%s\n", fname);
int lineNo = 0;
while ((read_len = getline(&line, &line_len, f)) != -1) {
while ((read_len = taosGetLineFile(pFile, &line)) != -1) {
++lineNo;
if (read_len >= tsMaxSQLStringLen) continue;
line[--read_len] = '\0';
@ -215,9 +215,9 @@ static void shellSourceFile(TAOS *con, char *fptr) {
}
free(cmd);
if (line) free(line);
if(line != NULL) free(line);
wordfree(&full_path);
fclose(f);
taosCloseFile(&pFile);
}
void* shellImportThreadFp(void *arg)

74
tools/shell/src/shellEngine.c
View File

@ -459,44 +459,44 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) {
return buf;
}
static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) {
static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int precision) {
if (val == NULL) {
fprintf(fp, "%s", TSDB_DATA_NULL_STR);
taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR);
return;
}
char buf[TSDB_MAX_BYTES_PER_ROW];
switch (field->type) {
case TSDB_DATA_TYPE_BOOL:
fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
break;
case TSDB_DATA_TYPE_TINYINT:
fprintf(fp, "%d", *((int8_t *)val));
taosFprintfFile(pFile, "%d", *((int8_t *)val));
break;
case TSDB_DATA_TYPE_SMALLINT:
fprintf(fp, "%d", *((int16_t *)val));
taosFprintfFile(pFile, "%d", *((int16_t *)val));
break;
case TSDB_DATA_TYPE_INT:
fprintf(fp, "%d", *((int32_t *)val));
taosFprintfFile(pFile, "%d", *((int32_t *)val));
break;
case TSDB_DATA_TYPE_BIGINT:
fprintf(fp, "%" PRId64, *((int64_t *)val));
taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val));
break;
case TSDB_DATA_TYPE_FLOAT:
fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val));
break;
case TSDB_DATA_TYPE_DOUBLE:
fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
taosFprintfFile(pFile, "%.9f", GET_DOUBLE_VAL(val));
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
memcpy(buf, val, length);
buf[length] = 0;
fprintf(fp, "\'%s\'", buf);
taosFprintfFile(pFile, "\'%s\'", buf);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
formatTimestamp(buf, *(int64_t *)val, precision);
fprintf(fp, "'%s'", buf);
taosFprintfFile(pFile, "'%s'", buf);
break;
default:
break;
@ -516,8 +516,9 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) {
return -1;
}
FILE *fp = fopen(full_path.we_wordv[0], "w");
if (fp == NULL) {
// FILE *fp = fopen(full_path.we_wordv[0], "w");
TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]);
wordfree(&full_path);
return -1;
@ -531,29 +532,29 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) {
for (int col = 0; col < num_fields; col++) {
if (col > 0) {
fprintf(fp, ",");
taosFprintfFile(pFile, ",");
}
fprintf(fp, "%s", fields[col].name);
taosFprintfFile(pFile, "%s", fields[col].name);
}
fputc('\n', fp);
taosFprintfFile(pFile, "\n");
int numOfRows = 0;
do {
int32_t *length = taos_fetch_lengths(tres);
for (int i = 0; i < num_fields; i++) {
if (i > 0) {
fputc(',', fp);
taosFprintfFile(pFile, "\n");
}
dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision);
dumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision);
}
fputc('\n', fp);
taosFprintfFile(pFile, "\n");
numOfRows++;
row = taos_fetch_row(tres);
} while (row != NULL);
result = 0;
fclose(fp);
taosCloseFile(&pFile);
return numOfRows;
}
@ -897,14 +898,14 @@ void read_history() {
history.hstart = 0;
history.hend = 0;
char *line = NULL;
size_t line_size = 0;
int read_size = 0;
char f_history[TSDB_FILENAME_LEN];
get_history_path(f_history);
FILE *f = fopen(f_history, "r");
if (f == NULL) {
// FILE *f = fopen(f_history, "r");
TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ);
if (pFile == NULL) {
#ifndef WINDOWS
if (errno != ENOENT) {
fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno));
@ -913,7 +914,7 @@ void read_history() {
return;
}
while ((read_size = tgetline(&line, &line_size, f)) != -1) {
while ((read_size = taosGetLineFile(pFile, &line)) != -1) {
line[read_size - 1] = '\0';
history.hist[history.hend] = strdup(line);
@ -924,16 +925,17 @@ void read_history() {
}
}
free(line);
fclose(f);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
void write_history() {
char f_history[TSDB_FILENAME_LEN];
get_history_path(f_history);
FILE *f = fopen(f_history, "w");
if (f == NULL) {
// FILE *f = fopen(f_history, "w");
TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
#ifndef WINDOWS
fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno));
#endif
@ -942,12 +944,12 @@ void write_history() {
for (int i = history.hstart; i != history.hend;) {
if (history.hist[i] != NULL) {
fprintf(f, "%s\n", history.hist[i]);
taosFprintfFile(pFile, "%s\n", history.hist[i]);
tfree(history.hist[i]);
}
i = (i + 1) % MAX_HISTORY_SIZE;
}
fclose(f);
taosCloseFile(&pFile);
}
void taos_error(TAOS_RES *tres, int64_t st) {
@ -969,7 +971,6 @@ void source_file(TAOS *con, char *fptr) {
char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1);
size_t cmd_len = 0;
char *line = NULL;
size_t line_len = 0;
if (wordexp(fptr, &full_path, 0) != 0) {
fprintf(stderr, "ERROR: illegal file name\n");
@ -989,15 +990,16 @@ void source_file(TAOS *con, char *fptr) {
}
*/
FILE *f = fopen(fname, "r");
if (f == NULL) {
// FILE *f = fopen(fname, "r");
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) {
fprintf(stderr, "ERROR: failed to open file %s\n", fname);
wordfree(&full_path);
free(cmd);
return;
}
while ((read_len = tgetline(&line, &line_len, f)) != -1) {
while ((read_len = taosGetLineFile(pFile, &line)) != -1) {
if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue;
line[--read_len] = '\0';
@ -1020,9 +1022,9 @@ void source_file(TAOS *con, char *fptr) {
}
free(cmd);
if (line) free(line);
if(line != NULL) free(line);
wordfree(&full_path);
fclose(f);
taosCloseFile(&pFile);
}
void shellGetGrantInfo(void *con) {