[td-13039] merge 3.0

include/common/ttszip.h

@@ -73,7 +73,7 @@ typedef struct STSGroupBlockInfoEx {
 } STSGroupBlockInfoEx;
 
 typedef struct STSBuf {
-  FILE*    f;
+  TdFilePtr pFile;
   char      path[PATH_MAX];
   uint32_t  fileSize;
 

include/libs/nodes/nodes.h

@@ -62,7 +62,6 @@ typedef enum ENodeType {
   QUERY_NODE_NODE_LIST,
   QUERY_NODE_FILL,
   QUERY_NODE_RAW_EXPR, // Only be used in parser module.
-  QUERY_NODE_COLUMN_REF,
   QUERY_NODE_TARGET,
   QUERY_NODE_TUPLE_DESC,
   QUERY_NODE_SLOT_DESC,
@@ -81,7 +80,9 @@ typedef enum ENodeType {
   // physical plan node
   QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN,
   QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
-  QUERY_NODE_PHYSICAL_PLAN_PROJECT
+  QUERY_NODE_PHYSICAL_PLAN_PROJECT,
+  QUERY_NODE_PHYSICAL_PLAN_JOIN,
+  QUERY_NODE_PHYSICAL_PLAN_AGG
 } ENodeType;
 
 /**

include/libs/nodes/plannodes.h

@@ -69,8 +69,6 @@ typedef struct SSlotDescNode {
   ENodeType type;
   int16_t slotId;
   SDataType dataType;
-  int16_t srcTupleId;
-  int16_t srcSlotId;
   bool reserve;
   bool output;
 } SSlotDescNode;
@@ -115,6 +113,20 @@ typedef struct SProjectPhysiNode {
   SNodeList* pProjections;
 } SProjectPhysiNode;
 
+typedef struct SJoinPhysiNode {
+  SPhysiNode node;
+  EJoinType joinType;
+  SNode* pOnConditions; // in or out tuple ?
+  SNodeList* pTargets;
+} SJoinPhysiNode;
+
+typedef struct SAggPhysiNode {
+  SPhysiNode node;
+  SNodeList* pExprs;   // these are expression list of group_by_clause and parameter expression of aggregate function
+  SNodeList* pGroupKeys; // SColumnRefNode list
+  SNodeList* pAggFuncs;
+} SAggPhysiNode;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/querynodes.h

@@ -58,15 +58,17 @@ typedef struct SColumnNode {
   char tableAlias[TSDB_TABLE_NAME_LEN];
   char colName[TSDB_COL_NAME_LEN];
   SNode* pProjectRef;
-} SColumnNode;
-
-typedef struct SColumnRefNode {
-  ENodeType type;
-  SDataType dataType;
   int16_t tupleId;
   int16_t slotId;
-  int16_t columnId;
+} SColumnNode;
-} SColumnRefNode;
+
+// typedef struct SColumnRefNode {
+//   ENodeType type;
+//   SDataType dataType;
+//   int16_t tupleId;
+//   int16_t slotId;
+//   int16_t columnId;
+// } SColumnRefNode;
 
 typedef struct STargetNode {
   ENodeType type;

include/libs/wal/wal.h

@@ -120,8 +120,8 @@ typedef struct SWal {
   int32_t fsyncSeq;
   // meta
   SWalVer vers;
-  int64_t writeLogTfd;
+  TdFilePtr pWriteLogTFile;
-  int64_t writeIdxTfd;
+  TdFilePtr pWriteIdxTFile;
   int32_t writeCur;
   SArray *fileInfoSet;
   // status
@@ -138,8 +138,8 @@ typedef struct SWal {
 
 typedef struct SWalReadHandle {
   SWal     *pWal;
-  int64_t   readLogTfd;
+  TdFilePtr pReadLogTFile;
-  int64_t   readIdxTfd;
+  TdFilePtr pReadIdxTFile;
   int64_t   curFileFirstVer;
   int64_t   curVersion;
   int64_t   capacity;

include/os/osFile.h

@@ -22,49 +22,63 @@ extern "C" {
 
 #include "osSocket.h"
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifndef ALLOW_FORBID_FUNC
-typedef int32_t FileFd;
+    #define open OPEN_FUNC_TAOS_FORBID
-#else
+    #define fopen FOPEN_FUNC_TAOS_FORBID
-typedef int32_t FileFd;
+    // #define close CLOSE_FUNC_TAOS_FORBID
+    // #define fclose FCLOSE_FUNC_TAOS_FORBID
 #endif
 
-#define FD_INITIALIZER ((int32_t)-1)
-
 #ifndef PATH_MAX
 #define PATH_MAX 256
 #endif
 
-int32_t taosLockFile(FileFd fd);
+typedef struct TdFile *TdFilePtr;
-int32_t taosUnLockFile(FileFd fd);
  
-int32_t taosUmaskFile(FileFd fd);
+#define TD_FILE_CTEATE    0x0001
+#define TD_FILE_WRITE     0x0002
+#define TD_FILE_READ      0x0004
+#define TD_FILE_TRUNC     0x0008
+#define TD_FILE_APPEND    0x0010
+#define TD_FILE_TEXT      0x0020
+#define TD_FILE_AUTO_DEL  0x0040
+#define TD_FILE_EXCL      0x0080
+ 
+int32_t taosLockFile(TdFilePtr pFile);
+int32_t taosUnLockFile(TdFilePtr pFile);
+ 
+int32_t taosUmaskFile(int32_t maskVal);
  
 int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
-int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime);
+int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
  
-FileFd taosOpenFileWrite(const char *path);
+TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
-FileFd taosOpenFileCreateWrite(const char *path);
-FileFd taosOpenFileCreateWriteTrunc(const char *path);
-FileFd taosOpenFileCreateWriteAppend(const char *path);
-FileFd taosOpenFileRead(const char *path);
-FileFd taosOpenFileReadWrite(const char *path);
  
-int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence);
+int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
-int32_t taosFtruncateFile(FileFd fd, int64_t length);
+int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length);
-int32_t taosFsyncFile(FileFd fd);
+int32_t taosFsyncFile(TdFilePtr pFile);
  
-int64_t taosReadFile(FileFd fd, void *buf, int64_t count);
+int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
-int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count);
+int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
+int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
+void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
+size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
+int32_t taosEOFFile(TdFilePtr pFile);
  
-void taosCloseFile(FileFd fd);
+int64_t taosCloseFile(TdFilePtr *ppFile);
  
 int32_t taosRenameFile(const char *oldName, const char *newName);
 int64_t taosCopyFile(const char *from, const char *to);
  
 void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
  
-int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size);
+int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
-int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);
+int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
+
+void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
+bool taosValidFile(TdFilePtr pFile);
+
+int taosGetErrorFile(TdFilePtr pFile);
 
 #ifdef __cplusplus
 }

include/util/tfile.h

@@ -30,20 +30,20 @@ void tfCleanup();
 
 // the same syntax as UNIX standard open/close/read/write
 // but FD is int64_t and will never be reused
-int64_t tfOpenRead(const char *pathname);
+// int64_t tfOpenRead(const char *pathname);
-int64_t tfOpenReadWrite(const char *pathname);
+// int64_t tfOpenReadWrite(const char *pathname);
-int64_t tfOpenCreateWrite(const char *pathname);
+// int64_t tfOpenCreateWrite(const char *pathname);
-int64_t tfOpenCreateWriteAppend(const char *pathname);
+// int64_t tfOpenCreateWriteAppend(const char *pathname);
 
-int64_t tfClose(int64_t tfd);
+// int64_t tfClose(int64_t tfd);
-int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
+// int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
-int64_t tfRead(int64_t tfd, void *buf, int64_t count);
+// int64_t tfRead(int64_t tfd, void *buf, int64_t count);
-int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset);
+// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset);
-int32_t tfFsync(int64_t tfd);
+// int32_t tfFsync(int64_t tfd);
-bool    tfValid(int64_t tfd);
+// bool    tfValid(int64_t tfd);
-int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
+// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
-int32_t tfFtruncate(int64_t tfd, int64_t length);
+// int32_t tfFtruncate(int64_t tfd, int64_t length);
-void *  tfMmapReadOnly(int64_t tfd, int64_t length);
+// void *  tfMmapReadOnly(int64_t tfd, int64_t length);
 #ifdef __cplusplus
 }
 #endif

source/client/test/clientTests.cpp

@@ -53,8 +53,10 @@ TEST(testCase, driverInit_Test) {
 //  taos_init();
 }
 
-#if 0
+#if 1
 TEST(testCase, connect_Test) {
+//  taos_options(TSDB_OPTION_CONFIGDIR, "/home/ubuntu/first/cfg");
+
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   if (pConn == NULL) {
     printf("failed to connect to server, reason:%s\n", taos_errstr(NULL));

source/common/src/ttszip.c

@@ -24,8 +24,9 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
   pTSBuf->autoDelete = autoDelete;
   
   taosGetTmpfilePath(osTempDir(), "join", pTSBuf->path);
-  pTSBuf->f = fopen(pTSBuf->path, "wb+");
+  // pTSBuf->pFile = fopen(pTSBuf->path, "wb+");
-  if (pTSBuf->f == NULL) {
+  pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
+  if (pTSBuf->pFile == NULL) {
     free(pTSBuf);
     return NULL;
   }
@@ -60,8 +61,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
   
   tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path));
   
-  pTSBuf->f = fopen(pTSBuf->path, "rb+");
+  // pTSBuf->pFile = fopen(pTSBuf->path, "rb+");
-  if (pTSBuf->f == NULL) {
+  pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ);
+  if (pTSBuf->pFile == NULL) {
     free(pTSBuf);
     return NULL;
   }
@@ -72,9 +74,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
   
   // validate the file magic number
   STSBufFileHeader header = {0};
-  int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET);
+  int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET);
   UNUSED(ret);
-  size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f);
+  size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader));
   UNUSED(sz);
 
   // invalid file
@@ -112,8 +114,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
     return NULL; 
   } 
   
-  //int64_t pos = ftell(pTSBuf->f); //pos not used
+  //int64_t pos = ftell(pTSBuf->pFile); //pos not used
-  sz = fread(buf, infoSize, 1, pTSBuf->f);
+  sz = taosReadFile(pTSBuf->pFile, buf, infoSize);
   UNUSED(sz);
   
   // the length value for each vnode is not kept in file, so does not set the length value
@@ -123,22 +125,22 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
   }
   free(buf);
   
-  ret = fseek(pTSBuf->f, 0, SEEK_END);
+  ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END);
   UNUSED(ret);
   
-  struct stat fileStat;
+  int64_t file_size;
-  if (fstat(fileno(pTSBuf->f), &fileStat) != 0) {
+  if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) {
     tsBufDestroy(pTSBuf);
     return NULL;
   }
   
-  pTSBuf->fileSize = (uint32_t)fileStat.st_size;
+  pTSBuf->fileSize = (uint32_t)file_size;
   tsBufResetPos(pTSBuf);
   
   // ascending by default
   pTSBuf->cur.order = TSDB_ORDER_ASC;
   
-//  tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f),
+//  tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->pFile),
 //           pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete);
   
   return pTSBuf;
@@ -156,7 +158,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
   tfree(pTSBuf->block.payload);
 
   if (!pTSBuf->remainOpen) {
-    fclose(pTSBuf->f);
+    taosCloseFile(&pTSBuf->pFile);
   }
   
   if (pTSBuf->autoDelete) {
@@ -253,7 +255,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
       tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize,
           TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize);
   
-  int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET);
+  int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET);
   assert(r == 0);
   
   /*
@@ -264,30 +266,30 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
    * both side has the compressed length is used to support load data forwards/backwords.
    */
   int32_t metaLen = 0;
-  metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f);
+  metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
 
   int32_t trueLen = pBlock->tag.nLen;
   if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
-    metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
-    metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f);
+    metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
   } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
-    metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
     float tfloat = (float)pBlock->tag.d;
-    metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);  
+    metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen);  
   } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
-    metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
-    metaLen += (int32_t)fwrite(&pBlock->tag.i, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
+    metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen);
   } else {
     trueLen = 0;
-    metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen));
   }
 
-  fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
+  taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem));
-  fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
+  taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
-  fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
+  taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen);
-  fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
+  taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
 
-  metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+  metaLen += (int32_t) taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen));
   assert(metaLen == getTagAreaLength(&pBlock->tag));
 
   int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen;
@@ -332,20 +334,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
      * the end of each comp data block
      */
     int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen));
-    int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR);
+    int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR);
-    size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f);
+    size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding));
-    sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
     UNUSED(sz); 
 
     pBlock->compLen = pBlock->padding;
 
     offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
-    ret = fseek(pTSBuf->f, -offset, SEEK_CUR);
+    ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR);
     UNUSED(ret);
   }
 
-  int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f);
+  int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
-  ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
+  ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
 
   // NOTE: mix types tags are not supported
   size_t sz = 0;
@@ -356,23 +358,23 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
     memset(tp, 0, pBlock->tag.nLen + 1);
     pBlock->tag.pz = tp;
 
-    sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
+    sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
     UNUSED(sz);
   } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
     float tfloat = 0;
-    sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
+    sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen);
     pBlock->tag.d = (double)tfloat;
     UNUSED(sz);
   } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value
-    sz = fread(&pBlock->tag.i, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
+    sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen);
     UNUSED(sz);
   }
 
-  sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
+  sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem));
   UNUSED(sz);
-  sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
+  sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
   UNUSED(sz);
-  sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
+  sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen);
 
   if (decomp) {
     pTSBuf->tsData.len =
@@ -381,11 +383,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
   }
   
   // read the comp length at the length of comp block
-  sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+  sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding));
   assert(pBlock->padding == pBlock->compLen);
 
   int32_t n = 0;
-  sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
+  sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen));
   if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) {
     assert(n == 0);
   } else {
@@ -396,7 +398,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
   
   // for backwards traverse, set the start position at the end of previous block
   if (order == TSDB_ORDER_DESC) {
-    int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
+    int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR);
     UNUSED(r);
   }
   
@@ -512,7 +514,7 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num
 
 // todo opt performance by cache blocks info
 static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) {
-  if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
+  if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) {
     return -1;
   }
   
@@ -531,7 +533,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int
     STSBlock* pBlock = &pTSBuf->block;
     int32_t   compBlockSize =
         pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
-    int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
+    int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR);
     UNUSED(ret);
   }
   
@@ -548,7 +550,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo
     offset = pBlockInfo->offset + pBlockInfo->compLen;
   }
   
-  if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
+  if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) {
     return -1;
   }
   
@@ -618,11 +620,11 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn
     return -1;
   }
 
-  if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
+  if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) {
     return -1;
   }
 
-  fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f);
+  taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo));
   return 0;
 }
 
@@ -636,19 +638,19 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) {
 }
 
 int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
-  if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
+  if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
     return -1;
   }
 
   assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC);
 
-  int32_t r = fseek(pTSBuf->f, 0, SEEK_SET);
+  int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET);
   if (r != 0) {
 //    qError("fseek failed, errno:%d", errno);
     return -1;
   }
 
-  size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f);
+  size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader));
   if (ws != 1) {    
 //    qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader));
     return -1;
@@ -823,12 +825,12 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
     pBlockInfoEx->info.id = id;
   }
   
-  int32_t r = fseek(pDestBuf->f, 0, SEEK_END);
+  int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END);
   assert(r == 0);
   
   int64_t offset = getDataStartOffset();
   int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
-  int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
+  int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size);
   
   if (written == -1 || written != size) {
     return -1;
@@ -839,17 +841,18 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
   int32_t oldSize = pDestBuf->fileSize;
 
   // file meta data may be cached, close and reopen the file for accurate file size.
-  fclose(pDestBuf->f);
+  taosCloseFile(&pDestBuf->pFile);
-  pDestBuf->f = fopen(pDestBuf->path, "rb+");
+  // pDestBuf->pFile = fopen(pDestBuf->path, "rb+");
-  if (pDestBuf->f == NULL) {
+  pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ);
+  if (pDestBuf->pFile == NULL) {
     return -1;
   }
 
-  struct stat fileStat;
+  int64_t file_size;
-  if (fstat(fileno(pDestBuf->f), &fileStat) != 0) {
+  if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) {
     return -1;  
   }
-  pDestBuf->fileSize = (uint32_t)fileStat.st_size;
+  pDestBuf->fileSize = (uint32_t)file_size;
 
   assert(pDestBuf->fileSize == oldSize + size);
 
@@ -868,13 +871,13 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
   // update prev vnode length info in file
   TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo);
   
-  int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
+  int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET);
   if (ret == -1) {
 //    qError("fseek failed, errno:%d", errno);
     tsBufDestroy(pTSBuf);
     return NULL;
   }
-  size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f);
+  size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len);
   if (sz != len) {
 //    qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len);
     tsBufDestroy(pTSBuf);
@@ -893,7 +896,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
   }
 
   // TODO taosFsync??
-//  if (taosFsync(fileno(pTSBuf->f)) == -1) {
+//  if (taosFsync(fileno(pTSBuf->pFile)) == -1) {
 ////    qError("fsync failed, errno:%d", errno);
 //    tsBufDestroy(pTSBuf);
 //    return NULL;
@@ -1071,15 +1074,15 @@ int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, in
   *len = 0;
   *numOfBlocks = 0;
 
-  if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
+  if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) {
-    int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+    int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile));
 //    qError("%p: fseek failed: %s", pSql, tstrerror(code));
     return code;
   }
 
-  size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f);
+  size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen);
   if (s != pBlockInfo->compLen) {
-    int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+    int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile));
 //    tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
     return code;
   }

source/dnode/mgmt/impl/inc/dndEnv.h

@@ -125,7 +125,7 @@ typedef struct SDnode {
   EStat        stat;
   SDnodeObjCfg cfg;
   SDnodeDir    dir;
-  FileFd       lockFd;
+  TdFilePtr    pLockFile;
   SDnodeMgmt   dmgmt;
   SMnodeMgmt   mmgmt;
   SQnodeMgmt   qmgmt;

source/dnode/mgmt/impl/src/dndBnode.c

@@ -62,14 +62,15 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) {
   char file[PATH_MAX + 20];
   snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
 
-  FILE *fp = fopen(file, "r");
+  // FILE *fp = fopen(file, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
     dDebug("file %s not exist", file);
     code = 0;
     goto PRASE_BNODE_OVER;
   }
 
-  len = (int32_t)fread(content, 1, maxLen, fp);
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
   if (len <= 0) {
     dError("failed to read %s since content is null", file);
     goto PRASE_BNODE_OVER;
@@ -102,7 +103,7 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) {
 PRASE_BNODE_OVER:
   if (content != NULL) free(content);
   if (root != NULL) cJSON_Delete(root);
-  if (fp != NULL) fclose(fp);
+  if (pFile != NULL) taosCloseFile(&pFile);
 
   terrno = code;
   return code;
@@ -114,8 +115,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) {
   char file[PATH_MAX + 20];
   snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
 
-  FILE *fp = fopen(file, "w");
+  // FILE *fp = fopen(file, "w");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
     terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
     dError("failed to write %s since %s", file, terrstr());
     return -1;
@@ -130,9 +132,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) {
   len += snprintf(content + len, maxLen - len, "  \"dropped\": %d\n", pMgmt->dropped);
   len += snprintf(content + len, maxLen - len, "}\n");
 
-  fwrite(content, 1, len, fp);
+  taosWriteFile(pFile, content, len);
-  taosFsyncFile(fileno(fp));
+  taosFsyncFile(pFile);
-  fclose(fp);
+  taosCloseFile(&pFile);
   free(content);
 
   char realfile[PATH_MAX + 20];

source/dnode/mgmt/impl/src/dndEnv.c

@@ -59,31 +59,31 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
   pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
 }
 
-static FileFd dndCheckRunning(char *dataDir) {
+static TdFilePtr dndCheckRunning(char *dataDir) {
   char filepath[PATH_MAX] = {0};
   snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
 
-  FileFd fd = taosOpenFileCreateWriteTrunc(filepath);
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (fd < 0) {
+  if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to lock file:%s since %s, quit", filepath, terrstr());
-    return -1;
+    return NULL;
   }
 
-  int32_t ret = taosLockFile(fd);
+  int32_t ret = taosLockFile(pFile);
   if (ret != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to lock file:%s since %s, quit", filepath, terrstr());
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
-    return -1;
+    return NULL;
   }
 
-  return fd;
+  return pFile;
 }
 
 static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) {
-  pDnode->lockFd = dndCheckRunning(pCfg->dataDir);
+  pDnode->pLockFile = dndCheckRunning(pCfg->dataDir);
-  if (pDnode->lockFd < 0) {
+  if (pDnode->pLockFile == NULL) {
     return -1;
   }
 
@@ -147,10 +147,10 @@ static void dndCloseImp(SDnode *pDnode) {
   tfree(pDnode->dir.snode);
   tfree(pDnode->dir.bnode);
 
-  if (pDnode->lockFd >= 0) {
+  if (pDnode->pLockFile != NULL) {
-    taosUnLockFile(pDnode->lockFd);
+    taosUnLockFile(pDnode->pLockFile);
-    taosCloseFile(pDnode->lockFd);
+    taosCloseFile(&pDnode->pLockFile);
-    pDnode->lockFd = 0;
+    pDnode->pLockFile = NULL;
   }
 }
 

source/dnode/mgmt/impl/src/dndMgmt.c

@@ -185,16 +185,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
   int32_t maxLen = 256 * 1024;
   char   *content = calloc(1, maxLen + 1);
   cJSON  *root = NULL;
-  FILE   *fp = NULL;
 
-  fp = fopen(pMgmt->file, "r");
+  // fp = fopen(pMgmt->file, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ);
+  if (pFile == NULL) {
     dDebug("file %s not exist", pMgmt->file);
     code = 0;
     goto PRASE_DNODE_OVER;
   }
 
-  len = (int32_t)fread(content, 1, maxLen, fp);
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
   if (len <= 0) {
     dError("failed to read %s since content is null", pMgmt->file);
     goto PRASE_DNODE_OVER;
@@ -286,7 +286,7 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
 PRASE_DNODE_OVER:
   if (content != NULL) free(content);
   if (root != NULL) cJSON_Delete(root);
-  if (fp != NULL) fclose(fp);
+  if (pFile != NULL) taosCloseFile(&pFile);
 
   if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) {
     dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file);
@@ -309,8 +309,9 @@ PRASE_DNODE_OVER:
 static int32_t dndWriteDnodes(SDnode *pDnode) {
   SDnodeMgmt *pMgmt = &pDnode->dmgmt;
 
-  FILE *fp = fopen(pMgmt->file, "w");
+  // FILE *fp = fopen(pMgmt->file, "w");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
     dError("failed to write %s since %s", pMgmt->file, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -341,9 +342,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) {
   }
   len += snprintf(content + len, maxLen - len, "}\n");
 
-  fwrite(content, 1, len, fp);
+  taosWriteFile(pFile, content, len);
-  taosFsyncFile(fileno(fp));
+  taosFsyncFile(pFile);
-  fclose(fp);
+  taosCloseFile(&pFile);
   free(content);
   terrno = 0;
 

source/dnode/mgmt/impl/src/dndMnode.c

@@ -63,14 +63,15 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
   char file[PATH_MAX + 20];
   snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
 
-  FILE *fp = fopen(file, "r");
+  // FILE *fp = fopen(file, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
     dDebug("file %s not exist", file);
     code = 0;
     goto PRASE_MNODE_OVER;
   }
 
-  len = (int32_t)fread(content, 1, maxLen, fp);
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
   if (len <= 0) {
     dError("failed to read %s since content is null", file);
     goto PRASE_MNODE_OVER;
@@ -143,7 +144,7 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
 PRASE_MNODE_OVER:
   if (content != NULL) free(content);
   if (root != NULL) cJSON_Delete(root);
-  if (fp != NULL) fclose(fp);
+  if (pFile != NULL) taosCloseFile(&pFile);
 
   terrno = code;
   return code;
@@ -155,8 +156,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
   char file[PATH_MAX + 20];
   snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode);
 
-  FILE *fp = fopen(file, "w");
+  // FILE *fp = fopen(file, "w");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
     terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
     dError("failed to write %s since %s", file, terrstr());
     return -1;
@@ -184,9 +186,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
   }
   len += snprintf(content + len, maxLen - len, "}\n");
 
-  fwrite(content, 1, len, fp);
+  taosWriteFile(pFile, content, len);
-  taosFsyncFile(fileno(fp));
+  taosFsyncFile(pFile);
-  fclose(fp);
+  taosCloseFile(&pFile);
   free(content);
 
   char realfile[PATH_MAX + 20];

source/dnode/mgmt/impl/src/dndQnode.c

@@ -62,14 +62,15 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
   char file[PATH_MAX + 20];
   snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
 
-  FILE *fp = fopen(file, "r");
+  // FILE *fp = fopen(file, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
     dDebug("file %s not exist", file);
     code = 0;
     goto PRASE_QNODE_OVER;
   }
 
-  len = (int32_t)fread(content, 1, maxLen, fp);
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
   if (len <= 0) {
     dError("failed to read %s since content is null", file);
     goto PRASE_QNODE_OVER;
@@ -102,7 +103,7 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
 PRASE_QNODE_OVER:
   if (content != NULL) free(content);
   if (root != NULL) cJSON_Delete(root);
-  if (fp != NULL) fclose(fp);
+  if (pFile != NULL) taosCloseFile(&pFile);
 
   terrno = code;
   return code;
@@ -114,8 +115,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
   char file[PATH_MAX + 20];
   snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
 
-  FILE *fp = fopen(file, "w");
+  // FILE *fp = fopen(file, "w");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
     terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
     dError("failed to write %s since %s", file, terrstr());
     return -1;
@@ -130,9 +132,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
   len += snprintf(content + len, maxLen - len, "  \"dropped\": %d\n", pMgmt->dropped);
   len += snprintf(content + len, maxLen - len, "}\n");
 
-  fwrite(content, 1, len, fp);
+  taosWriteFile(pFile, content, len);
-  taosFsyncFile(fileno(fp));
+  taosFsyncFile(pFile);
-  fclose(fp);
+  taosCloseFile(&pFile);
   free(content);
 
   char realfile[PATH_MAX + 20];

source/dnode/mgmt/impl/src/dndSnode.c

@@ -62,14 +62,15 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) {
   char file[PATH_MAX + 20];
   snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
 
-  FILE *fp = fopen(file, "r");
+  // FILE *fp = fopen(file, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
     dDebug("file %s not exist", file);
     code = 0;
     goto PRASE_SNODE_OVER;
   }
 
-  len = (int32_t)fread(content, 1, maxLen, fp);
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
   if (len <= 0) {
     dError("failed to read %s since content is null", file);
     goto PRASE_SNODE_OVER;
@@ -102,7 +103,7 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) {
 PRASE_SNODE_OVER:
   if (content != NULL) free(content);
   if (root != NULL) cJSON_Delete(root);
-  if (fp != NULL) fclose(fp);
+  if (pFile != NULL) taosCloseFile(&pFile);
 
   terrno = code;
   return code;
@@ -114,8 +115,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
   char file[PATH_MAX + 20];
   snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
 
-  FILE *fp = fopen(file, "w");
+  // FILE *fp = fopen(file, "w");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
     terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
     dError("failed to write %s since %s", file, terrstr());
     return -1;
@@ -130,9 +132,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
   len += snprintf(content + len, maxLen - len, "  \"dropped\": %d\n", pMgmt->dropped);
   len += snprintf(content + len, maxLen - len, "}\n");
 
-  fwrite(content, 1, len, fp);
+  taosWriteFile(pFile, content, len);
-  taosFsyncFile(fileno(fp));
+  taosFsyncFile(pFile);
-  fclose(fp);
+  taosCloseFile(&pFile);
   free(content);
 
   char realfile[PATH_MAX + 20];

source/dnode/mgmt/impl/src/dndVnodes.c

@@ -219,14 +219,15 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
 
   snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
 
-  fp = fopen(file, "r");
+  // fp = fopen(file, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
     dDebug("file %s not exist", file);
     code = 0;
     goto PRASE_VNODE_OVER;
   }
 
-  len = (int32_t)fread(content, 1, maxLen, fp);
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
   if (len <= 0) {
     dError("failed to read %s since content is null", file);
     goto PRASE_VNODE_OVER;
@@ -304,7 +305,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
 PRASE_VNODE_OVER:
   if (content != NULL) free(content);
   if (root != NULL) cJSON_Delete(root);
-  if (fp != NULL) fclose(fp);
+  if (pFile != NULL) taosCloseFile(&pFile);
 
   return code;
 }
@@ -315,8 +316,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
   snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes);
   snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
 
-  FILE *fp = fopen(file, "w");
+  // FILE *fp = fopen(file, "w");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to write %s since %s", file, terrstr());
     return -1;
@@ -347,9 +349,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
   len += snprintf(content + len, maxLen - len, "  ]\n");
   len += snprintf(content + len, maxLen - len, "}\n");
 
-  fwrite(content, 1, len, fp);
+  taosWriteFile(pFile, content, len);
-  taosFsyncFile(fileno(fp));
+  taosFsyncFile(pFile);
-  fclose(fp);
+  taosCloseFile(&pFile);
   free(content);
   terrno = 0;
 

source/dnode/mnode/impl/src/mndTelem.c

@@ -62,12 +62,13 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
   size_t  size = 0;
   int32_t done = 0;
 
-  FILE* fp = fopen("/proc/cpuinfo", "r");
+  // FILE* fp = fopen("/proc/cpuinfo", "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ);
+  if (pFile == NULL) {
     return;
   }
 
-  while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) {
+  while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
     line[size - 1] = '\0';
     if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
       const char* v = strchr(line, ':') + 2;
@@ -83,20 +84,21 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
     }
   }
 
-  free(line);
+  if(line != NULL) free(line);
-  fclose(fp);
+  taosCloseFile(&pFile);
 }
 
 static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
   char*  line = NULL;
   size_t size = 0;
 
-  FILE* fp = fopen("/etc/os-release", "r");
+  // FILE* fp = fopen("/etc/os-release", "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ);
+  if (pFile == NULL) {
     return;
   }
 
-  while ((size = tgetline(&line, &size, fp)) != -1) {
+  while ((size = taosGetLineFile(pFile, &line)) != -1) {
     line[size - 1] = '\0';
     if (strncmp(line, "PRETTY_NAME", 11) == 0) {
       const char* p = strchr(line, '=') + 1;
@@ -109,20 +111,21 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
     }
   }
 
-  free(line);
+  if(line != NULL) free(line);
-  fclose(fp);
+  taosCloseFile(&pFile);
 }
 
 static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
   char*  line = NULL;
   size_t size = 0;
 
-  FILE* fp = fopen("/proc/meminfo", "r");
+  // FILE* fp = fopen("/proc/meminfo", "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ);
+  if (pFile == NULL) {
     return;
   }
 
-  while ((size = tgetline(&line, &size, fp)) != -1) {
+  while ((size = taosGetLineFile(pFile, &line)) != -1) {
     line[size - 1] = '\0';
     if (strncmp(line, "MemTotal", 8) == 0) {
       const char* p = strchr(line, ':') + 1;
@@ -132,8 +135,8 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
     }
   }
 
-  free(line);
+  if(line != NULL) free(line);
-  fclose(fp);
+  taosCloseFile(&pFile);
 }
 
 static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) {
@@ -252,16 +255,16 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
 static void mndGetEmail(SMnode* pMnode, char* filepath) {
   STelemMgmt* pMgmt = &pMnode->telemMgmt;
 
-  int32_t fd = taosOpenFileRead(filepath);
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
-  if (fd < 0) {
+  if (pFile == NULL) {
     return;
   }
 
-  if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
+  if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
     mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
   }
 
-  taosCloseFile(fd);
+  taosCloseFile(&pFile);
 }
 
 int32_t mndInitTelem(SMnode* pMnode) {

source/dnode/mnode/impl/test/trans/trans.cpp

@@ -27,25 +27,25 @@ class MndTestTrans : public ::testing::Test {
 
   static void KillThenRestartServer() {
     char    file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data";
-    FileFd  fd = taosOpenFileRead(file);
+    TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
     int32_t size = 3 * 1024 * 1024;
     void*   buffer = malloc(size);
-    int32_t readLen = taosReadFile(fd, buffer, size);
+    int32_t readLen = taosReadFile(pFile, buffer, size);
     if (readLen < 0 || readLen == size) {
       ASSERT(1);
     }
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
 
     test.ServerStop();
 
-    fd = taosOpenFileCreateWriteTrunc(file);
+    pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-    int32_t writeLen = taosWriteFile(fd, buffer, readLen);
+    int32_t writeLen = taosWriteFile(pFile, buffer, readLen);
     if (writeLen < 0 || writeLen == readLen) {
       ASSERT(1);
     }
     free(buffer);
-    taosFsyncFile(fd);
+    taosFsyncFile(pFile);
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
     taosMsleep(1000);
 
     test.ServerStart();

source/dnode/mnode/sdb/src/sdbFile.c

@@ -37,8 +37,8 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) {
   return 0;
 }
 
-static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
+static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
-  int32_t ret = taosReadFile(fd, &pSdb->curVer, sizeof(int64_t));
+  int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t));
   if (ret < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
 
   for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
     int64_t maxId = -1;
-    ret = taosReadFile(fd, &maxId, sizeof(int64_t));
+    ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
     if (ret < 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return -1;
@@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
 
   for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
     int64_t ver = -1;
-    ret = taosReadFile(fd, &ver, sizeof(int64_t));
+    ret = taosReadFile(pFile, &ver, sizeof(int64_t));
     if (ret < 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return -1;
@@ -81,7 +81,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
   }
 
   char reserve[SDB_RESERVE_SIZE] = {0};
-  ret = taosReadFile(fd, reserve, sizeof(reserve));
+  ret = taosReadFile(pFile, reserve, sizeof(reserve));
   if (ret < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -94,8 +94,8 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
   return 0;
 }
 
-static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
+static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
-  if (taosWriteFile(fd, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) {
+  if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -105,7 +105,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
     if (i < SDB_MAX) {
       maxId = pSdb->maxId[i];
     }
-    if (taosWriteFile(fd, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
+    if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return -1;
     }
@@ -116,14 +116,14 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
     if (i < SDB_MAX) {
       ver = pSdb->tableVer[i];
     }
-    if (taosWriteFile(fd, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
+    if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return -1;
     }
   }
 
   char reserve[SDB_RESERVE_SIZE] = {0};
-  if (taosWriteFile(fd, reserve, sizeof(reserve)) != sizeof(reserve)) {
+  if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -148,25 +148,25 @@ int32_t sdbReadFile(SSdb *pSdb) {
   snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
   mDebug("start to read file:%s", file);
 
-  FileFd fd = taosOpenFileRead(file);
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
-  if (fd <= 0) {
+  if (pFile == NULL) {
     free(pRaw);
     terrno = TAOS_SYSTEM_ERROR(errno);
     mError("failed to read file:%s since %s", file, terrstr());
     return 0;
   }
 
-  if (sdbReadFileHead(pSdb, fd) != 0) {
+  if (sdbReadFileHead(pSdb, pFile) != 0) {
     mError("failed to read file:%s head since %s", file, terrstr());
     pSdb->curVer = -1;
     free(pRaw);
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
     return -1;
   }
 
   while (1) {
     readLen = sizeof(SSdbRaw);
-    ret = taosReadFile(fd, pRaw, readLen);
+    ret = taosReadFile(pFile, pRaw, readLen);
     if (ret == 0) break;
 
     if (ret < 0) {
@@ -182,7 +182,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
     }
 
     readLen = pRaw->dataLen + sizeof(int32_t);
-    ret = taosReadFile(fd, pRaw->pData, readLen);
+    ret = taosReadFile(pFile, pRaw->pData, readLen);
     if (ret < 0) {
       code = TAOS_SYSTEM_ERROR(errno);
       mError("failed to read file:%s since %s", file, tstrerror(code));
@@ -214,7 +214,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
   mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer);
 
 PARSE_SDB_DATA_ERROR:
-  taosCloseFile(fd);
+  taosCloseFile(&pFile);
   sdbFreeRaw(pRaw);
 
   terrno = code;
@@ -232,16 +232,16 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
   mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer,
          pSdb->lastCommitVer);
 
-  FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile);
+  TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (fd <= 0) {
+  if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     mError("failed to open file:%s for write since %s", tmpfile, terrstr());
     return -1;
   }
 
-  if (sdbWriteFileHead(pSdb, fd) != 0) {
+  if (sdbWriteFileHead(pSdb, pFile) != 0) {
     mError("failed to write file:%s head since %s", tmpfile, terrstr());
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
     return -1;
   }
 
@@ -269,7 +269,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
       if (pRaw != NULL) {
         pRaw->status = pRow->status;
         int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen;
-        if (taosWriteFile(fd, pRaw, writeLen) != writeLen) {
+        if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) {
           code = TAOS_SYSTEM_ERROR(errno);
           taosHashCancelIterate(hash, ppRow);
           sdbFreeRaw(pRaw);
@@ -277,7 +277,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
         }
 
         int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen);
-        if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
+        if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
           code = TAOS_SYSTEM_ERROR(errno);
           taosHashCancelIterate(hash, ppRow);
           sdbFreeRaw(pRaw);
@@ -296,14 +296,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
   }
 
   if (code == 0) {
-    code = taosFsyncFile(fd);
+    code = taosFsyncFile(pFile);
     if (code != 0) {
       code = TAOS_SYSTEM_ERROR(errno);
       mError("failed to write file:%s since %s", tmpfile, tstrerror(code));
     }
   }
 
-  taosCloseFile(fd);
+  taosCloseFile(&pFile);
 
   if (code == 0) {
     code = taosRenameFile(tmpfile, curfile);

source/dnode/vnode/src/inc/tqInt.h

@@ -141,9 +141,9 @@ typedef struct {
   STqMetaList* unconnectTopic;
 
   // TODO:temporaral use, to be replaced by unified tfile
-  int fileFd;
+  TdFilePtr pFile;
   // TODO:temporaral use, to be replaced by unified tfile
-  int idxFd;
+  TdFilePtr pIdxFile;
 
   char*          dirPath;
   int32_t        tqConfigFlag;

source/dnode/vnode/src/inc/tsdbFile.h

@@ -28,17 +28,17 @@
 
 #define TSDB_FILE_INFO(tf) (&((tf)->info))
 #define TSDB_FILE_F(tf) (&((tf)->f))
-#define TSDB_FILE_FD(tf) ((tf)->fd)
+#define TSDB_FILE_PFILE(tf) ((tf)->pFile)
 #define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname)
-#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
+#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL)
 #define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
-#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
+#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL)
 #define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level)
 #define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id)
 #define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did)
 #define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname)
 #define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname)
-#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf))
+#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf))
 #define TSDB_FILE_STATE(tf) ((tf)->state)
 #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
 #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
@@ -180,7 +180,7 @@ typedef struct {
 typedef struct {
   SDFInfo   info;
   STfsFile  f;
-  int      fd;
+  TdFilePtr pFile;
   uint8_t   state;
 } SDFile;
 
@@ -198,8 +198,8 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi
 static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
   ASSERT(!TSDB_FILE_OPENED(pDFile));
 
-  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags);
+  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
-  if (pDFile->fd < 0) {
+  if (pDFile->pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -209,7 +209,7 @@ static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
 
 static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
   if (TSDB_FILE_OPENED(pDFile)) {
-    close(pDFile->fd);
+    taosCloseFile(&pDFile->pFile);
     TSDB_FILE_SET_CLOSED(pDFile);
   }
 }
@@ -217,7 +217,7 @@ static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
 static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
   ASSERT(TSDB_FILE_OPENED(pDFile));
 
-  int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence);
+  int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
   if (loffset < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -229,7 +229,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh
 static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
   ASSERT(TSDB_FILE_OPENED(pDFile));
 
-  int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte);
+  int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
   if (nwrite < nbyte) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -271,7 +271,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(T
 static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
   ASSERT(TSDB_FILE_OPENED(pDFile));
 
-  int64_t nread = taosReadFile(pDFile->fd, buf, nbyte);
+  int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
   if (nread < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;

source/dnode/vnode/src/meta/metaBDBImpl.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define ALLOW_FORBID_FUNC
 #include "db.h"
 
 #include "metaDef.h"

source/dnode/vnode/src/tq/tqMetaStore.c

@@ -34,11 +34,11 @@ static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
   }
 }
 
-static inline int tqSeekLastPage(int fd) {
+static inline int64_t tqSeekLastPage(TdFilePtr pFile) {
-  int offset = lseek(fd, 0, SEEK_END);
+  int offset = taosLSeekFile(pFile, 0, SEEK_END);
   int pageNo = offset / TQ_PAGE_SIZE;
   int curPageOffset = pageNo * TQ_PAGE_SIZE;
-  return lseek(fd, curPageOffset, SEEK_SET);
+  return taosLSeekFile(pFile, curPageOffset, SEEK_SET);
 }
 
 // TODO: the struct is tightly coupled with index entry
@@ -52,10 +52,10 @@ typedef struct STqIdxPageBuf {
   char           buffer[TQ_IDX_PAGE_BODY_SIZE];
 } STqIdxPageBuf;
 
-static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
+static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) {
-  int offset = tqSeekLastPage(fd);
+  int offset = tqSeekLastPage(pFile);
   int nBytes;
-  if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
+  if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -65,7 +65,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
   }
   ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset);
 
-  return lseek(fd, offset, SEEK_SET);
+  return taosLSeekFile(pFile, offset, SEEK_SET);
 }
 
 STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer,
@@ -95,15 +95,15 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
     tqError("failed to create dir:%s since %s ", name, terrstr());
   }
   strcat(name, "/" TQ_IDX_NAME);
-  int idxFd = open(name, O_RDWR | O_CREAT, 0755);
+  TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
-  if (idxFd < 0) {
+  if (pIdxFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     tqError("failed to open file:%s since %s ", name, terrstr());
     // free memory
     return NULL;
   }
 
-  pMeta->idxFd = idxFd;
+  pMeta->pIdxFile = pIdxFile;
   pMeta->unpersistHead = calloc(1, sizeof(STqMetaList));
   if (pMeta->unpersistHead == NULL) {
     terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@@ -113,14 +113,14 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
 
   strcpy(name, path);
   strcat(name, "/" TQ_META_NAME);
-  int fileFd = open(name, O_RDWR | O_CREAT, 0755);
+  TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
-  if (fileFd < 0) {
+  if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     tqError("failed to open file:%s since %s", name, terrstr());
     return NULL;
   }
 
-  pMeta->fileFd = fileFd;
+  pMeta->pFile = pFile;
 
   pMeta->pSerializer = serializer;
   pMeta->pDeserializer = deserializer;
@@ -136,7 +136,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
   int  idxRead;
   int  allocated = TQ_PAGE_SIZE;
   bool readEnd = false;
-  while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
+  while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) {
     if (idxRead == -1) {
       // TODO: handle error
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -152,7 +152,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
       }
       memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
 
-      lseek(fileFd, pNode->handle.offset, SEEK_SET);
+      taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET);
       if (allocated < pNode->handle.serializedSize) {
         void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
         if (ptr == NULL) {
@@ -163,7 +163,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
         allocated = pNode->handle.serializedSize;
       }
       serializedObj->ssize = pNode->handle.serializedSize;
-      if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
+      if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
         // TODO: read error
       }
       if (serializedObj->action == TQ_ACTION_INUSE) {
@@ -237,8 +237,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
   // commit data and idx
   tqStorePersist(pMeta);
   ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
-  close(pMeta->fileFd);
+  taosCloseFile(&pMeta->pFile);
-  close(pMeta->idxFd);
+  taosCloseFile(&pMeta->pIdxFile);
   // free memory
   for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
     STqMetaList* pNode = pMeta->bucket[i];
@@ -263,8 +263,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
 }
 
 int32_t tqStoreDelete(STqMetaStore* pMeta) {
-  close(pMeta->fileFd);
+  taosCloseFile(&pMeta->pFile);
-  close(pMeta->idxFd);
+  taosCloseFile(&pMeta->pIdxFile);
   // free memory
   for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
     STqMetaList* pNode = pMeta->bucket[i];
@@ -302,12 +302,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
   pSHead->checksum = 0;
   pSHead->ssize = sizeof(STqSerializedHead);
   /*int allocatedSize = sizeof(STqSerializedHead);*/
-  int offset = lseek(pMeta->fileFd, 0, SEEK_CUR);
+  int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR);
 
-  tqReadLastPage(pMeta->idxFd, &idxBuf);
+  tqReadLastPage(pMeta->pIdxFile, &idxBuf);
 
   if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
-    lseek(pMeta->idxFd, 0, SEEK_END);
+    taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END);
     memset(&idxBuf, 0, TQ_PAGE_SIZE);
     idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
   } else {
@@ -329,7 +329,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
       } else {
         pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
       }
-      nBytes = write(pMeta->fileFd, pSHead, pSHead->ssize);
+      nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
       ASSERT(nBytes == pSHead->ssize);
     }
 
@@ -340,7 +340,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
       } else {
         pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
       }
-      int nBytesTxn = write(pMeta->fileFd, pSHead, pSHead->ssize);
+      int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
       ASSERT(nBytesTxn == pSHead->ssize);
       nBytes += nBytesTxn;
     }
@@ -355,7 +355,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
     idxBuf.head.writeOffset += TQ_IDX_SIZE;
 
     if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
-      nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE);
+      nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE);
       // TODO: handle error with tfile
       ASSERT(nBytes == TQ_PAGE_SIZE);
       memset(&idxBuf, 0, TQ_PAGE_SIZE);
@@ -391,13 +391,13 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
   free(pSHead);
   // TODO: write new version in tfile
   if ((char*)bufPtr != idxBuf.buffer) {
-    int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset);
+    int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset);
     // TODO: handle error in tfile
     ASSERT(nBytes == idxBuf.head.writeOffset);
   }
   // TODO: using fsync in tfile
-  fsync(pMeta->idxFd);
+  taosFsyncFile(pMeta->pIdxFile);
-  fsync(pMeta->fileFd);
+  taosFsyncFile(pMeta->pFile);
   return 0;
 }
 

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -416,8 +416,8 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
   tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname);
   tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname);
 
-  int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+  TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (fd < 0) {
+  if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -436,9 +436,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
 
   taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE);
 
-  if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    close(fd);
+    taosCloseFile(&pFile);
     remove(tfname);
     return -1;
   }
@@ -446,7 +446,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
   // Encode file status and write to file
   if (fsheader.len > 0) {
     if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) {
-      close(fd);
+      taosCloseFile(&pFile);
       remove(tfname);
       return -1;
     }
@@ -455,9 +455,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
     tsdbEncodeFSStatus(&ptr, pStatus);
     taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len);
 
-    if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) {
+    if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) {
       terrno = TAOS_SYSTEM_ERROR(errno);
-      close(fd);
+      taosCloseFile(&pFile);
       (void)remove(tfname);
       taosTZfree(pBuf);
       return -1;
@@ -465,15 +465,15 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
   }
 
   // fsync, close and rename
-  if (taosFsyncFile(fd) < 0) {
+  if (taosFsyncFile(pFile) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    close(fd);
+    taosCloseFile(&pFile);
     remove(tfname);
     taosTZfree(pBuf);
     return -1;
   }
 
-  (void)close(fd);
+  (void)taosCloseFile(&pFile);
   (void)taosRenameFile(tfname, cfname);
   taosTZfree(pBuf);
 
@@ -652,7 +652,7 @@ static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
 
 static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
   STsdbFS * pfs = REPO_FS(pRepo);
-  int       fd = -1;
+  TdFilePtr pFile = NULL;
   void *    buffer = NULL;
   SFSHeader fsheader;
   char      current[TSDB_FILENAME_LEN] = "\0";
@@ -661,8 +661,8 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
   tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current);
 
   // current file exists, try to recover
-  fd = open(current, O_RDONLY | O_BINARY);
+  pFile = taosOpenFile(current, TD_FILE_READ);
-  if (fd < 0) {
+  if (pFile == NULL) {
     tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -672,7 +672,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
     goto _err;
   }
 
-  int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE);
+  int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE);
   if (nread < 0) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current,
               strerror(errno));
@@ -706,7 +706,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
       goto _err;
     }
 
-    nread = (int)taosReadFile(fd, buffer, fsheader.len);
+    nread = (int)taosReadFile(pFile, buffer, fsheader.len);
     if (nread < 0) {
       tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno));
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -732,13 +732,13 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
   }
 
   taosTZfree(buffer);
-  close(fd);
+  taosCloseFile(&pFile);
 
   return 0;
 
 _err:
-  if (fd >= 0) {
+  if (pFile != NULL) {
-    close(fd);
+    taosCloseFile(&pFile);
   }
   taosTZfree(buffer);
   return -1;
@@ -1244,18 +1244,17 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
       }
 
       if (tsdbForceKeepFile) {
-        struct stat tfstat;
+        int64_t file_size;
-
         // Get real file size
-        if (fstat(pDFile->fd, &tfstat) < 0) {
+        if (taosFStatFile(pDFile->pFile, &file_size, NULL) < 0) {
           terrno = TAOS_SYSTEM_ERROR(errno);
           taosArrayDestroy(fArray);
           return -1;
         }
 
-        if (pDFile->info.size != tfstat.st_size) {
+        if (pDFile->info.size != file_size) {
           int64_t tfsize = pDFile->info.size;
-          pDFile->info.size = tfstat.st_size;
+          pDFile->info.size = file_size;
           tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo),
                    TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size);
         }

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -355,8 +355,8 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
 int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
   ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
 
-  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pDFile->fd < 0) {
+  if (pDFile->pFile < 0) {
     if (errno == ENOENT) {
       // Try to create directory recursively
       char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
@@ -366,8 +366,8 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
       }
       tfree(s);
 
-      pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+      pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-      if (pDFile->fd < 0) {
+      if (pDFile->pFile < 0) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
       }
@@ -460,7 +460,7 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
       return -1;
     }
 
-    if (taosFtruncateFile(df.fd, df.info.size) < 0) {
+    if (taosFtruncateFile(df.pFile, df.info.size) < 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       tsdbCloseDFile(&df);
       return -1;
@@ -541,7 +541,7 @@ static int tsdbRollBackDFile(SDFile *pDFile) {
     return -1;
   }
 
-  if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) {
+  if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     tsdbCloseDFile(&df);
     return -1;

source/libs/CMakeLists.txt

@@ -1,3 +1,4 @@
+add_definitions("-D ALLOW_FORBID_FUNC")
 add_subdirectory(transport)
 add_subdirectory(sync)
 add_subdirectory(tdb)

source/libs/function/src/taggfunction.c

@@ -3961,14 +3961,14 @@ static void ts_comp_finalize(SqlFunctionCtx *pCtx) {
 //  qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal);
 
   // TODO refactor transfer ownership of current file
-  *(FILE **)pCtx->pOutput = pTSbuf->f;
+  *(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile;
 
   pResInfo->complete = true;
 
   // get the file size
-  struct stat fStat;
+  int64_t file_size;
-  if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) {
+  if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) {
-    pResInfo->numOfRes = fStat.st_size;
+    pResInfo->numOfRes = (uint32_t )file_size;
   }
 
   pTSbuf->remainOpen = true;

source/libs/index/inc/index_fst_counting_writer.h

@@ -38,7 +38,7 @@ typedef struct WriterCtx {
   WriterType type;
   union {
     struct {
-      int  fd;
+      TdFilePtr pFile;
       bool readOnly;
       char buf[256];
       int  size;

source/libs/index/inc/index_util.h

@@ -15,19 +15,21 @@
 #ifndef __INDEX_UTIL_H__
 #define __INDEX_UTIL_H__
 
+#include "tarray.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #define SERIALIZE_MEM_TO_BUF(buf, key, mem)                     \
   do {                                                          \
-    memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \
+    memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \
     buf += sizeof(key->mem);                                    \
   } while (0)
 
 #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
   do {                                               \
-    memcpy((void*)buf, (void*)key->mem, len);        \
+    memcpy((void *)buf, (void *)key->mem, len);      \
     buf += len;                                      \
   } while (0)
 
@@ -35,16 +37,24 @@ extern "C" {
   do {                                          \
     type c = var;                               \
     assert(sizeof(type) == sizeof(c));          \
-    memcpy((void*)buf, (void*)&c, sizeof(c)); \
+    memcpy((void *)buf, (void *)&c, sizeof(c)); \
     buf += sizeof(c);                           \
   } while (0)
 
 #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
   do {                                          \
-    memcpy((void*)buf, (void*)var, len);        \
+    memcpy((void *)buf, (void *)var, len);      \
     buf += len;                                 \
   } while (0)
 
+/* multi sorted result intersection
+ * input: [1, 2, 4, 5]
+ *        [2, 3, 4, 5]
+ *        [1, 4, 5]
+ * output:[4, 5]
+ */
+void iIntersection(SArray *interResults, SArray *finalResult);
+void iUnion(SArray *interResults, SArray *finalResult);
 #ifdef __cplusplus
 }
 #endif

source/libs/index/src/index.c

@@ -370,22 +370,23 @@ static void indexInterResultsDestroy(SArray* results) {
   }
   taosArrayDestroy(results);
 }
+
 static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) {
   // refactor, merge interResults into fResults by oType
-  SArray* first = taosArrayGetP(interResults, 0);
+
-  taosArraySort(first, uidCompare);
+  for (int i = 0; i < taosArrayGetSize(interResults); i--) {
-  taosArrayRemoveDuplicate(first, uidCompare, NULL);
+    SArray* t = taosArrayGetP(interResults, i);
+    taosArraySort(t, uidCompare);
+    taosArrayRemoveDuplicate(t, uidCompare, NULL);
+  }
 
   if (oType == MUST) {
-    // just one column index, enhance later
+    iIntersection(interResults, fResults);
-    taosArrayAddAll(fResults, first);
   } else if (oType == SHOULD) {
-    // just one column index, enhance later
+    iUnion(interResults, fResults);
-    taosArrayAddAll(fResults, first);
-    // tag1 condistion || tag2 condition
   } else if (oType == NOT) {
     // just one column index, enhance later
-    taosArrayAddAll(fResults, first);
+    taosArrayAddAll(fResults, interResults);
     // not use currently
   }
   return 0;

source/libs/index/src/index_fst_counting_writer.c

@@ -19,7 +19,7 @@
 
 static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
   if (ctx->type == TFile) {
-    assert(len == tfWrite(ctx->file.fd, buf, len));
+    assert(len == taosWriteFile(ctx->file.pFile, buf, len));
   } else {
     memcpy(ctx->mem.buf + ctx->offset, buf, len);
   }
@@ -33,7 +33,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
     nRead = len < ctx->file.size ? len : ctx->file.size;
     memcpy(buf, ctx->file.ptr, nRead);
 #else
-    nRead = tfRead(ctx->file.fd, buf, len);
+    nRead = taosReadFile(ctx->file.pFile, buf, len);
 #endif
   } else {
     memcpy(buf, ctx->mem.buf + ctx->offset, len);
@@ -45,13 +45,13 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
 static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) {
   int nRead = 0;
   if (ctx->type == TFile) {
-    // tfLseek(ctx->file.fd, offset, 0);
+    // tfLseek(ctx->file.pFile, offset, 0);
 #ifdef USE_MMAP
     int32_t last = ctx->file.size - offset;
     nRead = last >= len ? len : last;
     memcpy(buf, ctx->file.ptr + offset, nRead);
 #else
-    nRead = tfPread(ctx->file.fd, buf, len, offset);
+    nRead = taosPReadFile(ctx->file.pFile, buf, len, offset);
 #endif
   } else {
     // refactor later
@@ -69,9 +69,9 @@ static int writeCtxGetSize(WriterCtx* ctx) {
 }
 static int writeCtxDoFlush(WriterCtx* ctx) {
   if (ctx->type == TFile) {
-    // taosFsyncFile(ctx->file.fd);
+    // taosFsyncFile(ctx->file.pFile);
-    tfFsync(ctx->file.fd);
+    taosFsyncFile(ctx->file.pFile);
-    // tfFlush(ctx->file.fd);
+    // tfFlush(ctx->file.pFile);
   } else {
     // do nothing
   }
@@ -87,25 +87,25 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
     // ugly code, refactor later
     ctx->file.readOnly = readOnly;
     if (readOnly == false) {
-      // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
+      // ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
-      ctx->file.fd = tfOpenCreateWriteAppend(path);
+      ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
-      tfFtruncate(ctx->file.fd, 0);
+      taosFtruncateFile(ctx->file.pFile, 0);
-      struct stat fstat;
+      int64_t file_size;
-      stat(path, &fstat);
+      taosStatFile(path, &file_size, NULL);
-      ctx->file.size = fstat.st_size;
+      ctx->file.size = (int)file_size;
     } else {
-      // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
+      // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
-      ctx->file.fd = tfOpenRead(path);
+      ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
 
       struct stat fstat;
       stat(path, &fstat);
       ctx->file.size = fstat.st_size;
 #ifdef USE_MMAP
-      ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size);
+      ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
 #endif
     }
     memcpy(ctx->file.buf, path, strlen(path));
-    if (ctx->file.fd < 0) {
+    if (ctx->file.pFile < 0) {
       indexError("failed to open file, error %d", errno);
       goto END;
     }
@@ -133,7 +133,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
     free(ctx->mem.buf);
   } else {
     ctx->flush(ctx);
-    tfClose(ctx->file.fd);
+    taosCloseFile(&ctx->file.pFile);
     if (ctx->file.readOnly) {
 #ifdef USE_MMAP
       munmap(ctx->file.ptr, ctx->file.size);

source/libs/index/src/index_tfile.c

@@ -371,7 +371,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
     return ret;
   }
 
-  IndexTFile* pTfile = (IndexTFile*)tfile;
+  IndexTFile* pTfile = tfile;
 
   SIndexTerm* term = query->term;
   ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
@@ -586,11 +586,11 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
 
   int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0);
   if (nread == -1) {
-    indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
+    indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf),
-               errno, reader->ctx->file.fd, reader->ctx->file.buf);
+               errno, reader->ctx->file.buf);
   } else {
-    indexInfo("actual Read: %d, to read: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
+    indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf),
-              reader->ctx->file.fd, reader->ctx->file.buf);
+              reader->ctx->file.buf);
   }
   // assert(nread == sizeof(buf));
   memcpy(&reader->header, buf, sizeof(buf));

source/libs/index/src/index_util.c

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "index_util.h"
+#include "index.h"
+typedef struct MergeIndex {
+  int idx;
+  int len;
+} MergeIndex;
+
+static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) {
+  uint64_t v;
+  int32_t  m;
+  while (s <= e) {
+    m = s + (e - s) / 2;
+    v = *(uint64_t *)taosArrayGet(arr, m);
+    if (v >= k) {
+      e = m - 1;
+    } else {
+      s = m + 1;
+    }
+  }
+  return s;
+}
+
+void iIntersection(SArray *inters, SArray *final) {
+  int32_t sz = taosArrayGetSize(inters);
+  if (sz <= 0) {
+    return;
+  }
+  MergeIndex *mi = calloc(sz, sizeof(MergeIndex));
+  for (int i = 0; i < sz; i++) {
+    SArray *t = taosArrayGetP(inters, i);
+    mi[i].len = taosArrayGetSize(t);
+    mi[i].idx = 0;
+  }
+
+  SArray *base = taosArrayGetP(inters, 0);
+  for (int i = 0; i < taosArrayGetSize(base); i++) {
+    uint64_t tgt = *(uint64_t *)taosArrayGet(base, i);
+    bool     has = true;
+    for (int j = 1; j < taosArrayGetSize(inters); j++) {
+      SArray *oth = taosArrayGetP(inters, j);
+      int     mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt);
+      if (mid >= 0 && mid < mi[j].len) {
+        uint64_t val = *(uint64_t *)taosArrayGet(oth, mid);
+        has = (val == tgt ? true : false);
+        mi[j].idx = mid;
+      } else {
+        has = false;
+      }
+    }
+    if (has == true) {
+      taosArrayPush(final, &tgt);
+    }
+  }
+  tfree(mi);
+}
+void iUnion(SArray *inters, SArray *final) {
+  int32_t sz = taosArrayGetSize(inters);
+  if (sz <= 0) {
+    return;
+  }
+  MergeIndex *mi = calloc(sz, sizeof(MergeIndex));
+  for (int i = 0; i < sz; i++) {
+    SArray *t = taosArrayGetP(inters, i);
+    mi[i].len = taosArrayGetSize(t);
+    mi[i].idx = 0;
+  }
+  while (1) {
+    uint64_t mVal = UINT_MAX;
+    int      mIdx = -1;
+
+    for (int j = 0; j < sz; j++) {
+      SArray *t = taosArrayGetP(inters, j);
+      if (mi[j].idx >= mi[j].len) {
+        continue;
+      }
+      uint64_t cVal = *(uint64_t *)taosArrayGet(t, mi[j].idx);
+      if (cVal < mVal) {
+        mVal = cVal;
+        mIdx = j;
+      }
+    }
+    if (mIdx != -1) {
+      mi[mIdx].idx++;
+      if (taosArrayGetSize(final) > 0) {
+        uint64_t lVal = *(uint64_t *)taosArrayGetLast(final);
+        if (lVal == mVal) {
+          continue;
+        }
+      }
+      taosArrayPush(final, &mVal);
+    } else {
+      break;
+    }
+  }
+
+  tfree(mi);
+}

source/libs/index/test/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_executable(indexTest "")
 add_executable(fstTest "")
 add_executable(fstUT "")
+add_executable(UtilUT "")
 
 target_sources(indexTest
   PRIVATE 
@@ -15,6 +16,11 @@ target_sources(fstUT
   PRIVATE 
   "fstUT.cc" 
 )
+target_sources(UtilUT
+  PRIVATE 
+  "utilUT.cc" 
+)
+
 target_include_directories ( indexTest
   PUBLIC
   "${CMAKE_SOURCE_DIR}/include/libs/index" 
@@ -31,6 +37,12 @@ target_include_directories ( fstUT
   "${CMAKE_SOURCE_DIR}/include/libs/index" 
   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 ) 
+
+target_include_directories ( UtilUT 
+  PUBLIC
+  "${CMAKE_SOURCE_DIR}/include/libs/index" 
+  "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+) 
 target_link_libraries (indexTest
   os  
   util
@@ -53,6 +65,14 @@ target_link_libraries (fstUT
   index
 )
 
+target_link_libraries (UtilUT
+  os  
+  util
+  common
+  gtest_main
+  index
+)
+
 
 #add_test(
 #  NAME index_test

source/libs/index/test/utilUT.cc

@@ -0,0 +1,203 @@
+#include <gtest/gtest.h>
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <thread>
+#include <vector>
+#include "index.h"
+#include "indexInt.h"
+#include "index_cache.h"
+#include "index_fst.h"
+#include "index_fst_counting_writer.h"
+#include "index_fst_util.h"
+#include "index_tfile.h"
+#include "index_util.h"
+#include "tglobal.h"
+#include "tskiplist.h"
+#include "tutil.h"
+
+class UtilEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() {
+    src = (SArray *)taosArrayInit(2, sizeof(void *));
+    for (int i = 0; i < 3; i++) {
+      SArray *m = taosArrayInit(10, sizeof(uint64_t));
+      taosArrayPush(src, &m);
+    }
+
+    rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t));
+  }
+  virtual void TearDown() {
+    for (int i = 0; i < taosArrayGetSize(src); i++) {
+      SArray *m = (SArray *)taosArrayGetP(src, i);
+      taosArrayDestroy(m);
+    }
+    taosArrayDestroy(src);
+  }
+
+  SArray *src;
+  SArray *rslt;
+};
+
+static void clearSourceArray(SArray *p) {
+  for (int i = 0; i < taosArrayGetSize(p); i++) {
+    SArray *m = (SArray *)taosArrayGetP(p, i);
+    taosArrayClear(m);
+  }
+}
+static void clearFinalArray(SArray *p) { taosArrayClear(p); }
+TEST_F(UtilEnv, intersectionSimpleResult) {
+  SArray *f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 0; i < 10; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < 10; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  f = (SArray *)taosArrayGetP(src, 2);
+  for (int i = 0; i < 10; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  iIntersection(src, rslt);
+  assert(taosArrayGetSize(rslt) == 10);
+
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+}
+TEST_F(UtilEnv, intersectMultiEmptyResult) {
+  SArray *f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 10; i < 20; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < 10; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  f = (SArray *)taosArrayGetP(src, 2);
+  for (int i = 20; i < 30; i++) {
+    uint64_t val = i;
+    taosArrayPush(f, &val);
+  }
+  // empty source
+  iIntersection(src, rslt);
+  assert(taosArrayGetSize(rslt) == 0);
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+}
+TEST_F(UtilEnv, intersectSimpleEmpty) {
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+
+  iIntersection(src, rslt);
+  assert(taosArrayGetSize(rslt) == 0);
+}
+TEST_F(UtilEnv, intersect01) {
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+
+  uint64_t arr1[] = {2, 3, 4, 5};
+  SArray * f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
+    taosArrayPush(f, &arr1[i]);
+  }
+
+  uint64_t arr2[] = {1, 2, 3, 5};
+  f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
+    taosArrayPush(f, &arr2[i]);
+  }
+
+  uint64_t arr3[] = {3, 5, 10, 11};
+  f = (SArray *)taosArrayGetP(src, 2);
+  for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
+    taosArrayPush(f, &arr3[i]);
+  }
+
+  iIntersection(src, rslt);
+  assert(taosArrayGetSize(rslt) == 2);
+}
+TEST_F(UtilEnv, intersect02) {
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+
+  uint64_t arr1[] = {13, 14, 15};
+  SArray * f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
+    taosArrayPush(f, &arr1[i]);
+  }
+
+  uint64_t arr2[] = {8, 10, 12, 13};
+  f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
+    taosArrayPush(f, &arr2[i]);
+  }
+
+  uint64_t arr3[] = {9, 10, 11};
+  f = (SArray *)taosArrayGetP(src, 2);
+  for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
+    taosArrayPush(f, &arr3[i]);
+  }
+
+  iIntersection(src, rslt);
+  assert(taosArrayGetSize(rslt) == 0);
+}
+TEST_F(UtilEnv, 01union) {
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+
+  uint64_t arr1[] = {13, 14, 15};
+  SArray * f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
+    taosArrayPush(f, &arr1[i]);
+  }
+  iUnion(src, rslt);
+  assert(taosArrayGetSize(rslt) == 3);
+}
+TEST_F(UtilEnv, 02union) {
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+
+  uint64_t arr1[] = {13, 14, 15};
+  SArray * f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
+    taosArrayPush(f, &arr1[i]);
+  }
+
+  uint64_t arr2[] = {13, 14, 15};
+  f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
+    taosArrayPush(f, &arr2[i]);
+  }
+  iUnion(src, rslt);
+  assert(taosArrayGetSize(rslt) == 3);
+}
+TEST_F(UtilEnv, 03union) {
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+
+  uint64_t arr1[] = {13, 16, 18, 20};
+  SArray * f = (SArray *)taosArrayGetP(src, 0);
+  for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
+    taosArrayPush(f, &arr1[i]);
+  }
+
+  uint64_t arr2[] = {0, 12, 13, 20, 23};
+  f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
+    taosArrayPush(f, &arr2[i]);
+  }
+
+  uint64_t arr3[] = {1, 12, 13, 16, 17};
+  f = (SArray *)taosArrayGetP(src, 2);
+  for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
+    taosArrayPush(f, &arr3[i]);
+  }
+  iUnion(src, rslt);
+  assert(taosArrayGetSize(rslt) == 9);
+}

source/libs/nodes/src/nodesCloneFuncs.c

@@ -142,14 +142,6 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
   return (SNode*)pDst;
 }
 
-static SNode* columnRefNodeCopy(const SColumnRefNode* pSrc, SColumnRefNode* pDst) {
-  dataTypeCopy(&pSrc->dataType, &pDst->dataType);
-  COPY_SCALAR_FIELD(tupleId);
-  COPY_SCALAR_FIELD(slotId);
-  COPY_SCALAR_FIELD(columnId);
-  return (SNode*)pDst;
-}
-
 static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) {
   COPY_SCALAR_FIELD(tupleId);
   COPY_SCALAR_FIELD(slotId);
@@ -183,8 +175,6 @@ SNode* nodesCloneNode(const SNode* pNode) {
       return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst);
     case QUERY_NODE_FUNCTION:
       return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst);
-    case QUERY_NODE_COLUMN_REF:
-      return columnRefNodeCopy((const SColumnRefNode*)pNode, (SColumnRefNode*)pDst);
     case QUERY_NODE_TARGET:
       return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst);
     case QUERY_NODE_REAL_TABLE:

source/libs/nodes/src/nodesCodeFuncs.c

@@ -55,8 +55,6 @@ static char* nodeName(ENodeType type) {
       return "NodeList";
     case QUERY_NODE_FILL:
       return "Fill";
-    case QUERY_NODE_COLUMN_REF:
-      return "ColumnRef";
     case QUERY_NODE_TARGET:
       return "Target";
     case QUERY_NODE_RAW_EXPR:
@@ -503,28 +501,6 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) {
   return code;
 }
 
-static const char* jkColumnRefDataType = "DataType";
-static const char* jkColumnRefTupleId = "TupleId";
-static const char* jkColumnRefSlotId = "SlotId";
-static const char* jkColumnRefColumnId = "ColumnId";
-
-static int32_t columnRefNodeToJson(const void* pObj, SJson* pJson) {
-  const SColumnRefNode* pNode = (const SColumnRefNode*)pObj;
-
-  int32_t code = tjsonAddObject(pJson, jkColumnRefDataType, dataTypeToJson, &pNode->dataType);
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkColumnRefTupleId, pNode->tupleId);
-  }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkColumnRefSlotId, pNode->slotId);
-  }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkColumnRefColumnId, pNode->columnId);
-  }
-
-  return code;
-}
-
 static const char* jkTargetTupleId = "TupleId";
 static const char* jkTargetSlotId = "SlotId";
 static const char* jkTargetExpr = "Expr";
@@ -646,8 +622,6 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
     case QUERY_NODE_INTERVAL_WINDOW:
     case QUERY_NODE_NODE_LIST:
     case QUERY_NODE_FILL:
-    case QUERY_NODE_COLUMN_REF:
-      return columnRefNodeToJson(pObj, pJson);
     case QUERY_NODE_TARGET:
       return targetNodeToJson(pObj, pJson);
     case QUERY_NODE_RAW_EXPR:

source/libs/nodes/src/nodesUtilFuncs.c

@@ -63,8 +63,6 @@ SNode* nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SNodeListNode));
     case QUERY_NODE_FILL:
       return makeNode(type, sizeof(SFillNode));
-    case QUERY_NODE_COLUMN_REF:
-      return makeNode(type, sizeof(SColumnRefNode));
     case QUERY_NODE_RAW_EXPR:
       return makeNode(type, sizeof(SRawExprNode));
     case QUERY_NODE_SET_OPERATOR:
@@ -93,6 +91,10 @@ SNode* nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(STableScanPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
       return makeNode(type, sizeof(SProjectPhysiNode));
+    case QUERY_NODE_PHYSICAL_PLAN_JOIN:
+      return makeNode(type, sizeof(SJoinPhysiNode));
+    case QUERY_NODE_PHYSICAL_PLAN_AGG:
+      return makeNode(type, sizeof(SAggPhysiNode));
     default:
       break;
   }

source/libs/planner/src/plannerImpl.c

@@ -55,6 +55,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
       SNode* pExpr;
       int32_t index = 0;
       FOREACH(pExpr, pCxt->pExprs) {
+        if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) {
+          pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0);
+        }
         if (nodesEqualNode(pExpr, *pNode)) {
           SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
           CHECK_ALLOC(pCol, DEAL_RES_ERROR);
@@ -406,22 +409,13 @@ typedef struct SPhysiPlanContext {
 
 static int32_t getSlotKey(SNode* pNode, char* pKey) {
   if (QUERY_NODE_COLUMN == nodeType(pNode)) {
-    return sprintf(pKey, "%s.%s", ((SColumnNode*)pNode)->tableAlias, ((SColumnNode*)pNode)->colName);
+    SColumnNode* pCol = (SColumnNode*)pNode;
-  } else {
+    if ('\0' == pCol->tableAlias[0]) {
+      return sprintf(pKey, "%s", pCol->colName);
+    }
+    return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName);
+  }
   return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName);
-  }
-}
-
-static SNode* createColumnRef(SNode* pNode, int16_t tupleId, int16_t slotId) {
-  SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF);
-  if (NULL == pCol) {
-    return NULL;
-  }
-  pCol->dataType = ((SExprNode*)pNode)->resType;
-  pCol->tupleId = tupleId;
-  pCol->slotId = slotId;
-  pCol->columnId = (QUERY_NODE_COLUMN == nodeType(pNode) ? ((SColumnNode*)pNode)->colId : -1);
-  return (SNode*)pCol;
 }
 
 static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) {
@@ -429,10 +423,8 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_
   CHECK_ALLOC(pSlot, NULL);
   pSlot->slotId = slotId;
   pSlot->dataType = ((SExprNode*)pNode)->resType;
-  pSlot->srcTupleId = -1;
-  pSlot->srcSlotId = -1;
   pSlot->reserve = false;
-  pSlot->output = true;
+  pSlot->output = false;
   return (SNode*)pSlot;
 }
 
@@ -443,17 +435,11 @@ static SNode* createTarget(SNode* pNode, int16_t tupleId, int16_t slotId) {
   }
   pTarget->tupleId = tupleId;
   pTarget->slotId = slotId;
-  pTarget->pExpr = nodesCloneNode(pNode);
+  pTarget->pExpr = pNode;
-  if (NULL == pTarget->pExpr) {
-    nodesDestroyNode((SNode*)pTarget);
-    return NULL;
-  }
   return (SNode*)pTarget;
 }
 
-static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple, SNodeList** pOutput) {
+static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple) {
-  pTuple->tupleId = pCxt->nextTupleId++;
-
   SHashObj* pHash = NULL;
   if (NULL == pTuple->pSlots) {
     pTuple->pSlots = nodesMakeList();
@@ -469,11 +455,8 @@ static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDes
     pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId);
   }
   
-  *pOutput = nodesMakeList();
-  CHECK_ALLOC(*pOutput, TSDB_CODE_OUT_OF_MEMORY);
-
   SNode* pNode = NULL;
-  int16_t slotId = 0;
+  int16_t slotId = taosHashGetSize(pHash);
   FOREACH(pNode, pList) {
     SNode* pSlot = createSlotDesc(pCxt, pNode, slotId);
     CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY);
@@ -482,48 +465,50 @@ static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDes
       return TSDB_CODE_OUT_OF_MEMORY;
     }
 
-    SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId);
-    CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY);
-    if (TSDB_CODE_SUCCESS != nodesListAppend(*pOutput, pTarget)) {
-      nodesDestroyNode(pTarget);
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
     SSlotIndex index = { .tupleId = pTuple->tupleId, .slotId = slotId };
     char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
     int32_t len = getSlotKey(pNode, name);
     CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY);
 
+    SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId);
+    CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY);
+    REPLACE_NODE(pTarget);
+  
     ++slotId;
   }
   return TSDB_CODE_SUCCESS;
 }
 
-typedef struct STransformCxt {
+typedef struct SSetSlotIdCxt {
   int32_t errCode;
-  SHashObj* pHash;
+  SHashObj* pLeftHash;
-} STransformCxt;
+  SHashObj* pRightHash;
+} SSetSlotIdCxt;
 
-static EDealRes doTransform(SNode** pNode, void* pContext) {
+static EDealRes doSetSlotId(SNode* pNode, void* pContext) {
-  if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
+  if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) {
-    STransformCxt* pCxt = (STransformCxt*)pContext;
+    SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext;
     char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
-    int32_t len = getSlotKey(*pNode, name);
+    int32_t len = getSlotKey(pNode, name);
-    SSlotIndex* pIndex = taosHashGet(pCxt->pHash, name, len);
+    SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len);
-    if (NULL != pIndex) {
+    if (NULL == pIndex) {
-      *pNode = createColumnRef(*pNode, pIndex->tupleId, pIndex->slotId);
+      pIndex = taosHashGet(pCxt->pRightHash, name, len);
-      CHECK_ALLOC(*pNode, DEAL_RES_ERROR);
-      return DEAL_RES_IGNORE_CHILD;
     }
+    // pIndex is definitely not NULL, otherwise it is a bug
+    ((SColumnNode*)pNode)->tupleId = pIndex->tupleId;
+    ((SColumnNode*)pNode)->slotId = pIndex->slotId;
+    CHECK_ALLOC(pNode, DEAL_RES_ERROR);
+    return DEAL_RES_IGNORE_CHILD;
   }
   return DEAL_RES_CONTINUE;
 }
 
-static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNode* pNode) {
+static SNode* setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNode* pNode) {
   SNode* pRes = nodesCloneNode(pNode);
   CHECK_ALLOC(pRes, NULL);
-  STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) };
+  SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId),
-  nodesRewriteNode(&pRes, doTransform, &cxt);
+      .pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) };
+  nodesWalkNode(pRes, doSetSlotId, &cxt);
   if (TSDB_CODE_SUCCESS != cxt.errCode) {
     nodesDestroyNode(pRes);
     return NULL;
@@ -531,11 +516,12 @@ static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SN
   return pRes;
 }
 
-static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNodeList* pList) {
+static SNodeList* setListSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNodeList* pList) {
   SNodeList* pRes = nodesCloneList(pList);
   CHECK_ALLOC(pRes, NULL);
-  STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) };
+  SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId),
-  nodesRewriteList(pRes, doTransform, &cxt);
+      .pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) };
+  nodesWalkList(pRes, doSetSlotId, &cxt);
   if (TSDB_CODE_SUCCESS != cxt.errCode) {
     nodesDestroyList(pRes);
     return NULL;
@@ -543,22 +529,48 @@ static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tup
   return pRes;
 }
 
-static SPhysiNode* makePhysiNode(ENodeType type) {
+static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) {
   SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type);
   if (NULL == pPhysiNode) {
     return NULL;
   }
+  pPhysiNode->outputTuple.tupleId = pCxt->nextTupleId++;
   pPhysiNode->outputTuple.type = QUERY_NODE_TUPLE_DESC;
   return pPhysiNode;
 }
 
-static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) {
+static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pLogicNode, SPhysiNode* pPhysiNode) {
-  CHECK_CODE(addTupleDesc(pCxt, pScanLogicNode->pScanCols, &pScanPhysiNode->node.outputTuple, &pScanPhysiNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY);
+  if (NULL != pLogicNode->pConditions) {
-
+    pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->outputTuple.tupleId, -1, pLogicNode->pConditions);
-  if (NULL != pScanLogicNode->node.pConditions) {
+    CHECK_ALLOC(pPhysiNode->pConditions, TSDB_CODE_OUT_OF_MEMORY);
-    pScanPhysiNode->node.pConditions = transformForPhysiPlan(pCxt, pScanPhysiNode->node.outputTuple.tupleId, pScanLogicNode->node.pConditions);
-    CHECK_ALLOC(pScanPhysiNode->node.pConditions, TSDB_CODE_OUT_OF_MEMORY);
   }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, STupleDescNode* pTuple) {
+  SHashObj* pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId);
+  char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
+  SNode* pNode;
+  FOREACH(pNode, pTargets) {
+    int32_t len = getSlotKey(pNode, name);
+    SSlotIndex* pIndex = taosHashGet(pHash, name, len);
+    ((SSlotDescNode*)nodesListGetNode(pTuple->pSlots, pIndex->slotId))->output = true;
+  }
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) {
+  if (NULL != pScanLogicNode->pScanCols) {
+    pScanPhysiNode->pScanCols = nodesCloneList(pScanLogicNode->pScanCols);
+    CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY);
+  }
+  // Tuple describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t
+  CHECK_CODE(addTupleDesc(pCxt, pScanPhysiNode->pScanCols, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY);
+
+  CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY);
+
+  CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY);
 
   pScanPhysiNode->uid = pScanLogicNode->pMeta->uid;
   pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType;
@@ -570,14 +582,14 @@ static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanL
 }
 
 static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
-  STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
+  STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
   CHECK_ALLOC(pTagScan, NULL);
   CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan);
   return (SPhysiNode*)pTagScan;
 }
 
 static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
-  STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
+  STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
   CHECK_ALLOC(pTableScan, NULL);
   CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan);
   pTableScan->scanFlag = pScanLogicNode->scanFlag;
@@ -597,35 +609,205 @@ static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode*
     default:
       break;
   }
+  return NULL;
 }
 
-static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SProjectLogicNode* pProjectLogicNode) {
+static SNodeList* createJoinOutputCols(SPhysiPlanContext* pCxt, STupleDescNode* pLeftTuple, STupleDescNode* pRightTuple) {
-  SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_PROJECT);
+  SNodeList* pCols = nodesMakeList();
+  CHECK_ALLOC(pCols, NULL);
+  SNode* pNode;
+  FOREACH(pNode, pLeftTuple->pSlots) {
+    SSlotDescNode* pSlot = (SSlotDescNode*)pNode;
+    SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
+    if (NULL == pCol) {
+      goto error;
+    }
+    pCol->node.resType = pSlot->dataType;
+    pCol->tupleId = pLeftTuple->tupleId;
+    pCol->slotId = pSlot->slotId;
+    pCol->colId = -1;
+    if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) {
+      goto error;
+    }
+  }
+  FOREACH(pNode, pRightTuple->pSlots) {
+    SSlotDescNode* pSlot = (SSlotDescNode*)pNode;
+    SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
+    if (NULL == pCol) {
+      goto error;
+    }
+    pCol->node.resType = pSlot->dataType;
+    pCol->tupleId = pRightTuple->tupleId;
+    pCol->slotId = pSlot->slotId;
+    pCol->colId = -1;
+    if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) {
+      goto error;
+    }
+  }
+  return pCols;
+error:
+  nodesDestroyList(pCols);
+  return NULL;
+}
+
+static SPhysiNode* createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode) {
+  SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN);
+  CHECK_ALLOC(pJoin, NULL);
+
+  STupleDescNode* pLeftTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple;
+  STupleDescNode* pRightTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 1))->outputTuple;
+  pJoin->pOnConditions = setNodeSlotId(pCxt, pLeftTuple->tupleId, pRightTuple->tupleId, pJoinLogicNode->pOnConditions);
+  CHECK_ALLOC(pJoin->pOnConditions, (SPhysiNode*)pJoin);
+
+  pJoin->pTargets = createJoinOutputCols(pCxt, pLeftTuple, pRightTuple);
+  CHECK_ALLOC(pJoin->pTargets, (SPhysiNode*)pJoin);
+  CHECK_CODE(addTupleDesc(pCxt, pJoin->pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin);
+
+  CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin), (SPhysiNode*)pJoin);
+
+  CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin);
+
+  return (SPhysiNode*)pJoin;
+}
+
+typedef struct SRewritePrecalcExprsCxt {
+  int32_t errCode;
+  int32_t planNodeId;
+  int32_t rewriteId;
+  SNodeList* pPrecalcExprs;
+} SRewritePrecalcExprsCxt;
+
+static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) {
+  SNode* pExpr = nodesCloneNode(*pNode);
+  CHECK_ALLOC(pExpr, DEAL_RES_ERROR);
+  if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) {
+    nodesDestroyNode(pExpr);
+    return DEAL_RES_ERROR;
+  }
+  SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
+  if (NULL == pCol) {
+    nodesDestroyNode(pExpr);
+    return DEAL_RES_ERROR;
+  }
+  SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode);
+  pCol->node.resType = pToBeRewrittenExpr->resType;
+  strcpy(pCol->colName, pToBeRewrittenExpr->aliasName);
+  nodesDestroyNode(*pNode);
+  *pNode = (SNode*)pCol;
+  return DEAL_RES_IGNORE_CHILD;
+}
+
+static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) {
+  SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext;
+  switch (nodeType(*pNode)) {
+    case QUERY_NODE_OPERATOR:
+    case QUERY_NODE_LOGIC_CONDITION: {
+      return collectAndRewrite(pContext, pNode);
+    }
+    case QUERY_NODE_FUNCTION: {
+      if (!fmIsAggFunc(((SFunctionNode*)(*pNode))->funcId)) {
+        return collectAndRewrite(pContext, pNode);
+      }
+    }
+    default:
+      break;
+  }
+  return DEAL_RES_CONTINUE;
+}
+
+static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SNodeList** pPrecalcExprs, SNodeList** pRewrittenList) {
+  if (NULL == pList) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (NULL == *pPrecalcExprs) {
+    *pPrecalcExprs = nodesMakeList();
+    CHECK_ALLOC(*pPrecalcExprs, TSDB_CODE_OUT_OF_MEMORY);
+  }
+  if (NULL == *pRewrittenList) {
+    *pRewrittenList = nodesMakeList();
+    CHECK_ALLOC(*pRewrittenList, TSDB_CODE_OUT_OF_MEMORY);
+  }
+  SNode* pNode = NULL;
+  FOREACH(pNode, pList) {
+    SNode* pNew = NULL;
+    if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) {
+      pNew = nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0));
+    } else {
+      pNew = nodesCloneNode(pNode);
+    }
+    CHECK_ALLOC(pNew, TSDB_CODE_OUT_OF_MEMORY);
+    CHECK_CODE(nodesListAppend(*pRewrittenList, pNew), TSDB_CODE_OUT_OF_MEMORY);
+  }
+  SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs };
+  nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt);
+  if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) {
+    nodesDestroyList(cxt.pPrecalcExprs);
+    *pPrecalcExprs = NULL;
+  }
+  return cxt.errCode;
+}
+
+static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode) {
+  SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG);
+  CHECK_ALLOC(pAgg, NULL);
+
+  SNodeList* pPrecalcExprs = NULL;
+  SNodeList* pGroupKeys = NULL;
+  SNodeList* pAggFuncs = NULL;
+  CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys), (SPhysiNode*)pAgg);
+  CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs), (SPhysiNode*)pAgg);
+
+  STupleDescNode* pChildTupe = &(((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple);
+  // push down expression to outputTuple of child node
+  if (NULL != pPrecalcExprs) {
+    pAgg->pExprs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pPrecalcExprs);
+    CHECK_ALLOC(pAgg->pExprs, (SPhysiNode*)pAgg);
+    CHECK_CODE(addTupleDesc(pCxt, pAgg->pExprs, pChildTupe), (SPhysiNode*)pAgg);
+  }
+
+  if (NULL != pGroupKeys) {
+    pAgg->pGroupKeys = setListSlotId(pCxt, pChildTupe->tupleId, -1, pGroupKeys);
+    CHECK_ALLOC(pAgg->pGroupKeys, (SPhysiNode*)pAgg);
+    CHECK_CODE(addTupleDesc(pCxt, pAgg->pGroupKeys, &pAgg->node.outputTuple), (SPhysiNode*)pAgg);
+  }
+
+  if (NULL != pAggFuncs) {
+    pAgg->pAggFuncs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pAggFuncs);
+    CHECK_ALLOC(pAgg->pAggFuncs, (SPhysiNode*)pAgg);
+    CHECK_CODE(addTupleDesc(pCxt, pAgg->pAggFuncs, &pAgg->node.outputTuple), (SPhysiNode*)pAgg);
+  }
+
+  CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg), (SPhysiNode*)pAgg);
+
+  CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, &pAgg->node.outputTuple), (SPhysiNode*)pAgg);
+
+  return (SPhysiNode*)pAgg;
+}
+
+static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode) {
+  SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT);
   CHECK_ALLOC(pProject, NULL);
 
-  SNodeList* pProjections = transformListForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->pProjections);
+  pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple.tupleId, -1, pProjectLogicNode->pProjections);
-  CHECK_ALLOC(pProjections, (SPhysiNode*)pProject);
+  CHECK_ALLOC(pProject->pProjections, (SPhysiNode*)pProject);
-  CHECK_CODE(addTupleDesc(pCxt, pProjections, &pProject->node.outputTuple, &pProject->pProjections), (SPhysiNode*)pProject);
+  CHECK_CODE(addTupleDesc(pCxt, pProject->pProjections, &pProject->node.outputTuple), (SPhysiNode*)pProject);
-  nodesDestroyList(pProjections);
 
-  if (NULL != pProjectLogicNode->node.pConditions) {
+  CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject), (SPhysiNode*)pProject);
-    pProject->node.pConditions = transformForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->node.pConditions);
-    CHECK_ALLOC(pProject->node.pConditions, (SPhysiNode*)pProject);
-  }
 
   return (SPhysiNode*)pProject;
 }
 
 static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) {
-  SNodeList* pChildern = nodesMakeList();
+  SNodeList* pChildren = nodesMakeList();
-  CHECK_ALLOC(pChildern, NULL);
+  CHECK_ALLOC(pChildren, NULL);
 
   SNode* pLogicChild;
   FOREACH(pLogicChild, pLogicPlan->pChildren) {
     SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild);
-    if (TSDB_CODE_SUCCESS != nodesListAppend(pChildern, pChildPhyNode)) {
+    if (TSDB_CODE_SUCCESS != nodesListAppend(pChildren, pChildPhyNode)) {
       pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
-      nodesDestroyList(pChildern);
+      nodesDestroyList(pChildren);
       return NULL;
     }
   }
@@ -636,23 +818,23 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPl
       pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan);
       break;
     case QUERY_NODE_LOGIC_PLAN_JOIN:
+      pPhyNode = createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicPlan);
       break;
     case QUERY_NODE_LOGIC_PLAN_AGG:
+      pPhyNode = createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicPlan);
       break;
     case QUERY_NODE_LOGIC_PLAN_PROJECT:
-      pPhyNode = createProjectPhysiNode(pCxt, (SProjectLogicNode*)pLogicPlan);
+      pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan);
       break;
     default:
       break;
   }
 
-  if (NULL != pPhyNode) {
+  pPhyNode->pChildren = pChildren;
-    pPhyNode->pChildren = pChildern;
   SNode* pChild;
   FOREACH(pChild, pPhyNode->pChildren) {
     ((SPhysiNode*)pChild)->pParent = pPhyNode;
   }
-  }
 
   return pPhyNode;
 }

source/libs/planner/test/newPlannerTest.cpp

@@ -123,8 +123,8 @@ TEST_F(NewPlannerTest, simple) {
 TEST_F(NewPlannerTest, groupBy) {
   setDatabase("root", "test");
 
-  bind("SELECT count(*) FROM t1");
+  // bind("SELECT count(*) FROM t1");
-  ASSERT_TRUE(run());
+  // ASSERT_TRUE(run());
 
   bind("SELECT c1, count(*) FROM t1 GROUP BY c1");
   ASSERT_TRUE(run());

source/libs/scalar/inc/filterInt.h

@@ -301,11 +301,11 @@ typedef struct SFilterInfo {
 
 #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
 #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
-#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type)
+#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnNode *)((fi)->desc))->node.resType.type)
-#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes)
+#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnNode *)((fi)->desc))->node.resType.bytes)
-#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId)
+#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId)
-#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId)
+#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId)
-#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc))
+#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnNode *)((fi)->desc))
 #define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri)))
 #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type)
 #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)

source/libs/scalar/src/filter.c

@@ -915,14 +915,14 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f
     FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
   }
   
-  if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) {
+  if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) {
     FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
   }
   
   int32_t type;
   void *v;
 
-  if (nodeType(node) == QUERY_NODE_COLUMN_REF) {
+  if (nodeType(node) == QUERY_NODE_COLUMN) {
     type = FLD_TYPE_COLUMN;
     v = node;
   } else {
@@ -1455,7 +1455,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
       qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
       for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
         SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i];
-        SColumnRefNode *refNode = (SColumnRefNode *)field->desc;
+        SColumnNode *refNode = (SColumnNode *)field->desc;
         qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId);
       }
 
@@ -1484,7 +1484,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
         char str[512] = {0};
         
         SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
-        SColumnRefNode *refNode = (SColumnRefNode *)left->desc;
+        SColumnNode *refNode = (SColumnNode *)left->desc;
         if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){
           len = sprintf(str, "UNIT[%d] => [%d][%d]  %s  [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str);
         }
@@ -3488,7 +3488,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
     return DEAL_RES_CONTINUE;
   }
 
-  if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN_REF == nodeType(*pNode)) {
+  if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
     return DEAL_RES_CONTINUE;
   }
 
@@ -3511,7 +3511,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
         return DEAL_RES_ERROR;
       }
       
-      if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) {
+      if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
         stat->scalarMode = true;
         return DEAL_RES_CONTINUE;
       }
@@ -3522,12 +3522,12 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
         return DEAL_RES_CONTINUE;
       }
     } else {
-      if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) {
+      if ((QUERY_NODE_COLUMN != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) {
         stat->scalarMode = true;
         return DEAL_RES_CONTINUE;
       }
 
-      if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) {
+      if ((QUERY_NODE_COLUMN != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) {
         stat->scalarMode = true;
         return DEAL_RES_CONTINUE;
       }      
@@ -3537,7 +3537,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
         return DEAL_RES_CONTINUE;
       }
 
-      if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) {
+      if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
         SNode *t = node->pLeft;
         node->pLeft = node->pRight;
         node->pRight = t;
@@ -3550,10 +3550,10 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
       }
 
       if (OP_TYPE_IN != node->opType) {
-        SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft;
+        SColumnNode *refNode = (SColumnNode *)node->pLeft;
         SValueNode *valueNode = (SValueNode *)node->pRight;
-        int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type);
+        int32_t type = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type);
-        if (0 != type && type != refNode->dataType.type) {
+        if (0 != type && type != refNode->node.resType.type) {
           stat->scalarMode = true;
           return DEAL_RES_CONTINUE;
         }

source/libs/scalar/src/scalar.c

@@ -134,13 +134,13 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
       
       break;
     }
-    case QUERY_NODE_COLUMN_REF: {
+    case QUERY_NODE_COLUMN: {
       if (NULL == ctx) {
         sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx);
         SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
       }
       
-      SColumnRefNode *ref = (SColumnRefNode *)node;
+      SColumnNode *ref = (SColumnNode *)node;
       if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) {
         sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock));
         SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
@@ -281,7 +281,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu
   SScalarFuncExecFuncs ffpSet = {0};
   int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
   if (code) {
-    sclError(
"fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
+    sclError(
+"fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
     SCL_ERR_RET(code);
   }
 
@@ -299,7 +300,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu
   for (int32_t i = 0; i < rowNum; ++i) {
     code = (*ffpSet.process)(params, node->pParameterList->length, output);
     if (code) {
-      sclError(
"scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
+      sclError(
+"scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
       SCL_ERR_JRET(code);    
     }
 
@@ -599,7 +601,7 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) {
 
 
 EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
-  if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN_REF == nodeType(pNode)) {
+  if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
     return DEAL_RES_CONTINUE;
   }
     

source/libs/scalar/test/filter/filterTests.cpp

@@ -76,15 +76,15 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
 }
 
 void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
-  SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF);
+  SNode *node = nodesMakeNode(QUERY_NODE_COLUMN);
-  SColumnRefNode *rnode = (SColumnRefNode *)node;
+  SColumnNode *rnode = (SColumnNode *)node;
-  rnode->dataType.type = dataType;
+  rnode->node.resType.type = dataType;
-  rnode->dataType.bytes = dataBytes;
+  rnode->node.resType.bytes = dataBytes;
   rnode->tupleId = 0;
 
   if (NULL == block) {
     rnode->slotId = 2;
-    rnode->columnId = 55;
+    rnode->colId = 55;
     *pNode = (SNode *)rnode;
 
     return;
@@ -120,7 +120,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
     taosArrayPush(res->pDataBlock, &idata);
 
     rnode->slotId = 2;
-    rnode->columnId = 55;
+    rnode->colId = 55;
 
     *block = res;
   } else {
@@ -137,7 +137,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
     res->info.numOfCols++;
     
     rnode->slotId = idx;
-    rnode->columnId = 55 + idx;
+    rnode->colId = 55 + idx;
   }
 
   *pNode = (SNode *)rnode;
@@ -259,7 +259,7 @@ TEST(columnTest, smallint_column_greater_double_value) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  stat.colId = ((SColumnRefNode *)pLeft)->columnId;
+  stat.colId = ((SColumnNode *)pLeft)->colId;
   stat.max = 10;
   stat.min = 5;
   stat.numOfNull = 0;
@@ -310,7 +310,7 @@ TEST(columnTest, int_column_greater_smallint_value) {
   ASSERT_EQ(code, 0);
 
   SColumnDataAgg stat = {0};
-  stat.colId = ((SColumnRefNode *)pLeft)->columnId;
+  stat.colId = ((SColumnNode *)pLeft)->colId;
   stat.max = 10;
   stat.min = 5;
   stat.numOfNull = 0;

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -75,10 +75,10 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
 }
 
 void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
-  SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF);
+  SNode *node = nodesMakeNode(QUERY_NODE_COLUMN);
-  SColumnRefNode *rnode = (SColumnRefNode *)node;
+  SColumnNode *rnode = (SColumnNode *)node;
-  rnode->dataType.type = dataType;
+  rnode->node.resType.type = dataType;
-  rnode->dataType.bytes = dataBytes;
+  rnode->node.resType.bytes = dataBytes;
   rnode->tupleId = 0;
 
   if (NULL == *block) {
@@ -111,7 +111,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
     taosArrayPush(res->pDataBlock, &idata);
 
     rnode->slotId = 2;
-    rnode->columnId = 55;
+    rnode->colId = 55;
 
     *block = res;
   } else {
@@ -126,7 +126,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
     taosArrayPush(res->pDataBlock, &idata);
     
     rnode->slotId = idx;
-    rnode->columnId = 55 + idx;
+    rnode->colId = 55 + idx;
   }
 
   *pNode = (SNode *)rnode;

source/libs/tdb/src/db/tdbEnv.c

@@ -18,7 +18,7 @@
 struct STDbEnv {
   char *      rootDir;    // root directory of the environment
   char *      jname;      // journal file name
-  int         jfd;        // journal file fd
+  TdFilePtr   jpFile;        // journal file fd
   pgsz_t      pgSize;     // page size
   cachesz_t   cacheSize;  // total cache size
   STDbList    dbList;     // TDB List
@@ -55,7 +55,7 @@ int tdbEnvCreate(TENV **ppEnv, const char *rootDir) {
 
   pEnv->rootDir = (char *)(&pEnv[1]);
   pEnv->jname = pEnv->rootDir + slen + 1;
-  pEnv->jfd = -1;
+  pEnv->jpFile = NULL;
   pEnv->pgSize = TDB_DEFAULT_PGSIZE;
   pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE;
 
@@ -139,8 +139,8 @@ static int tdbEnvDestroy(TENV *pEnv) {
 }
 
 int tdbEnvBeginTxn(TENV *pEnv) {
-  pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755);
+  pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
-  if (pEnv->jfd < 0) {
+  if (pEnv->jpFile < 0) {
     return -1;
   }
 
@@ -149,8 +149,8 @@ int tdbEnvBeginTxn(TENV *pEnv) {
 
 int tdbEnvCommit(TENV *pEnv) {
   /* TODO */
-  close(pEnv->jfd);
+  taosCloseFile(&pEnv->jpFile);
-  pEnv->jfd = -1;
+  pEnv->jpFile = NULL;
   return 0;
 }
 

source/libs/tdb/src/db/tdbPgFile.c

@@ -51,10 +51,10 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
   pPgFile->fname = (char *)(&(pPgFile[1]));
   memcpy(pPgFile->fname, fname, fnameLen);
   pPgFile->fname[fnameLen] = '\0';
-  pPgFile->fd = -1;
+  pPgFile->pFile = NULL;
 
-  pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755);
+  pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
-  if (pPgFile->fd < 0) {
+  if (pPgFile->pFile == NULL) {
     // TODO: handle error
     return -1;
   }
@@ -95,8 +95,8 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
 
 int pgFileClose(SPgFile *pPgFile) {
   if (pPgFile) {
-    if (pPgFile->fd >= 0) {
+    if (pPgFile->pFile >= 0) {
-      close(pPgFile->fd);
+      taosCloseFile(&pPgFile->pFile);
     }
 
     tfree(pPgFile->fname);
@@ -201,7 +201,7 @@ static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) {
   pTData = pData;
   szToRead = pgSize;
   for (; szToRead > 0;) {
-    rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize);
+    rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize);
     if (rsize < 0) {
       if (errno == EINTR) {
         continue;

source/libs/tdb/src/inc/tdbPgFile.h

@@ -38,7 +38,7 @@ struct SPgFile {
   uint8_t         fileid[TDB_FILE_ID_LEN];  // file id
   pgno_t          lsize;                    // page file logical size (for count)
   pgno_t          fsize;                    // real file size on disk (for rollback)
-  int             fd;
+  TdFilePtr       pFile;
   SPgFileListNode envHash;
   SPgFileListNode envPgfList;
 };

source/libs/tfs/test/tfsTest.cpp

@@ -230,10 +230,11 @@ TEST_F(TfsTest, 04_File) {
 
     EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0);
 
-    FILE *fp = fopen(f1.aname, "w");
+    // FILE *fp = fopen(f1.aname, "w");
-    ASSERT_NE(fp, nullptr);
+    TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-    fwrite("12345678", 1, 5, fp);
+    ASSERT_NE(pFile, nullptr);
-    fclose(fp);
+    taosWriteFile(pFile, "12345678", 5);
+    taosCloseFile(&pFile);
 
     char base[128] = {0};
     tfsBasename(&f1, base);
@@ -638,10 +639,11 @@ TEST_F(TfsTest, 05_MultiDisk) {
 
       EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0);
 
-      FILE *fp = fopen(f1.aname, "w");
+      // FILE *fp = fopen(f1.aname, "w");
-      ASSERT_NE(fp, nullptr);
+      TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-      fwrite("12345678", 1, 5, fp);
+      ASSERT_NE(pFile, nullptr);
-      fclose(fp);
+      taosWriteFile(pFile, "12345678", 5);
+      taosCloseFile(&pFile);
 
       char base[128] = {0};
       tfsBasename(&f1, base);

source/libs/transport/inc/rpcUdp.h

@@ -30,7 +30,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
 
 void  taosFreeMsgHdr(void *hdr);
 int   taosMsgHdrSize(void *hdr);
-void  taosSendMsgHdr(void *hdr, int fd);
+void  taosSendMsgHdr(void *hdr, TdFilePtr pFile);
 void  taosInitMsgHdr(void **hdr, void *dest, int maxPkts);
 void  taosSetMsgHdrData(void *hdr, char *data, int dataLen);
 

source/libs/transport/test/pushServer.c

@@ -22,7 +22,7 @@
 
 int         msgSize = 128;
 int         commit = 0;
-int         dataFd = -1;
+TdFilePtr   pDataFile = NULL;
 STaosQueue *qhandle = NULL;
 STaosQset * qset = NULL;
 
@@ -43,8 +43,8 @@ void processShellMsg() {
     for (int i = 0; i < numOfMsgs; ++i) {
       taosGetQitem(qall, (void **)&pRpcMsg);
 
-      if (dataFd >= 0) {
+      if (pDataFile != NULL) {
-        if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
+        if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
           tInfo("failed to write data file, reason:%s", strerror(errno));
         }
       }
@@ -52,7 +52,7 @@ void processShellMsg() {
 
     if (commit >= 2) {
       num += numOfMsgs;
-      // if (taosFsync(dataFd) < 0) {
+      // if (taosFsync(pDataFile) < 0) {
       //  tInfo("failed to flush data to file, reason:%s", strerror(errno));
       //}
 
@@ -181,8 +181,8 @@ int main(int argc, char *argv[]) {
   tInfo("RPC server is running, ctrl-c to exit");
 
   if (commit) {
-    dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
+    pDataFile = taosOpenFile(dataName, TD_FILE_APPEND | TD_FILE_CTEATE | TD_FILE_WRITE);
-    if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno));
+    if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno));
   }
   qhandle = taosOpenQueue();
   qset = taosOpenQset();
@@ -190,8 +190,8 @@ int main(int argc, char *argv[]) {
 
   processShellMsg();
 
-  if (dataFd >= 0) {
+  if (pDataFile != NULL) {
-    close(dataFd);
+    taosCloseFile(&pDataFile);
     remove(dataName);
   }
 

source/libs/transport/test/rserver.c

@@ -22,7 +22,7 @@
 
 int         msgSize = 128;
 int         commit = 0;
-int         dataFd = -1;
+TdFilePtr   pDataFile = NULL;
 STaosQueue *qhandle = NULL;
 STaosQset * qset = NULL;
 
@@ -43,8 +43,8 @@ void processShellMsg() {
     for (int i = 0; i < numOfMsgs; ++i) {
       taosGetQitem(qall, (void **)&pRpcMsg);
 
-      if (dataFd >= 0) {
+      if (pDataFile >= 0) {
-        if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
+        if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
           tInfo("failed to write data file, reason:%s", strerror(errno));
         }
       }
@@ -52,7 +52,7 @@ void processShellMsg() {
 
     if (commit >= 2) {
       num += numOfMsgs;
-      // if (taosFsync(dataFd) < 0) {
+      // if (taosFsync(pDataFile) < 0) {
       //  tInfo("failed to flush data to file, reason:%s", strerror(errno));
       //}
 
@@ -170,8 +170,8 @@ int main(int argc, char *argv[]) {
   tInfo("RPC server is running, ctrl-c to exit");
 
   if (commit) {
-    dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
+    pDataFile = taosOpenFile(dataName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
-    if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno));
+    if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno));
   }
   qhandle = taosOpenQueue();
   qset = taosOpenQset();
@@ -179,8 +179,8 @@ int main(int argc, char *argv[]) {
 
   processShellMsg();
 
-  if (dataFd >= 0) {
+  if (pDataFile != NULL) {
-    close(dataFd);
+    taosCloseFile(&pDataFile);
     remove(dataName);
   }
 

source/libs/wal/src/walMeta.c

@@ -69,8 +69,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
   int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size);
   pLastFileInfo->fileSize = statbuf.st_size;
 
-  FileFd fd = taosOpenFileRead(fnameStr);
+  TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
-  if (fd < 0) {
+  if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -79,15 +79,15 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
 
   char* buf = malloc(readSize + 5);
   if (buf == NULL) {
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
     terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
     return -1;
   }
 
-  taosLSeekFile(fd, -readSize, SEEK_END);
+  taosLSeekFile(pFile, -readSize, SEEK_END);
-  if (readSize != taosReadFile(fd, buf, readSize)) {
+  if (readSize != taosReadFile(pFile, buf, readSize)) {
     free(buf);
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -108,12 +108,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
     if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) {
       // file has to be deleted
       free(buf);
-      taosCloseFile(fd);
+      taosCloseFile(&pFile);
       terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
       return -1;
     }
   }
-  taosCloseFile(fd);
+  taosCloseFile(&pFile);
   SWalHead *lastEntry = (SWalHead*)found;
 
   return lastEntry->head.version;
@@ -364,18 +364,18 @@ int walSaveMeta(SWal* pWal) {
   int  metaVer = walFindCurMetaVer(pWal);
   char fnameStr[WAL_FILE_LEN];
   walBuildMetaName(pWal, metaVer + 1, fnameStr);
-  FileFd metaFd = taosOpenFileCreateWrite(fnameStr);
+  TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE);
-  if (metaFd < 0) {
+  if (pMataFile == NULL) {
     return -1;
   }
   char* serialized = walMetaSerialize(pWal);
   int   len = strlen(serialized);
-  if (len != taosWriteFile(metaFd, serialized, len)) {
+  if (len != taosWriteFile(pMataFile, serialized, len)) {
     // TODO:clean file
     return -1;
   }
 
-  taosCloseFile(metaFd);
+  taosCloseFile(&pMataFile);
   // delete old file
   if (metaVer > -1) {
     walBuildMetaName(pWal, metaVer, fnameStr);
@@ -404,20 +404,20 @@ int walLoadMeta(SWal* pWal) {
     return -1;
   }
   memset(buf, 0, size + 5);
-  FileFd fd = taosOpenFileRead(fnameStr);
+  TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
-  if (fd < 0) {
+  if (pFile == NULL) {
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }
-  if (taosReadFile(fd, buf, size) != size) {
+  if (taosReadFile(pFile, buf, size) != size) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
     free(buf);
     return -1;
   }
   // load into fileInfoSet
   int code = walMetaDeserialize(pWal, buf);
-  taosCloseFile(fd);
+  taosCloseFile(&pFile);
   free(buf);
   return code;
 }

source/libs/wal/src/walMgmt.c

@@ -89,8 +89,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
 
   // open meta
   walResetVer(&pWal->vers);
-  pWal->writeLogTfd = -1;
+  pWal->pWriteLogTFile = NULL;
-  pWal->writeIdxTfd = -1;
+  pWal->pWriteIdxTFile = NULL;
   pWal->writeCur = -1;
   pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo));
   if (pWal->fileInfoSet == NULL) {
@@ -164,10 +164,10 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
 
 void walClose(SWal *pWal) {
   pthread_mutex_lock(&pWal->mutex);
-  tfClose(pWal->writeLogTfd);
+  taosCloseFile(&pWal->pWriteLogTFile);
-  pWal->writeLogTfd = -1;
+  pWal->pWriteLogTFile = NULL;
-  tfClose(pWal->writeIdxTfd);
+  taosCloseFile(&pWal->pWriteIdxTFile);
-  pWal->writeIdxTfd = -1;
+  pWal->pWriteIdxTFile = NULL;
   walSaveMeta(pWal);
   taosArrayDestroy(pWal->fileInfoSet);
   pWal->fileInfoSet = NULL;
@@ -207,7 +207,7 @@ static void walFsyncAll() {
     if (walNeedFsync(pWal)) {
       wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq,
              atomic_load_32(&tsWal.seq));
-      int32_t code = tfFsync(pWal->writeLogTfd);
+      int32_t code = taosFsyncFile(pWal->pWriteLogTFile);
       if (code != 0) {
         wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
                strerror(code));

source/libs/wal/src/walRead.c

@@ -25,8 +25,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
   }
 
   pRead->pWal = pWal;
-  pRead->readIdxTfd = -1;
+  pRead->pReadIdxTFile = NULL;
-  pRead->readLogTfd = -1;
+  pRead->pReadLogTFile = NULL;
   pRead->curVersion = -1;
   pRead->curFileFirstVer = -1;
   pRead->capacity = 0;
@@ -41,8 +41,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
 }
 
 void walCloseReadHandle(SWalReadHandle *pRead) {
-  tfClose(pRead->readIdxTfd);
+  taosCloseFile(&pRead->pReadIdxTFile);
-  tfClose(pRead->readLogTfd);
+  taosCloseFile(&pRead->pReadLogTFile);
   tfree(pRead->pHead);
   free(pRead);
 }
@@ -52,24 +52,24 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; }
 static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
   int code = 0;
 
-  int64_t idxTfd = pRead->readIdxTfd;
+  TdFilePtr pIdxTFile = pRead->pReadIdxTFile;
-  int64_t logTfd = pRead->readLogTfd;
+  TdFilePtr pLogTFile = pRead->pReadLogTFile;
 
   // seek position
   int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
-  code = tfLseek(idxTfd, offset, SEEK_SET);
+  code = taosLSeekFile(pIdxTFile, offset, SEEK_SET);
   if (code < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
   SWalIdxEntry entry;
-  if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
+  if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }
   // TODO:deserialize
   ASSERT(entry.ver == ver);
-  code = tfLseek(logTfd, entry.offset, SEEK_SET);
+  code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
   if (code < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -80,24 +80,24 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
 static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
   char fnameStr[WAL_FILE_LEN];
 
-  tfClose(pRead->readIdxTfd);
+  taosCloseFile(&pRead->pReadIdxTFile);
-  tfClose(pRead->readLogTfd);
+  taosCloseFile(&pRead->pReadLogTFile);
 
   walBuildLogName(pRead->pWal, fileFirstVer, fnameStr);
-  int64_t logTfd = tfOpenRead(fnameStr);
+  TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ);
-  if (logTfd < 0) {
+  if (pLogTFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
   walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
-  int64_t idxTfd = tfOpenRead(fnameStr);
+  TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ);
-  if (idxTfd < 0) {
+  if (pIdxTFile == NULL) {
     return -1;
   }
 
-  pRead->readLogTfd = logTfd;
+  pRead->pReadLogTFile = pLogTFile;
-  pRead->readIdxTfd = idxTfd;
+  pRead->pReadIdxTFile = pIdxTFile;
   return 0;
 }
 
@@ -145,9 +145,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
     }
   }
 
-  if (!tfValid(pRead->readLogTfd)) return -1;
+  if (!taosValidFile(pRead->pReadLogTFile)) return -1;
 
-  code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead));
+  code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead));
   if (code != sizeof(SWalHead)) {
     return -1;
   }
@@ -165,7 +165,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
     pRead->pHead = ptr;
     pRead->capacity = pRead->pHead->head.len;
   }
-  if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) {
+  if (pRead->pHead->head.len != taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.len)) {
     return -1;
   }
 
@@ -202,7 +202,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
     }
     *ppHead = ptr;
   }
-  if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
+  if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
     return -1;
   }
   // TODO: endian compatibility processing after read
@@ -215,7 +215,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
     *ppHead = NULL;
     return -1;
   }
-  if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
+  if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
     return -1;
   }
   // TODO: endian compatibility processing after read

source/libs/wal/src/walSeek.c

@@ -23,25 +23,25 @@
 static int walSeekWritePos(SWal* pWal, int64_t ver) {
   int code = 0;
 
-  int64_t idxTfd = pWal->writeIdxTfd;
+  TdFilePtr pIdxTFile = pWal->pWriteIdxTFile;
-  int64_t logTfd = pWal->writeLogTfd;
+  TdFilePtr pLogTFile = pWal->pWriteLogTFile;
 
   // seek position
   int64_t idxOff = walGetVerIdxOffset(pWal, ver);
-  code = tfLseek(idxTfd, idxOff, SEEK_SET);
+  code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET);
   if (code != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
   SWalIdxEntry entry;
   // TODO:deserialize
-  code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry));
+  code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry));
   if (code != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
   ASSERT(entry.ver == ver);
-  code = tfLseek(logTfd, entry.offset, SEEK_SET);
+  code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
   if (code < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -50,43 +50,43 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) {
 }
 
 int walSetWrite(SWal* pWal) {
-  int64_t       idxTfd, logTfd;
+  TdFilePtr     pIdxTFile, pLogTFile;
   SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
   ASSERT(pRet != NULL);
   int64_t fileFirstVer = pRet->firstVer;
 
   char fnameStr[WAL_FILE_LEN];
   walBuildIdxName(pWal, fileFirstVer, fnameStr);
-  idxTfd = tfOpenCreateWriteAppend(fnameStr);
+  pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
-  if (idxTfd < 0) {
+  if (pIdxTFile < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
   walBuildLogName(pWal, fileFirstVer, fnameStr);
-  logTfd = tfOpenCreateWriteAppend(fnameStr);
+  pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
-  if (logTfd < 0) {
+  if (pLogTFile < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
   // switch file
-  pWal->writeIdxTfd = idxTfd;
+  pWal->pWriteIdxTFile = pIdxTFile;
-  pWal->writeLogTfd = logTfd;
+  pWal->pWriteLogTFile = pLogTFile;
   return 0;
 }
 
 int walChangeWrite(SWal* pWal, int64_t ver) {
   int     code = 0;
-  int64_t idxTfd, logTfd;
+  TdFilePtr pIdxTFile, pLogTFile;
   char    fnameStr[WAL_FILE_LEN];
-  if (pWal->writeLogTfd != -1) {
+  if (pWal->pWriteLogTFile != NULL) {
-    code = tfClose(pWal->writeLogTfd);
+    code = taosCloseFile(&pWal->pWriteLogTFile);
     if (code != 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return -1;
     }
   }
-  if (pWal->writeIdxTfd != -1) {
+  if (pWal->pWriteIdxTFile != NULL) {
-    code = tfClose(pWal->writeIdxTfd);
+    code = taosCloseFile(&pWal->pWriteIdxTFile);
     if (code != 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return -1;
@@ -103,23 +103,23 @@ int walChangeWrite(SWal* pWal, int64_t ver) {
 
   int64_t fileFirstVer = pFileInfo->firstVer;
   walBuildIdxName(pWal, fileFirstVer, fnameStr);
-  idxTfd = tfOpenCreateWriteAppend(fnameStr);
+  pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
-  if (idxTfd < 0) {
+  if (pIdxTFile < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    pWal->writeIdxTfd = -1;
+    pWal->pWriteIdxTFile = NULL;
     return -1;
   }
   walBuildLogName(pWal, fileFirstVer, fnameStr);
-  logTfd = tfOpenCreateWriteAppend(fnameStr);
+  pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
-  if (logTfd < 0) {
+  if (pLogTFile < 0) {
-    tfClose(idxTfd);
+    taosCloseFile(&pIdxTFile);
     terrno = TAOS_SYSTEM_ERROR(errno);
-    pWal->writeLogTfd = -1;
+    pWal->pWriteLogTFile = NULL;
     return -1;
   }
 
-  pWal->writeLogTfd = logTfd;
+  pWal->pWriteLogTFile = pLogTFile;
-  pWal->writeIdxTfd = idxTfd;
+  pWal->pWriteIdxTFile = pIdxTFile;
   pWal->writeCur = idx;
   return fileFirstVer;
 }

source/libs/wal/src/walWrite.c

@@ -65,15 +65,15 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
   }
 
   walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
-  int64_t idxTfd = tfOpenReadWrite(fnameStr);
+  TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
 
   // TODO:change to deserialize function
-  if (idxTfd < 0) {
+  if (pIdxTFile == NULL) {
     pthread_mutex_unlock(&pWal->mutex);
     return -1;
   }
   int64_t idxOff = walGetVerIdxOffset(pWal, ver);
-  code = tfLseek(idxTfd, idxOff, SEEK_SET);
+  code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET);
   if (code < 0) {
     pthread_mutex_unlock(&pWal->mutex);
     return -1;
@@ -81,20 +81,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
   // read idx file and get log file pos
   // TODO:change to deserialize function
   SWalIdxEntry entry;
-  if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
+  if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
     pthread_mutex_unlock(&pWal->mutex);
     return -1;
   }
   ASSERT(entry.ver == ver);
 
   walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
-  int64_t logTfd = tfOpenReadWrite(fnameStr);
+  TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
-  if (logTfd < 0) {
+  if (pLogTFile < 0) {
     // TODO
     pthread_mutex_unlock(&pWal->mutex);
     return -1;
   }
-  code = tfLseek(logTfd, entry.offset, SEEK_SET);
+  code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
   if (code < 0) {
     // TODO
     pthread_mutex_unlock(&pWal->mutex);
@@ -102,8 +102,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
   }
   // validate offset
   SWalHead head;
-  ASSERT(tfValid(logTfd));
+  ASSERT(taosValidFile(pLogTFile));
-  int size = tfRead(logTfd, &head, sizeof(SWalHead));
+  int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead));
   if (size != sizeof(SWalHead)) {
     return -1;
   }
@@ -118,11 +118,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
     return -1;
   }
   // truncate old files
-  code = tfFtruncate(logTfd, entry.offset);
+  code = taosFtruncateFile(pLogTFile, entry.offset);
   if (code < 0) {
     return -1;
   }
-  code = tfFtruncate(idxTfd, idxOff);
+  code = taosFtruncateFile(pIdxTFile, idxOff);
   if (code < 0) {
     return -1;
   }
@@ -203,31 +203,31 @@ int32_t walEndSnapshot(SWal *pWal) {
 
 int walRoll(SWal *pWal) {
   int code = 0;
-  if (pWal->writeIdxTfd != -1) {
+  if (pWal->pWriteIdxTFile != NULL) {
-    code = tfClose(pWal->writeIdxTfd);
+    code = taosCloseFile(&pWal->pWriteIdxTFile);
     if (code != 0) {
       return -1;
     }
   }
-  if (pWal->writeLogTfd != -1) {
+  if (pWal->pWriteLogTFile != NULL) {
-    code = tfClose(pWal->writeLogTfd);
+    code = taosCloseFile(&pWal->pWriteLogTFile);
     if (code != 0) {
       return -1;
     }
   }
-  int64_t idxTfd, logTfd;
+  TdFilePtr pIdxTFile, pLogTFile;
   // create new file
   int64_t newFileFirstVersion = pWal->vers.lastVer + 1;
   char    fnameStr[WAL_FILE_LEN];
   walBuildIdxName(pWal, newFileFirstVersion, fnameStr);
-  idxTfd = tfOpenCreateWriteAppend(fnameStr);
+  pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
-  if (idxTfd < 0) {
+  if (pIdxTFile < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
   walBuildLogName(pWal, newFileFirstVersion, fnameStr);
-  logTfd = tfOpenCreateWriteAppend(fnameStr);
+  pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
-  if (logTfd < 0) {
+  if (pLogTFile < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -237,8 +237,8 @@ int walRoll(SWal *pWal) {
   }
 
   // switch file
-  pWal->writeIdxTfd = idxTfd;
+  pWal->pWriteIdxTFile = pIdxTFile;
-  pWal->writeLogTfd = logTfd;
+  pWal->pWriteLogTFile = pLogTFile;
   pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
   ASSERT(pWal->writeCur >= 0);
 
@@ -248,7 +248,7 @@ int walRoll(SWal *pWal) {
 
 static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
   SWalIdxEntry entry = {.ver = ver, .offset = offset};
-  int          size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry));
+  int          size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry));
   if (size != sizeof(SWalIdxEntry)) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     // TODO truncate
@@ -282,16 +282,16 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
     // must truncate explicitly first
     return -1;
   }
-  /*if (!tfValid(pWal->writeLogTfd)) return -1;*/
+  /*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/
 
   ASSERT(pWal->writeCur >= 0);
 
   pthread_mutex_lock(&pWal->mutex);
 
-  if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) {
+  if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) {
     walSetWrite(pWal);
-    tfLseek(pWal->writeLogTfd, 0, SEEK_END);
+    taosLSeekFile(pWal->pWriteLogTFile, 0, SEEK_END);
-    tfLseek(pWal->writeIdxTfd, 0, SEEK_END);
+    taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END);
   }
 
   pWal->writeHead.head.version = index;
@@ -302,14 +302,14 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
   pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead);
   pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen);
 
-  if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
+  if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
     // ftruncate
     code = TAOS_SYSTEM_ERROR(errno);
     wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
            strerror(errno));
   }
 
-  if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) {
+  if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) {
     // ftruncate
     code = TAOS_SYSTEM_ERROR(errno);
     wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
@@ -336,7 +336,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
 void walFsync(SWal *pWal, bool forceFsync) {
   if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
     wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
-    if (tfFsync(pWal->writeLogTfd) < 0) {
+    if (taosFsyncFile(pWal->pWriteLogTFile) < 0) {
       wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
              strerror(errno));
     }

source/os/src/osFile.c

@@ -12,38 +12,45 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-
+#define ALLOW_FORBID_FUNC
 #include "os.h"
 
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-#include <io.h>
+  #include <io.h>
 
-#if defined(_MSDOS)
+  #if defined(_MSDOS)
-#define open _open
+    #define open _open
-#endif
+  #endif
 
-#if defined(_WIN32)
+  #if defined(_WIN32)
-extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */
+    extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */
-extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */
+    extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */
-#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
+    #if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
-#define open openU
+      #define open openU
-#else /* _ANSI_SOURCE */
+    #else /* _ANSI_SOURCE */
-#define open openA
+      #define open openA
-#endif /* defined(_UTF8_SOURCE) */
+    #endif /* defined(_UTF8_SOURCE) */
-#endif /* defined(_WIN32) */
+  #endif /* defined(_WIN32) */
 
 #else
-#include <fcntl.h>
+  #include <fcntl.h>
-#include <sys/file.h>
+  #include <sys/file.h>
-#include <sys/sendfile.h>
+  #include <sys/sendfile.h>
-#include <sys/stat.h>
+  #include <sys/stat.h>
-#include <unistd.h>
+  #include <unistd.h>
+  #define LINUX_FILE_NO_TEXT_OPTION 0
+  #define O_TEXT LINUX_FILE_NO_TEXT_OPTION
 #endif
 
-void taosCloseFile(FileFd fd) {
+typedef int32_t FileFd;
-  close(fd);
+
-  fd = FD_INITIALIZER;
+typedef struct TdFile {
-}
+ int refId;
+ FileFd  fd;
+ FILE *fp;
+}*TdFilePtr,TdFile;
+
+
 
 void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, char *dstPath) {
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
@@ -96,13 +103,166 @@ void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, ch
 #endif
 }
 
-int64_t taosReadFile(FileFd fd, void *buf, int64_t count) {
+int64_t taosCopyFile(const char *from, const char *to) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return 0;
+#else
+  char    buffer[4096];
+  int64_t size = 0;
+  int64_t bytes;
+
+  // fidfrom = open(from, O_RDONLY);
+  TdFilePtr pFileFrom = taosOpenFile(from,TD_FILE_READ);
+  if (pFileFrom == NULL) goto _err;
+
+  // fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755);
+  TdFilePtr pFileTo = taosOpenFile(to,TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_EXCL);
+  if (pFileTo == NULL) goto _err;
+
+  while (true) {
+    bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer));
+    if (bytes < 0) goto _err;
+    if (bytes == 0) break;
+
+    size += bytes;
+
+    if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) goto _err;
+    if (bytes < sizeof(buffer)) break;
+  }
+
+  taosFsyncFile(pFileTo);
+
+  taosCloseFile(&pFileFrom);
+  taosCloseFile(&pFileTo);
+  return size;
+
+_err:
+  if (pFileFrom != NULL) taosCloseFile(&pFileFrom);
+  if (pFileTo != NULL) taosCloseFile(&pFileTo);
+  remove(to);
+  return -1;
+#endif
+}
+
+int32_t taosRenameFile(const char *oldName, const char *newName) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
+  if (code < 0) {
+    //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
+  }
+
+  return code;
+#else
+  int32_t code = rename(oldName, newName);
+  if (code < 0) {
+    //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
+  }
+
+  return code;
+#endif
+}
+
+int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return 0;
+#else
+  struct stat fileStat;
+  int32_t     code = stat(path, &fileStat);
+  if (code < 0) {
+    return code;
+  }
+
+  if (size != NULL) {
+    *size = fileStat.st_size;
+  }
+
+  if (mtime != NULL) {
+    *mtime = fileStat.st_mtime;
+  }
+
+  return 0;
+#endif
+}
+
+void autoDelFileListAdd(const char *path) {
+  return;
+}
+
+TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return NULL;
+#else
+  int access = O_BINARY;
+  char *mode = NULL;
+  access |= (tdFileOptions & TD_FILE_CTEATE) ? O_CREAT : 0;
+  if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ))
+  {
+    access |= O_RDWR;
+    mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+";
+  }else if(tdFileOptions & TD_FILE_WRITE) {
+    access |= O_WRONLY;
+    mode = (tdFileOptions & TD_FILE_TEXT) ? "wt" : "wb";
+  }else if(tdFileOptions & TD_FILE_READ) {
+    access |= O_RDONLY;
+    mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb";
+  }
+  access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0;
+  access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0;
+  access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0;
+  access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0;
+  if(tdFileOptions & TD_FILE_AUTO_DEL) {
+    autoDelFileListAdd(path);
+  }
+  int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO);
+  if(fd == -1) {
+    return NULL;
+  }
+  FILE* fp = fdopen(fd, mode);
+  if (fp == NULL) {
+    close(fd);
+    return NULL;
+  }
+  TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile));
+  if (pFile == NULL) {
+    close(fd);
+    fclose(fp);
+    return NULL;
+  }
+  pFile->fd = fd;
+  pFile->fp = fp;
+  pFile->refId = 0;
+  return pFile;
+#endif
+}
+
+int64_t taosCloseFile(TdFilePtr *ppFile) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return 0;
+#else
+  if(ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) {
+    return 0;
+  }
+  fsync((*ppFile)->fd);
+  close((*ppFile)->fd);
+  (*ppFile)->fd = -1;
+  (*ppFile)->fp = NULL;
+  (*ppFile)->refId = 0;
+  free(*ppFile);
+  *ppFile = NULL;
+  return 0;
+#endif
+}
+
+int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
+  if(pFile == NULL) {
+    return 0;
+  }
   int64_t leftbytes = count;
   int64_t readbytes;
   char *  tbuf = (char *)buf;
 
   while (leftbytes > 0) {
-    readbytes = read(fd, (void *)tbuf, (uint32_t)leftbytes);
+    readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes);
     if (readbytes < 0) {
       if (errno == EINTR) {
         continue;
@@ -120,13 +280,20 @@ int64_t taosReadFile(FileFd fd, void *buf, int64_t count) {
   return count;
 }
 
-int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) {
+int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) {
-  int64_t nleft = n;
+  if(pFile == NULL) {
+    return 0;
+  }
+  return pread(pFile->fd, buf, count, offset);
+}
+
+int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
+  int64_t nleft = count;
   int64_t nwritten = 0;
   char *  tbuf = (char *)buf;
 
   while (nleft > 0) {
-    nwritten = write(fd, (void *)tbuf, (uint32_t)nleft);
+    nwritten = write(pFile->fd, (void *)tbuf, (uint32_t)nleft);
     if (nwritten < 0) {
       if (errno == EINTR) {
         continue;
@@ -137,48 +304,119 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) {
     tbuf += nwritten;
   }
 
-  return n;
+  return count;
 }
 
-int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); }
+int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { 
+  return (int64_t)lseek(pFile->fd, (long)offset, whence); 
+}
 
-int64_t taosCopyFile(const char *from, const char *to) {
+int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   return 0;
 #else
-  char    buffer[4096];
+  struct stat fileStat;
-  int     fidto = -1, fidfrom = -1;
+  int32_t     code = fstat(pFile->fd, &fileStat);
-  int64_t size = 0;
+  if (code < 0) {
-  int64_t bytes;
+    return code;
-
-  fidfrom = open(from, O_RDONLY);
-  if (fidfrom < 0) goto _err;
-
-  fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755);
-  if (fidto < 0) goto _err;
-
-  while (true) {
-    bytes = taosReadFile(fidfrom, buffer, sizeof(buffer));
-    if (bytes < 0) goto _err;
-    if (bytes == 0) break;
-
-    size += bytes;
-
-    if (taosWriteFile(fidto, (void *)buffer, bytes) < bytes) goto _err;
-    if (bytes < sizeof(buffer)) break;
   }
 
-  taosFsyncFile(fidto);
+  if (size != NULL) {
+    *size = fileStat.st_size;
+  }
 
-  taosCloseFile(fidfrom);
+  if (mtime != NULL) {
-  taosCloseFile(fidto);
+    *mtime = fileStat.st_mtime;
-  return size;
+  }
 
-_err:
+  return 0;
-  if (fidfrom >= 0) taosCloseFile(fidfrom);
+#endif
-  if (fidto >= 0) taosCloseFile(fidto);
+}
-  remove(to);
+
+int32_t taosLockFile(TdFilePtr pFile) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return 0;
+#else
+  return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB);
+#endif
+}
+
+int32_t taosUnLockFile(TdFilePtr pFile) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return 0;
+#else
+  return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB);
+#endif
+}
+
+int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  if (pFile->fd < 0) {
+    errno = EBADF;
+    uError("%s\n", "fd arg was negative");
     return -1;
+  }
+
+  HANDLE h = (HANDLE)_get_osfhandle(pFile->fd);
+
+  LARGE_INTEGER li_0;
+  li_0.QuadPart = (int64_t)0;
+  BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT);
+  if (!cur) {
+    uError("SetFilePointerEx Error getting current position in file.\n");
+    return -1;
+  }
+
+  LARGE_INTEGER li_size;
+  li_size.QuadPart = l_size;
+  BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN);
+  if (cur2 == 0) {
+    int error = GetLastError();
+    uError("SetFilePointerEx GetLastError is: %d\n", error);
+    switch (error) {
+      case ERROR_INVALID_HANDLE:
+        errno = EBADF;
+        break;
+      default:
+        errno = EIO;
+        break;
+    }
+    return -1;
+  }
+
+  if (!SetEndOfFile(h)) {
+    int error = GetLastError();
+    uError("SetEndOfFile GetLastError is:%d", error);
+    switch (error) {
+      case ERROR_INVALID_HANDLE:
+        errno = EBADF;
+        break;
+      default:
+        errno = EIO;
+        break;
+    }
+    return -1;
+  }
+
+  return 0;
+#else
+  return ftruncate(pFile->fd, l_size);
+#endif
+}
+
+int32_t taosFsyncFile(TdFilePtr pFile) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  if (pFile->fd < 0) {
+    errno = EBADF;
+    uError("%s\n", "fd arg was negative");
+    return -1;
+  }
+
+  HANDLE h = (HANDLE)_get_osfhandle(pFile->fd);
+
+  return FlushFileBuffers(h);
+#else
+  return fsync(pFile->fd);
 #endif
 }
 
@@ -301,12 +539,12 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) {
 
 #else
 
-int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) {
+int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size)  {
   int64_t leftbytes = size;
   int64_t sentbytes;
 
   while (leftbytes > 0) {
-    sentbytes = sendfile(dfd, sfd, offset, leftbytes);
+    sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes);
     if (sentbytes == -1) {
       if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
         continue;
@@ -323,215 +561,49 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) {
   return size;
 }
 
-int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) {
+int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) {
-  return taosSendFile(fileno(outfile), fileno(infile), offset, size);
+  return taosSendFile(pFileOut->fd, pFileIn, offset, size);
 }
 
 #endif
 
-int32_t taosFtruncateFile(FileFd fd, int64_t l_size) {
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  if (fd < 0) {
-    errno = EBADF;
-    uError("%s\n", "fd arg was negative");
-    return -1;
-  }
 
-  HANDLE h = (HANDLE)_get_osfhandle(fd);
+#ifdef __GNUC__
-
+ __attribute__((format(printf, 2, 3)))
-  LARGE_INTEGER li_0;
-  li_0.QuadPart = (int64_t)0;
-  BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT);
-  if (!cur) {
-    uError("SetFilePointerEx Error getting current position in file.\n");
-    return -1;
-  }
-
-  LARGE_INTEGER li_size;
-  li_size.QuadPart = l_size;
-  BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN);
-  if (cur2 == 0) {
-    int error = GetLastError();
-    uError("SetFilePointerEx GetLastError is: %d\n", error);
-    switch (error) {
-      case ERROR_INVALID_HANDLE:
-        errno = EBADF;
-        break;
-      default:
-        errno = EIO;
-        break;
-    }
-    return -1;
-  }
-
-  if (!SetEndOfFile(h)) {
-    int error = GetLastError();
-    uError("SetEndOfFile GetLastError is:%d", error);
-    switch (error) {
-      case ERROR_INVALID_HANDLE:
-        errno = EBADF;
-        break;
-      default:
-        errno = EIO;
-        break;
-    }
-    return -1;
-  }
-
-  return 0;
-#else
-  return ftruncate(fd, l_size);
 #endif
+void taosFprintfFile(TdFilePtr pFile, const char *format, ...) {
+  va_list        ap;
+  va_start(ap, format);
+  fprintf(pFile->fp, format, ap);
+  va_end(ap);
+  fflush(pFile->fp);
 }
 
-int32_t taosFsyncFile(FileFd fd) {
+void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) {
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  if (pFile == NULL) return NULL;
-  if (fd < 0) {
-    errno = EBADF;
-    uError("%s\n", "fd arg was negative");
-    return -1;
-  }
 
-  HANDLE h = (HANDLE)_get_osfhandle(fd);
+  void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0);
-
+  return ptr;
-  return FlushFileBuffers(h);
-#else
-  return fsync(fd);
-#endif
 }
 
-int32_t taosRenameFile(const char *oldName, const char *newName) {
+bool taosValidFile(TdFilePtr pFile) {
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return pFile != NULL;
-  int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
-  if (code < 0) {
-    //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
-  }
-
-  return code;
-#else
-  int32_t code = rename(oldName, newName);
-  if (code < 0) {
-    //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
-  }
-
-  return code;
-#endif
 }
 
-int32_t taosLockFile(int32_t fd) {
+int32_t taosUmaskFile(int32_t maskVal) {
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   return 0;
 #else
-  return (int32_t)flock(fd, LOCK_EX | LOCK_NB);
+  return umask(maskVal);
 #endif
 }
 
-int32_t taosUnLockFile(int32_t fd) {
+int taosGetErrorFile(TdFilePtr pFile) {
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return errno;
-  return 0;
-#else
-  return (int32_t)flock(fd, LOCK_UN | LOCK_NB);
-#endif
 }
-
+size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf) {
-int32_t taosUmaskFile(int32_t val) {
+  return getline(ptrBuf, NULL, pFile->fp);
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  return 0;
-#else
-  return umask(val);
-#endif
 }
-
+int32_t taosEOFFile(TdFilePtr pFile) {
-int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
+  return feof(pFile->fp);
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  return 0;
-#else
-  struct stat fileStat;
-  int32_t     code = stat(path, &fileStat);
-  if (code < 0) {
-    return code;
-  }
-
-  if (size != NULL) {
-    *size = fileStat.st_size;
-  }
-
-  if (mtime != NULL) {
-    *mtime = fileStat.st_mtime;
-  }
-
-  return 0;
-#endif
 }
-
-int32_t taosFStatFile(int32_t fd, int64_t *size, int32_t *mtime) {
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  return 0;
-#else
-  struct stat fileStat;
-  int32_t     code = fstat(fd, &fileStat);
-  if (code < 0) {
-    return code;
-  }
-
-  if (size != NULL) {
-    *size = fileStat.st_size;
-  }
-
-  if (mtime != NULL) {
-    *mtime = fileStat.st_mtime;
-  }
-
-  return 0;
-#endif
-}
-
-int32_t taosOpenFileWrite(const char *path) {
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  return 0;
-#else
-  return open(path, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
-#endif
-}
-
-int32_t taosOpenFileCreateWrite(const char *path) {
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  return 0;
-#else
-  return open(path, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
-#endif
-}
-
-int32_t taosOpenFileCreateWriteTrunc(const char *path) {
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  return 0;
-#else
-  return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
-#endif
-}
-
-int32_t taosOpenFileCreateWriteAppend(const char *path) {
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  return 0;
-#else
-  return open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
-#endif
-}
-
-FileFd taosOpenFileRead(const char *path) {
-  #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  return 0;
-#else
-  return open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
-#endif
-}
-
-FileFd taosOpenFileReadWrite(const char *path) {
-  #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  return 0;
-#else
-  return open(path, O_RDWR, S_IRWXU | S_IRWXG | S_IRWXO);
-#endif
-}
-
-

source/os/src/osRand.c

@@ -24,18 +24,18 @@
 uint32_t taosRand(void) { return rand(); }
 
 uint32_t taosSafeRand(void) {
-  int fd;
+  TdFilePtr pFile;
   int seed;
 
-  fd = open("/dev/urandom", 0);
+  pFile = taosOpenFile("/dev/urandom", TD_FILE_READ);
-  if (fd < 0) {
+  if (pFile == NULL) {
     seed = (int)time(0);
   } else {
-    int len = read(fd, &seed, sizeof(seed));
+    int len = taosReadFile(pFile, &seed, sizeof(seed));
     if (len < 0) {
       seed = (int)time(0);
     }
-    close(fd);
+    taosCloseFile(&pFile);
   }
 
   return (uint32_t)seed;

source/os/src/osSocket.c

@@ -14,6 +14,7 @@
  */
 
 #define _DEFAULT_SOURCE
+#define ALLOW_FORBID_FUNC
 #include "os.h"
 
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)

source/os/src/osSysinfo.c

@@ -408,19 +408,17 @@ bool taosGetSysMemory(float *memoryUsedMB) {
 }
 
 bool taosGetProcMemory(float *memoryUsedMB) {
-  FILE *fp = fopen(tsProcMemFile, "r");
+  // FILE *fp = fopen(tsProcMemFile, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ);
+  if (pFile == NULL) {
     //printf("open file:%s failed", tsProcMemFile);
     return false;
   }
 
   ssize_t _bytes = 0;
-  size_t  len;
   char *  line = NULL;
-  while (!feof(fp)) {
+  while (!taosEOFFile(pFile)) {
-    tfree(line);
+    _bytes = taosGetLineFile(pFile, &line);
-    len = 0;
-    _bytes = getline(&line, &len, fp);
     if ((_bytes < 0) || (line == NULL)) {
       break;
     }
@@ -431,7 +429,7 @@ bool taosGetProcMemory(float *memoryUsedMB) {
 
   if (line == NULL) {
     //printf("read file:%s failed", tsProcMemFile);
-    fclose(fp);
+    taosCloseFile(&pFile);
     return false;
   }
 
@@ -440,24 +438,24 @@ bool taosGetProcMemory(float *memoryUsedMB) {
   sscanf(line, "%s %" PRId64, tmp, &memKB);
   *memoryUsedMB = (float)((double)memKB / 1024);
 
-  tfree(line);
+  if(line != NULL) tfree(line);
-  fclose(fp);
+  taosCloseFile(&pFile);
   return true;
 }
 
 static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
-  FILE *fp = fopen(tsSysCpuFile, "r");
+  // FILE *fp = fopen(tsSysCpuFile, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ);
+  if (pFile == NULL) {
     //printf("open file:%s failed", tsSysCpuFile);
     return false;
   }
 
-  size_t  len;
   char *  line = NULL;
-  ssize_t _bytes = getline(&line, &len, fp);
+  ssize_t _bytes = taosGetLineFile(pFile, &line);
   if ((_bytes < 0) || (line == NULL)) {
     //printf("read file:%s failed", tsSysCpuFile);
-    fclose(fp);
+    taosCloseFile(&pFile);
     return false;
   }
 
@@ -465,24 +463,24 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
   sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system,
          &cpuInfo->idle);
 
-  tfree(line);
+  if(line != NULL) tfree(line);
-  fclose(fp);
+  taosCloseFile(&pFile);
   return true;
 }
 
 static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
-  FILE *fp = fopen(tsProcCpuFile, "r");
+  // FILE *fp = fopen(tsProcCpuFile, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ);
+  if (pFile == NULL) {
     //printf("open file:%s failed", tsProcCpuFile);
     return false;
   }
 
-  size_t  len = 0;
   char *  line = NULL;
-  ssize_t _bytes = getline(&line, &len, fp);
+  ssize_t _bytes = taosGetLineFile(pFile, &line);
   if ((_bytes < 0) || (line == NULL)) {
     //printf("read file:%s failed", tsProcCpuFile);
-    fclose(fp);
+    taosCloseFile(&pFile);
     return false;
   }
 
@@ -495,8 +493,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
     }
   }
 
-  tfree(line);
+  if(line != NULL) tfree(line);
-  fclose(fp);
+  taosCloseFile(&pFile);
   return true;
 }
 
@@ -556,19 +554,17 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) {
 
 bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
   *bytes = 0;
-  FILE *fp = fopen(tsSysNetFile, "r");
+  // FILE *fp = fopen(tsSysNetFile, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ);
+  if (pFile == NULL) {
     //printf("open file:%s failed", tsSysNetFile);
     return false;
   }
 
   ssize_t _bytes = 0;
-  size_t  len = 2048;
+  char *  line = NULL;
-  char *  line = calloc(1, len);
-
-  while (!feof(fp)) {
-    memset(line, 0, len);
 
+  while (!taosEOFFile(pFile)) {
     int64_t o_rbytes = 0;
     int64_t rpackts = 0;
     int64_t o_tbytes = 0;
@@ -581,12 +577,12 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
     int64_t nouse6 = 0;
     char    nouse0[200] = {0};
 
-    _bytes = getline(&line, &len, fp);
+    _bytes = taosGetLineFile(pFile, &line);
     if (_bytes < 0) {
       break;
     }
 
-    line[len - 1] = 0;
+    line[_bytes - 1] = 0;
 
     if (strstr(line, "lo:") != NULL) {
       continue;
@@ -601,8 +597,8 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
     *bytes += (o_rbytes + o_tbytes);
   }
 
-  tfree(line);
+  if(line != NULL) tfree(line);
-  fclose(fp);
+  taosCloseFile(&pFile);
 
   return true;
 }
@@ -644,22 +640,20 @@ bool taosGetBandSpeed(float *bandSpeedKb) {
 }
 
 bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
-  FILE *fp = fopen(tsProcIOFile, "r");
+  // FILE *fp = fopen(tsProcIOFile, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ);
+  if (pFile == NULL) {
     //printf("open file:%s failed", tsProcIOFile);
     return false;
   }
 
   ssize_t _bytes = 0;
-  size_t  len;
   char *  line = NULL;
   char    tmp[10];
   int     readIndex = 0;
 
-  while (!feof(fp)) {
+  while (!taosEOFFile(pFile)) {
-    tfree(line);
+    _bytes = taosGetLineFile(pFile, &line);
-    len = 0;
-    _bytes = getline(&line, &len, fp);
     if ((_bytes < 0) || (line == NULL)) {
       break;
     }
@@ -675,8 +669,8 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
     if (readIndex >= 2) break;
   }
 
-  tfree(line);
+  if(line != NULL) tfree(line);
-  fclose(fp);
+  taosCloseFile(&pFile);
 
   if (readIndex < 2) {
     //printf("read file:%s failed", tsProcIOFile);
@@ -839,15 +833,15 @@ void taosSetCoreDump(bool enable) {
 }
 
 int32_t taosGetSystemUUID(char *uid, int32_t uidlen) {
-  int fd;
   int len = 0;
 
-  fd = open("/proc/sys/kernel/random/uuid", 0);
+  // fd = open("/proc/sys/kernel/random/uuid", 0);
-  if (fd < 0) {
+  TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ);
+  if (pFile == NULL) {
     return -1;
   } else {
-    len = read(fd, uid, uidlen);
+    len = taosReadFile(pFile, uid, uidlen);
-    close(fd);
+    taosCloseFile(&pFile);
   }
 
   if (len >= 36) {
@@ -862,16 +856,17 @@ char *taosGetCmdlineByPID(int pid) {
   static char cmdline[1024];
   sprintf(cmdline, "/proc/%d/cmdline", pid);
 
-  int fd = open(cmdline, O_RDONLY);
+  // int fd = open(cmdline, O_RDONLY);
-  if (fd >= 0) {
+  TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ);
-    int n = read(fd, cmdline, sizeof(cmdline) - 1);
+  if (pFile != NULL) {
+    int n = taosReadFile(pFile, cmdline, sizeof(cmdline) - 1);
     if (n < 0) n = 0;
 
     if (n > 0 && cmdline[n - 1] == '\n') --n;
 
     cmdline[n] = 0;
 
-    close(fd);
+    taosCloseFile(&pFile);
   } else {
     cmdline[0] = 0;
   }

source/os/src/osTimezone.c

@@ -150,17 +150,18 @@ void taosGetSystemTimezone(char *outTimezone) {
   localtime_r(&tx1, &tm1);
 
   /* load time zone string from /etc/timezone */
-  FILE *f = fopen("/etc/timezone", "r");
+  // FILE *f = fopen("/etc/timezone", "r");
+  TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ);
   char  buf[68] = {0};
-  if (f != NULL) {
+  if (pFile != NULL) {
-    int len = fread(buf, 64, 1, f);
+    int len = taosReadFile(pFile, buf, 64);
-    if (len < 64 && ferror(f)) {
+    if (len < 64 && taosGetErrorFile(pFile)) {
-      fclose(f);
+      taosCloseFile(&pFile);
       // printf("read /etc/timezone error, reason:%s", strerror(errno));
       return;
     }
 
-    fclose(f);
+    taosCloseFile(&pFile);
 
     buf[sizeof(buf) - 1] = 0;
     char *lineEnd = strstr(buf, "\n");

source/util/src/tconfig.c

@@ -571,31 +571,28 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) {
 }
 
 int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
-  char   *line, *name, *value, *value2, *value3;
+  char   *line = NULL, *name, *value, *value2, *value3;
   int     olen, vlen, vlen2, vlen3;
   ssize_t _bytes = 0;
-  size_t  len = 1024;
 
-  FILE *fp = fopen(filepath, "r");
+  // FILE *fp = fopen(filepath, "r");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
+  if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
-  line = malloc(len);
+  while (!taosEOFFile(pFile)) {
-
-  while (!feof(fp)) {
-    memset(line, 0, len);
 
     name = value = value2 = value3 = NULL;
     olen = vlen = vlen2 = vlen3 = 0;
 
-    _bytes = tgetline(&line, &len, fp);
+    _bytes = taosGetLineFile(pFile, &line);
     if (_bytes < 0) {
       break;
     }
 
-    line[len - 1] = 0;
+    line[_bytes - 1] = 0;
 
     paGetToken(line, &name, &olen);
     if (olen == 0) continue;
@@ -616,8 +613,8 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
     // taosReadConfigOption(name, value, value2, value3);
   }
 
-  fclose(fp);
+  taosCloseFile(&pFile);
-  tfree(line);
+  if(line != NULL) tfree(line);
 
   uInfo("load from cfg file %s success", filepath);
   return 0;

source/util/src/tfile.c

@@ -24,147 +24,147 @@ static int32_t tsFileRsetId = -1;
 
 static int8_t tfInited = 0;
 
-static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); }
+// static void tfCloseFile(void *p) { taosCloseFile((TdFilePtr)(uintptr_t)p); }
 
 int32_t tfInit() {
-  int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1);
+  // int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1);
-  if (old == 1) return 0;
+  // if (old == 1) return 0;
-  tsFileRsetId = taosOpenRef(2000, tfCloseFile);
+  // tsFileRsetId = taosOpenRef(2000, tfCloseFile);
-  if (tsFileRsetId > 0) {
+  // if (tsFileRsetId > 0) {
-    return 0;
+  //   return 0;
-  } else {
+  // } else {
-    atomic_store_8(&tfInited, 0);
+  //   atomic_store_8(&tfInited, 0);
-    return -1;
+  //   return -1;
-  }
+  // }
 }
 
 void tfCleanup() {
-  atomic_store_8(&tfInited, 0);
+  // atomic_store_8(&tfInited, 0);
-  if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
+  // if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
-  tsFileRsetId = -1;
+  // tsFileRsetId = -1;
 }
 
-static int64_t tfOpenImp(int32_t fd) {
+// static int64_t tfOpenImp(TdFilePtr pFile) {
-  if (fd < 0) {
+//   if (pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
+//     terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
+//     return -1;
-  }
+//   }
 
-  void *  p = (void *)(int64_t)fd;
+//   void *  p = (void *)(int64_t)pFile;
-  int64_t rid = taosAddRef(tsFileRsetId, p);
+//   int64_t rid = taosAddRef(tsFileRsetId, p);
-  if (rid < 0) taosCloseFile(fd);
+//   if (rid < 0) taosCloseFile(&pFile);
 
-  return rid;
+//   return rid;
-}
+// }
 
-int64_t tfOpenRead(const char *pathname, int32_t flags) {
+// int64_t tfOpenRead(const char *pathname, int32_t flags) {
-  int32_t fd = taosOpenFileRead(pathname);
+//   int32_t pFile = taosOpenFile(pathname, TD_FILE_READ);
-  return tfOpenImp(fd);
+//   return tfOpenImp(fd);
-}
+// }
 
-int64_t tfOpenReadWrite(const char *pathname, int32_t flags) {
+// int64_t tfOpenReadWrite(const char *pathname, int32_t flags) {
-  int32_t fd = taosOpenFileReadWrite(pathname);
+//   int32_t pFile = taosOpenFile(pathname, TD_FILE_READ | TD_FILE_WRITE);
-  return tfOpenImp(fd);
+//   return tfOpenImp(fd);
-}
+// }
 
-int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) {
+// int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) {
-  int32_t fd = taosOpenFileCreateWrite(pathname);
+//   int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE);
-  return tfOpenImp(fd);
+//   return tfOpenImp(fd);
-}
+// }
 
-int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) {
+// int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) {
-  int32_t fd = taosOpenFileCreateWriteAppend(pathname);
+//   int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
-  return tfOpenImp(fd);
+//   return tfOpenImp(fd);
-}
+// }
 
-int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); }
+// int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); }
 
-int64_t tfWrite(int64_t tfd, void *buf, int64_t count) {
+// int64_t tfWrite(int64_t tfd, void *buf, int64_t count) {
-  void *p = taosAcquireRef(tsFileRsetId, tfd);
+//   void *p = taosAcquireRef(tsFileRsetId, tfd);
-  if (p == NULL) return -1;
+//   if (p == NULL) return -1;
 
-  int32_t fd = (int32_t)(uintptr_t)p;
+//   int32_t pFile = (TdFilePtr)(uintptr_t)p;
 
-  int64_t ret = taosWriteFile(fd, buf, count);
+//   int64_t ret = taosWriteFile(pFile, buf, count);
-  if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
+//   if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
 
-  taosReleaseRef(tsFileRsetId, tfd);
+//   taosReleaseRef(tsFileRsetId, tfd);
-  return ret;
+//   return ret;
-}
+// }
 
-int64_t tfRead(int64_t tfd, void *buf, int64_t count) {
+// int64_t tfRead(int64_t tfd, void *buf, int64_t count) {
-  void *p = taosAcquireRef(tsFileRsetId, tfd);
+//   void *p = taosAcquireRef(tsFileRsetId, tfd);
-  if (p == NULL) return -1;
+//   if (p == NULL) return -1;
 
-  int32_t fd = (int32_t)(uintptr_t)p;
+//   int32_t pFile = (TdFilePtr)(uintptr_t)p;
 
-  int64_t ret = taosReadFile(fd, buf, count);
+//   int64_t ret = taosReadFile(pFile, buf, count);
-  if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
+//   if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
 
-  taosReleaseRef(tsFileRsetId, tfd);
+//   taosReleaseRef(tsFileRsetId, tfd);
-  return ret;
+//   return ret;
-}
+// }
 
-int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) {
+// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) {
-  void *p = taosAcquireRef(tsFileRsetId, tfd);
+//   void *p = taosAcquireRef(tsFileRsetId, tfd);
-  if (p == NULL) return -1;
+//   if (p == NULL) return -1;
 
-  int32_t fd = (int32_t)(uintptr_t)p;
+//   int32_t pFile = (TdFilePtr)(uintptr_t)p;
 
-  int64_t ret = pread(fd, buf, count, offset);
+//   int64_t ret = pread(fd, buf, count, offset);
-  if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
+//   if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
 
-  taosReleaseRef(tsFileRsetId, tfd);
+//   taosReleaseRef(tsFileRsetId, tfd);
-  return ret;
+//   return ret;
-}
+// }
 
-int32_t tfFsync(int64_t tfd) {
+// int32_t tfFsync(int64_t tfd) {
-  void *p = taosAcquireRef(tsFileRsetId, tfd);
+//   void *p = taosAcquireRef(tsFileRsetId, tfd);
-  if (p == NULL) return -1;
+//   if (p == NULL) return -1;
 
-  int32_t fd = (int32_t)(uintptr_t)p;
+//   int32_t pFile = (TdFilePtr)(uintptr_t)p;
-  int32_t code = taosFsyncFile(fd);
+//   int32_t code = taosFsyncFile(pFile);
 
-  taosReleaseRef(tsFileRsetId, tfd);
+//   taosReleaseRef(tsFileRsetId, tfd);
-  return code;
+//   return code;
-}
+// }
 
-bool tfValid(int64_t tfd) {
+// bool tfValid(int64_t tfd) {
-  void *p = taosAcquireRef(tsFileRsetId, tfd);
+//   void *p = taosAcquireRef(tsFileRsetId, tfd);
-  if (p == NULL) return false;
+//   if (p == NULL) return false;
 
-  taosReleaseRef(tsFileRsetId, tfd);
+//   taosReleaseRef(tsFileRsetId, tfd);
-  return true;
+//   return true;
-}
+// }
 
-int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) {
+// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) {
-  void *p = taosAcquireRef(tsFileRsetId, tfd);
+//   void *p = taosAcquireRef(tsFileRsetId, tfd);
-  if (p == NULL) return -1;
+//   if (p == NULL) return -1;
 
-  int32_t fd = (int32_t)(uintptr_t)p;
+//   int32_t pFile = (TdFilePtr)(uintptr_t)p;
-  int64_t ret = taosLSeekFile(fd, offset, whence);
+//   int64_t ret = taosLSeekFile(fd, offset, whence);
 
-  taosReleaseRef(tsFileRsetId, tfd);
+//   taosReleaseRef(tsFileRsetId, tfd);
-  return ret;
+//   return ret;
-}
+// }
 
-int32_t tfFtruncate(int64_t tfd, int64_t length) {
+// int32_t tfFtruncate(int64_t tfd, int64_t length) {
-  void *p = taosAcquireRef(tsFileRsetId, tfd);
+//   void *p = taosAcquireRef(tsFileRsetId, tfd);
-  if (p == NULL) return -1;
+//   if (p == NULL) return -1;
 
-  int32_t fd = (int32_t)(uintptr_t)p;
+//   int32_t pFile = (TdFilePtr)(uintptr_t)p;
-  int32_t code = taosFtruncateFile(fd, length);
+//   int32_t code = taosFtruncateFile(fd, length);
 
-  taosReleaseRef(tsFileRsetId, tfd);
+//   taosReleaseRef(tsFileRsetId, tfd);
-  return code;
+//   return code;
-}
+// }
 
-void *tfMmapReadOnly(int64_t tfd, int64_t length) {
+// void *tfMmapReadOnly(int64_t tfd, int64_t length) {
-  void *p = taosAcquireRef(tsFileRsetId, tfd);
+//   void *p = taosAcquireRef(tsFileRsetId, tfd);
-  if (p == NULL) return NULL;
+//   if (p == NULL) return NULL;
-  int32_t fd = (int32_t)(uintptr_t)p;
+//   int32_t pFile = (TdFilePtr)(uintptr_t)p;
 
-  void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0);
+//   void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0);
-  taosReleaseRef(tsFileRsetId, tfd);
+//   taosReleaseRef(tsFileRsetId, tfd);
-  return ptr;
+//   return ptr;
-}
+// }

source/util/src/tlog.c

@@ -47,7 +47,7 @@ typedef struct {
   int32_t         buffEnd;
   int32_t         buffSize;
   int32_t         minBuffSize;
-  int32_t         fd;
+  TdFilePtr       pFile;
   int32_t         stop;
   pthread_t       asyncThread;
   pthread_mutex_t buffMutex;
@@ -101,7 +101,7 @@ static SLogObj   tsLogObj = {.fileNum = 1};
 static void     *taosAsyncOutputLog(void *param);
 static int32_t   taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen);
 static SLogBuff *taosLogBuffNew(int32_t bufSize);
-static void      taosCloseLogByFd(int32_t oldFd);
+static void      taosCloseLogByFd(TdFilePtr pFile);
 static int32_t   taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum);
 extern void      taosPrintCfg();
 static int32_t   taosCompressFile(char *srcFileName, char *destFileName);
@@ -150,11 +150,11 @@ void taosCloseLog() {
   // taosCloseLog();
 }
 
-static bool taosLockLogFile(int32_t fd) {
+static bool taosLockLogFile(TdFilePtr pFile) {
-  if (fd < 0) return false;
+  if (pFile == NULL) return false;
 
   if (tsLogObj.fileNum > 1) {
-    int32_t ret = taosLockFile(fd);
+    int32_t ret = taosLockFile(pFile);
     if (ret == 0) {
       return true;
     }
@@ -163,11 +163,11 @@ static bool taosLockLogFile(int32_t fd) {
   return false;
 }
 
-static void taosUnLockLogFile(int32_t fd) {
+static void taosUnLockLogFile(TdFilePtr pFile) {
-  if (fd < 0) return;
+  if (pFile == NULL) return;
 
   if (tsLogObj.fileNum > 1) {
-    taosUnLockFile(fd);
+    taosUnLockFile(pFile);
   }
 }
 
@@ -201,22 +201,22 @@ static void *taosThreadToOpenNewFile(void *param) {
 
   taosUmaskFile(0);
 
-  int32_t fd = taosOpenFileCreateWriteTrunc(name);
+  TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (fd < 0) {
+  if (pFile == NULL) {
     tsLogObj.openInProgress = 0;
     tsLogObj.lines = tsLogObj.maxLines - 1000;
-    uError("open new log file fail! fd:%d reason:%s, reuse lastlog", fd, strerror(errno));
+    uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno));
     return NULL;
   }
 
-  taosLockLogFile(fd);
+  taosLockLogFile(pFile);
-  (void)taosLSeekFile(fd, 0, SEEK_SET);
+  (void)taosLSeekFile(pFile, 0, SEEK_SET);
 
-  int32_t oldFd = tsLogObj.logHandle->fd;
+  TdFilePtr pOldFile = tsLogObj.logHandle->pFile;
-  tsLogObj.logHandle->fd = fd;
+  tsLogObj.logHandle->pFile = pFile;
   tsLogObj.lines = 0;
   tsLogObj.openInProgress = 0;
-  taosCloseLogByFd(oldFd);
+  taosCloseLogByFd(pOldFile);
 
   uInfo("   new log file:%d is opened", tsLogObj.flag);
   uInfo("==================================");
@@ -262,8 +262,8 @@ void taosResetLog() {
 }
 
 static bool taosCheckFileIsOpen(char *logFileName) {
-  int32_t fd = taosOpenFileWrite(logFileName);
+  TdFilePtr pFile = taosOpenFile(logFileName, TD_FILE_WRITE);
-  if (fd < 0) {
+  if (pFile == NULL) {
     if (errno == ENOENT) {
       return false;
     } else {
@@ -272,12 +272,12 @@ static bool taosCheckFileIsOpen(char *logFileName) {
     }
   }
 
-  if (taosLockLogFile(fd)) {
+  if (taosLockLogFile(pFile)) {
-    taosUnLockLogFile(fd);
+    taosUnLockLogFile(pFile);
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
     return false;
   } else {
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
     return true;
   }
 }
@@ -350,31 +350,31 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
   pthread_mutex_init(&tsLogObj.logMutex, NULL);
 
   taosUmaskFile(0);
-  tsLogObj.logHandle->fd = taosOpenFileCreateWrite(fileName);
+  tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE);
 
-  if (tsLogObj.logHandle->fd < 0) {
+  if (tsLogObj.logHandle->pFile == NULL) {
     printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
     return -1;
   }
-  taosLockLogFile(tsLogObj.logHandle->fd);
+  taosLockLogFile(tsLogObj.logHandle->pFile);
 
   // only an estimate for number of lines
   int64_t filesize = 0;
-  if (taosFStatFile(tsLogObj.logHandle->fd, &filesize, NULL) < 0) {
+  if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) {
     printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno));
     return -1;
   }
   size = (int32_t)filesize;
   tsLogObj.lines = size / 60;
 
-  taosLSeekFile(tsLogObj.logHandle->fd, 0, SEEK_END);
+  taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END);
 
   sprintf(name, "==================================================\n");
-  taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
   sprintf(name, "                new log file                      \n");
-  taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
   sprintf(name, "==================================================\n");
-  taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name));
 
   return 0;
 }
@@ -416,11 +416,11 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
   buffer[len++] = '\n';
   buffer[len] = 0;
 
-  if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) {
+  if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) {
     if (tsAsyncLog) {
       taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
     } else {
-      taosWriteFile(tsLogObj.logHandle->fd, buffer, len);
+      taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
     }
 
     if (tsLogObj.maxLines > 0) {
@@ -430,7 +430,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
     }
   }
 
-  if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len);
+  if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len);
 }
 
 void taosDumpData(unsigned char *msg, int32_t len) {
@@ -445,7 +445,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
     pos += 3;
     if (c >= 16) {
       temp[pos++] = '\n';
-      taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
+      taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos);
       c = 0;
       pos = 0;
     }
@@ -453,7 +453,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
 
   temp[pos++] = '\n';
 
-  taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
+  taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos);
 }
 
 void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) {
@@ -483,11 +483,11 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
   buffer[len++] = '\n';
   buffer[len] = 0;
 
-  if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) {
+  if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) {
     if (tsAsyncLog) {
       taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
     } else {
-      taosWriteFile(tsLogObj.logHandle->fd, buffer, len);
+      taosWriteFile(tsLogObj.logHandle->pFile, buffer, len);
     }
 
     if (tsLogObj.maxLines > 0) {
@@ -497,19 +497,19 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
     }
   }
 
-  if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len);
+  if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len);
 }
 
 #if 0
 void taosCloseLog() { 
-  taosCloseLogByFd(tsLogObj.logHandle->fd); 
+  taosCloseLogByFd(tsLogObj.logHandle->pFile); 
 }
 #endif
 
-static void taosCloseLogByFd(int32_t fd) {
+static void taosCloseLogByFd(TdFilePtr pFile) {
-  if (fd >= 0) {
+  if (pFile != NULL) {
-    taosUnLockLogFile(fd);
+    taosUnLockLogFile(pFile);
-    taosCloseFile(fd);
+    taosCloseFile(&pFile);
   }
 }
 
@@ -645,12 +645,12 @@ static void taosWriteLog(SLogBuff *tLogBuff) {
     }
 
     if (start < end) {
-      taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize);
+      taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, pollSize);
     } else {
       int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start;
-      taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize);
+      taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, tsize);
 
-      taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end);
+      taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff), end);
     }
 
     dbgWN++;
@@ -707,17 +707,17 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
   int32_t ret = 0;
   int32_t len = 0;
   char   *data = malloc(compressSize);
-  FILE   *srcFp = NULL;
 //  gzFile  dstFp = NULL;
 
-  srcFp = fopen(srcFileName, "r");
+  // srcFp = fopen(srcFileName, "r");
-  if (srcFp == NULL) {
+  TdFilePtr pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ);
+  if (pSrcFile == NULL) {
     ret = -1;
     goto cmp_end;
   }
 
-  int32_t fd = taosOpenFileCreateWriteTrunc(destFileName);
+  TdFilePtr pFile = taosOpenFile(destFileName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (fd < 0) {
+  if (pFile == NULL) {
     ret = -2;
     goto cmp_end;
   }
@@ -735,8 +735,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
 //  }
 
 cmp_end:
-  if (srcFp) {
+  if (pSrcFile) {
-    fclose(srcFp);
+    taosCloseFile(&pSrcFile);
   }
 //  if (dstFp) {
 //    gzclose(dstFp);

source/util/src/tpagedbuf.c

@@ -27,7 +27,7 @@ struct SDiskbasedBuf {
   int32_t   numOfPages;
   int64_t   totalBufSize;
   uint64_t  fileSize;            // disk file size
-  FILE*     file;
+  TdFilePtr pFile;
   int32_t   allocateId;          // allocated page id
   char*     path;                // file path
   int32_t   pageSize;            // current used page size
@@ -48,8 +48,9 @@ struct SDiskbasedBuf {
 };
 
 static int32_t createDiskFile(SDiskbasedBuf* pBuf) {
-  pBuf->file = fopen(pBuf->path, "wb+");
+  // pBuf->file = fopen(pBuf->path, "wb+");
-  if (pBuf->file == NULL) {
+  pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
+  if (pBuf->pFile == NULL) {
 //    qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
   }
@@ -140,13 +141,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
       pg->offset = allocatePositionInFile(pBuf, size);
       pBuf->nextPos += size;
 
-      int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
+    int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
     if (ret != 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return NULL;
     }
 
-      ret = (int32_t)fwrite(t, 1, size, pBuf->file);
+    ret = (int32_t) taosWriteFile(pBuf->pFile, t, size);
     if (ret != size) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return NULL;
@@ -171,13 +172,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
       }
 
     // 3. write to disk.
-      int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
+    int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
     if (ret != 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return NULL;
     }
 
-      ret = (int32_t)fwrite(t, 1, size, pBuf->file);
+    ret = (int32_t) taosWriteFile(pBuf->pFile, t, size);
     if (ret != size) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return NULL;
@@ -210,7 +211,7 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
   int32_t ret = TSDB_CODE_SUCCESS;
   assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages);
 
-  if (pBuf->file == NULL) {
+  if (pBuf->pFile == NULL) {
     if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) {
       terrno = ret;
       return NULL;
@@ -226,14 +227,14 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
 
 // load file block data in disk
 static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
-  int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
+  int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
   if (ret != 0) {
     ret = TAOS_SYSTEM_ERROR(errno);
     return ret;
   }
 
   void* pPage = (void*) GET_DATA_PAYLOAD(pg);
-  ret = (int32_t)fread(pPage, 1, pg->length, pBuf->file);
+  ret = (int32_t)taosReadFile(pBuf->pFile, pPage, pg->length);
   if (ret != pg->length) {
     ret = TAOS_SYSTEM_ERROR(errno);
     return ret;
@@ -373,7 +374,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem
   pPBuf->inMemPages   = inMemBufSize/pagesize;    // maximum allowed pages, it is a soft limit.
   pPBuf->allocateId   = -1;
   pPBuf->comp         = true;
-  pPBuf->file         = NULL;
+  pPBuf->pFile        = NULL;
   pPBuf->qId          = qId;
   pPBuf->fileSize     = 0;
   pPBuf->pFree        = taosArrayInit(4, sizeof(SFreeListItem));
@@ -538,12 +539,12 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
 
   dBufPrintStatis(pBuf);
 
-  if (pBuf->file != NULL) {
+  if (pBuf->pFile != NULL) {
   uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n",
       pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
       listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId);
 
-  fclose(pBuf->file);
+  taosCloseFile(&pBuf->pFile);
   } else {
     uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId);
   }

tools/shell/src/backup/shellCheck.c

@@ -116,7 +116,7 @@ static void *shellCheckThreadFp(void *arg) {
   char file[32] = {0};
   snprintf(file, 32, "tb%d.txt", pThread->threadIndex);
 
-  FILE *fp = fopen(file, "w");
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (!fp) {
     fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno));
     return NULL;
@@ -133,7 +133,7 @@ static void *shellCheckThreadFp(void *arg) {
     int32_t   code = taos_errno(pSql);
     if (code != 0) {
       int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname);
-      fwrite(sql, 1, len, fp);
+      taosWriteFile(pFile, sql, len);
       atomic_add_fetch_32(&errorNum, 1);
     }
 
@@ -145,8 +145,8 @@ static void *shellCheckThreadFp(void *arg) {
     taos_free_result(pSql);
   }
 
-  taosFsync(fileno(fp));
+  taosFsync(pFile);
-  fclose(fp);
+  taosCloseFile(&pFile);
 
   return NULL;
 }

tools/shell/src/backup/shellImport.c

@@ -138,7 +138,6 @@ static void shellSourceFile(TAOS *con, char *fptr) {
   char *    cmd = malloc(tsMaxSQLStringLen);
   size_t    cmd_len = 0;
   char *    line = NULL;
-  size_t    line_len = 0;
 
   if (wordexp(fptr, &full_path, 0) != 0) {
     fprintf(stderr, "ERROR: illegal file name\n");
@@ -171,8 +170,9 @@ static void shellSourceFile(TAOS *con, char *fptr) {
   }
   */
 
-  FILE *f = fopen(fname, "r");
+  // FILE *f = fopen(fname, "r");
-  if (f == NULL) {
+  TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
+  if (pFile == NULL) {
     fprintf(stderr, "ERROR: failed to open file %s\n", fname);
     wordfree(&full_path);
     free(cmd);
@@ -182,7 +182,7 @@ static void shellSourceFile(TAOS *con, char *fptr) {
   fprintf(stdout, "begin import file:%s\n", fname);
 
   int lineNo = 0;
-  while ((read_len = getline(&line, &line_len, f)) != -1) {
+  while ((read_len = taosGetLineFile(pFile, &line)) != -1) {
     ++lineNo;
     if (read_len >= tsMaxSQLStringLen) continue;
     line[--read_len] = '\0';
@@ -215,9 +215,9 @@ static void shellSourceFile(TAOS *con, char *fptr) {
   }
 
   free(cmd);
-  if (line) free(line);
+  if(line != NULL) free(line);
   wordfree(&full_path);
-  fclose(f);
+  taosCloseFile(&pFile);
 }
 
 void* shellImportThreadFp(void *arg)

tools/shell/src/shellEngine.c

@@ -459,44 +459,44 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) {
   return buf;
 }
 
-static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) {
+static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int precision) {
   if (val == NULL) {
-    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
+    taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR);
     return;
   }
 
   char buf[TSDB_MAX_BYTES_PER_ROW];
   switch (field->type) {
     case TSDB_DATA_TYPE_BOOL:
-      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
+      taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
       break;
     case TSDB_DATA_TYPE_TINYINT:
-      fprintf(fp, "%d", *((int8_t *)val));
+      taosFprintfFile(pFile, "%d", *((int8_t *)val));
       break;
     case TSDB_DATA_TYPE_SMALLINT:
-      fprintf(fp, "%d", *((int16_t *)val));
+      taosFprintfFile(pFile, "%d", *((int16_t *)val));
       break;
     case TSDB_DATA_TYPE_INT:
-      fprintf(fp, "%d", *((int32_t *)val));
+      taosFprintfFile(pFile, "%d", *((int32_t *)val));
       break;
     case TSDB_DATA_TYPE_BIGINT:
-      fprintf(fp, "%" PRId64, *((int64_t *)val));
+      taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val));
       break;
     case TSDB_DATA_TYPE_FLOAT:
-      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
+      taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val));
       break;
     case TSDB_DATA_TYPE_DOUBLE:
-      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
+      taosFprintfFile(pFile, "%.9f", GET_DOUBLE_VAL(val));
       break;
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
       memcpy(buf, val, length);
       buf[length] = 0;
-      fprintf(fp, "\'%s\'", buf);
+      taosFprintfFile(pFile, "\'%s\'", buf);
       break;
     case TSDB_DATA_TYPE_TIMESTAMP:
       formatTimestamp(buf, *(int64_t *)val, precision);
-      fprintf(fp, "'%s'", buf);
+      taosFprintfFile(pFile, "'%s'", buf);
       break;
     default:
       break;
@@ -516,8 +516,9 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) {
     return -1;
   }
 
-  FILE *fp = fopen(full_path.we_wordv[0], "w");
+  // FILE *fp = fopen(full_path.we_wordv[0], "w");
-  if (fp == NULL) {
+  TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
     fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]);
     wordfree(&full_path);
     return -1;
@@ -531,29 +532,29 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) {
 
   for (int col = 0; col < num_fields; col++) {
     if (col > 0) {
-      fprintf(fp, ",");
+      taosFprintfFile(pFile, ",");
     }
-    fprintf(fp, "%s", fields[col].name);
+    taosFprintfFile(pFile, "%s", fields[col].name);
   }
-  fputc('\n', fp);
+  taosFprintfFile(pFile, "\n");
 
   int numOfRows = 0;
   do {
     int32_t *length = taos_fetch_lengths(tres);
     for (int i = 0; i < num_fields; i++) {
       if (i > 0) {
-        fputc(',', fp);
+        taosFprintfFile(pFile, "\n");
       }
-      dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision);
+      dumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision);
     }
-    fputc('\n', fp);
+    taosFprintfFile(pFile, "\n");
 
     numOfRows++;
     row = taos_fetch_row(tres);
   } while (row != NULL);
 
   result = 0;
-  fclose(fp);
+  taosCloseFile(&pFile);
 
   return numOfRows;
 }
@@ -897,14 +898,14 @@ void read_history() {
   history.hstart = 0;
   history.hend = 0;
   char  *line = NULL;
-  size_t line_size = 0;
   int    read_size = 0;
 
   char f_history[TSDB_FILENAME_LEN];
   get_history_path(f_history);
 
-  FILE *f = fopen(f_history, "r");
+  // FILE *f = fopen(f_history, "r");
-  if (f == NULL) {
+  TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ);
+  if (pFile == NULL) {
 #ifndef WINDOWS
     if (errno != ENOENT) {
       fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno));
@@ -913,7 +914,7 @@ void read_history() {
     return;
   }
 
-  while ((read_size = tgetline(&line, &line_size, f)) != -1) {
+  while ((read_size = taosGetLineFile(pFile, &line)) != -1) {
     line[read_size - 1] = '\0';
     history.hist[history.hend] = strdup(line);
 
@@ -924,16 +925,17 @@ void read_history() {
     }
   }
 
-  free(line);
+  if(line != NULL) free(line);
-  fclose(f);
+  taosCloseFile(&pFile);
 }
 
 void write_history() {
   char f_history[TSDB_FILENAME_LEN];
   get_history_path(f_history);
 
-  FILE *f = fopen(f_history, "w");
+  // FILE *f = fopen(f_history, "w");
-  if (f == NULL) {
+  TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
 #ifndef WINDOWS
     fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno));
 #endif
@@ -942,12 +944,12 @@ void write_history() {
 
   for (int i = history.hstart; i != history.hend;) {
     if (history.hist[i] != NULL) {
-      fprintf(f, "%s\n", history.hist[i]);
+      taosFprintfFile(pFile, "%s\n", history.hist[i]);
       tfree(history.hist[i]);
     }
     i = (i + 1) % MAX_HISTORY_SIZE;
   }
-  fclose(f);
+  taosCloseFile(&pFile);
 }
 
 void taos_error(TAOS_RES *tres, int64_t st) {
@@ -969,7 +971,6 @@ void source_file(TAOS *con, char *fptr) {
   char     *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1);
   size_t    cmd_len = 0;
   char     *line = NULL;
-  size_t    line_len = 0;
 
   if (wordexp(fptr, &full_path, 0) != 0) {
     fprintf(stderr, "ERROR: illegal file name\n");
@@ -989,15 +990,16 @@ void source_file(TAOS *con, char *fptr) {
   }
   */
 
-  FILE *f = fopen(fname, "r");
+  // FILE *f = fopen(fname, "r");
-  if (f == NULL) {
+  TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
+  if (pFile == NULL) {
     fprintf(stderr, "ERROR: failed to open file %s\n", fname);
     wordfree(&full_path);
     free(cmd);
     return;
   }
 
-  while ((read_len = tgetline(&line, &line_len, f)) != -1) {
+  while ((read_len = taosGetLineFile(pFile, &line)) != -1) {
     if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue;
     line[--read_len] = '\0';
 
@@ -1020,9 +1022,9 @@ void source_file(TAOS *con, char *fptr) {
   }
 
   free(cmd);
-  if (line) free(line);
+  if(line != NULL) free(line);
   wordfree(&full_path);
-  fclose(f);
+  taosCloseFile(&pFile);
 }
 
 void shellGetGrantInfo(void *con) {