TD-4088

cmake/define.inc

@@ -57,7 +57,7 @@ IF (TD_LINUX_64)
   ADD_DEFINITIONS(-D_M_X64)
   ADD_DEFINITIONS(-D_TD_LINUX_64)
   MESSAGE(STATUS "linux64 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
   ADD_DEFINITIONS(-DUSE_LIBICONV)
 ENDIF ()
 
@@ -65,7 +65,7 @@ IF (TD_LINUX_32)
   ADD_DEFINITIONS(-D_TD_LINUX_32)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "linux32 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
 ENDIF ()
 
 IF (TD_ARM_64)
@@ -73,7 +73,7 @@ IF (TD_ARM_64)
   ADD_DEFINITIONS(-D_TD_ARM_)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "arm64 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
 ENDIF ()
 
 IF (TD_ARM_32)
@@ -81,7 +81,7 @@ IF (TD_ARM_32)
   ADD_DEFINITIONS(-D_TD_ARM_)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "arm32 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ")
+  SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ")
 ENDIF ()
 
 IF (TD_MIPS_64)
@@ -89,7 +89,7 @@ IF (TD_MIPS_64)
   ADD_DEFINITIONS(-D_TD_MIPS_64)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "mips64 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
 ENDIF ()
 
 IF (TD_MIPS_32)
@@ -97,7 +97,7 @@ IF (TD_MIPS_32)
   ADD_DEFINITIONS(-D_TD_MIPS_32)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "mips32 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-Wall -Werror -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
 ENDIF ()
 
 IF (TD_APLHINE)
@@ -138,7 +138,7 @@ IF (TD_DARWIN_64)
   ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "darwin64 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
   SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG")
   SET(RELEASE_FLAGS "-Og")
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)

cmake/env.inc

@@ -32,6 +32,7 @@ ENDIF ()
 #
 
 # Set compiler options
+SET(COMMON_C_FLAGS "${COMMON_FLAGS} -std=gnu99")
 SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_FLAGS} ${DEBUG_FLAGS}")
 SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FLAGS}")
 

src/dnode/src/dnodeCfg.c

@@ -158,7 +158,7 @@ static int32_t dnodeWriteCfg() {
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
-  fsync(fileno(fp));
+  taosFsync(fileno(fp));
   fclose(fp);
   free(content);
   terrno = 0;

src/dnode/src/dnodeEps.c

@@ -277,7 +277,7 @@ static int32_t dnodeWriteEps() {
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
-  fsync(fileno(fp));
+  taosFsync(fileno(fp));
   fclose(fp);
   free(content);
   terrno = 0;

src/dnode/src/dnodeMInfos.c

@@ -286,7 +286,7 @@ static int32_t dnodeWriteMInfos() {
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
-  fsync(fileno(fp));
+  taosFsync(fileno(fp));
   fclose(fp);
   free(content);
   terrno = 0;

src/kit/shell/src/shellCheck.c

@@ -142,7 +142,7 @@ static void *shellCheckThreadFp(void *arg) {
     taos_free_result(pSql);
   }
 
-  fsync(fileno(fp));
+  taosFsync(fileno(fp));
   fclose(fp);
 
   return NULL;

src/mnode/src/mnodeUser.c

@@ -123,7 +123,7 @@ static void mnodePrintUserAuth() {
     mnodeDecUserRef(pUser);
   }
 
-  fsync(fileno(fp));
+  taosFsync(fileno(fp));
   fclose(fp);
 }
 

src/os/inc/os.h

@@ -67,6 +67,7 @@ extern "C" {
 #include "osRand.h"
 #include "osSemphone.h"
 #include "osSignal.h"
+#include "osSleep.h"
 #include "osSocket.h"
 #include "osString.h"
 #include "osSysinfo.h"

src/os/inc/osDef.h

@@ -26,8 +26,6 @@ extern "C" {
   #endif
 #endif
 
-#define FD_VALID(x) ((x) > STDERR_FILENO)
-#define FD_INITIALIZER  ((int32_t)-1)
 
 // #define WCHAR wchar_t
 

src/os/inc/osFile.h

@@ -22,48 +22,37 @@ extern "C" {
 
 #include "osSocket.h"
 
-int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
+#define FD_VALID(x) ((x) > STDERR_FILENO)
-int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
+#define FD_INITIALIZER  ((int32_t)-1)
-int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
-int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath);
-int64_t taosCopy(char *from, char *to);
 
-#define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
+#ifndef PATH_MAX
-#define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)
+  #define PATH_MAX 256
-#define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence)
-#define taosClose(fd)      \
-  {                        \
-    if (FD_VALID(fd)) {    \
-      close(fd);           \
-      fd = FD_INITIALIZER; \
-    }                      \
-  }
-
-// TAOS_OS_FUNC_FILE_SENDIFLE
-int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size);
-int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);
-
-#ifdef TAOS_RANDOM_FILE_FAIL
-  void taosSetRandomFileFailFactor(int32_t factor);
-  void taosSetRandomFileFailOutput(const char *path);
-  #ifdef TAOS_RANDOM_FILE_FAIL_TEST
-    int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line);
-    int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line);
-    int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line);
-    #undef taosRead
-    #undef taosWrite
-    #undef taosLSeek
-    #define taosRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
-    #define taosWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
-    #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__)
-  #endif  
 #endif
 
-// TAOS_OS_FUNC_FILE_GETTMPFILEPATH
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath);
+typedef int32_t FileFd;
+typedef SOCKET  SocketFd;
+#else
+typedef int32_t FileFd;
+typedef int32_t SocketFd;
+#endif
+
+int64_t taosRead(FileFd fd, void *buf, int64_t count);
+int64_t taosWrite(FileFd fd, void *buf, int64_t count);
+
+int64_t taosLSeek(FileFd fd, int64_t offset, int32_t whence);
+int32_t taosFtruncate(FileFd fd, int64_t length);
+int32_t taosFsync(FileFd fd);
+
+int32_t taosRename(char* oldName, char *newName);
+int64_t taosCopy(char *from, char *to);
+
+int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size);
+int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);
+
+void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath);
+void taosClose(FileFd fd);
 
-// TAOS_OS_FUNC_FILE_FTRUNCATE
-int32_t taosFtruncate(int32_t fd, int64_t length);
 #ifdef __cplusplus
 }
 #endif

src/os/inc/osSleep.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_SLEEP_H
+#define TDENGINE_OS_SLEEP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void taosMsleep(int32_t ms);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/os/inc/osTimer.h

@@ -21,7 +21,6 @@ extern "C" {
 #endif
 
 // TAOS_OS_FUNC_TIMER
-void taosMsleep(int mseconds);
 int taosInitTimer(void (*callback)(int), int ms);
 void taosUninitTimer();
 

src/os/inc/osWindows.h

@@ -133,7 +133,6 @@ char *stpncpy (char *dest, const char *src, size_t n);
     #define MILLISECOND_PER_SECOND (1000i64)
   #endif
 
-#define TAOS_OS_FUNC_TIMER_SLEEP
 #define TAOS_OS_FUNC_TIMER
 
 // specific
@@ -154,7 +153,6 @@ char *     strptime(const char *buf, const char *fmt, struct tm *tm);
 char *     strsep(char **stringp, const char *delim);
 char *     getpass(const char *prefix);
 int        flock(int fd, int option);
-int        fsync(int filedes);
 char *     strndup(const char *s, size_t n);
 int        gettimeofday(struct timeval *ptv, void *pTimeZone);
 

src/os/src/darwin/darwinFile.c

@@ -17,49 +17,5 @@
 #include "os.h"
 #include "tulog.h"
 
-int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
-  int r = 0;
-  if (offset) {
-    r = fseek(in_file, *offset, SEEK_SET);
-    if (r==-1) return -1;
-  }
-  off_t len = count;
-  while (len>0) {
-    char buf[1024*16];
-    off_t n = sizeof(buf);
-    if (len<n) n = len;
-    size_t m = fread(buf, 1, n, in_file);
-    if (m<n) {
-      int e = ferror(in_file);
-      if (e) return -1;
-    }
-    if (m==0) break;
-    if (m!=fwrite(buf, 1, m, out_file)) {
-      return -1;
-    }
-    len -= m;
-  }
-  return count - len;
-}
 
-int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) {
-  int r = 0;
-  if (offset) {
-    r = lseek(sfd, *offset, SEEK_SET);
-    if (r==-1) return -1;
-  }
-  off_t len = count;
-  while (len>0) {
-    char buf[1024*16];
-    off_t n = sizeof(buf);
-    if (len<n) n = len;
-    size_t m = read(sfd, buf, n);
-    if (m==-1) return -1;
-    if (m==0) break;
-    size_t l = write(dfd, buf, m);
-    if (l==-1) return -1;
-    len -= l;
-  }
-  return count - len;
-}
 

src/os/src/detail/osFile.c

@@ -16,19 +16,50 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tglobal.h"
+#include "tulog.h"
 
-#ifndef TAOS_OS_FUNC_FILE_GETTMPFILEPATH
+void taosClose(FileFd fd) {
+  close(fd);
+  fd = FD_INITIALIZER;
+}
+
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+
+void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
+  const char *tdengineTmpFileNamePrefix = "tdengine-";
+  char        tmpPath[PATH_MAX];
+
+  int32_t len = (int32_t)strlen(tsTempDir);
+  memcpy(tmpPath, tsTempDir, len);
+
+  if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') {
+    tmpPath[len++] = '\\';
+  }
+
+  strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
+  strcat(tmpPath, tdengineTmpFileNamePrefix);
+  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
+    strcat(tmpPath, fileNamePrefix);
+    strcat(tmpPath, "-%d-%s");
+  }
+
+  char rand[8] = {0};
+  taosRandStr(rand, tListLen(rand) - 1);
+  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
+}
+
+#else
 
 void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
   const char *tdengineTmpFileNamePrefix = "tdengine-";
 
-  char  tmpPath[PATH_MAX];
+  char    tmpPath[PATH_MAX];
   int32_t len = strlen(tsTempDir);
   memcpy(tmpPath, tsTempDir, len);
   static uint64_t seqId = 0;
 
   if (tmpPath[len - 1] != '/') {
-      tmpPath[len++] = '/';
+    tmpPath[len++] = '/';
   }
 
   strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
@@ -39,42 +70,14 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
 
   char rand[32] = {0};
 
-  sprintf(rand, "%"PRIu64, atomic_add_fetch_64(&seqId, 1));
+  sprintf(rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1));
 
   snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
 }
 
 #endif
 
-int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath) {
+int64_t taosRead(FileFd fd, void *buf, int64_t count) {
-  int32_t ts = taosGetTimestampSec();
-
-  char fname[PATH_MAX] = {0};  // max file name length must be less than 255
-
-  char *delimiterPos = strrchr(fullPath, delimiter);
-  if (delimiterPos == NULL) return -1;
-
-  int32_t fileNameLen = 0;
-  if (suffix) {
-    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix);
-  } else {
-    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts);
-  }
-
-  int32_t len = (int32_t)((delimiterPos - fullPath) + fileNameLen + 1);
-  if (*dstPath == NULL) {
-    *dstPath = calloc(1, len + 1);
-    if (*dstPath == NULL) return -1;
-  }
-
-  strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1));
-  strncat(*dstPath, fname, (size_t)fileNameLen);
-  (*dstPath)[len] = 0;
-
-  return rename(fullPath, *dstPath);
-}
-
-int64_t taosReadImp(int32_t fd, void *buf, int64_t count) {
   int64_t leftbytes = count;
   int64_t readbytes;
   char *  tbuf = (char *)buf;
@@ -98,7 +101,7 @@ int64_t taosReadImp(int32_t fd, void *buf, int64_t count) {
   return count;
 }
 
-int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) {
+int64_t taosWrite(FileFd fd, void *buf, int64_t n) {
   int64_t nleft = n;
   int64_t nwritten = 0;
   char *  tbuf = (char *)buf;
@@ -118,9 +121,7 @@ int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) {
   return n;
 }
 
-int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
+int64_t taosLSeek(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); }
-  return (int64_t)lseek(fd, (long)offset, whence);
-}
 
 int64_t taosCopy(char *from, char *to) {
   char    buffer[4096];
@@ -145,22 +146,139 @@ int64_t taosCopy(char *from, char *to) {
     if (bytes < sizeof(buffer)) break;
   }
 
-  fsync(fidto);
+  taosFsync(fidto);
 
-  close(fidfrom);
+  taosClose(fidfrom);
-  close(fidto);
+  taosClose(fidto);
   return size;
 
 _err:
-  if (fidfrom >= 0) close(fidfrom);
+  if (fidfrom >= 0) taosClose(fidfrom);
-  if (fidto >= 0) close(fidto);
+  if (fidto >= 0) taosClose(fidto);
   remove(to);
   return -1;
 }
 
-#ifndef TAOS_OS_FUNC_FILE_SENDIFLE
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
 
-int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) {
+#define _SEND_FILE_STEP_ 1000
+
+int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
+  fseek(in_file, (int32_t)(*offset), 0);
+  int64_t writeLen = 0;
+  uint8_t buffer[_SEND_FILE_STEP_] = {0};
+
+  for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
+    size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
+    if (rlen <= 0) {
+      return writeLen;
+    } else if (rlen < _SEND_FILE_STEP_) {
+      fwrite(buffer, 1, rlen, out_file);
+      return (int64_t)(writeLen + rlen);
+    } else {
+      fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
+      writeLen += _SEND_FILE_STEP_;
+    }
+  }
+
+  int64_t remain = count - writeLen;
+  if (remain > 0) {
+    size_t rlen = fread(buffer, 1, (size_t)remain, in_file);
+    if (rlen <= 0) {
+      return writeLen;
+    } else {
+      fwrite(buffer, 1, (size_t)remain, out_file);
+      writeLen += remain;
+    }
+  }
+
+  return writeLen;
+}
+
+int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) {
+  if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0);
+
+  int64_t writeLen = 0;
+  uint8_t buffer[_SEND_FILE_STEP_] = {0};
+
+  for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
+    int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_);
+    if (rlen <= 0) {
+      return writeLen;
+    } else if (rlen < _SEND_FILE_STEP_) {
+      taosWriteSocket(dfd, buffer, rlen);
+      return (int64_t)(writeLen + rlen);
+    } else {
+      taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_);
+      writeLen += _SEND_FILE_STEP_;
+    }
+  }
+
+  int64_t remain = count - writeLen;
+  if (remain > 0) {
+    int32_t rlen = read(sfd, buffer, (int32_t)remain);
+    if (rlen <= 0) {
+      return writeLen;
+    } else {
+      taosWriteSocket(sfd, buffer, (int32_t)remain);
+      writeLen += remain;
+    }
+  }
+
+  return writeLen;
+}
+
+#elif defined(_TD_DARWIN_64)
+
+int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
+  int r = 0;
+  if (offset) {
+    r = fseek(in_file, *offset, SEEK_SET);
+    if (r == -1) return -1;
+  }
+  off_t len = count;
+  while (len > 0) {
+    char  buf[1024 * 16];
+    off_t n = sizeof(buf);
+    if (len < n) n = len;
+    size_t m = fread(buf, 1, n, in_file);
+    if (m < n) {
+      int e = ferror(in_file);
+      if (e) return -1;
+    }
+    if (m == 0) break;
+    if (m != fwrite(buf, 1, m, out_file)) {
+      return -1;
+    }
+    len -= m;
+  }
+  return count - len;
+}
+
+int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) {
+  int r = 0;
+  if (offset) {
+    r = lseek(sfd, *offset, SEEK_SET);
+    if (r == -1) return -1;
+  }
+  off_t len = count;
+  while (len > 0) {
+    char  buf[1024 * 16];
+    off_t n = sizeof(buf);
+    if (len < n) n = len;
+    size_t m = read(sfd, buf, n);
+    if (m == -1) return -1;
+    if (m == 0) break;
+    size_t l = write(dfd, buf, m);
+    if (l == -1) return -1;
+    len -= l;
+  }
+  return count - len;
+}
+
+#else
+
+int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) {
   int64_t leftbytes = size;
   int64_t sentbytes;
 
@@ -188,10 +306,96 @@ int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size
 
 #endif
 
-#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
 
-int32_t taosFtruncate(int32_t fd, int64_t length) {
+int32_t taosFtruncate(int32_t fd, int64_t l_size) {
-  return ftruncate(fd, length);
+  if (fd < 0) {
+    errno = EBADF;
+    uError("%s\n", "fd arg was negative");
+    return -1;
+  }
+
+  HANDLE h = (HANDLE)_get_osfhandle(fd);
+
+  LARGE_INTEGER li_0;
+  li_0.QuadPart = (int64_t)0;
+  BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT);
+  if (!cur) {
+    uError("SetFilePointerEx Error getting current position in file.\n");
+    return -1;
+  }
+
+  LARGE_INTEGER li_size;
+  li_size.QuadPart = l_size;
+  BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN);
+  if (cur2 == 0) {
+    int error = GetLastError();
+    uError("SetFilePointerEx GetLastError is: %d\n", error);
+    switch (error) {
+      case ERROR_INVALID_HANDLE:
+        errno = EBADF;
+        break;
+      default:
+        errno = EIO;
+        break;
+    }
+    return -1;
+  }
+
+  if (!SetEndOfFile(h)) {
+    int error = GetLastError();
+    uError("SetEndOfFile GetLastError is:%d", error);
+    switch (error) {
+      case ERROR_INVALID_HANDLE:
+        errno = EBADF;
+        break;
+      default:
+        errno = EIO;
+        break;
+    }
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t taosFsync(FileFd fd) {
+  if (fd < 0) {
+    errno = EBADF;
+    uError("%s\n", "fd arg was negative");
+    return -1;
+  }
+
+  HANDLE h = (HANDLE)_get_osfhandle(fd);
+
+  return FlushFileBuffers(h);
+}
+
+int32_t taosRename(char *oldName, char *newName) {
+  int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
+  if (code < 0) {
+    uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
+  } else {
+    uTrace("successfully to rename file %s to %s", oldName, newName);
+  }
+
+  return code;
+}
+
+#else
+
+int32_t taosFtruncate(FileFd fd, int64_t length) { return ftruncate(fd, length); }
+int32_t taosFsync(FileFd fd) { return fsync(fd); }
+
+int32_t taosRename(char *oldName, char *newName) {
+  int32_t code = rename(oldName, newName);
+  if (code < 0) {
+    uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
+  } else {
+    uTrace("successfully to rename file %s to %s", oldName, newName);
+  }
+
+  return code;
 }
 
 #endif

src/os/src/detail/osSleep.c

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+
+void taosMsleep(int32_t ms) { Sleep(ms); }
+
+#else
+
+/*
+  to make taosMsleep work,
+   signal SIGALRM shall be blocked in the calling thread,
+
+  sigset_t set;
+  sigemptyset(&set);
+  sigaddset(&set, SIGALRM);
+  pthread_sigmask(SIG_BLOCK, &set, NULL);
+*/
+void taosMsleep(int32_t mseconds) {
+#ifdef _TD_DARWIN_64
+  taos_block_sigalrm();
+#endif
+#if 1
+  usleep(mseconds * 1000);
+#else
+  struct timeval timeout;
+  int32_t        seconds, useconds;
+
+  seconds = mseconds / 1000;
+  useconds = (mseconds % 1000) * 1000;
+  timeout.tv_sec = seconds;
+  timeout.tv_usec = useconds;
+
+  /* sigset_t set; */
+  /* sigemptyset(&set); */
+  /* sigaddset(&set, SIGALRM); */
+  /* pthread_sigmask(SIG_BLOCK, &set, NULL); */
+
+  select(0, NULL, NULL, NULL, &timeout);
+
+/* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */
+#endif
+}
+
+#endif

src/os/src/detail/osTimer.c

@@ -104,42 +104,3 @@ void taosUninitTimer() {
 }
 
 #endif
-
-#ifndef TAOS_OS_FUNC_TIMER_SLEEP
-/*
-  to make taosMsleep work,
-   signal SIGALRM shall be blocked in the calling thread,
-
-  sigset_t set;
-  sigemptyset(&set);
-  sigaddset(&set, SIGALRM);
-  pthread_sigmask(SIG_BLOCK, &set, NULL);
-*/
-void taosMsleep(int mseconds) {
-#ifdef __APPLE__
-  taos_block_sigalrm();
-#endif // __APPLE__
-#if 1
-  usleep(mseconds * 1000);
-#else
-  struct timeval timeout;
-  int            seconds, useconds;
-
-  seconds = mseconds / 1000;
-  useconds = (mseconds % 1000) * 1000;
-  timeout.tv_sec = seconds;
-  timeout.tv_usec = useconds;
-
-  /* sigset_t set; */
-  /* sigemptyset(&set); */
-  /* sigaddset(&set, SIGALRM); */
-  /* pthread_sigmask(SIG_BLOCK, &set, NULL); */
-
-  select(0, NULL, NULL, NULL, &timeout);
-
-/* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */
-#endif
-
-}
-
-#endif

src/os/src/windows/wFile.c

@@ -19,166 +19,8 @@
 #include "tglobal.h"
 #include "tulog.h"
 
-void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
-  const char *tdengineTmpFileNamePrefix = "tdengine-";
-  char        tmpPath[PATH_MAX];
 
-  int32_t len = (int32_t)strlen(tsTempDir);
-  memcpy(tmpPath, tsTempDir, len);
 
-  if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') {
-    tmpPath[len++] = '\\';
-  }
 
-  strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
-  strcat(tmpPath, tdengineTmpFileNamePrefix);
-  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
-    strcat(tmpPath, fileNamePrefix);
-    strcat(tmpPath, "-%d-%s");
-  }
 
-  char rand[8] = {0};
-  taosRandStr(rand, tListLen(rand) - 1);
-  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
-}
 
-#define _SEND_FILE_STEP_ 1000
-
-int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
-  fseek(in_file, (int32_t)(*offset), 0);
-  int64_t writeLen = 0;
-  uint8_t buffer[_SEND_FILE_STEP_] = {0};
-
-  for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
-    size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
-    if (rlen <= 0) {
-      return writeLen;
-    } else if (rlen < _SEND_FILE_STEP_) {
-      fwrite(buffer, 1, rlen, out_file);
-      return (int64_t)(writeLen + rlen);
-    } else {
-      fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
-      writeLen += _SEND_FILE_STEP_;
-    }
-  }
-
-  int64_t remain = count - writeLen;
-  if (remain > 0) {
-    size_t rlen = fread(buffer, 1, (size_t)remain, in_file);
-    if (rlen <= 0) {
-      return writeLen;
-    } else {
-      fwrite(buffer, 1, (size_t)remain, out_file);
-      writeLen += remain;
-    }
-  }
-
-  return writeLen;
-}
-
-int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) {
-  if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0);
-
-  int64_t writeLen = 0;
-  uint8_t buffer[_SEND_FILE_STEP_] = {0};
-
-  for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
-    int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_);
-    if (rlen <= 0) {
-      return writeLen;
-    } else if (rlen < _SEND_FILE_STEP_) {
-      taosWriteSocket(dfd, buffer, rlen);
-      return (int64_t)(writeLen + rlen);
-    } else {
-      taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_);
-      writeLen += _SEND_FILE_STEP_;
-    }
-  }
-
-  int64_t remain = count - writeLen;
-  if (remain > 0) {
-    int32_t rlen = read(sfd, buffer, (int32_t)remain);
-    if (rlen <= 0) {
-      return writeLen;
-    } else {
-      taosWriteSocket(sfd, buffer, (int32_t)remain);
-      writeLen += remain;
-    }
-  }
-
-  return writeLen;
-}
-
-int32_t taosFtruncate(int32_t fd, int64_t l_size) {
-  if (fd < 0) {
-    errno = EBADF;
-    uError("%s\n", "fd arg was negative");
-    return -1;
-  }
-
-  HANDLE h = (HANDLE)_get_osfhandle(fd);
-
-  LARGE_INTEGER li_0;
-  li_0.QuadPart = (int64_t)0;
-  BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT);
-  if (!cur) {
-    uError("SetFilePointerEx Error getting current position in file.\n");
-    return -1;
-  }
-
-  LARGE_INTEGER li_size;
-  li_size.QuadPart = l_size;
-  BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN);
-  if (cur2 == 0) {
-    int error = GetLastError();
-    uError("SetFilePointerEx GetLastError is: %d\n", error);
-    switch (error) {
-      case ERROR_INVALID_HANDLE:
-        errno = EBADF;
-        break;
-      default:
-        errno = EIO;
-        break;
-    }
-    return -1;
-  }
-
-  if (!SetEndOfFile(h)) {
-    int error = GetLastError();
-    uError("SetEndOfFile GetLastError is:%d", error);
-    switch (error) {
-      case ERROR_INVALID_HANDLE:
-        errno = EBADF;
-        break;
-      default:
-        errno = EIO;
-        break;
-    }
-    return -1;
-  }
-
-  return 0;
-}
-
-int fsync(int filedes) { 
-  if (filedes < 0) {
-    errno = EBADF;
-    uError("%s\n", "fd arg was negative");
-    return -1;
-  }
-
-  HANDLE h = (HANDLE)_get_osfhandle(filedes);
-  
-  return FlushFileBuffers(h);
-}
-
-int32_t taosRename(char* oldName, char *newName) {
-  int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
-  if (code < 0) {
-    uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
-  } else {
-    uTrace("successfully to rename file %s to %s", oldName, newName);
-  }
-
-  return code;
-}

src/os/src/windows/wTimer.c

@@ -43,7 +43,3 @@ int taosInitTimer(win_timer_f callback, int ms) {
 void taosUninitTimer() {
   timeKillEvent(timerId);
 }
-
-void taosMsleep(int mseconds) {
-  Sleep(mseconds);
-}

src/query/src/qTsbuf.c

@@ -484,7 +484,7 @@ void tsBufFlush(STSBuf* pTSBuf) {
       .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
   STSBufUpdateHeader(pTSBuf, &header);
   
-  fsync(fileno(pTSBuf->f));
+  taosFsync(fileno(pTSBuf->f));
 }
 
 static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t numOfGroups, int32_t id) {
@@ -868,7 +868,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
       .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
   STSBufUpdateHeader(pTSBuf, &header);
   
-  fsync(fileno(pTSBuf->f));
+  taosFsync(fileno(pTSBuf->f));
   
   return pTSBuf;
 }

src/rpc/test/rserver.c

@@ -52,7 +52,7 @@ void processShellMsg() {
 
     if (commit >=2) {
       num += numOfMsgs;
-      if ( fsync(dataFd) < 0 ) {
+      if ( taosFsync(dataFd) < 0 ) {
         tInfo("failed to flush data to file, reason:%s", strerror(errno));
       }
 

src/tsdb/inc/tsdbFile.h

@@ -32,7 +32,7 @@
 #define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
 #define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf))
 #define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf))
-#define TSDB_FILE_FSYNC(tf) fsync(TSDB_FILE_FD(tf))
+#define TSDB_FILE_FSYNC(tf) taosFsync(TSDB_FILE_FD(tf))
 #define TSDB_FILE_STATE(tf) ((tf)->state)
 #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
 #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)

src/tsdb/src/tsdbFS.c

@@ -390,7 +390,7 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
   }
 
   // fsync, close and rename
-  if (fsync(fd) < 0) {
+  if (taosFsync(fd) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     close(fd);
     remove(tfname);

src/util/src/tcache.c

@@ -661,11 +661,7 @@ void* taosCacheTimedRefresh(void *handle) {
 
   int64_t count = 0;
   while(1) {
-#if defined LINUX
-    usleep(500*1000);
-#else
     taosMsleep(500);
-#endif
 
     // check if current cache object will be deleted every 500ms.
     if (pCacheObj->deleting) {

src/util/src/tfile.c

@@ -98,7 +98,7 @@ int32_t tfFsync(int64_t tfd) {
   if (p == NULL) return -1;
 
   int32_t fd = (int32_t)(uintptr_t)p;
-  int32_t code = fsync(fd);
+  int32_t code = taosFsync(fd);
 
   taosReleaseRef(tsFileRsetId, tfd);
   return code;

src/vnode/src/vnodeCfg.c

@@ -362,7 +362,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
-  fsync(fileno(fp));
+  taosFsync(fileno(fp));
   fclose(fp);
   free(content);
   terrno = 0;

src/vnode/src/vnodeVersion.c

@@ -90,7 +90,7 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
-  fsync(fileno(fp));
+  taosFsync(fileno(fp));
   fclose(fp);
   free(content);
   terrno = 0;