Merge pull request #5128 from taosdata/feature/mac

Feature/mac

src/dnode/src/dnodeVWrite.c

@@ -53,7 +53,7 @@ void dnodeCleanupVWrite() {
   for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
     SVWriteWorker *pWorker = tsVWriteWP.worker + i;
     if (taosCheckPthreadValid(pWorker->thread)) {
-      taosQsetThreadResume(pWorker->qset);
+      if (pWorker->qset) taosQsetThreadResume(pWorker->qset);
     }
   }
 

src/os/inc/osDarwin.h

@@ -85,6 +85,7 @@ extern "C" {
 #define TAOS_OS_FUNC_STRING_STR2INT64
 #define TAOS_OS_FUNC_SYSINFO
 #define TAOS_OS_FUNC_TIMER
+#define TAOS_OS_FUNC_SEMPHONE_PTHREAD
 
 // specific
 #define htobe64 htonll

src/os/src/darwin/darwinEnv.c

@@ -17,17 +17,29 @@
 #include "os.h"
 #include "tglobal.h"
 
+static const char* expand_like_shell(const char *path) {
+  static __thread char buf[TSDB_FILENAME_LEN];
+  buf[0] = '\0';
+  wordexp_t we;
+  if (wordexp(path, &we, 0)) return "/tmp/taosd";
+  if (sizeof(buf)<=snprintf(buf, sizeof(buf), "%s", we.we_wordv[0])) return "/tmp/taosd";
+  wordfree(&we);
+  return buf;
+}
+
 void osInit() {
   if (configDir[0] == 0) {
-    strcpy(configDir, "~/TDengine/cfg");
+    strcpy(configDir, expand_like_shell("~/TDengine/cfg"));
   }
 
   strcpy(tsVnodeDir, "");
   strcpy(tsDnodeDir, "");
   strcpy(tsMnodeDir, "");
-  strcpy(tsDataDir, "~/TDengine/data");
+
-  strcpy(tsLogDir, "~/TDengine/log");
+  strcpy(tsDataDir,   expand_like_shell("~/TDengine/data"));
-  strcpy(tsScriptDir, "~/TDengine/cfg");
+  strcpy(tsLogDir,    expand_like_shell("~/TDengine/log"));
+  strcpy(tsScriptDir, expand_like_shell("~/TDengine/cfg"));
+
   strcpy(tsOsName, "Darwin");
 }
 

src/os/src/darwin/darwinFile.c

@@ -17,71 +17,49 @@
 #include "os.h"
 #include "tulog.h"
 
-#define _SEND_FILE_STEP_ 1000
-
 int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
-  fseek(in_file, (int32_t)(*offset), 0);
+  int r = 0;
-  int     writeLen = 0;
+  if (offset) {
-  uint8_t buffer[_SEND_FILE_STEP_] = {0};
+    r = fseek(in_file, *offset, SEEK_SET);
-
+    if (r==-1) return -1;
-  for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
-    size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
-    if (rlen <= 0) {
-      return writeLen;
-    } else if (rlen < _SEND_FILE_STEP_) {
-      fwrite(buffer, 1, rlen, out_file);
-      return (int)(writeLen + rlen);
-    } else {
-      fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
-      writeLen += _SEND_FILE_STEP_;
-    }
   }
-
+  off_t len = count;
-  int remain = count - writeLen;
+  while (len>0) {
-  if (remain > 0) {
+    char buf[1024*16];
-    size_t rlen = fread(buffer, 1, remain, in_file);
+    off_t n = sizeof(buf);
-    if (rlen <= 0) {
+    if (len<n) n = len;
-      return writeLen;
+    size_t m = fread(buf, 1, n, in_file);
-    } else {
+    if (m<n) {
-      fwrite(buffer, 1, remain, out_file);
+      int e = ferror(in_file);
-      writeLen += remain;
+      if (e) return -1;
     }
+    if (m==0) break;
+    if (m!=fwrite(buf, 1, m, out_file)) {
+      return -1;
+    }
+    len -= m;
   }
-
+  return count - len;
-  return writeLen;
 }
 
 int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) {
-  lseek(sfd, (int32_t)(*offset), 0);
+  int r = 0;
-  int64_t writeLen = 0;
+  if (offset) {
-  uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
+    r = lseek(sfd, *offset, SEEK_SET);
-  
+    if (r==-1) return -1;
-  for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
-    int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_);
-    if (rlen <= 0) {
-      return writeLen;
-    }
-    else if (rlen < _SEND_FILE_STEP_) {
-      taosWriteSocket(dfd, buffer, rlen);
-      return (int64_t)(writeLen + rlen);
-    }
-    else {
-      taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_);
-      writeLen += _SEND_FILE_STEP_;
-    }
   }
-
+  off_t len = count;
-  int64_t remain = count - writeLen;
+  while (len>0) {
-  if (remain > 0) {
+    char buf[1024*16];
-    int32_t rlen = read(sfd, buffer, (int32_t)remain);
+    off_t n = sizeof(buf);
-    if (rlen <= 0) {
+    if (len<n) n = len;
-      return writeLen;
+    size_t m = read(sfd, buf, n);
-    }
+    if (m==-1) return -1;
-    else {
+    if (m==0) break;
-      taosWriteSocket(sfd, buffer, (int32_t)remain);
+    size_t l = write(dfd, buf, m);
-      writeLen += remain;
+    if (l==-1) return -1;
-    }
+    len -= l;
   }
-
+  return count - len;
-  return writeLen;
 }
+

src/os/src/darwin/darwinSemphone.c

@@ -21,6 +21,8 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 
+#include <libproc.h>
+
 // #define SEM_USE_PTHREAD
 // #define SEM_USE_POSIX
 #define SEM_USE_SEM
@@ -279,3 +281,41 @@ int tsem_destroy(tsem_t *sem) {
   return 0;
 }
 
+bool taosCheckPthreadValid(pthread_t thread) {
+  uint64_t id = 0;
+  int r = pthread_threadid_np(thread, &id);
+  return r ? false : true;
+}
+
+int64_t taosGetSelfPthreadId() {
+  return (int64_t)pthread_self();
+}
+
+int64_t taosGetPthreadId(pthread_t thread) {
+  return (int64_t)thread;
+}
+
+void taosResetPthread(pthread_t* thread) {
+  *thread = NULL;
+}
+
+bool taosComparePthread(pthread_t first, pthread_t second) {
+  return pthread_equal(first, second) ? true : false;
+}
+
+int32_t taosGetPId() {
+  return (int32_t)getpid();
+}
+
+int32_t taosGetCurrentAPPName(char* name, int32_t* len) {
+  char buf[PATH_MAX+1];
+  buf[0] = '\0';
+  proc_name(getpid(), buf, sizeof(buf)-1);
+  buf[PATH_MAX] = '\0';
+  size_t n = strlen(buf);
+  if (len) *len = n;
+  if (name) strcpy(name, buf);
+  return 0;
+}
+
+

src/os/src/darwin/darwinSysInfo.c

@@ -24,42 +24,134 @@
 
 
 static void taosGetSystemTimezone() {
-  // get and set default timezone
   SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone");
-  if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
+  if (cfg_timezone == NULL) return;
-    char *tz = getenv("TZ");
+  if (cfg_timezone->cfgStatus >= TAOS_CFG_CSTATUS_DEFAULT) {
-    if (tz == NULL || strlen(tz) == 0) {
+    return;
-      strcpy(tsTimezone, "not configured");
-    }
-    else {
-      strcpy(tsTimezone, tz);
-    }
-    cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
-    uInfo("timezone not configured, use default");
   }
+
+  /* load time zone string from /etc/localtime */
+  char buf[4096];
+  char *tz = NULL; {
+    int n = readlink("/etc/localtime", buf, sizeof(buf));
+    if (n<0) {
+      uError("read /etc/localtime error, reason:%s", strerror(errno));
+      return;
+    }
+    buf[n] = '\0';
+    for(int i=n-1; i>=0; --i) {
+      if (buf[i]=='/') {
+        if (tz) {
+          tz = buf + i + 1;
+          break;
+        }
+        tz = buf + i + 1;
+      }
+    }
+    if (!tz || 0==strchr(tz, '/')) {
+      uError("parsing /etc/localtime failed");
+      return;
+    }
+
+    setenv("TZ", tz, 1);
+    tzset();
+  }
+
+  /*
+   * NOTE: do not remove it.
+   * Enforce set the correct daylight saving time(DST) flag according
+   * to current time
+   */
+  time_t    tx1 = time(NULL);
+  struct tm tm1;
+  localtime_r(&tx1, &tm1);
+
+  /*
+   * format example:
+   *
+   * Asia/Shanghai   (CST, +0800)
+   * Europe/London   (BST, +0100)
+   */
+  snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)",
+           tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone/3600);
+
+  // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
+  uWarn("timezone not configured, set to system default:%s", tsTimezone);
 }
 
-static void taosGetSystemLocale() {
+/*
-  // get and set default locale
+ * originally from src/os/src/detail/osSysinfo.c
+ * POSIX format locale string:
+ * (Language Strings)_(Country/Region Strings).(code_page)
+ *
+ * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
+ *
+ * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
+ *
+ * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
+ * for libiconv that is employed to convert string in this system. This program will automatically use
+ * UTF-8 instead as the charset.
+ *
+ * In case of windows client, the locale string is not valid POSIX format, user needs to set the
+ * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
+ * CP936, CP437 for English charset.
+ *
+ */
+static void taosGetSystemLocale() {  // get and set default locale
+  char  sep = '.';
+  char *locale = NULL;
+
   SGlobalCfg *cfg_locale = taosGetConfigOption("locale");
   if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
-    char *locale = setlocale(LC_CTYPE, "chs");
+    locale = setlocale(LC_CTYPE, "");
-    if (locale != NULL) {
+    if (locale == NULL) {
+      uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
+      strcpy(tsLocale, "en_US.UTF-8");
+    } else {
       tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
-      cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
+      uWarn("locale not configured, set to system default:%s", tsLocale);
-      uInfo("locale not configured, set to default:%s", tsLocale);
     }
   }
 
+  /* if user does not specify the charset, extract it from locale */
   SGlobalCfg *cfg_charset = taosGetConfigOption("charset");
   if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
-    strcpy(tsCharset, "cp936");
+    char *str = strrchr(tsLocale, sep);
-    cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
+    if (str != NULL) {
-    uInfo("charset not configured, set to default:%s", tsCharset);
+      str++;
+
+      char *revisedCharset = taosCharsetReplace(str);
+      tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN);
+
+      free(revisedCharset);
+      uWarn("charset not configured, set to system default:%s", tsCharset);
+    } else {
+      strcpy(tsCharset, "UTF-8");
+      uWarn("can't get locale and charset from system, set it to UTF-8");
+    }
   }
 }
 
-void taosPrintOsInfo() {}
+void taosPrintOsInfo() {
+  uInfo(" os pageSize:            %" PRId64 "(KB)", tsPageSize / 1024);
+  // uInfo(" os openMax:             %" PRId64, tsOpenMax);
+  // uInfo(" os streamMax:           %" PRId64, tsStreamMax);
+  uInfo(" os numOfCores:          %d", tsNumOfCores);
+  uInfo(" os totalDisk:           %f(GB)", tsTotalDataDirGB);
+  uInfo(" os totalMemory:         %d(MB)", tsTotalMemoryMB);
+
+  struct utsname buf;
+  if (uname(&buf)) {
+    uInfo(" can't fetch os info");
+    return;
+  }
+  uInfo(" os sysname:             %s", buf.sysname);
+  uInfo(" os nodename:            %s", buf.nodename);
+  uInfo(" os release:             %s", buf.release);
+  uInfo(" os version:             %s", buf.version);
+  uInfo(" os machine:             %s", buf.machine);
+  uInfo("==================================");
+}
 
 void taosKillSystem() {
   uError("function taosKillSystem, exit!");
@@ -67,6 +159,22 @@ void taosKillSystem() {
 }
 
 void taosGetSystemInfo() {
+  // taosGetProcInfos();
+
+  tsNumOfCores        = sysconf(_SC_NPROCESSORS_ONLN);
+  long physical_pages = sysconf(_SC_PHYS_PAGES);
+  long page_size      = sysconf(_SC_PAGESIZE);
+  tsTotalMemoryMB     = physical_pages * page_size / (1024 * 1024);
+  tsPageSize          = page_size;
+
+  // float tmp1, tmp2;
+  // taosGetSysMemory(&tmp1);
+  // taosGetProcMemory(&tmp2);
+  // taosGetDisk();
+  // taosGetBandSpeed(&tmp1);
+  // taosGetCpuUsage(&tmp1, &tmp2);
+  // taosGetProcIO(&tmp1, &tmp2);
+
   taosGetSystemTimezone();
   taosGetSystemLocale();
 }
@@ -121,7 +229,6 @@ int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
 char cmdline[1024];
 
 char *taosGetCmdlineByPID(int pid) {
-
   errno = 0;
 
   if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) {
@@ -136,6 +243,7 @@ bool taosGetSystemUid(char *uid) {
   uuid_t uuid = {0};
   uuid_generate(uuid);
   // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null
-  uuid_unparse(uuid, uid);
+  uuid_unparse_lower(uuid, uid);
   return true;
 }
+

src/plugins/http/src/httpSystem.c

@@ -92,6 +92,11 @@ void httpStopSystem() {
   tsHttpServer.stop = 1;
 #ifdef WINDOWS
   closesocket(tsHttpServer.fd);
+#elif __APPLE__
+  if (tsHttpServer.fd!=-1) {
+    close(tsHttpServer.fd);
+    tsHttpServer.fd = -1;
+  }
 #else
   shutdown(tsHttpServer.fd, SHUT_RD);
 #endif

src/rpc/src/rpcTcp.c

@@ -197,6 +197,11 @@ void taosStopTcpServer(void *handle) {
   if (pServerObj->fd >= 0) {
 #ifdef WINDOWS
     closesocket(pServerObj->fd);
+#elif defined(__APPLE__)
+    if (pServerObj->fd!=-1) {
+      close(pServerObj->fd);
+      pServerObj->fd = -1;
+    }
 #else
     shutdown(pServerObj->fd, SHUT_RD);
 #endif

src/sync/src/syncTcp.c

@@ -103,6 +103,11 @@ void syncCloseTcpThreadPool(void *param) {
 
 #ifdef WINDOWS
   closesocket(pPool->acceptFd);
+#elif defined(__APPLE__)
+  if (pPool->acceptFd!=-1) {
+    close(pPool->acceptFd);
+    pPool->acceptFd = -1;
+  }
 #else
   shutdown(pPool->acceptFd, SHUT_RD);
 #endif

src/util/src/tsocket.c

@@ -32,14 +32,27 @@ int32_t taosGetFqdn(char *fqdn) {
 
   struct addrinfo  hints = {0};
   struct addrinfo *result = NULL;
+#ifdef __APPLE__
+  // on macosx, hostname -f has the form of xxx.local
+  // which will block getaddrinfo for a few seconds if AI_CANONNAME is set
+  // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return
+  // immediately
+  hints.ai_family = AF_INET;
+#else // __APPLE__
   hints.ai_flags = AI_CANONNAME;
+#endif // __APPLE__
   int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
   if (!result) {
     uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
     return -1;
   }
 
+#ifdef __APPLE__
+  // refer to comments above
+  strcpy(fqdn, hostname);
+#else // __APPLE__
   strcpy(fqdn, result->ai_canonname);
+#endif // __APPLE__
   freeaddrinfo(result);
   return 0;
 }

tests/examples/c/epoll.c

@@ -38,6 +38,8 @@
 #include <string.h>
 #include <arpa/inet.h>
 #include <libgen.h>
+#include <locale.h>
+#include <netdb.h>
 
 #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__)
 #define A(statement, fmt, ...) do {                                     \
@@ -56,6 +58,8 @@
           ##__VA_ARGS__);                                               \
 } while (0)
 
+#include "os.h"
+
 typedef struct ep_s            ep_t;
 struct ep_s {
   int                    ep;
@@ -214,7 +218,6 @@ static void* routine(void* arg) {
         A(0==pthread_mutex_lock(&ep->lock), "");
         ep->wakenup = 0;
         A(0==pthread_mutex_unlock(&ep->lock), "");
-        D("........");
         continue;
       }
       A(ev->data.ptr, "internal logic error");