Merge branch 'develop' into feature/query

deps/CMakeLists.txt

@@ -9,6 +9,7 @@ ADD_SUBDIRECTORY(lz4)
 ADD_SUBDIRECTORY(cJson)
 ADD_SUBDIRECTORY(wepoll)
 ADD_SUBDIRECTORY(MsvcLibX)
+ADD_SUBDIRECTORY(rmonotonic)
 
 IF (TD_LINUX AND TD_MQTT)
   ADD_SUBDIRECTORY(MQTT-C)

deps/MsvcLibX/include/msvcTime.h

@@ -38,6 +38,7 @@
 typedef int clockid_t;
 /* Supported values for clockid_t */
 #define CLOCK_REALTIME 0
+#define CLOCK_MONOTONIC 1
 
 int clock_gettime(clockid_t clock_id, struct timespec *tp);
 

deps/MsvcLibX/src/clock_gettime.c

@@ -34,15 +34,56 @@
 #include "msvcTime.h"
 #include "sys/msvcStat.h" /* For MsvcLibX's Filetime2Timespec */
 
-int clock_gettime(clockid_t clock_id, struct timespec *pTS) {
-  FILETIME ft;
-  if (clock_id != CLOCK_REALTIME) {
-    errno = EINVAL;
-    return -1;
+#define MS_PER_SEC      1000ULL     // MS = milliseconds
+#define US_PER_MS       1000ULL     // US = microseconds
+#define HNS_PER_US      10ULL       // HNS = hundred-nanoseconds (e.g., 1 hns = 100 ns)
+#define NS_PER_US       1000ULL
+
+#define HNS_PER_SEC     (MS_PER_SEC * US_PER_MS * HNS_PER_US)
+#define NS_PER_HNS      (100ULL)    // NS = nanoseconds
+#define NS_PER_SEC      (MS_PER_SEC * US_PER_MS * NS_PER_US)
+
+int clock_gettime_monotonic(struct timespec *tv) {
+  static LARGE_INTEGER ticksPerSec;
+  LARGE_INTEGER ticks;
+  double seconds;
+
+  if (!ticksPerSec.QuadPart) {
+    QueryPerformanceFrequency(&ticksPerSec);
+    if (!ticksPerSec.QuadPart) {
+      errno = ENOTSUP;
+      return -1;
+    }
   }
-  GetSystemTimeAsFileTime(&ft);
-  Filetime2Timespec(&ft, pTS);
+
+  QueryPerformanceCounter(&ticks);
+
+  seconds = (double) ticks.QuadPart / (double) ticksPerSec.QuadPart;
+  tv->tv_sec = (time_t)seconds;
+  tv->tv_nsec = (long)((ULONGLONG)(seconds * NS_PER_SEC) % NS_PER_SEC);
+
   return 0;
 }
 
+int clock_gettime_realtime(struct timespec *pTS) {
+  FILETIME ft;
+
+  GetSystemTimeAsFileTime(&ft);
+  Filetime2Timespec(&ft, pTS);
+
+  return 0;
+}
+
+int clock_gettime(clockid_t clock_id, struct timespec *pTS) {
+  if (clock_id == CLOCK_MONOTONIC) {
+    return clock_gettime_monotonic(pTS);
+  } else if (clock_id == CLOCK_REALTIME) {
+    return clock_gettime_realtime(pTS);
+  }
+
+  errno = ENOTSUP;
+
+  return -1;
+}
+
 #endif /* defined(_WIN32) */

deps/rmonotonic/CMakeLists.txt

@@ -0,0 +1,11 @@
+AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST)
+
+add_definitions(-DUSE_PROCESSOR_CLOCK)
+
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../MsvcLibX/include)
+
+ADD_LIBRARY(rmonotonic ${SOURCE_LIST})
+TARGET_INCLUDE_DIRECTORIES(rmonotonic PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc)
+IF (TD_WINDOWS)
+  TARGET_LINK_LIBRARIES(rmonotonic MsvcLibXw)
+ENDIF ()

deps/rmonotonic/inc/monotonic.h

@@ -0,0 +1,56 @@
+#ifndef __MONOTONIC_H
+#define __MONOTONIC_H
+/* The monotonic clock is an always increasing clock source.  It is unrelated to
+ * the actual time of day and should only be used for relative timings.  The
+ * monotonic clock is also not guaranteed to be chronologically precise; there
+ * may be slight skew/shift from a precise clock.
+ *
+ * Depending on system architecture, the monotonic time may be able to be
+ * retrieved much faster than a normal clock source by using an instruction
+ * counter on the CPU.  On x86 architectures (for example), the RDTSC
+ * instruction is a very fast clock source for this purpose.
+ */
+
+//#include "fmacros.h"
+#include <stdint.h>
+//#include <unistd.h>
+
+#if defined(_WIN32) || defined(_WIN64)
+  #define inline
+#endif
+
+/* A counter in micro-seconds.  The 'monotime' type is provided for variables
+ * holding a monotonic time.  This will help distinguish & document that the
+ * variable is associated with the monotonic clock and should not be confused
+ * with other types of time.*/
+typedef uint64_t monotime;
+
+/* Retrieve counter of micro-seconds relative to an arbitrary point in time.  */
+extern monotime (*getMonotonicUs)(void);
+
+
+/* Call once at startup to initialize the monotonic clock.  Though this only
+ * needs to be called once, it may be called additional times without impact.
+ * Returns a printable string indicating the type of clock initialized.
+ * (The returned string is static and doesn't need to be freed.)  */
+const char * monotonicInit();
+
+
+/* Functions to measure elapsed time.  Example:
+ *     monotime myTimer;
+ *     elapsedStart(&myTimer);
+ *     while (elapsedMs(myTimer) < 10) {} // loops for 10ms
+ */
+static inline void elapsedStart(monotime *start_time) {
+    *start_time = getMonotonicUs();
+}
+
+static inline uint64_t elapsedUs(monotime start_time) {
+    return getMonotonicUs() - start_time;
+}
+
+static inline uint64_t elapsedMs(monotime start_time) {
+    return elapsedUs(start_time) / 1000;
+}
+
+#endif

deps/rmonotonic/src/monotonic.c

@@ -0,0 +1,173 @@
+#include "monotonic.h"
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+
+#undef NDEBUG
+#include <assert.h>
+
+#if defined(_WIN32) || defined(_WIN64)
+#include "msvcTime.h"
+#include "msvcStdio.h"
+#endif
+
+/* The function pointer for clock retrieval.  */
+monotime (*getMonotonicUs)(void) = NULL;
+
+static char monotonic_info_string[32];
+
+
+/* Using the processor clock (aka TSC on x86) can provide improved performance
+ * throughout Redis wherever the monotonic clock is used.  The processor clock
+ * is significantly faster than calling 'clock_getting' (POSIX).  While this is
+ * generally safe on modern systems, this link provides additional information
+ * about use of the x86 TSC: http://oliveryang.net/2015/09/pitfalls-of-TSC-usage
+ *
+ * To use the processor clock, either uncomment this line, or build with
+ *   CFLAGS="-DUSE_PROCESSOR_CLOCK"
+#define USE_PROCESSOR_CLOCK
+ */
+
+
+#if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__)
+#include <regex.h>
+#include <x86intrin.h>
+
+static long mono_ticksPerMicrosecond = 0;
+
+static monotime getMonotonicUs_x86() {
+    return __rdtsc() / mono_ticksPerMicrosecond;
+}
+
+static void monotonicInit_x86linux() {
+    const int bufflen = 256;
+    char buf[bufflen];
+    regex_t cpuGhzRegex, constTscRegex;
+    const size_t nmatch = 2;
+    regmatch_t pmatch[nmatch];
+    int constantTsc = 0;
+    int rc;
+
+    /* Determine the number of TSC ticks in a micro-second.  This is
+     * a constant value matching the standard speed of the processor.
+     * On modern processors, this speed remains constant even though
+     * the actual clock speed varies dynamically for each core.  */
+    rc = regcomp(&cpuGhzRegex, "^model name\\s+:.*@ ([0-9.]+)GHz", REG_EXTENDED);
+    assert(rc == 0);
+
+    /* Also check that the constant_tsc flag is present.  (It should be
+     * unless this is a really old CPU.  */
+    rc = regcomp(&constTscRegex, "^flags\\s+:.* constant_tsc", REG_EXTENDED);
+    assert(rc == 0);
+
+    FILE *cpuinfo = fopen("/proc/cpuinfo", "r");
+    if (cpuinfo != NULL) {
+        while (fgets(buf, bufflen, cpuinfo) != NULL) {
+            if (regexec(&cpuGhzRegex, buf, nmatch, pmatch, 0) == 0) {
+                buf[pmatch[1].rm_eo] = '\0';
+                double ghz = atof(&buf[pmatch[1].rm_so]);
+                mono_ticksPerMicrosecond = (long)(ghz * 1000);
+                break;
+            }
+        }
+        while (fgets(buf, bufflen, cpuinfo) != NULL) {
+            if (regexec(&constTscRegex, buf, nmatch, pmatch, 0) == 0) {
+                constantTsc = 1;
+                break;
+            }
+        }
+
+        fclose(cpuinfo);
+    }
+    regfree(&cpuGhzRegex);
+    regfree(&constTscRegex);
+
+    if (mono_ticksPerMicrosecond == 0) {
+        fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate");
+        return;
+    }
+    if (!constantTsc) {
+        fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present");
+        return;
+    }
+
+    snprintf(monotonic_info_string, sizeof(monotonic_info_string),
+            "X86 TSC @ %ld ticks/us", mono_ticksPerMicrosecond);
+    getMonotonicUs = getMonotonicUs_x86;
+}
+#endif
+
+
+#if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__)
+static long mono_ticksPerMicrosecond = 0;
+
+/* Read the clock value.  */
+static inline uint64_t __cntvct() {
+    uint64_t virtual_timer_value;
+    __asm__ volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value));
+    return virtual_timer_value;
+}
+
+/* Read the Count-timer Frequency.  */
+static inline uint32_t cntfrq_hz() {
+    uint64_t virtual_freq_value;
+    __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(virtual_freq_value));
+    return (uint32_t)virtual_freq_value;    /* top 32 bits are reserved */
+}
+
+static monotime getMonotonicUs_aarch64() {
+    return __cntvct() / mono_ticksPerMicrosecond;
+}
+
+static void monotonicInit_aarch64() {
+    mono_ticksPerMicrosecond = (long)cntfrq_hz() / 1000L / 1000L;
+    if (mono_ticksPerMicrosecond == 0) {
+        fprintf(stderr, "monotonic: aarch64, unable to determine clock rate");
+        return;
+    }
+
+    snprintf(monotonic_info_string, sizeof(monotonic_info_string),
+            "ARM CNTVCT @ %ld ticks/us", mono_ticksPerMicrosecond);
+    getMonotonicUs = getMonotonicUs_aarch64;
+}
+#endif
+
+static monotime getMonotonicUs_posix(void) {
+    /* clock_gettime() is specified in POSIX.1b (1993).  Even so, some systems
+     * did not support this until much later.  CLOCK_MONOTONIC is technically
+     * optional and may not be supported - but it appears to be universal.
+     * If this is not supported, provide a system-specific alternate version.  */
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000;
+}
+
+static void monotonicInit_posix() {
+    /* Ensure that CLOCK_MONOTONIC is supported.  This should be supported
+     * on any reasonably current OS.  If the assertion below fails, provide
+     * an appropriate alternate implementation.  */
+    struct timespec ts;
+    int rc = clock_gettime(CLOCK_MONOTONIC, &ts);
+    assert(rc == 0);
+
+    snprintf(monotonic_info_string, sizeof(monotonic_info_string),
+            "POSIX clock_gettime");
+    getMonotonicUs = getMonotonicUs_posix;
+}
+
+
+
+const char * monotonicInit() {
+    #if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__)
+    if (getMonotonicUs == NULL) monotonicInit_x86linux();
+    #endif
+
+    #if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__)
+    if (getMonotonicUs == NULL) monotonicInit_aarch64();
+    #endif
+
+    if (getMonotonicUs == NULL) monotonicInit_posix();
+
+    return monotonic_info_string;
+}

packaging/tools/install.sh

@@ -168,6 +168,7 @@ function install_main_path() {
     if [ "$verMode" == "cluster" ]; then
         ${csudo} mkdir -p ${nginx_dir}
     fi
+    ${csudo} cp ${script_dir}/email ${install_main_dir}/ ||: 
 }
 
 function install_bin() {

src/client/src/tscSystem.c

@@ -65,7 +65,7 @@ void tscReleaseRpc(void *param)  {
     return;
   }
   pthread_mutex_lock(&rpcObjMutex);
-  taosCacheRelease(tscRpcCache, (void *)&param, false); 
+  taosCacheRelease(tscRpcCache, (void *)&param, true); 
   pthread_mutex_unlock(&rpcObjMutex);
 } 
 
@@ -216,7 +216,6 @@ void taos_cleanup(void) {
   taosCloseRef(id);
 
   taosCleanupKeywordsTable();
-  taosCloseLog();
 
   p = tscRpcCache; 
   tscRpcCache = NULL;
@@ -226,7 +225,10 @@ void taos_cleanup(void) {
     pthread_mutex_destroy(&rpcObjMutex);
   }
 
-  if (tscEmbedded == 0) rpcCleanup();
+  if (tscEmbedded == 0) {
+    rpcCleanup();
+    taosCloseLog();
+  };
 
   p = tscTmr;
   tscTmr = NULL;

src/os/src/darwin/darwinSysInfo.c

@@ -18,6 +18,9 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
+#include <errno.h>
+#include <libproc.h>
+
 
 static void taosGetSystemTimezone() {
   // get and set default timezone
@@ -103,8 +106,18 @@ int taosSystem(const char *cmd) {
 
 void taosSetCoreDump() {}
 
+char cmdline[1024];
+
 char *taosGetCmdlineByPID(int pid) {
-  return "[not supported yet]";
+
+  errno = 0;
+
+  if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) {
+    fprintf(stderr, "PID is %d, %s", pid, strerror(errno));
+    return strerror(errno);
+  }
+
+  return cmdline;
 }
 
 bool taosGetSystemUid(char *uid) {

src/rpc/src/rpcTcp.c

@@ -364,10 +364,11 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
 }
 
 void taosStopTcpClient(void *chandle) {
-  SThreadObj *pThreadObj = chandle;
-  if (pThreadObj == NULL) return;
+  SClientObj *pClientObj = chandle;
 
-  tDebug ("%s TCP client is stopped", pThreadObj->label);
+  if (pClientObj == NULL) return;
+
+  tDebug ("%s TCP client is stopped", pClientObj->label);
 }
 
 void taosCleanUpTcpClient(void *chandle) {

src/util/CMakeLists.txt

@@ -3,9 +3,10 @@ PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/rmonotonic/inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 ADD_LIBRARY(tutil ${SRC})
-TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z)
+TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z rmonotonic)
 
 IF (TD_LINUX)
   TARGET_LINK_LIBRARIES(tutil m rt)

src/util/src/ttimer.c

@@ -18,6 +18,7 @@
 #include "tsched.h"
 #include "ttimer.h"
 #include "tutil.h"
+#include "monotonic.h"
 
 extern int8_t tscEmbedded;
 
@@ -186,6 +187,10 @@ static void removeTimer(uintptr_t id) {
   unlockTimerList(list);
 }
 
+static int64_t getMonotonicMs(void) {
+  return (int64_t) getMonotonicUs() / 1000;
+}
+
 static void addToWheel(tmr_obj_t* timer, uint32_t delay) {
   timerAddRef(timer);
   // select a wheel for the timer, we are not an accurate timer,
@@ -201,7 +206,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) {
 
   time_wheel_t* wheel = wheels + timer->wheel;
   timer->prev = NULL;
-  timer->expireAt = taosGetTimestampMs() + delay;
+  timer->expireAt = getMonotonicMs() + delay;
 
   pthread_mutex_lock(&wheel->mutex);
 
@@ -334,7 +339,7 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle
 }
 
 static void taosTimerLoopFunc(int signo) {
-  int64_t now = taosGetTimestampMs();
+  int64_t now = getMonotonicMs();
 
   for (int i = 0; i < tListLen(wheels); i++) {
     // `expried` is a temporary expire list.
@@ -501,7 +506,7 @@ static void taosTmrModuleInit(void) {
 
   pthread_mutex_init(&tmrCtrlMutex, NULL);
 
-  int64_t now = taosGetTimestampMs();
+  int64_t now = getMonotonicMs();
   for (int i = 0; i < tListLen(wheels); i++) {
     time_wheel_t* wheel = wheels + i;
     if (pthread_mutex_init(&wheel->mutex, NULL) != 0) {
@@ -532,6 +537,8 @@ static void taosTmrModuleInit(void) {
 }
 
 void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* label) {
+  tmrInfo("ttimer monotonic clock source:%s", monotonicInit());
+
   pthread_once(&tmrModuleInit, taosTmrModuleInit);
 
   pthread_mutex_lock(&tmrCtrlMutex);

tests/pytest/fulltest.sh

@@ -20,6 +20,7 @@ python3 insert/retentionpolicy.py
 python3 ./test.py -f insert/alterTableAndInsert.py
 python3 ./test.py -f insert/insertIntoTwoTables.py
 #python3 ./test.py -f insert/before_1970.py
+python3 ./test.py -f insert/metadataUpdate.py
 python3 bug2265.py
 
 #table

tests/pytest/insert/metadataUpdate.py

@@ -0,0 +1,67 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+from util.dnodes import tdDnodes
+from multiprocessing import  Process
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.ts = 1538548685000
+
+    def updateMetadata(self):
+        self.host = "127.0.0.1"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = tdDnodes.getSimCfgPath()
+
+        self.conn = taos.connect(host = self.host, user = self.user, password = self.password, config = self.config)
+        self.cursor = self.conn.cursor()    
+        self.cursor.execute("alter table db.tb add column col2 int")
+        print("alter table done")
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+        tdSql.execute("create table if not exists tb (ts timestamp, col1 int)")
+        tdSql.execute("insert into tb values(%d, 1)" % self.ts)
+
+        print("==============step2")
+        tdSql.query("select * from tb")
+        tdSql.checkRows(1)
+
+        p = Process(target=self.updateMetadata, args=())
+        p.start()        
+        p.join()
+        p.terminate()
+        
+        tdSql.execute("insert into tb values(%d, 1, 2)" % (self.ts + 1))
+
+        print("==============step2")
+        tdSql.query("select * from tb")
+        tdSql.checkRows(2)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())