Merge pull request #3677 from taosdata/feature/ad

Feature/ad

deps/zlib-1.2.11/CMakeLists.txt

@@ -1,8 +1,11 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF (TD_LINUX)
+IF (TD_WINDOWS)
-  INCLUDE_DIRECTORIES(inc)
+  SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /WX-")
-  AUX_SOURCE_DIRECTORY(src SRC)
+  SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /WX-")
-  ADD_LIBRARY(z ${SRC})
+ENDIF()
-ENDIF ()
+
+INCLUDE_DIRECTORIES(inc)
+AUX_SOURCE_DIRECTORY(src SRC)
+ADD_LIBRARY(z ${SRC})

src/common/src/tglobal.c

@@ -1014,7 +1014,7 @@ static void doInitGlobalConfig(void) {
   cfg.ptr = &tsLogKeepDays;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
-  cfg.minValue = 0;
+  cfg.minValue = -365000;
   cfg.maxValue = 365000;
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;

src/os/inc/osDir.h

@@ -25,6 +25,7 @@ void taosRemoveDir(char *rootDir);
 int  taosMkDir(const char *pathname, mode_t mode); 
 void taosRename(char* oldName, char *newName);
 void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays);
+int32_t taosCompressFile(char *srcFileName, char *destFileName);
 
 #ifdef __cplusplus
 }

src/os/src/detail/CMakeLists.txt

@@ -2,6 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(.)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
 AUX_SOURCE_DIRECTORY(. SRC)
 SET_SOURCE_FILES_PROPERTIES(osSysinfo.c PROPERTIES COMPILE_FLAGS -w)
 SET_SOURCE_FILES_PROPERTIES(osCoredump.c PROPERTIES COMPILE_FLAGS -w)

src/os/src/detail/osDir.c

@@ -17,6 +17,9 @@
 #include "os.h"
 #include "tglobal.h"
 #include "tulog.h"
+#include "zlib.h"
+
+#define COMPRESS_STEP_SIZE 163840
 
 void taosRemoveDir(char *rootDir) {
   DIR *dir = opendir(rootDir);
@@ -73,11 +76,11 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) {
     if (de->d_type & DT_DIR) {
       continue;
     } else {
-      // struct stat fState;
-      // if (stat(fname, &fState) < 0) {
-      //   continue;
-      // }
       int32_t len = (int32_t)strlen(filename);
+      if (len > 3 && strcmp(filename + len - 3, ".gz") == 0) {
+        len -= 3;
+      }
+
       int64_t fileSec = 0;
       for (int i = len - 1; i >= 0; i--) {
         if (filename[i] == '.') {
@@ -100,3 +103,45 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) {
   closedir(dir);
   rmdir(rootDir);
 }
+
+int32_t taosCompressFile(char *srcFileName, char *destFileName) {
+  int32_t ret = 0;
+  int32_t len = 0;
+  char *  data = malloc(COMPRESS_STEP_SIZE);
+  FILE *  srcFp = NULL;
+  gzFile  dstFp = NULL;
+
+  srcFp = fopen(srcFileName, "r");
+  if (srcFp == NULL) {
+    ret = -1;
+    goto cmp_end;
+  }
+
+  int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
+  if (fd < 0) {
+    ret = -2;
+    goto cmp_end;
+  }
+
+  dstFp = gzdopen(fd, "wb6f");
+  if (dstFp == NULL) {
+    ret = -3;
+    goto cmp_end;
+  }
+
+  while (!feof(srcFp)) {
+    len = (int32_t)fread(data, 1, COMPRESS_STEP_SIZE, srcFp);
+    gzwrite(dstFp, data, len);
+  }
+
+cmp_end:
+  if (srcFp) {
+    fclose(srcFp);
+  }
+  if (dstFp) {
+    gzclose(dstFp);
+  }
+  free(data);
+
+  return ret;
+}

src/rpc/src/rpcMain.c

@@ -323,14 +323,14 @@ void *rpcMallocCont(int contLen) {
     tError("failed to malloc msg, size:%d", size);
     return NULL;
   } else {
-    tDebug("malloc mem: %p", start);
+    tDebug("malloc msg: %p", start);
   }
 
   return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
 }
 
 void rpcFreeCont(void *cont) {
-  if ( cont ) {
+  if (cont) {
     char *temp = ((char *)cont) - sizeof(SRpcHead) - sizeof(SRpcReqContext);
     free(temp);
     tDebug("free mem: %p", temp);
@@ -553,7 +553,7 @@ static void rpcFreeMsg(void *msg) {
   if ( msg ) {
     char *temp = (char *)msg - sizeof(SRpcReqContext);
     free(temp);
-    tDebug("free mem: %p", temp);
+    tDebug("free msg: %p", temp);
   }
 }
 

src/rpc/src/rpcTcp.c

@@ -421,7 +421,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
 
   msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
   buffer = malloc(msgLen + tsRpcOverhead);
-  if ( NULL == buffer) {
+  if (NULL == buffer) {
     tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
     return -1;
   } else {

src/util/CMakeLists.txt

@@ -3,7 +3,7 @@ PROJECT(TDengine)
 
 AUX_SOURCE_DIRECTORY(src SRC)
 ADD_LIBRARY(tutil ${SRC})
-TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4)
+TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z)
   
 IF (TD_LINUX)
   TARGET_LINK_LIBRARIES(tutil m rt)

src/util/src/tlog.c

@@ -139,14 +139,22 @@ static void taosUnLockFile(int32_t fd) {
 }
 
 static void taosKeepOldLog(char *oldName) {
-  if (tsLogKeepDays <= 0) return;
+  if (tsLogKeepDays == 0) return;
 
   int64_t fileSec = taosGetTimestampSec();
   char    fileName[LOG_FILE_NAME_LEN + 20];
   snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec);
 
   taosRename(oldName, fileName);
-  taosRemoveOldLogFiles(tsLogDir, tsLogKeepDays);
+  if (tsLogKeepDays < 0) {
+    char compressFileName[LOG_FILE_NAME_LEN + 20];
+    snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec);
+    if (taosCompressFile(fileName, compressFileName) == 0) {
+      (void)remove(fileName);
+    }
+  }
+
+  taosRemoveOldLogFiles(tsLogDir, ABS(tsLogKeepDays));
 }
 
 static void *taosThreadToOpenNewFile(void *param) {

src/vnode/src/vnodeWrite.c

@@ -130,8 +130,15 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
   int code = TSDB_CODE_SUCCESS;
 
   STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont);
-  if (pCfg == NULL) return terrno;
+  if (pCfg == NULL) {
-  if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) code = terrno;
+    ASSERT(terrno != 0);
+    return terrno;
+  }
+
+  if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) {
+    code = terrno;
+    ASSERT(code != 0);
+  }
 
   tsdbClearTableCfg(pCfg);
   return code;

tests/test/c/CMakeLists.txt

@@ -22,8 +22,8 @@ IF (TD_LINUX)
   #add_executable(importOneRow importOneRow.c)
   #target_link_libraries(importOneRow taos_static pthread)
 
-  add_executable(importPerTable importPerTable.c)
+  #add_executable(importPerTable importPerTable.c)
-  target_link_libraries(importPerTable taos_static pthread)
+  #target_link_libraries(importPerTable taos_static pthread)
 
   #add_executable(hashPerformance hashPerformance.c)
   #target_link_libraries(hashPerformance taos_static tutil common pthread)

tests/test/c/createNormalTable.c

@@ -50,7 +50,9 @@ void  createDbAndSTable();
 int main(int argc, char *argv[]) {
   shellParseArgument(argc, argv);
   taos_init();
-  createDbAndSTable();
+  if (replica != 0) {
+    createDbAndSTable();
+  }
 
   pPrint("%d threads are spawned to create table", numOfThreads);
   
@@ -134,14 +136,31 @@ void *threadFunc(void *param) {
 
   int64_t startMs = taosGetTimestampMs();
 
-  for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) {
+  if (replica != 0) {
-    sprintf(qstr, "create table %s%d (ts timestamp, i int)", stableName, t);
+    for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) {
-    TAOS_RES *pSql = taos_query(con, qstr);
+      sprintf(qstr, "create table %s%d (ts timestamp, i int)", stableName, t);
-    code = taos_errno(pSql);
+      TAOS_RES *pSql = taos_query(con, qstr);
-    if (code != 0) {
+      code = taos_errno(pSql);
-      pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code));
+      if (code != 0) {
+        pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code));
+      }
+      taos_free_result(pSql);
+    }
+  } else {
+    for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) {
+      sprintf(qstr, "insert into %s%d values(now, 1)", stableName, t);
+      TAOS_RES *pSql = taos_query(con, qstr);
+      code = taos_errno(pSql);
+      if (code != 0) {
+        if (code != TSDB_CODE_MND_INVALID_TABLE_NAME) {
+          pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code));
+        }
+        if (code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
+          exit(0);
+        }
+      }
+      taos_free_result(pSql);
     }
-    taos_free_result(pSql);
   }
 
   float createTableSpeed = 0;