update telemetry formater to cjson

include/os/osSysinfo.h

@@ -33,21 +33,26 @@ typedef struct {
   SDiskSize size;
 } SDiskSpace;
 
-int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
-int32_t taosGetCpuCores();
 void    taosGetSystemInfo();
+bool    taosGetEmail(char *email, int32_t maxLen);
+bool    taosGetOsReleaseName(char *releaseName, int32_t maxLen);
+bool    taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores);
+int32_t taosGetCpuCores();
+bool    taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
+bool    taosGetTotalSysMemoryKB(uint64_t *kb);
+bool    taosGetProcMemory(float *memoryUsedMB);  //
+bool    taosGetSysMemory(float *memoryUsedMB);   //
+void    taosGetDisk();
+int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
 bool    taosReadProcIO(int64_t *rchars, int64_t *wchars);
 bool    taosGetProcIO(float *readKB, float *writeKB);
 bool    taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
 bool    taosGetBandSpeed(float *bandSpeedKb);
-void    taosGetDisk();
+
-bool    taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
+int32_t taosSystem(const char *cmd);
-bool    taosGetProcMemory(float *memoryUsedMB);
-bool    taosGetSysMemory(float *memoryUsedMB);
-int     taosSystem(const char *cmd);
 void    taosKillSystem();
 int32_t taosGetSystemUUID(char *uid, int32_t uidlen);
-char *  taosGetCmdlineByPID(int pid);
+char   *taosGetCmdlineByPID(int32_t pid);
 void    taosSetCoreDump(bool enable);
 
 typedef struct {

source/dnode/mnode/impl/src/mndTelem.c

@@ -18,186 +18,84 @@
 #include "mndCluster.h"
 #include "mndSync.h"
 #include "tbuffer.h"
-#include "tversion.h"
+#include "tjson.h"
 
-#define TELEMETRY_SERVER "telemetry.taosdata.com"
+#define TELEMETRY_SERVER "localhost"
 #define TELEMETRY_PORT   80
-#define REPORT_INTERVAL  86400
 
-static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); }
+static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) {
-
-static void mndCloseObject(SBufferWriter* bw) {
-  size_t len = tbufTell(bw);
-  if (tbufGetData(bw, false)[len - 1] == ',') {
-    tbufWriteCharAt(bw, len - 1, '}');
-  } else {
-    tbufWriteChar(bw, '}');
-  }
-}
-
-static void mndWriteString(SBufferWriter* bw, const char* str) {
-  tbufWriteChar(bw, '"');
-  tbufWrite(bw, str, strlen(str));
-  tbufWriteChar(bw, '"');
-}
-
-static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) {
-  mndWriteString(bw, k);
-  tbufWriteChar(bw, ':');
-  char buf[32] = {0};
-  sprintf(buf, "%" PRId64, v);
-  tbufWrite(bw, buf, strlen(buf));
-  tbufWriteChar(bw, ',');
-}
-
-static void mndAddStringField(SBufferWriter* bw, const char* k, const char* v) {
-  mndWriteString(bw, k);
-  tbufWriteChar(bw, ':');
-  mndWriteString(bw, v);
-  tbufWriteChar(bw, ',');
-}
-
-static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
-  char*   line = NULL;
-  size_t  size = 0;
-  int32_t done = 0;
-
-  // FILE* fp = fopen("/proc/cpuinfo", "r");
-  TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM);
-  if (pFile == NULL) {
-    return;
-  }
-
-  while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
-    line[size - 1] = '\0';
-    if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
-      const char* v = strchr(line, ':') + 2;
-      mndAddStringField(bw, "cpuModel", v);
-      done |= 1;
-    } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) {
-      const char* v = strchr(line, ':') + 2;
-      mndWriteString(bw, "numOfCpu");
-      tbufWriteChar(bw, ':');
-      tbufWrite(bw, v, strlen(v));
-      tbufWriteChar(bw, ',');
-      done |= 2;
-    }
-  }
-
-  if(line != NULL) free(line);
-  taosCloseFile(&pFile);
-}
-
-static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
-  char*  line = NULL;
-  size_t size = 0;
-
-  // FILE* fp = fopen("/etc/os-release", "r");
-  TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM);
-  if (pFile == NULL) {
-    return;
-  }
-
-  while ((size = taosGetLineFile(pFile, &line)) != -1) {
-    line[size - 1] = '\0';
-    if (strncmp(line, "PRETTY_NAME", 11) == 0) {
-      const char* p = strchr(line, '=') + 1;
-      if (*p == '"') {
-        p++;
-        line[size - 2] = 0;
-      }
-      mndAddStringField(bw, "os", p);
-      break;
-    }
-  }
-
-  if(line != NULL) free(line);
-  taosCloseFile(&pFile);
-}
-
-static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
-  char*  line = NULL;
-  size_t size = 0;
-
-  // FILE* fp = fopen("/proc/meminfo", "r");
-  TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM);
-  if (pFile == NULL) {
-    return;
-  }
-
-  while ((size = taosGetLineFile(pFile, &line)) != -1) {
-    line[size - 1] = '\0';
-    if (strncmp(line, "MemTotal", 8) == 0) {
-      const char* p = strchr(line, ':') + 1;
-      while (*p == ' ') p++;
-      mndAddStringField(bw, "memory", p);
-      break;
-    }
-  }
-
-  if(line != NULL) free(line);
-  taosCloseFile(&pFile);
-}
-
-static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) {
-  STelemMgmt* pMgmt = &pMnode->telemMgmt;
-  mndAddStringField(bw, "version", version);
-  mndAddStringField(bw, "buildInfo", buildinfo);
-  mndAddStringField(bw, "gitInfo", gitinfo);
-  mndAddStringField(bw, "email", pMgmt->email);
-}
-
-static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) {
   SMnodeLoad load = {0};
-  if (mndGetLoad(pMnode, &load) != 0) {
+  if (mndGetLoad(pMnode, &load) != 0) return;
-    return;
-  }
 
-  mndAddIntField(bw, "numOfDnode", load.numOfDnode);
+  tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode);
-  mndAddIntField(bw, "numOfMnode", load.numOfMnode);
+  tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode);
-  mndAddIntField(bw, "numOfVgroup", load.numOfVgroup);
+  tjsonAddDoubleToObject(pJson, "numOfVgroup", load.numOfVgroup);
-  mndAddIntField(bw, "numOfDatabase", load.numOfDatabase);
+  tjsonAddDoubleToObject(pJson, "numOfDatabase", load.numOfDatabase);
-  mndAddIntField(bw, "numOfSuperTable", load.numOfSuperTable);
+  tjsonAddDoubleToObject(pJson, "numOfSuperTable", load.numOfSuperTable);
-  mndAddIntField(bw, "numOfChildTable", load.numOfChildTable);
+  tjsonAddDoubleToObject(pJson, "numOfChildTable", load.numOfChildTable);
-  mndAddIntField(bw, "numOfColumn", load.numOfColumn);
+  tjsonAddDoubleToObject(pJson, "numOfColumn", load.numOfColumn);
-  mndAddIntField(bw, "numOfPoint", load.totalPoints);
+  tjsonAddDoubleToObject(pJson, "numOfPoint", load.totalPoints);
-  mndAddIntField(bw, "totalStorage", load.totalStorage);
+  tjsonAddDoubleToObject(pJson, "totalStorage", load.totalStorage);
-  mndAddIntField(bw, "compStorage", load.compStorage);
+  tjsonAddDoubleToObject(pJson, "compStorage", load.compStorage);
 }
 
-static void mndSendTelemetryReport(SMnode* pMnode) {
+static char* mndBuildTelemetryReport(SMnode* pMnode) {
-  STelemMgmt*   pMgmt = &pMnode->telemMgmt;
+  char        tmp[4096] = {0};
-  SBufferWriter bw = tbufInitWriter(NULL, false);
+  STelemMgmt* pMgmt = &pMnode->telemMgmt;
-  int32_t       code = -1;
+
-  char          buf[128] = {0};
+  SJson* pJson = tjsonCreateObject();
-  SOCKET        fd = 0;
+  if (pJson == NULL) return NULL;
 
   char clusterName[64] = {0};
-  if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) {
+  mndGetClusterName(pMnode, clusterName, sizeof(clusterName));
-    goto SEND_OVER;
+  tjsonAddStringToObject(pJson, "instanceId", clusterName);
+  tjsonAddDoubleToObject(pJson, "reportVersion", 1);
+
+  if (taosGetOsReleaseName(tmp, sizeof(tmp))) {
+    tjsonAddStringToObject(pJson, "os", tmp);
   }
 
-  mndBeginObject(&bw);
+  int32_t numOfCores = 0;
-  mndAddStringField(&bw, "instanceId", clusterName);
+  if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores)) {
-  mndAddIntField(&bw, "reportVersion", 1);
+    tjsonAddStringToObject(pJson, "cpuModel", tmp);
-  mndAddOsInfo(pMnode, &bw);
+    tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores);
-  mndAddCpuInfo(pMnode, &bw);
+  } else {
-  mndAddMemoryInfo(pMnode, &bw);
+    tjsonAddDoubleToObject(pJson, "numOfCpu", taosGetCpuCores());
-  mndAddVersionInfo(pMnode, &bw);
+  }
-  mndAddRuntimeInfo(pMnode, &bw);
+
-  mndCloseObject(&bw);
+  uint64_t memoryKB = 0;
+  if (taosGetTotalSysMemoryKB(&memoryKB)) {
+    snprintf(tmp, sizeof(tmp), "%" PRIu64 " kB", memoryKB);
+    tjsonAddStringToObject(pJson, "memory", tmp);
+  }
+
+  tjsonAddStringToObject(pJson, "version", version);
+  tjsonAddStringToObject(pJson, "buildInfo", buildinfo);
+  tjsonAddStringToObject(pJson, "gitInfo", gitinfo);
+  tjsonAddStringToObject(pJson, "email", pMgmt->email);
+
+  mndBuildRuntimeInfo(pMnode, pJson);
+
+  char* pCont = tjsonToString(pJson);
+  tjsonDelete(pJson);
+  return pCont;
+}
+
+static void mndSendTelemetryReport(const char* pCont) {
+  int32_t code = -1;
+  char    buf[128] = {0};
+  SOCKET  fd = 0;
+  int32_t contLen = strlen(pCont);
 
   uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER);
   if (ip == 0xffffffff) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
+    mError("failed to get telemetry server ip");
-    mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr());
     goto SEND_OVER;
   }
 
   fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0);
   if (fd < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
+    mError("failed to create telemetry socket");
-    mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr());
     goto SEND_OVER;
   }
 
@@ -208,24 +106,24 @@ static void mndSendTelemetryReport(SMnode* pMnode) {
       "Content-Type: application/json\n"
       "Content-Length: ";
   if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) {
+    mError("failed to send telemetry header");
     goto SEND_OVER;
   }
 
-  int32_t contLen = (int32_t)(tbufTell(&bw));
+  snprintf(buf, sizeof(buf), "%d\n\n", contLen);
-  sprintf(buf, "%d\n\n", contLen);
   if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) {
+    mError("failed to send telemetry contlen");
     goto SEND_OVER;
   }
 
-  const char* pCont = tbufGetData(&bw, false);
   if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) {
+    mError("failed to send telemetry content");
     goto SEND_OVER;
   }
 
   // read something to avoid nginx error 499
   if (taosReadSocket(fd, buf, 10) < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
+    mError("failed to receive telemetry response");
-    mError("failed to receive response since %s", terrstr());
     goto SEND_OVER;
   }
 
@@ -233,12 +131,10 @@ static void mndSendTelemetryReport(SMnode* pMnode) {
   code = 0;
 
 SEND_OVER:
-  tbufCloseWriter(&bw);
-  taosCloseSocket(fd);
-
   if (code != 0) {
     mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr());
   }
+  taosCloseSocket(fd);
 }
 
 static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
@@ -247,33 +143,23 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
   if (!pMgmt->enable) return 0;
 
   taosWLockLatch(&pMgmt->lock);
-  mndSendTelemetryReport(pMnode);
+  char* pCont = mndBuildTelemetryReport(pMnode);
+  if (pCont != NULL) {
+    mndSendTelemetryReport(pCont);
+    free(pCont);
+  }
   taosWUnLockLatch(&pMgmt->lock);
   return 0;
 }
 
-static void mndGetEmail(SMnode* pMnode, char* filepath) {
-  STelemMgmt* pMgmt = &pMnode->telemMgmt;
-
-  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
-  if (pFile == NULL) {
-    return;
-  }
-
-  if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
-    mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
-  }
-
-  taosCloseFile(&pFile);
-}
-
 int32_t mndInitTelem(SMnode* pMnode) {
   STelemMgmt* pMgmt = &pMnode->telemMgmt;
-  pMgmt->enable = tsEnableTelemetryReporting;
-  taosInitRWLatch(&pMgmt->lock);
-  mndGetEmail(pMnode, "/usr/local/taos/email");
 
+  taosInitRWLatch(&pMgmt->lock);
+  pMgmt->enable = tsEnableTelemetryReporting;
+  taosGetEmail(pMgmt->email, sizeof(pMgmt->email));
   mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
+
   mDebug("mnode telemetry is initialized");
   return 0;
 }

source/dnode/mnode/impl/src/mnode.c

@@ -132,7 +132,7 @@ static int32_t mndInitTimer(SMnode *pMnode) {
     return -1;
   }
 
-  if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) {
+  if (taosTmrReset(mndPullupTelem, 300, pMnode, pMnode->timer, &pMnode->telemTimer)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }

source/os/src/osSysinfo.c

@@ -888,4 +888,99 @@ SysNameInfo taosGetSysNameInfo() {
   return info;
 }
 
-#endif
+bool taosGetEmail(char *email, int32_t maxLen) {
+  const char *filepath = "/usr/local/taos/email";
+
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
+  if (pFile == NULL) return false;
+
+  if (taosReadFile(pFile, (void *)email, maxLen) < 0) {
+    taosCloseFile(&pFile);
+    return false;
+  }
+
+  taosCloseFile(&pFile);
+  return true;
+}
+
+bool taosGetOsReleaseName(char *releaseName, int32_t maxLen) {
+  char  *line = NULL;
+  size_t size = 0;
+  bool   ret = false;
+
+  TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM);
+  if (pFile == NULL) return false;
+
+  while ((size = taosGetLineFile(pFile, &line)) != -1) {
+    line[size - 1] = '\0';
+    if (strncmp(line, "PRETTY_NAME", 11) == 0) {
+      const char *p = strchr(line, '=') + 1;
+      if (*p == '"') {
+        p++;
+        line[size - 2] = 0;
+      }
+      tstrncpy(releaseName, p, maxLen);
+      ret = true;
+      break;
+    }
+  }
+
+  if (line != NULL) free(line);
+  taosCloseFile(&pFile);
+  return ret;
+}
+
+bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) {
+  char   *line = NULL;
+  size_t  size = 0;
+  int32_t done = 0;
+  bool    ret = false;
+
+  TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM);
+  if (pFile == NULL) return false;
+
+  while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
+    line[size - 1] = '\0';
+    if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
+      const char *v = strchr(line, ':') + 2;
+      tstrncpy(cpuModel, v, maxLen);
+      ret = true;
+      done |= 1;
+    } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) {
+      const char *v = strchr(line, ':') + 2;
+      *numOfCores = atoi(v);
+      done |= 2;
+    }
+  }
+
+  if (line != NULL) free(line);
+  taosCloseFile(&pFile);
+
+  return ret;
+}
+
+bool taosGetTotalSysMemoryKB(uint64_t *kb) {
+  char  *line = NULL;
+  size_t size = 0;
+  bool   ret = false;
+
+  TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM);
+  if (pFile == NULL) return false;
+
+  while ((size = taosGetLineFile(pFile, &line)) != -1) {
+    line[size - 1] = '\0';
+    if (strncmp(line, "MemTotal", 8) == 0) {
+      const char *p = strchr(line, ':') + 1;
+      while (*p == ' ') p++;
+      ret = true;
+      *kb = atoll(p);
+      break;
+    }
+  }
+
+  if (line != NULL) free(line);
+  taosCloseFile(&pFile);
+  return ret;
+}
+
+#endif

source/util/src/tjson.c

@@ -18,36 +18,73 @@
 #include "cJSON.h"
 #include "taoserror.h"
 
-SJson* tjsonCreateObject() { return cJSON_CreateObject(); }
+SJson* tjsonCreateObject() {
+  SJson* pJson = cJSON_CreateObject();
+  if (pJson == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return pJson;
+}
 
 void tjsonDelete(SJson* pJson) { cJSON_Delete((cJSON*)pJson); }
 
 int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number) {
   char tmp[40] = {0};
-  snprintf(tmp, tListLen(tmp), "%" PRId64, number);
+  snprintf(tmp, sizeof(tmp), "%" PRId64, number);
   return tjsonAddStringToObject(pJson, pName, tmp);
 }
 
 int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) {
-  return (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
+  if (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number)) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_FAILED;
+  }
+
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean) {
-  return (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
+  if (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean)) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_FAILED;
+  }
+
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) {
-  return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
+  if (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal)) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_FAILED;
+  }
+
+  return TSDB_CODE_SUCCESS;
 }
 
-SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) { return cJSON_AddArrayToObject((cJSON*)pJson, pName); }
+SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) {
+  SJson* ret = (SJson*)cJSON_AddArrayToObject((cJSON*)pJson, pName);
+  if (ret == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return ret;
+}
 
 int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem) {
-  return (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED);
+  if (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  terrno = TSDB_CODE_OUT_OF_MEMORY;
+  return TSDB_CODE_FAILED;
 }
 
 int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) {
-  return (cJSON_AddItemToArray((cJSON*)pJson, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED);
+  if (cJSON_AddItemToArray((cJSON*)pJson, pItem)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  terrno = TSDB_CODE_OUT_OF_MEMORY;
+  return TSDB_CODE_FAILED;
 }
 
 int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj) {