xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

update telemetry formater to cjson

This commit is contained in:
Shengliang Guan 2022-03-01 17:13:12 +08:00
parent 0febebafb2
commit bbcfc1b7b2
5 changed files with 229 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/os/osSysinfo.h
View File

@ -33,21 +33,26 @@ typedef struct {
SDiskSize size;
} SDiskSpace;
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
int32_t taosGetCpuCores();
void taosGetSystemInfo();
bool taosGetEmail(char *email, int32_t maxLen);
bool taosGetOsReleaseName(char *releaseName, int32_t maxLen);
bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores);
int32_t taosGetCpuCores();
bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
bool taosGetTotalSysMemoryKB(uint64_t *kb);
bool taosGetProcMemory(float *memoryUsedMB); //
bool taosGetSysMemory(float *memoryUsedMB); //
void taosGetDisk();
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
bool taosReadProcIO(int64_t *rchars, int64_t *wchars);
bool taosGetProcIO(float *readKB, float *writeKB);
bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
bool taosGetBandSpeed(float *bandSpeedKb);
void taosGetDisk();
bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
bool taosGetProcMemory(float *memoryUsedMB);
bool taosGetSysMemory(float *memoryUsedMB);
int taosSystem(const char *cmd);
int32_t taosSystem(const char *cmd);
void taosKillSystem();
int32_t taosGetSystemUUID(char *uid, int32_t uidlen);
char * taosGetCmdlineByPID(int pid);
char *taosGetCmdlineByPID(int32_t pid);
void taosSetCoreDump(bool enable);
typedef struct {

262
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -18,186 +18,84 @@
#include "mndCluster.h"
#include "mndSync.h"
#include "tbuffer.h"
#include "tversion.h"
#include "tjson.h"
#define TELEMETRY_SERVER "telemetry.taosdata.com"
#define TELEMETRY_SERVER "localhost"
#define TELEMETRY_PORT 80
#define REPORT_INTERVAL 86400
static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); }
static void mndCloseObject(SBufferWriter* bw) {
size_t len = tbufTell(bw);
if (tbufGetData(bw, false)[len - 1] == ',') {
tbufWriteCharAt(bw, len - 1, '}');
} else {
tbufWriteChar(bw, '}');
}
}
static void mndWriteString(SBufferWriter* bw, const char* str) {
tbufWriteChar(bw, '"');
tbufWrite(bw, str, strlen(str));
tbufWriteChar(bw, '"');
}
static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) {
mndWriteString(bw, k);
tbufWriteChar(bw, ':');
char buf[32] = {0};
sprintf(buf, "%" PRId64, v);
tbufWrite(bw, buf, strlen(buf));
tbufWriteChar(bw, ',');
}
static void mndAddStringField(SBufferWriter* bw, const char* k, const char* v) {
mndWriteString(bw, k);
tbufWriteChar(bw, ':');
mndWriteString(bw, v);
tbufWriteChar(bw, ',');
}
static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
int32_t done = 0;
// FILE* fp = fopen("/proc/cpuinfo", "r");
TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) {
return;
}
while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
const char* v = strchr(line, ':') + 2;
mndAddStringField(bw, "cpuModel", v);
done |= 1;
} else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) {
const char* v = strchr(line, ':') + 2;
mndWriteString(bw, "numOfCpu");
tbufWriteChar(bw, ':');
tbufWrite(bw, v, strlen(v));
tbufWriteChar(bw, ',');
done |= 2;
}
}
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
// FILE* fp = fopen("/etc/os-release", "r");
TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) {
return;
}
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "PRETTY_NAME", 11) == 0) {
const char* p = strchr(line, '=') + 1;
if (*p == '"') {
p++;
line[size - 2] = 0;
}
mndAddStringField(bw, "os", p);
break;
}
}
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
// FILE* fp = fopen("/proc/meminfo", "r");
TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) {
return;
}
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "MemTotal", 8) == 0) {
const char* p = strchr(line, ':') + 1;
while (*p == ' ') p++;
mndAddStringField(bw, "memory", p);
break;
}
}
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
mndAddStringField(bw, "version", version);
mndAddStringField(bw, "buildInfo", buildinfo);
mndAddStringField(bw, "gitInfo", gitinfo);
mndAddStringField(bw, "email", pMgmt->email);
}
static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) {
static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) {
SMnodeLoad load = {0};
if (mndGetLoad(pMnode, &load) != 0) {
return;
}
if (mndGetLoad(pMnode, &load) != 0) return;
mndAddIntField(bw, "numOfDnode", load.numOfDnode);
mndAddIntField(bw, "numOfMnode", load.numOfMnode);
mndAddIntField(bw, "numOfVgroup", load.numOfVgroup);
mndAddIntField(bw, "numOfDatabase", load.numOfDatabase);
mndAddIntField(bw, "numOfSuperTable", load.numOfSuperTable);
mndAddIntField(bw, "numOfChildTable", load.numOfChildTable);
mndAddIntField(bw, "numOfColumn", load.numOfColumn);
mndAddIntField(bw, "numOfPoint", load.totalPoints);
mndAddIntField(bw, "totalStorage", load.totalStorage);
mndAddIntField(bw, "compStorage", load.compStorage);
tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode);
tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode);
tjsonAddDoubleToObject(pJson, "numOfVgroup", load.numOfVgroup);
tjsonAddDoubleToObject(pJson, "numOfDatabase", load.numOfDatabase);
tjsonAddDoubleToObject(pJson, "numOfSuperTable", load.numOfSuperTable);
tjsonAddDoubleToObject(pJson, "numOfChildTable", load.numOfChildTable);
tjsonAddDoubleToObject(pJson, "numOfColumn", load.numOfColumn);
tjsonAddDoubleToObject(pJson, "numOfPoint", load.totalPoints);
tjsonAddDoubleToObject(pJson, "totalStorage", load.totalStorage);
tjsonAddDoubleToObject(pJson, "compStorage", load.compStorage);
}
static void mndSendTelemetryReport(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
SBufferWriter bw = tbufInitWriter(NULL, false);
int32_t code = -1;
char buf[128] = {0};
SOCKET fd = 0;
static char* mndBuildTelemetryReport(SMnode* pMnode) {
char tmp[4096] = {0};
STelemMgmt* pMgmt = &pMnode->telemMgmt;
SJson* pJson = tjsonCreateObject();
if (pJson == NULL) return NULL;
char clusterName[64] = {0};
if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) {
goto SEND_OVER;
mndGetClusterName(pMnode, clusterName, sizeof(clusterName));
tjsonAddStringToObject(pJson, "instanceId", clusterName);
tjsonAddDoubleToObject(pJson, "reportVersion", 1);
if (taosGetOsReleaseName(tmp, sizeof(tmp))) {
tjsonAddStringToObject(pJson, "os", tmp);
}
mndBeginObject(&bw);
mndAddStringField(&bw, "instanceId", clusterName);
mndAddIntField(&bw, "reportVersion", 1);
mndAddOsInfo(pMnode, &bw);
mndAddCpuInfo(pMnode, &bw);
mndAddMemoryInfo(pMnode, &bw);
mndAddVersionInfo(pMnode, &bw);
mndAddRuntimeInfo(pMnode, &bw);
mndCloseObject(&bw);
int32_t numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores)) {
tjsonAddStringToObject(pJson, "cpuModel", tmp);
tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores);
} else {
tjsonAddDoubleToObject(pJson, "numOfCpu", taosGetCpuCores());
}
uint64_t memoryKB = 0;
if (taosGetTotalSysMemoryKB(&memoryKB)) {
snprintf(tmp, sizeof(tmp), "%" PRIu64 " kB", memoryKB);
tjsonAddStringToObject(pJson, "memory", tmp);
}
tjsonAddStringToObject(pJson, "version", version);
tjsonAddStringToObject(pJson, "buildInfo", buildinfo);
tjsonAddStringToObject(pJson, "gitInfo", gitinfo);
tjsonAddStringToObject(pJson, "email", pMgmt->email);
mndBuildRuntimeInfo(pMnode, pJson);
char* pCont = tjsonToString(pJson);
tjsonDelete(pJson);
return pCont;
}
static void mndSendTelemetryReport(const char* pCont) {
int32_t code = -1;
char buf[128] = {0};
SOCKET fd = 0;
int32_t contLen = strlen(pCont);
uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER);
if (ip == 0xffffffff) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr());
mError("failed to get telemetry server ip");
goto SEND_OVER;
}
fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0);
if (fd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr());
mError("failed to create telemetry socket");
goto SEND_OVER;
}
@ -208,24 +106,24 @@ static void mndSendTelemetryReport(SMnode* pMnode) {
"Content-Type: application/json\n"
"Content-Length: ";
if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) {
mError("failed to send telemetry header");
goto SEND_OVER;
}
int32_t contLen = (int32_t)(tbufTell(&bw));
sprintf(buf, "%d\n\n", contLen);
snprintf(buf, sizeof(buf), "%d\n\n", contLen);
if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) {
mError("failed to send telemetry contlen");
goto SEND_OVER;
}
const char* pCont = tbufGetData(&bw, false);
if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) {
mError("failed to send telemetry content");
goto SEND_OVER;
}
// read something to avoid nginx error 499
if (taosReadSocket(fd, buf, 10) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to receive response since %s", terrstr());
mError("failed to receive telemetry response");
goto SEND_OVER;
}
@ -233,12 +131,10 @@ static void mndSendTelemetryReport(SMnode* pMnode) {
code = 0;
SEND_OVER:
tbufCloseWriter(&bw);
taosCloseSocket(fd);
if (code != 0) {
mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr());
}
taosCloseSocket(fd);
}
static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
@ -247,33 +143,23 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
if (!pMgmt->enable) return 0;
taosWLockLatch(&pMgmt->lock);
mndSendTelemetryReport(pMnode);
char* pCont = mndBuildTelemetryReport(pMnode);
if (pCont != NULL) {
mndSendTelemetryReport(pCont);
free(pCont);
}
taosWUnLockLatch(&pMgmt->lock);
return 0;
}
static void mndGetEmail(SMnode* pMnode, char* filepath) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
if (pFile == NULL) {
return;
}
if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
}
taosCloseFile(&pFile);
}
int32_t mndInitTelem(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
pMgmt->enable = tsEnableTelemetryReporting;
taosInitRWLatch(&pMgmt->lock);
mndGetEmail(pMnode, "/usr/local/taos/email");
taosInitRWLatch(&pMgmt->lock);
pMgmt->enable = tsEnableTelemetryReporting;
taosGetEmail(pMgmt->email, sizeof(pMgmt->email));
mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
mDebug("mnode telemetry is initialized");
return 0;
}

2
source/dnode/mnode/impl/src/mnode.c
View File

@ -132,7 +132,7 @@ static int32_t mndInitTimer(SMnode *pMnode) {
return -1;
}
if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) {
if (taosTmrReset(mndPullupTelem, 300, pMnode, pMnode->timer, &pMnode->telemTimer)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}

97
source/os/src/osSysinfo.c
View File

@ -888,4 +888,99 @@ SysNameInfo taosGetSysNameInfo() {
return info;
}
#endif
bool taosGetEmail(char *email, int32_t maxLen) {
const char *filepath = "/usr/local/taos/email";
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
if (pFile == NULL) return false;
if (taosReadFile(pFile, (void *)email, maxLen) < 0) {
taosCloseFile(&pFile);
return false;
}
taosCloseFile(&pFile);
return true;
}
bool taosGetOsReleaseName(char *releaseName, int32_t maxLen) {
char *line = NULL;
size_t size = 0;
bool ret = false;
TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) return false;
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "PRETTY_NAME", 11) == 0) {
const char *p = strchr(line, '=') + 1;
if (*p == '"') {
p++;
line[size - 2] = 0;
}
tstrncpy(releaseName, p, maxLen);
ret = true;
break;
}
}
if (line != NULL) free(line);
taosCloseFile(&pFile);
return ret;
}
bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) {
char *line = NULL;
size_t size = 0;
int32_t done = 0;
bool ret = false;
TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) return false;
while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
const char *v = strchr(line, ':') + 2;
tstrncpy(cpuModel, v, maxLen);
ret = true;
done |= 1;
} else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) {
const char *v = strchr(line, ':') + 2;
*numOfCores = atoi(v);
done |= 2;
}
}
if (line != NULL) free(line);
taosCloseFile(&pFile);
return ret;
}
bool taosGetTotalSysMemoryKB(uint64_t *kb) {
char *line = NULL;
size_t size = 0;
bool ret = false;
TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) return false;
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "MemTotal", 8) == 0) {
const char *p = strchr(line, ':') + 1;
while (*p == ' ') p++;
ret = true;
*kb = atoll(p);
break;
}
}
if (line != NULL) free(line);
taosCloseFile(&pFile);
return ret;
}
#endif

53
source/util/src/tjson.c
View File

@ -18,36 +18,73 @@
#include "cJSON.h"
#include "taoserror.h"
SJson* tjsonCreateObject() { return cJSON_CreateObject(); }
SJson* tjsonCreateObject() {
SJson* pJson = cJSON_CreateObject();
if (pJson == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return pJson;
}
void tjsonDelete(SJson* pJson) { cJSON_Delete((cJSON*)pJson); }
int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number) {
char tmp[40] = {0};
snprintf(tmp, tListLen(tmp), "%" PRId64, number);
snprintf(tmp, sizeof(tmp), "%" PRId64, number);
return tjsonAddStringToObject(pJson, pName, tmp);
}
int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) {
return (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
if (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean) {
return (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
if (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) {
return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
if (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) { return cJSON_AddArrayToObject((cJSON*)pJson, pName); }
SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) {
SJson* ret = (SJson*)cJSON_AddArrayToObject((cJSON*)pJson, pName);
if (ret == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return ret;
}
int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem) {
return (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED);
if (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem)) {
return TSDB_CODE_SUCCESS;
}
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) {
return (cJSON_AddItemToArray((cJSON*)pJson, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED);
if (cJSON_AddItemToArray((cJSON*)pJson, pItem)) {
return TSDB_CODE_SUCCESS;
}
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj) {