xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #753 from taosdata/feature/lihui 

[TBASE-1123]
This commit is contained in:
fangpanpan 2019-11-21 17:23:59 +08:00 committed by GitHub
parent 5473468a4f 03abae0c6d
commit a8a7ba3cfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 182 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/client/src/tscAsync.c
View File

@ -24,6 +24,7 @@
#include "tsocket.h"
#include "tsql.h"
#include "tutil.h"
#include "tnote.h"
void tscProcessFetchRow(SSchedMsg *pMsg);
void tscProcessAsyncRetrieve(void *param, TAOS_RES *tres, int numOfRows);
@ -55,6 +56,8 @@ void taos_query_a(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *,
return;
}
taosNotePrintTsc(sqlstr);
SSqlObj *pSql = (SSqlObj *)malloc(sizeof(SSqlObj));
if (pSql == NULL) {
tscError("failed to malloc sqlObj");

3
src/client/src/tscSql.c
View File

@ -27,6 +27,7 @@
#include "tsql.h"
#include "ttimer.h"
#include "tutil.h"
#include "tnote.h"
TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
void *param, void **taos) {
@ -274,6 +275,8 @@ int taos_query(TAOS *taos, const char *sqlstr) {
return pRes->code;
}
taosNotePrintTsc(sqlstr);
void *sql = realloc(pSql->sqlstr, sqlLen + 1);
if (sql == NULL) {
pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;

8
src/client/src/tscSystem.c
View File

@ -45,6 +45,10 @@ extern int tscEmbedded;
int tscNumOfThreads;
static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
extern int tsTscEnableRecordSql;
extern int tsNumOfLogLines;
void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
void tscCheckDiskUsage(void *para, void *unused) {
taosGetDisk();
taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr);
@ -85,6 +89,10 @@ void taos_init_imp() {
taosSetCoreDump();
if (tsTscEnableRecordSql != 0) {
taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note");
}
#ifdef CLUSTER
tscMgmtIpList.numOfIps = 2;
strcpy(tscMgmtIpList.ipstr[0], tsMasterIp);

2
src/inc/tglobalcfg.h
View File

@ -150,6 +150,8 @@ extern int tsHttpEnableRecordSql;
extern int tsTelegrafUseFieldNum;
extern int tsAdminRowLimit;
extern int tsTscEnableRecordSql;
extern char tsMonitorDbName[];
extern char tsInternalPass[];
extern int tsMonitorInterval;

64
src/inc/tnote.h Normal file
View File

@ -0,0 +1,64 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TNOTE_H
#define TDENGINE_TNOTE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "unistd.h"
#include "os.h"
#include "tutil.h"
#include "tglobalcfg.h"
#define MAX_NOTE_LINE_SIZE 66000
#define NOTE_FILE_NAME_LEN 300
typedef struct _taosNoteInfo {
int taosNoteFileNum ;
int taosNoteMaxLines;
int taosNoteLines;
char taosNoteName[NOTE_FILE_NAME_LEN];
int taosNoteFlag;
int taosNoteFd;
int taosNoteOpenInProgress;
pthread_mutex_t taosNoteMutex;
}taosNoteInfo;
void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...);
extern taosNoteInfo m_HttpNote;
extern taosNoteInfo m_TscNote;
extern int tsHttpEnableRecordSql;
extern int tsTscEnableRecordSql;
#define taosNotePrintHttp(...) \
if (tsHttpEnableRecordSql) { \
taosNotePrint(&m_HttpNote, __VA_ARGS__); \
}
#define taosNotePrintTsc(...) \
if (tsTscEnableRecordSql) { \
taosNotePrint(&m_TscNote, __VA_ARGS__); \
}
#ifdef __cplusplus
}
#endif
#endif

6
src/modules/http/src/httpSql.c
View File

@ -24,11 +24,11 @@
#include "httpResp.h"
#include "taos.h"
#include "tsclient.h"
#include "tnote.h"
void *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
void *param, void **taos);
void httpProcessMultiSql(HttpContext *pContext);
void taosNotePrint(const char * const format, ...);
void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
HttpContext *pContext = (HttpContext *)param;
@ -165,7 +165,7 @@ void httpProcessMultiSql(HttpContext *pContext) {
char *sql = httpGetCmdsString(pContext, cmd->sql);
httpDump("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd,
pContext->ipstr, pContext->user, multiCmds->pos, sql);
taosNotePrint(sql);
taosNotePrintHttp(sql);
taos_query_a(pContext->session->taos, sql, httpProcessMultiSqlCallBack, (void *)pContext);
}
@ -298,7 +298,7 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) {
httpDump("context:%p, fd:%d, ip:%s, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->ipstr,
pContext->user, sql);
taosNotePrint(sql);
taosNotePrintHttp(sql);
taos_query_a(pSession->taos, sql, httpProcessSingleSqlCallBack, (void *)pContext);
}

4
src/modules/http/src/httpSystem.c
View File

@ -42,7 +42,7 @@
#endif
static HttpServer *httpServer = NULL;
void taosInitNote(int numOfNoteLines, int maxNotes);
void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
int httpInitSystem() {
taos_init();
@ -61,7 +61,7 @@ int httpInitSystem() {
pthread_mutex_init(&httpServer->serverMutex, NULL);
if (tsHttpEnableRecordSql != 0) {
taosInitNote(tsNumOfLogLines / 10, 1);
taosInitNote(tsNumOfLogLines / 10, 1, (char*)"http_note");
}
restInitHandle(httpServer);
adminInitHandle(httpServer);

5
src/util/src/tglobalcfg.c
View File

@ -162,6 +162,8 @@ int tsHttpEnableRecordSql = 0;
int tsTelegrafUseFieldNum = 0;
int tsAdminRowLimit = 10240;
int tsTscEnableRecordSql = 0;
int tsRpcTimer = 300;
int tsRpcMaxTime = 600; // seconds;
@ -767,6 +769,9 @@ static void doInitGlobalConfig() {
0, 255, 0,
TSDB_CFG_UTYPE_NONE);
tsInitConfigOption(cfg++, "tscEnableRecordSql", &tsTscEnableRecordSql, TSDB_CFG_VTYPE_INT,
TSDB_CFG_CTYPE_B_CONFIG,
1, 100000, 0, TSDB_CFG_UTYPE_NONE);
// version info
tsInitConfigOption(cfg++, "gitinfo", gitinfo, TSDB_CFG_VTYPE_STRING,
TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT,

173
src/util/src/tnote.c
View File

@ -13,42 +13,50 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tutil.h"
#include "tglobalcfg.h"
#include "tnote.h"
#define MAX_NOTE_LINE_SIZE 66000
#define NOTE_FILE_NAME_LEN 300
taosNoteInfo m_HttpNote;
taosNoteInfo m_TscNote;
static int taosNoteFileNum = 1;
static int taosNoteMaxLines = 0;
static int taosNoteLines = 0;
static char taosNoteName[NOTE_FILE_NAME_LEN];
static int taosNoteFlag = 0;
static int taosNoteFd = -1;
static int taosNoteOpenInProgress = 0;
static pthread_mutex_t taosNoteMutex;
void taosNotePrint(const char * const format, ...);
int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum);
int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote);
void taosInitNote(int numOfNoteLines, int maxNotes)
void taosInitNote(int numOfNoteLines, int maxNotes, char* lable)
{
taosNoteInfo * pNote = NULL;
char temp[128] = { 0 };
sprintf(temp, "%s/taosnote", logDir);
if (taosOpenNoteWithMaxLines(temp, numOfNoteLines, maxNotes) < 0)
fprintf(stderr, "failed to init note file\n");
taosNotePrint("==================================================");
taosNotePrint("=================== new note ===================");
taosNotePrint("==================================================");
if (strcasecmp(lable, "http_note") == 0) {
pNote = &m_HttpNote;
sprintf(temp, "%s/httpnote", logDir);
} else if (strcasecmp(lable, "tsc_note") == 0) {
pNote = &m_TscNote;
sprintf(temp, "%s/tscnote-%d", logDir, getpid());
} else {
return;
}
void taosCloseNoteByFd(int oldFd);
bool taosLockNote(int fd)
memset(pNote, 0, sizeof(taosNoteInfo));
pNote->taosNoteFileNum = 1;
//pNote->taosNoteMaxLines = 0;
//pNote->taosNoteLines = 0;
//pNote->taosNoteFlag = 0;
pNote->taosNoteFd = -1;
//pNote->taosNoteOpenInProgress = 0;
if (taosOpenNoteWithMaxLines(temp, numOfNoteLines, maxNotes, pNote) < 0)
fprintf(stderr, "failed to init note file\n");
taosNotePrint(pNote, "==================================================");
taosNotePrint(pNote, "=================== new note ===================");
taosNotePrint(pNote, "==================================================");
}
void taosCloseNoteByFd(int oldFd, taosNoteInfo * pNote);
bool taosLockNote(int fd, taosNoteInfo * pNote)
{
if (fd < 0) return false;
if (taosNoteFileNum > 1) {
if (pNote->taosNoteFileNum > 1) {
int ret = (int)(flock(fd, LOCK_EX | LOCK_NB));
if (ret == 0) {
return true;
@ -58,11 +66,11 @@ bool taosLockNote(int fd)
return false;
}
void taosUnLockNote(int fd)
void taosUnLockNote(int fd, taosNoteInfo * pNote)
{
if (fd < 0) return;
if (taosNoteFileNum > 1) {
if (pNote->taosNoteFileNum > 1) {
flock(fd, LOCK_UN | LOCK_NB);
}
}
@ -70,50 +78,51 @@ void taosUnLockNote(int fd)
void *taosThreadToOpenNewNote(void *param)
{
char name[NOTE_FILE_NAME_LEN];
taosNoteInfo * pNote = (taosNoteInfo *)param;
taosNoteFlag ^= 1;
taosNoteLines = 0;
sprintf(name, "%s.%d", taosNoteName, taosNoteFlag);
pNote->taosNoteFlag ^= 1;
pNote->taosNoteLines = 0;
sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
umask(0);
int fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
taosLockNote(fd);
taosLockNote(fd, pNote);
lseek(fd, 0, SEEK_SET);
int oldFd = taosNoteFd;
taosNoteFd = fd;
taosNoteLines = 0;
taosNoteOpenInProgress = 0;
taosNotePrint("=============== new note is opened =============");
int oldFd = pNote->taosNoteFd;
pNote->taosNoteFd = fd;
pNote->taosNoteLines = 0;
pNote->taosNoteOpenInProgress = 0;
taosNotePrint(pNote, "=============== new note is opened =============");
taosCloseNoteByFd(oldFd);
taosCloseNoteByFd(oldFd, pNote);
return NULL;
}
int taosOpenNewNote()
int taosOpenNewNote(taosNoteInfo * pNote)
{
pthread_mutex_lock(&taosNoteMutex);
pthread_mutex_lock(&pNote->taosNoteMutex);
if (taosNoteLines > taosNoteMaxLines && taosNoteOpenInProgress == 0) {
taosNoteOpenInProgress = 1;
if (pNote->taosNoteLines > pNote->taosNoteMaxLines && pNote->taosNoteOpenInProgress == 0) {
pNote->taosNoteOpenInProgress = 1;
taosNotePrint("=============== open new note ==================");
taosNotePrint(pNote, "=============== open new note ==================");
pthread_t pattern;
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
pthread_create(&pattern, &attr, taosThreadToOpenNewNote, NULL);
pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void*)pNote);
pthread_attr_destroy(&attr);
}
pthread_mutex_unlock(&taosNoteMutex);
pthread_mutex_unlock(&pNote->taosNoteMutex);
return taosNoteFd;
return pNote->taosNoteFd;
}
bool taosCheckNoteIsOpen(char *noteName)
bool taosCheckNoteIsOpen(char *noteName, taosNoteInfo * pNote)
{
int exist = access(noteName, F_OK);
if (exist != 0) {
@ -126,8 +135,8 @@ bool taosCheckNoteIsOpen(char *noteName)
return true;
}
if (taosLockNote(fd)) {
taosUnLockNote(fd);
if (taosLockNote(fd, pNote)) {
taosUnLockNote(fd, pNote);
close(fd);
return false;
}
@ -137,80 +146,80 @@ bool taosCheckNoteIsOpen(char *noteName)
}
}
void taosGetNoteName(char *fn)
void taosGetNoteName(char *fn, taosNoteInfo * pNote)
{
if (taosNoteFileNum > 1) {
for (int i = 0; i < taosNoteFileNum; i++) {
if (pNote->taosNoteFileNum > 1) {
for (int i = 0; i < pNote->taosNoteFileNum; i++) {
char fileName[NOTE_FILE_NAME_LEN];
sprintf(fileName, "%s%d.0", fn, i);
bool file1open = taosCheckNoteIsOpen(fileName);
bool file1open = taosCheckNoteIsOpen(fileName, pNote);
sprintf(fileName, "%s%d.1", fn, i);
bool file2open = taosCheckNoteIsOpen(fileName);
bool file2open = taosCheckNoteIsOpen(fileName, pNote);
if (!file1open && !file2open) {
sprintf(taosNoteName, "%s%d", fn, i);
sprintf(pNote->taosNoteName, "%s%d", fn, i);
return;
}
}
}
strcpy(taosNoteName, fn);
strcpy(pNote->taosNoteName, fn);
}
int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum)
int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote)
{
char name[NOTE_FILE_NAME_LEN] = "\0";
struct stat notestat0, notestat1;
int size;
taosNoteMaxLines = maxLines;
taosNoteFileNum = maxNoteNum;
taosGetNoteName(fn);
pNote->taosNoteMaxLines = maxLines;
pNote->taosNoteFileNum = maxNoteNum;
taosGetNoteName(fn, pNote);
strcpy(name, fn);
strcat(name, ".0");
// if none of the note files exist, open 0, if both exists, open the old one
if (stat(name, &notestat0) < 0) {
taosNoteFlag = 0;
pNote->taosNoteFlag = 0;
}
else {
strcpy(name, fn);
strcat(name, ".1");
if (stat(name, &notestat1) < 0) {
taosNoteFlag = 1;
pNote->taosNoteFlag = 1;
}
else {
taosNoteFlag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1;
pNote->taosNoteFlag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1;
}
}
sprintf(name, "%s.%d", taosNoteName, taosNoteFlag);
pthread_mutex_init(&taosNoteMutex, NULL);
sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
pthread_mutex_init(&pNote->taosNoteMutex, NULL);
umask(0);
taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
pNote->taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
if (taosNoteFd < 0) {
if (pNote->taosNoteFd < 0) {
fprintf(stderr, "failed to open note file:%s reason:%s\n", name, strerror(errno));
return -1;
}
taosLockNote(taosNoteFd);
taosLockNote(pNote->taosNoteFd, pNote);
// only an estimate for number of lines
struct stat filestat;
fstat(taosNoteFd, &filestat);
fstat(pNote->taosNoteFd, &filestat);
size = (int)filestat.st_size;
taosNoteLines = size / 60;
pNote->taosNoteLines = size / 60;
lseek(taosNoteFd, 0, SEEK_END);
lseek(pNote->taosNoteFd, 0, SEEK_END);
return 0;
}
void taosNotePrint(const char * const format, ...)
void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...)
{
va_list argpointer;
char buffer[MAX_NOTE_LINE_SIZE];
@ -222,7 +231,7 @@ void taosNotePrint(const char * const format, ...)
gettimeofday(&timeSecs, NULL);
curTime = timeSecs.tv_sec;
ptm = localtime_r(&curTime, &Tm);
len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int)timeSecs.tv_usec);
len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %lx ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int)timeSecs.tv_usec, pthread_self());
va_start(argpointer, format);
len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer);
@ -233,26 +242,26 @@ void taosNotePrint(const char * const format, ...)
buffer[len++] = '\n';
buffer[len] = 0;
if (taosNoteFd >= 0) {
twrite(taosNoteFd, buffer, (unsigned int)len);
if (pNote->taosNoteFd >= 0) {
twrite(pNote->taosNoteFd, buffer, (unsigned int)len);
if (taosNoteMaxLines > 0) {
taosNoteLines++;
if ((taosNoteLines > taosNoteMaxLines) && (taosNoteOpenInProgress == 0))
taosOpenNewNote();
if (pNote->taosNoteMaxLines > 0) {
pNote->taosNoteLines++;
if ((pNote->taosNoteLines > pNote->taosNoteMaxLines) && (pNote->taosNoteOpenInProgress == 0))
taosOpenNewNote(pNote);
}
}
}
void taosCloseNote()
void taosCloseNote(taosNoteInfo * pNote)
{
taosCloseNoteByFd(taosNoteFd);
taosCloseNoteByFd(pNote->taosNoteFd, pNote);
}
void taosCloseNoteByFd(int fd)
void taosCloseNoteByFd(int fd, taosNoteInfo * pNote)
{
if (fd >= 0) {
taosUnLockNote(fd);
taosUnLockNote(fd, pNote);
close(fd);
}
}