xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

remove files

This commit is contained in:
Shengliang Guan 2021-11-02 18:26:58 +08:00
parent 99c7162816
commit cfae3b9452
64 changed files with 0 additions and 12298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/plugins/CMakeLists.txt
View File

@ -1,8 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
ADD_SUBDIRECTORY(monitor)
ADD_SUBDIRECTORY(http)
IF (TD_LINUX AND TD_MQTT)
ADD_SUBDIRECTORY(mqtt)
ENDIF ()

24
src/plugins/http/CMakeLists.txt
View File

@ -1,24 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(http ${SRC})
TARGET_LINK_LIBRARIES(http z)
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(http taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(http taos)
ENDIF ()
IF (TD_ADMIN)
TARGET_LINK_LIBRARIES(http admin)
ENDIF ()

23
src/plugins/http/inc/httpAuth.h
View File

@ -1,23 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_TOKEN_H
#define TDENGINE_HTTP_TOKEN_H
int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len);
int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len);
int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen);
#endif

34
src/plugins/http/inc/httpContext.h
View File

@ -1,34 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_CONTEXT_H
#define TDENGINE_HTTP_CONTEXT_H
#include "httpInt.h"
bool httpInitContexts();
void httpCleanupContexts();
const char *httpContextStateStr(HttpContextState state);
HttpContext *httpCreateContext(SOCKET fd);
bool httpInitContext(HttpContext *pContext);
HttpContext *httpGetContext(void * pContext);
void httpReleaseContext(HttpContext *pContext/*, bool clearRes*/);
void httpCloseContextByServer(HttpContext *pContext);
void httpCloseContextByApp(HttpContext *pContext);
void httpNotifyContextClose(HttpContext *pContext);
bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState);
#endif

33
src/plugins/http/inc/httpGcHandle.h
View File

@ -1,33 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_GC_HANDLE_H
#define TDENGINE_GC_HANDLE_H
#include "http.h"
#include "httpInt.h"
#include "httpUtil.h"
#include "httpResp.h"
#include "httpSql.h"
#define GC_ROOT_URL_POS 0
#define GC_ACTION_URL_POS 1
#define GC_USER_URL_POS 2
#define GC_PASS_URL_POS 3
void gcInitHandle(HttpServer* pServer);
bool gcProcessRequest(struct HttpContext* pContext);
#endif

31
src/plugins/http/inc/httpGcJson.h
View File

@ -1,31 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_GC_JSON_H
#define TDENGINE_GC_JSON_H
#include "../../../../include/client/taos.h"
#include "httpHandle.h"
#include "httpJson.h"
void gcInitQueryJson(HttpContext *pContext);
void gcCleanQueryJson(HttpContext *pContext);
void gcStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result);
void gcStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd);
bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows);
void gcSendHeartBeatResp(HttpContext *pContext, HttpSqlCmd *cmd);
#endif

43
src/plugins/http/inc/httpGzip.h
View File

@ -1,43 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef HTTP_GZIP_H
#define HTTP_GZIP_H
#define EHTTP_GZIP_CHUNK_SIZE_DEFAULT (1024*16)
typedef struct ehttp_gzip_s ehttp_gzip_t;
typedef struct ehttp_gzip_callbacks_s ehttp_gzip_callbacks_t;
typedef struct ehttp_gzip_conf_s ehttp_gzip_conf_t;
struct ehttp_gzip_callbacks_s {
void (*on_data)(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len);
};
struct ehttp_gzip_conf_s {
int32_t get_header:2; // 0: not fetching header info
int32_t chunk_size; // 0: fallback to default: EHTTP_GZIP_CHUNK_SIZE_DEFAULT
};
ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg);
ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg);
void ehttp_gzip_destroy(ehttp_gzip_t *gzip);
int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len);
int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip);
#endif // _ehttp_gzip_h_9196791b_ac2a_4d73_9979_f4b41abbc4c0_

23
src/plugins/http/inc/httpHandle.h
View File

@ -1,23 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_HANDLE_H
#define TDENGINE_HTTP_HANDLE_H
// http request handler
void httpProcessRequest(HttpContext *pContext);
bool httpProcessData(HttpContext *pContext);
#endif

204
src/plugins/http/inc/httpInt.h
View File

@ -1,204 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_INT_H
#define TDENGINE_HTTP_INT_H
#include "os.h"
#include <stdbool.h>
#include "pthread.h"
#include "semaphore.h"
#include "tmempool.h"
#include "taosdef.h"
#include "tutil.h"
#include "zlib.h"
#include "http.h"
#include "httpLog.h"
#include "httpJson.h"
#include "httpParser.h"
#define HTTP_MAX_CMD_SIZE 1024
#define HTTP_MAX_BUFFER_SIZE 1024*1024*8
#define HTTP_LABEL_SIZE 8
#define HTTP_MAX_EVENTS 10
#define HTTP_BUFFER_INIT 4096
#define HTTP_BUFFER_SIZE 8388608
#define HTTP_STEP_SIZE 4096 //http message get process step by step
#define HTTP_METHOD_SCANNER_SIZE 7 //http method fp size
#define HTTP_GC_TARGET_SIZE 512
#define HTTP_WRITE_RETRY_TIMES 500
#define HTTP_WRITE_WAIT_TIME_MS 5
#define HTTP_PASSWORD_LEN TSDB_UNI_LEN
#define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + HTTP_PASSWORD_LEN)
typedef enum HttpReqType {
HTTP_REQTYPE_OTHERS = 0,
HTTP_REQTYPE_LOGIN = 1,
HTTP_REQTYPE_HEARTBEAT = 2,
HTTP_REQTYPE_SINGLE_SQL = 3,
HTTP_REQTYPE_MULTI_SQL = 4
} HttpReqType;
typedef enum {
HTTP_SERVER_INIT,
HTTP_SERVER_RUNNING,
HTTP_SERVER_CLOSING,
HTTP_SERVER_CLOSED
} HttpServerStatus;
typedef enum {
HTTP_CONTEXT_STATE_READY,
HTTP_CONTEXT_STATE_HANDLING,
HTTP_CONTEXT_STATE_DROPPING,
HTTP_CONTEXT_STATE_CLOSED
} HttpContextState;
typedef enum {
HTTP_CMD_TYPE_UN_SPECIFIED,
HTTP_CMD_TYPE_CREATE_DB,
HTTP_CMD_TYPE_CREATE_STBALE,
HTTP_CMD_TYPE_INSERT
} HttpSqlCmdType;
typedef enum { HTTP_CMD_STATE_NOT_RUN_YET, HTTP_CMD_STATE_RUN_FINISHED } HttpSqlCmdState;
typedef enum { HTTP_CMD_RETURN_TYPE_WITH_RETURN, HTTP_CMD_RETURN_TYPE_NO_RETURN } HttpSqlCmdReturnType;
struct HttpContext;
struct HttpThread;
typedef struct {
char id[HTTP_SESSION_ID_LEN];
int32_t refCount;
void * taos;
} HttpSession;
typedef struct {
// used by single cmd
char *nativSql;
int32_t numOfRows;
int32_t code;
// these are the locations in the buffer
int32_t tagNames[TSDB_MAX_TAGS];
int32_t tagValues[TSDB_MAX_TAGS];
int32_t timestamp;
int32_t metric;
int32_t stable;
int32_t table;
int32_t values;
int32_t sql;
// used by multi-cmd
int8_t cmdType;
int8_t cmdReturnType;
int8_t cmdState;
int8_t tagNum;
} HttpSqlCmd;
typedef struct {
HttpSqlCmd *cmds;
int16_t pos;
int16_t size;
int16_t maxSize;
int32_t bufferPos;
int32_t bufferSize;
char * buffer;
} HttpSqlCmds;
typedef struct {
char *module;
bool (*fpDecode)(struct HttpContext *pContext);
} HttpDecodeMethod;
typedef struct {
void (*startJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result);
void (*stopJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd);
bool (*buildQueryJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows);
void (*buildAffectRowJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int affectRows);
void (*initJsonFp)(struct HttpContext *pContext);
void (*cleanJsonFp)(struct HttpContext *pContext);
bool (*checkFinishedFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
void (*setNextCmdFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
} HttpEncodeMethod;
typedef enum {
EHTTP_CONTEXT_PROCESS_FAILED = 0x01,
EHTTP_CONTEXT_PARSER_FAILED = 0x02
} EHTTP_CONTEXT_FAILED_CAUSE;
typedef struct HttpContext {
int32_t refCount;
SOCKET fd;
uint32_t accessTimes;
uint32_t lastAccessTime;
int32_t state;
uint8_t reqType;
uint8_t parsed;
uint8_t error;
char ipstr[22];
char user[TSDB_USER_LEN]; // parsed from auth token or login message
char pass[HTTP_PASSWORD_LEN];
char db[/*TSDB_ACCT_ID_LEN + */TSDB_DB_NAME_LEN];
TAOS * taos;
void * ppContext;
HttpSession *session;
z_stream gzipStream;
HttpParser *parser;
HttpSqlCmd singleCmd;
HttpSqlCmds *multiCmds;
JsonBuf * jsonBuf;
HttpEncodeMethod *encodeMethod;
HttpDecodeMethod *decodeMethod;
struct HttpThread *pThread;
} HttpContext;
typedef struct HttpThread {
pthread_t thread;
HttpContext * pHead;
pthread_mutex_t threadMutex;
bool stop;
EpollFd pollFd;
int32_t numOfContexts;
int32_t threadId;
char label[HTTP_LABEL_SIZE << 1];
bool (*processData)(HttpContext *pContext);
} HttpThread;
typedef struct HttpServer {
char label[HTTP_LABEL_SIZE];
uint32_t serverIp;
uint16_t serverPort;
int8_t stop;
int8_t reserve;
SOCKET fd;
int32_t numOfThreads;
int32_t methodScannerLen;
int32_t requestNum;
int32_t status;
pthread_t thread;
HttpThread * pThreads;
void * contextCache;
void * sessionCache;
pthread_mutex_t serverMutex;
HttpDecodeMethod *methodScanner[HTTP_METHOD_SCANNER_SIZE];
bool (*processData)(HttpContext *pContext);
} HttpServer;
extern const char *httpKeepAliveStr[];
extern const char *httpVersionStr[];
extern HttpServer tsHttpServer;
#endif

106
src/plugins/http/inc/httpJson.h
View File

@ -1,106 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_JSON_H
#define TDENGINE_HTTP_JSON_H
#include <stdint.h>
#include <stdbool.h>
#define JSON_BUFFER_SIZE 16384
struct HttpContext;
enum { JsonNumber, JsonString, JsonBoolean, JsonArray, JsonObject, JsonNull };
extern char JsonItmTkn;
extern char JsonObjStt;
extern char JsonObjEnd;
extern char JsonArrStt;
extern char JsonArrEnd;
extern char JsonStrStt;
extern char JsonStrEnd;
extern char JsonPairTkn;
extern char JsonNulTkn[];
extern char JsonTrueTkn[];
extern char JsonFalseTkn[];
typedef struct {
int32_t size;
int32_t total;
char* lst;
char buf[JSON_BUFFER_SIZE];
struct HttpContext* pContext;
} JsonBuf;
// http response
int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz);
int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz);
int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t sz);
// builder callback
typedef void (*httpJsonBuilder)(JsonBuf* buf, void* jsnHandle);
// buffer
void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext);
void httpWriteJsonBufHead(JsonBuf* buf);
int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast);
void httpWriteJsonBufEnd(JsonBuf* buf);
// value
void httpJsonString(JsonBuf* buf, char* sVal, int32_t len);
void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len);
void httpJsonStringForTransMean(JsonBuf* buf, char* SVal, int32_t maxLen);
void httpJsonInt64(JsonBuf* buf, int64_t num);
void httpJsonUInt64(JsonBuf* buf, uint64_t num);
void httpJsonTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision);
void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision);
void httpJsonInt(JsonBuf* buf, int32_t num);
void httpJsonUInt(JsonBuf* buf, uint32_t num);
void httpJsonFloat(JsonBuf* buf, float num);
void httpJsonDouble(JsonBuf* buf, double num);
void httpJsonNull(JsonBuf* buf);
void httpJsonBool(JsonBuf* buf, int32_t val);
// pair
void httpJsonPair(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen);
void httpJsonPairOriginString(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen);
void httpJsonPairHead(JsonBuf* buf, char* name, int32_t len);
void httpJsonPairIntVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num);
void httpJsonPairInt64Val(JsonBuf* buf, char* name, int32_t nNameLen, int64_t num);
void httpJsonPairBoolVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num);
void httpJsonPairFloatVal(JsonBuf* buf, char* name, int32_t nNameLen, float num);
void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int32_t nNameLen, double num);
void httpJsonPairNullVal(JsonBuf* buf, char* name, int32_t nNameLen);
// object
void httpJsonPairArray(JsonBuf* buf, char* name, int32_t nLen, httpJsonBuilder builder, void* dsHandle);
void httpJsonPairObject(JsonBuf* buf, char* name, int32_t nLen, httpJsonBuilder builder, void* dsHandle);
void httpJsonObject(JsonBuf* buf, httpJsonBuilder fnBuilder, void* dsHandle);
void httpJsonArray(JsonBuf* buf, httpJsonBuilder fnBuidler, void* jsonHandle);
// print
void httpJsonTestBuf(JsonBuf* buf, int32_t safety);
void httpJsonToken(JsonBuf* buf, char c);
void httpJsonItemToken(JsonBuf* buf);
void httpJsonPrint(JsonBuf* buf, const char* json, int32_t len);
// quick
void httpJsonPairStatus(JsonBuf* buf, int32_t code);
// http json printer
JsonBuf* httpMallocJsonBuf(struct HttpContext* pContext);
void httpFreeJsonBuf(struct HttpContext* pContext);
#endif

31
src/plugins/http/inc/httpLog.h
View File

@ -1,31 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_LOG_H
#define TDENGINE_HTTP_LOG_H
#include "tlog.h"
extern int32_t httpDebugFlag;
#define httpFatal(...) { if (httpDebugFlag & DEBUG_FATAL) { taosPrintLog("HTP FATAL ", 255, __VA_ARGS__); }}
#define httpError(...) { if (httpDebugFlag & DEBUG_ERROR) { taosPrintLog("HTP ERROR ", 255, __VA_ARGS__); }}
#define httpWarn(...) { if (httpDebugFlag & DEBUG_WARN) { taosPrintLog("HTP WARN ", 255, __VA_ARGS__); }}
#define httpInfo(...) { if (httpDebugFlag & DEBUG_INFO) { taosPrintLog("HTP ", 255, __VA_ARGS__); }}
#define httpDebug(...) { if (httpDebugFlag & DEBUG_DEBUG) { taosPrintLog("HTP ", httpDebugFlag, __VA_ARGS__); }}
#define httpTrace(...) { if (httpDebugFlag & DEBUG_TRACE) { taosPrintLog("HTP ", httpDebugFlag, __VA_ARGS__); }}
#define httpTraceL(...){ if (httpDebugFlag & DEBUG_TRACE) { taosPrintLongString("HTP ", httpDebugFlag, __VA_ARGS__); }}
#endif

27
src/plugins/http/inc/httpMetricsHandle.h
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTPMETRICSHANDLE_H
#define TDENGINE_HTTPMETRICSHANDLE_H
#include "http.h"
#include "httpInt.h"
#include "httpUtil.h"
#include "httpResp.h"
void metricsInitHandle(HttpServer* httpServer);
bool metricsProcessRequest(struct HttpContext* httpContext);
#endif // TDENGINE_HTTPMETRICHANDLE_H

184
src/plugins/http/inc/httpParser.h
View File

@ -1,184 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef HTTP_PARSER_H
#define HTTP_PARSER_H
#include "httpGzip.h"
#define HTTP_MAX_URL 6 // http url stack size
#define HTTP_CODE_CONTINUE 100
#define HTTP_CODE_SWITCHING_PROTOCOL 101
#define HTTP_CODE_PROCESSING 102
#define HTTP_CODE_EARLY_HINTS 103
#define HTTP_CODE_OK 200
#define HTTP_CODE_CREATED 201
#define HTTP_CODE_ACCEPTED 202
#define HTTP_CODE_NON_AUTHORITATIVE_INFO 203
#define HTTP_CODE_NO_CONTENT 204
#define HTTP_CODE_RESET_CONTENT 205
#define HTTP_CODE_PARTIAL_CONTENT 206
#define HTTP_CODE_MULTI_STATUS 207
#define HTTP_CODE_ALREADY_REPORTED 208
#define HTTP_CODE_IM_USED 226
#define HTTP_CODE_MULTIPLE_CHOICE 300
#define HTTP_CODE_MOVED_PERMANENTLY 301
#define HTTP_CODE_FOUND 302
#define HTTP_CODE_SEE_OTHER 303
#define HTTP_CODE_NOT_MODIFIED 304
#define HTTP_CODE_USE_PROXY 305
#define HTTP_CODE_UNUSED 306
#define HTTP_CODE_TEMPORARY_REDIRECT 307
#define HTTP_CODE_PERMANENT_REDIRECT 308
#define HTTP_CODE_BAD_REQUEST 400
#define HTTP_CODE_UNAUTHORIZED 401
#define HTTP_CODE_PAYMENT_REQUIRED 402
#define HTTP_CODE_FORBIDDEN 403
#define HTTP_CODE_NOT_FOUND 404
#define HTTP_CODE_METHOD_NOT_ALLOWED 405
#define HTTP_CODE_NOT_ACCEPTABLE 406
#define HTTP_CODE_PROXY_AUTH_REQUIRED 407
#define HTTP_CODE_REQUEST_TIMEOUT 408
#define HTTP_CODE_CONFLICT 409
#define HTTP_CODE_GONE 410
#define HTTP_CODE_LENGTH_REQUIRED 411
#define HTTP_CODE_PRECONDITION_FAILED 412
#define HTTP_CODE_PAYLOAD_TOO_LARGE 413
#define HTTP_CODE_URI_TOO_LARGE 414
#define HTTP_CODE_UNSUPPORTED_MEDIA_TYPE 415
#define HTTP_CODE_RANGE_NOT_SATISFIABLE 416
#define HTTP_CODE_EXPECTATION_FAILED 417
#define HTTP_CODE_IM_A_TEAPOT 418
#define HTTP_CODE_MISDIRECTED_REQUEST 421
#define HTTP_CODE_UNPROCESSABLE_ENTITY 422
#define HTTP_CODE_LOCKED 423
#define HTTP_CODE_FAILED_DEPENDENCY 424
#define HTTP_CODE_TOO_EARLY 425
#define HTTP_CODE_UPGRADE_REQUIRED 426
#define HTTP_CODE_PRECONDITION_REQUIRED 428
#define HTTP_CODE_TOO_MANY_REQUESTS 429
#define HTTP_CODE_REQ_HDR_FIELDS_TOO_LARGE 431
#define HTTP_CODE_UNAVAIL_4_LEGAL_REASONS 451
#define HTTP_CODE_INTERNAL_SERVER_ERROR 500
#define HTTP_CODE_NOT_IMPLEMENTED 501
#define HTTP_CODE_BAD_GATEWAY 502
#define HTTP_CODE_SERVICE_UNAVAILABLE 503
#define HTTP_CODE_GATEWAY_TIMEOUT 504
#define HTTP_CODE_HTTP_VER_NOT_SUPPORTED 505
#define HTTP_CODE_VARIANT_ALSO_NEGOTIATES 506
#define HTTP_CODE_INSUFFICIENT_STORAGE 507
#define HTTP_CODE_LOOP_DETECTED 508
#define HTTP_CODE_NOT_EXTENDED 510
#define HTTP_CODE_NETWORK_AUTH_REQUIRED 511
typedef enum HTTP_PARSER_STATE {
HTTP_PARSER_BEGIN,
HTTP_PARSER_REQUEST_OR_RESPONSE,
HTTP_PARSER_METHOD,
HTTP_PARSER_TARGET,
HTTP_PARSER_HTTP_VERSION,
HTTP_PARSER_SP,
HTTP_PARSER_STATUS_CODE,
HTTP_PARSER_REASON_PHRASE,
HTTP_PARSER_CRLF,
HTTP_PARSER_HEADER,
HTTP_PARSER_HEADER_KEY,
HTTP_PARSER_HEADER_VAL,
HTTP_PARSER_CHUNK_SIZE,
HTTP_PARSER_CHUNK,
HTTP_PARSER_END,
HTTP_PARSER_ERROR,
HTTP_PARSER_OPTIONAL_SP
} HTTP_PARSER_STATE;
typedef enum HTTP_AUTH_TYPE {
HTTP_INVALID_AUTH,
HTTP_BASIC_AUTH,
HTTP_TAOSD_AUTH
} HTTP_AUTH_TYPE;
typedef enum HTTP_VERSION {
HTTP_VERSION_10 = 0,
HTTP_VERSION_11 = 1,
HTTP_VERSION_12 = 2,
HTTP_INVALID_VERSION
} HTTP_VERSION;
typedef enum HTTP_KEEPALIVE {
HTTP_KEEPALIVE_NO_INPUT = 0,
HTTP_KEEPALIVE_ENABLE = 1,
HTTP_KEEPALIVE_DISABLE = 2
} HTTP_KEEPALIVE;
typedef struct HttpString {
char * str;
int32_t pos;
int32_t size;
} HttpString;
typedef struct HttpStatus {
int32_t code;
char * desc;
} HttpStatus;
typedef struct HttpStack{
int8_t *stacks;
int32_t pos;
int32_t size;
} HttpStack;
struct HttpContext;
typedef struct HttpParser {
struct HttpContext *pContext;
ehttp_gzip_t *gzip;
HttpStack stacks;
HttpString str;
HttpString body;
HttpString path[HTTP_MAX_URL];
char * method;
char * target;
char * version;
char * reasonPhrase;
char * key;
char * val;
char * authContent;
int8_t httpVersion;
int8_t acceptEncodingGzip;
int8_t acceptEncodingChunked;
int8_t contentLengthSpecified;
int8_t contentChunked;
int8_t transferGzip;
int8_t transferChunked;
int8_t keepAlive;
int8_t authType;
int32_t contentLength;
int32_t chunkSize;
int32_t receivedChunkSize;
int32_t receivedSize;
int32_t statusCode;
int8_t inited;
int8_t parsed;
int16_t httpCode;
int32_t parseCode;
} HttpParser;
void httpInitParser(HttpParser *parser);
HttpParser *httpCreateParser(struct HttpContext *pContext);
void httpClearParser(HttpParser *parser);
void httpDestroyParser(HttpParser *parser);
int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len);
char * httpGetStatusDesc(int32_t statusCode);
#endif

35
src/plugins/http/inc/httpQueue.h
View File

@ -1,35 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_QUEUE_H
#define TDENGINE_HTTP_QUEUE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
typedef void (*FHttpResultFp)(void *param, void *result, int32_t code, int32_t rows);
bool httpInitResultQueue();
void httpCleanupResultQueue();
void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp);
#ifdef __cplusplus
}
#endif
#endif

40
src/plugins/http/inc/httpResp.h
View File

@ -1,40 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_RESP_H
#define TDENGINE_HTTP_RESP_H
#include "httpInt.h"
enum _httpRespTempl {
HTTP_RESPONSE_JSON_OK,
HTTP_RESPONSE_JSON_ERROR,
HTTP_RESPONSE_OK,
HTTP_RESPONSE_ERROR,
HTTP_RESPONSE_CHUNKED_UN_COMPRESS,
HTTP_RESPONSE_CHUNKED_COMPRESS,
HTTP_RESPONSE_OPTIONS,
HTTP_RESPONSE_GRAFANA,
HTTP_RESP_END
};
extern const char *httpRespTemplate[];
void httpSendErrorResp(HttpContext *pContext, int32_t errNo);
void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg);
void httpSendSuccResp(HttpContext *pContext, char *desc);
void httpSendOptionResp(HttpContext *pContext, char *desc);
#endif

33
src/plugins/http/inc/httpRestHandle.h
View File

@ -1,33 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_REST_HANDLE_H
#define TDENGINE_REST_HANDLE_H
#include "http.h"
#include "httpInt.h"
#include "httpUtil.h"
#include "httpResp.h"
#include "httpSql.h"
#define REST_ROOT_URL_POS 0
#define REST_ACTION_URL_POS 1
#define REST_USER_USEDB_URL_POS 2
#define REST_PASS_URL_POS 3
void restInitHandle(HttpServer* pServer);
bool restProcessRequest(struct HttpContext* pContext);
#endif

56
src/plugins/http/inc/httpRestJson.h
View File

@ -1,56 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_REST_JSON_H
#define TDENGINE_REST_JSON_H
#include <stdbool.h>
#include "../../../../include/client/taos.h"
#include "httpHandle.h"
#include "httpJson.h"
#define REST_JSON_SUCCESS "succ"
#define REST_JSON_SUCCESS_LEN 4
#define REST_JSON_FAILURE "error"
#define REST_JSON_FAILURE_LEN 5
#define REST_JSON_STATUS "status"
#define REST_JSON_STATUS_LEN 6
#define REST_JSON_CODE "code"
#define REST_JSON_CODE_LEN 4
#define REST_JSON_DESC "desc"
#define REST_JSON_DESC_LEN 4
#define REST_JSON_DATA "data"
#define REST_JSON_DATA_LEN 4
#define REST_JSON_HEAD "head"
#define REST_JSON_HEAD_LEN 4
#define REST_JSON_HEAD_INFO "column_meta"
#define REST_JSON_HEAD_INFO_LEN 11
#define REST_JSON_ROWS "rows"
#define REST_JSON_ROWS_LEN 4
#define REST_JSON_AFFECT_ROWS "affected_rows"
#define REST_JSON_AFFECT_ROWS_LEN 13
#define REST_TIMESTAMP_FMT_LOCAL_STRING 0
#define REST_TIMESTAMP_FMT_TIMESTAMP 1
#define REST_TIMESTAMP_FMT_UTC_STRING 2
void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows);
void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result);
bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows);
bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows);
bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows);
void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd);
#endif

27
src/plugins/http/inc/httpServer.h
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_SERVER_H
#define TDENGINE_HTTP_SERVER_H
#include "httpInt.h"
bool httpInitConnect();
void httpCleanUpConnect();
void *httpInitServer(char *ip, uint16_t port, char *label, int32_t numOfThreads, void *fp, void *shandle);
void httpCleanUpServer(HttpServer *pServer);
#endif

27
src/plugins/http/inc/httpSession.h
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_SESSION_H
#define TDENGINE_HTTP_SESSION_H
bool httpInitSessions();
void httpCleanUpSessions();
// http session method
void httpCreateSession(HttpContext *pContext, void *taos);
void httpGetSession(HttpContext *pContext);
void httpReleaseSession(HttpContext *pContext);
#endif

41
src/plugins/http/inc/httpSql.h
View File

@ -1,41 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_SQL_H
#define TDENGINE_HTTP_SQL_H
int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...);
int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const format, ...);
int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int32_t mallocSize);
int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext);
bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferSize);
bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize);
bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize);
void httpFreeMultiCmds(HttpContext *pContext);
HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext);
HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext);
int32_t httpCurSqlCmdPos(HttpContext *pContext);
void httpTrimTableName(char *name);
int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name);
char * httpGetCmdsString(HttpContext *pContext, int32_t pos);
int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql);
void httpCheckFreeEscapedSql(char *oldSql, char *newSql);
#endif

34
src/plugins/http/inc/httpSystem.h
View File

@ -1,34 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_SYSTEM_H
#define TDENGINE_HTTP_SYSTEM_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
int32_t httpInitSystem();
int32_t httpStartSystem();
void httpStopSystem();
void httpCleanUpSystem();
#ifdef __cplusplus
}
#endif
#endif

35
src/plugins/http/inc/httpTgHandle.h
View File

@ -1,35 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TG_HANDLE_H
#define TDENGINE_TG_HANDLE_H
#include "http.h"
#include "httpInt.h"
#include "httpUtil.h"
#include "httpResp.h"
#include "httpSql.h"
#define TG_ROOT_URL_POS 0
#define TG_DB_URL_POS 1
#define TG_USER_URL_POS 2
#define TG_PASS_URL_POS 3
void tgInitHandle(HttpServer *pServer);
void tgCleanupHandle();
bool tgProcessRquest(struct HttpContext *pContext);
#endif

31
src/plugins/http/inc/httpTgJson.h
View File

@ -1,31 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TG_JSON_H
#define TDENGINE_TG_JSON_H
#include "../../../../include/client/taos.h"
#include "httpHandle.h"
#include "httpJson.h"
void tgInitQueryJson(HttpContext *pContext);
void tgCleanQueryJson(HttpContext *pContext);
void tgStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result);
void tgStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd);
void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows);
bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code);
void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code);
#endif

36
src/plugins/http/inc/httpUtil.h
View File

@ -1,36 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_UTIL_H
#define TDENGINE_HTTP_UTIL_H
bool httpCheckUsedbSql(char *sql);
bool httpCheckAlterSql(char *sql);
void httpTimeToString(int32_t t, char *buf, int32_t buflen);
bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp);
bool httpParseRequest(HttpContext *pContext);
int32_t httpCheckReadCompleted(HttpContext *pContext);
void httpReadDirtyData(HttpContext *pContext);
int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData);
int32_t httpGzipCompressInit(HttpContext *pContext);
int32_t httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLen,
char *outDestData, int32_t *outDestDataLen, bool isTheLast);
// http request parser
void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod);
#endif

117
src/plugins/http/src/httpAuth.c
View File

@ -1,117 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tkey.h"
#include "tutil.h"
#include "http.h"
#include "httpInt.h"
#include "httpAuth.h"
#define KEY_DES_4 4971256377704625728L
int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) {
token[len] = '\0';
int32_t outlen = 0;
char * base64 = (char *)base64_decode(token, len, &outlen);
if (base64 == NULL || outlen == 0) {
httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token);
free(base64);
return -1;
}
char *user = strstr(base64, ":");
if (user == NULL) {
httpError("context:%p, fd:%d, basic token:%s invalid format", pContext, pContext->fd, token);
free(base64);
return -1;
}
int32_t user_len = (int32_t)(user - base64);
if (user_len < 1 || user_len >= TSDB_USER_LEN) {
httpError("context:%p, fd:%d, basic token:%s parse user error", pContext, pContext->fd, token);
free(base64);
return -1;
}
strncpy(pContext->user, base64, (size_t)user_len);
pContext->user[user_len] = 0;
char * password = user + 1;
int32_t pass_len = (int32_t)((base64 + outlen) - password);
if (pass_len < 1 || pass_len >= HTTP_PASSWORD_LEN) {
httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token);
free(base64);
return -1;
}
strncpy(pContext->pass, password, (size_t)pass_len);
pContext->pass[pass_len] = 0;
free(base64);
httpDebug("context:%p, fd:%d, basic token parsed success, user:%s", pContext, pContext->fd, pContext->user);
return 0;
}
int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) {
token[len] = '\0';
int32_t outlen = 0;
unsigned char *base64 = base64_decode(token, len, &outlen);
if (base64 == NULL || outlen == 0) {
httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token);
if (base64) free(base64);
return 01;
}
if (outlen != (TSDB_USER_LEN + HTTP_PASSWORD_LEN)) {
httpError("context:%p, fd:%d, taosd token:%s length error", pContext, pContext->fd, token);
free(base64);
return -1;
}
char *descrypt = taosDesDecode(KEY_DES_4, (char *)base64, outlen);
if (descrypt == NULL) {
httpError("context:%p, fd:%d, taosd token:%s descrypt error", pContext, pContext->fd, token);
free(base64);
return -1;
} else {
tstrncpy(pContext->user, descrypt, sizeof(pContext->user));
tstrncpy(pContext->pass, descrypt + TSDB_USER_LEN, sizeof(pContext->pass));
httpDebug("context:%p, fd:%d, taosd token:%s parsed success, user:%s", pContext, pContext->fd, token,
pContext->user);
free(base64);
free(descrypt);
return 0;
}
}
int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) {
char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0};
size_t size = sizeof(pContext->user);
tstrncpy(buffer, pContext->user, size);
size = sizeof(pContext->pass);
tstrncpy(buffer + sizeof(pContext->user), pContext->pass, size);
char *encrypt = taosDesEncode(KEY_DES_4, buffer, TSDB_USER_LEN + HTTP_PASSWORD_LEN);
char *base64 = base64_encode((const unsigned char *)encrypt, TSDB_USER_LEN + HTTP_PASSWORD_LEN);
size_t len = strlen(base64);
tstrncpy(token, base64, len + 1);
free(encrypt);
free(base64);
httpDebug("context:%p, fd:%d, generate taosd token:%s", pContext, pContext->fd, token);
return 0;
}

240
src/plugins/http/src/httpContext.c
View File

@ -1,240 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosmsg.h"
#include "tsocket.h"
#include "tutil.h"
#include "ttimer.h"
#include "tglobal.h"
#include "tcache.h"
#include "hash.h"
#include "httpInt.h"
#include "httpResp.h"
#include "httpSql.h"
#include "httpSession.h"
#include "httpContext.h"
#include "httpParser.h"
static void httpDestroyContext(void *data);
static void httpRemoveContextFromEpoll(HttpContext *pContext) {
HttpThread *pThread = pContext->pThread;
if (pContext->fd >= 0) {
epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL);
#ifdef WINDOWS
SOCKET fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1);
#else
SOCKET fd = atomic_val_compare_exchange_64(&pContext->fd, pContext->fd, -1);
#endif
taosCloseSocket(fd);
}
}
static void httpDestroyContext(void *data) {
HttpContext *pContext = *(HttpContext **)data;
if (pContext->fd > 0) taosCloseSocket(pContext->fd);
HttpThread *pThread = pContext->pThread;
httpRemoveContextFromEpoll(pContext);
httpReleaseSession(pContext);
atomic_sub_fetch_32(&pThread->numOfContexts, 1);
httpDebug("context:%p, is destroyed, refCount:%d data:%p thread:%s numOfContexts:%d", pContext, pContext->refCount,
data, pContext->pThread->label, pContext->pThread->numOfContexts);
pContext->pThread = 0;
pContext->state = HTTP_CONTEXT_STATE_CLOSED;
// avoid double free
httpFreeJsonBuf(pContext);
httpFreeMultiCmds(pContext);
if (pContext->parser) {
httpDestroyParser(pContext->parser);
pContext->parser = NULL;
}
tfree(pContext);
}
bool httpInitContexts() {
tsHttpServer.contextCache = taosCacheInit(TSDB_CACHE_PTR_KEY, 2, true, httpDestroyContext, "restc");
if (tsHttpServer.contextCache == NULL) {
httpError("failed to init context cache");
return false;
}
return true;
}
void httpCleanupContexts() {
if (tsHttpServer.contextCache != NULL) {
SCacheObj *cache = tsHttpServer.contextCache;
httpInfo("context cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable));
taosCacheCleanup(tsHttpServer.contextCache);
tsHttpServer.contextCache = NULL;
}
}
const char *httpContextStateStr(HttpContextState state) {
switch (state) {
case HTTP_CONTEXT_STATE_READY:
return "ready";
case HTTP_CONTEXT_STATE_HANDLING:
return "handling";
case HTTP_CONTEXT_STATE_DROPPING:
return "dropping";
case HTTP_CONTEXT_STATE_CLOSED:
return "closed";
default:
return "unknown";
}
}
void httpNotifyContextClose(HttpContext *pContext) { shutdown(pContext->fd, SHUT_WR); }
bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) {
return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState);
}
HttpContext *httpCreateContext(SOCKET fd) {
HttpContext *pContext = calloc(1, sizeof(HttpContext));
if (pContext == NULL) return NULL;
pContext->fd = fd;
pContext->lastAccessTime = taosGetTimestampSec();
pContext->state = HTTP_CONTEXT_STATE_READY;
pContext->parser = httpCreateParser(pContext);
TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pContext;
HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &pContext,
sizeof(TSDB_CACHE_PTR_TYPE), 3000);
pContext->ppContext = ppContext;
httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext);
// set the ref to 0
taosCacheRelease(tsHttpServer.contextCache, (void **)&ppContext, false);
return pContext;
}
HttpContext *httpGetContext(void *ptr) {
TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)ptr;
HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE));
if (ppContext) {
HttpContext *pContext = *ppContext;
if (pContext) {
int32_t refCount = atomic_add_fetch_32(&pContext->refCount, 1);
httpTrace("context:%p, fd:%d, is accquired, data:%p refCount:%d", pContext, pContext->fd, ppContext, refCount);
return pContext;
}
}
return NULL;
}
void httpReleaseContext(HttpContext *pContext/*, bool clearRes*/) {
int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1);
if (refCount < 0) {
httpError("context:%p, is already released, refCount:%d", pContext, refCount);
return;
}
/*
if (clearRes) {
if (pContext->parser) {
httpClearParser(pContext->parser);
}
memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd));
}
*/
HttpContext **ppContext = pContext->ppContext;
httpTrace("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount);
if (tsHttpServer.contextCache != NULL) {
taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false);
} else {
httpDebug("context:%p, won't be destroyed for cache is already released", pContext);
// httpDestroyContext((void **)(&ppContext));
}
}
bool httpInitContext(HttpContext *pContext) {
pContext->accessTimes++;
pContext->lastAccessTime = taosGetTimestampSec();
pContext->reqType = HTTP_REQTYPE_OTHERS;
pContext->encodeMethod = NULL;
memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd));
httpTrace("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed);
return true;
}
void httpCloseContextByApp(HttpContext *pContext) {
HttpParser *parser = pContext->parser;
pContext->parsed = false;
bool keepAlive = true;
if (pContext->error == true) {
keepAlive = false;
} else if (parser && parser->httpVersion == HTTP_VERSION_10 && parser->keepAlive != HTTP_KEEPALIVE_ENABLE) {
keepAlive = false;
} else if (parser && parser->httpVersion != HTTP_VERSION_10 && parser->keepAlive == HTTP_KEEPALIVE_DISABLE) {
keepAlive = false;
}
if (keepAlive) {
if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) {
httpTrace("context:%p, fd:%d, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) {
httpRemoveContextFromEpoll(pContext);
httpTrace("context:%p, fd:%d, ast state:dropping, keepAlive:true, close connect", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
httpTrace("context:%p, fd:%d, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
httpRemoveContextFromEpoll(pContext);
httpTrace("context:%p, fd:%d, last state:ready, keepAlive:true, close connect", pContext, pContext->fd);
} else {
httpRemoveContextFromEpoll(pContext);
httpError("context:%p, fd:%d, last state:%s:%d, keepAlive:true, close connect", pContext, pContext->fd,
httpContextStateStr(pContext->state), pContext->state);
}
} else {
httpRemoveContextFromEpoll(pContext);
httpTrace("context:%p, fd:%d, ilast state:%s:%d, keepAlive:false, close context", pContext, pContext->fd,
httpContextStateStr(pContext->state), pContext->state);
}
httpReleaseContext(pContext/*, true*/);
}
void httpCloseContextByServer(HttpContext *pContext) {
if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) {
httpTrace("context:%p, fd:%d, epoll finished, still used by app", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) {
httpTrace("context:%p, fd:%d, epoll already finished, wait app finished", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) {
httpTrace("context:%p, fd:%d, epoll finished, close connect", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
httpTrace("context:%p, fd:%d, epoll finished, will be closed soon", pContext, pContext->fd);
} else {
httpError("context:%p, fd:%d, unknown state:%d", pContext, pContext->fd, pContext->state);
}
pContext->parsed = false;
httpRemoveContextFromEpoll(pContext);
httpReleaseContext(pContext/*, true*/);
}

304
src/plugins/http/src/httpGcHandle.c
View File

@ -1,304 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosdef.h"
#include "taoserror.h"
#include "cJSON.h"
#include "httpLog.h"
#include "httpGcHandle.h"
#include "httpGcJson.h"
static HttpDecodeMethod gcDecodeMethod = {"grafana", gcProcessRequest};
static HttpEncodeMethod gcHeartBeatMethod = {
.startJsonFp = NULL,
.stopJsonFp = gcSendHeartBeatResp,
.buildQueryJsonFp = NULL,
.buildAffectRowJsonFp = NULL,
.initJsonFp = NULL,
.cleanJsonFp = NULL,
.checkFinishedFp = NULL,
.setNextCmdFp = NULL
};
static HttpEncodeMethod gcQueryMethod = {
.startJsonFp = NULL,
.stopJsonFp = gcStopQueryJson,
.buildQueryJsonFp = gcBuildQueryJson,
.buildAffectRowJsonFp = NULL,
.initJsonFp = gcInitQueryJson,
.cleanJsonFp = gcCleanQueryJson,
.checkFinishedFp = NULL,
.setNextCmdFp = NULL
};
void gcInitHandle(HttpServer* pServer) { httpAddMethod(pServer, &gcDecodeMethod); }
bool gcGetUserFromUrl(HttpContext* pContext) {
HttpParser* pParser = pContext->parser;
if (pParser->path[GC_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[GC_USER_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->user, pParser->path[GC_USER_URL_POS].str, TSDB_USER_LEN);
return true;
}
bool gcGetPassFromUrl(HttpContext* pContext) {
HttpParser* pParser = pContext->parser;
if (pParser->path[GC_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].str, HTTP_PASSWORD_LEN);
return true;
}
bool gcProcessLoginRequest(HttpContext* pContext) {
httpDebug("context:%p, fd:%d, user:%s, process grafana login msg", pContext, pContext->fd, pContext->user);
pContext->reqType = HTTP_REQTYPE_LOGIN;
return true;
}
/**
* Process the query request
* @param fd for http send back
* @param context is taos conn
* @param filter, the request format is json, such as
*/
// https://github.com/grafana/grafana/blob/master/docs/sources/plugins/developing/datasources.md
// input
//[{
// "refId": "A",
// "alias" : "taosd",
// "sql" : "select first(taosd) from sys.mem where ts > now-6h and ts < now interval(20000a)"
//},
//{
// "refId": "B",
// "alias" : "system",
// "sql" : "select first(taosd) from sys.mem where ts > now-6h and ts < now interval(20000a)"
//}]
// output
//[{
// "datapoints": [[339.386719,
// 1537873132000],
// [339.656250,
// 1537873162400],
// [339.656250,
// 1537873192600],
// [339.656250,
// 1537873222800],
// [339.589844,
// 1537873253200],
// [339.964844,
// 1537873283400],
// [340.093750,
// 1537873313800],
// [340.093750,
// 1537873344000],
// [340.093750,
// 1537873374200],
// [340.093750,
// 1537873404600]],
// "refId": "A",
// "target" : "taosd"
//},
//{
// "datapoints": [[339.386719,
// 1537873132000],
// [339.656250,
// 1537873162400],
// [339.656250,
// 1537873192600],
// [339.656250,
// 1537873222800],
// [339.589844,
// 1537873253200],
// [339.964844,
// 1537873283400],
// [340.093750,
// 1537873313800],
// [340.093750,
// 1537873344000],
// [340.093750,
// 1537873374200],
// [340.093750,
// 1537873404600]],
// "refId": "B",
// "target" : "system"
//}]
bool gcProcessQueryRequest(HttpContext* pContext) {
httpDebug("context:%p, fd:%d, process grafana query msg", pContext, pContext->fd);
char* filter = pContext->parser->body.str;
if (filter == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_MSG_INPUT);
return false;
}
cJSON* root = cJSON_Parse(filter);
if (root == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR);
return false;
}
int32_t size = cJSON_GetArraySize(root);
if (size <= 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_QUERY_NULL);
cJSON_Delete(root);
return false;
}
if (size > 100) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_QUERY_SIZE);
cJSON_Delete(root);
return false;
}
if (!httpMallocMultiCmds(pContext, size, HTTP_BUFFER_SIZE)) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
#define ESCAPE_ERROR_PROC(code, context, root) \
do { \
if (code != TSDB_CODE_SUCCESS) { \
httpSendErrorResp(context, code); \
\
cJSON_Delete(root); \
return false; \
} \
} while (0)
for (int32_t i = 0; i < size; ++i) {
cJSON* query = cJSON_GetArrayItem(root, i);
if (query == NULL) continue;
cJSON* refId = cJSON_GetObjectItem(query, "refId");
if (refId == NULL || refId->valuestring == NULL || strlen(refId->valuestring) == 0) {
httpDebug("context:%p, fd:%d, user:%s, refId is null", pContext, pContext->fd, pContext->user);
continue;
}
char *newStr = NULL;
int32_t retCode = 0;
retCode = httpCheckAllocEscapeSql(refId->valuestring, &newStr);
ESCAPE_ERROR_PROC(retCode, pContext, root);
int32_t refIdBuffer = httpAddToSqlCmdBuffer(pContext, newStr);
httpCheckFreeEscapedSql(refId->valuestring, newStr);
if (refIdBuffer == -1) {
httpWarn("context:%p, fd:%d, user:%s, refId buffer is full", pContext, pContext->fd, pContext->user);
break;
}
cJSON* alias = cJSON_GetObjectItem(query, "alias");
int32_t aliasBuffer = -1;
if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) {
retCode = httpCheckAllocEscapeSql(alias->valuestring, &newStr);
ESCAPE_ERROR_PROC(retCode, pContext, root);
aliasBuffer = httpAddToSqlCmdBuffer(pContext, newStr);
httpCheckFreeEscapedSql(alias->valuestring, newStr);
if (aliasBuffer == -1) {
httpWarn("context:%p, fd:%d, user:%s, alias buffer is full", pContext, pContext->fd, pContext->user);
break;
}
}
if (aliasBuffer == -1) {
aliasBuffer = httpAddToSqlCmdBuffer(pContext, "");
}
cJSON* sql = cJSON_GetObjectItem(query, "sql");
if (sql == NULL || sql->valuestring == NULL || strlen(sql->valuestring) == 0) {
httpDebug("context:%p, fd:%d, user:%s, sql is null", pContext, pContext->fd, pContext->user);
continue;
}
retCode = httpCheckAllocEscapeSql(sql->valuestring, &newStr);
ESCAPE_ERROR_PROC(retCode, pContext, root);
int32_t sqlBuffer = httpAddToSqlCmdBuffer(pContext, newStr);
httpCheckFreeEscapedSql(sql->valuestring, newStr);
if (sqlBuffer == -1) {
httpWarn("context:%p, fd:%d, user:%s, sql buffer is full", pContext, pContext->fd, pContext->user);
break;
}
HttpSqlCmd* cmd = httpNewSqlCmd(pContext);
if (cmd == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
cmd->sql = sqlBuffer;
cmd->values = refIdBuffer;
cmd->table = aliasBuffer;
cmd->numOfRows = 0; // hack way as target flags
cmd->timestamp = httpAddToSqlCmdBufferWithSize(pContext, HTTP_GC_TARGET_SIZE + 1); // hack way
if (cmd->timestamp == -1) {
httpWarn("context:%p, fd:%d, user:%s, cant't malloc target size, sql buffer is full", pContext, pContext->fd,
pContext->user);
break;
}
}
#undef ESCAPE_ERROR_PROC
pContext->reqType = HTTP_REQTYPE_MULTI_SQL;
pContext->encodeMethod = &gcQueryMethod;
pContext->multiCmds->pos = 0;
return true;
}
bool gcProcessHeartbeatRequest(HttpContext* pContext) {
httpDebug("context:%p, fd:%d, process grafana heartbeat msg", pContext, pContext->fd);
pContext->reqType = HTTP_REQTYPE_HEARTBEAT;
pContext->encodeMethod = &gcHeartBeatMethod;
return true;
}
/**
* Process get/post/options msg, such as login and logout
*/
bool gcProcessRequest(struct HttpContext* pContext) {
if (httpUrlMatch(pContext, GC_ACTION_URL_POS, "login")) {
gcGetUserFromUrl(pContext);
gcGetPassFromUrl(pContext);
}
if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO);
return false;
}
if (httpUrlMatch(pContext, GC_ACTION_URL_POS, "query")) {
return gcProcessQueryRequest(pContext);
} else if (httpUrlMatch(pContext, GC_ACTION_URL_POS, "heartbeat")) {
return gcProcessHeartbeatRequest(pContext);
} else if (httpUrlMatch(pContext, GC_ACTION_URL_POS, "login")) {
return gcProcessLoginRequest(pContext);
} else {
return gcProcessHeartbeatRequest(pContext);
}
}

272
src/plugins/http/src/httpGcJson.c
View File

@ -1,272 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "httpGcHandle.h"
#include "httpGcJson.h"
#include "httpJson.h"
#include "httpResp.h"
unsigned char *base64_decode(const char *value, int inlen, int *outlen);
void gcInitQueryJson(HttpContext *pContext) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
httpInitJsonBuf(jsonBuf, pContext);
httpWriteJsonBufHead(jsonBuf);
// data array begin
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
}
void gcCleanQueryJson(HttpContext *pContext) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
// array end
httpJsonToken(jsonBuf, JsonArrEnd);
httpWriteJsonBufEnd(jsonBuf);
}
void gcWriteTargetStartJson(JsonBuf *jsonBuf, char *refId, char *target) {
if (strlen(target) == 0) {
target = refId;
}
// object begin
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonObjStt);
// target section
httpJsonPair(jsonBuf, "refId", 5, refId, (int32_t)strlen(refId));
httpJsonPair(jsonBuf, "target", 6, target, (int32_t)strlen(target));
// data begin
httpJsonPairHead(jsonBuf, "datapoints", 10);
// data array begin
httpJsonToken(jsonBuf, JsonArrStt);
}
void gcWriteTargetEndJson(JsonBuf *jsonBuf) {
// data array end
httpJsonToken(jsonBuf, JsonArrEnd);
// object end
httpJsonToken(jsonBuf, JsonObjEnd);
}
void gcStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
// write end of target
if (cmd->numOfRows != 0) {
gcWriteTargetEndJson(jsonBuf);
}
}
bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return false;
int32_t num_fields = taos_num_fields(result);
TAOS_FIELD *fields = taos_fetch_fields(result);
if (num_fields == 0) {
return false;
}
int32_t precision = taos_result_precision(result);
// such as select count(*) from sys.cpu
// such as select count(*) from sys.cpu group by ipaddr
// such as select count(*) from sys.cpu interval(1d)
// such as select count(*) from sys.cpu interval(1d) group by ipaddr
// such as select count(*) count(*) from sys.cpu group by ipaddr interval(1d)
int32_t dataFields = -1;
int32_t groupFields = -1;
bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP;
if (hasTimestamp) {
dataFields = 1;
if (num_fields > 2) groupFields = num_fields - 1;
} else {
dataFields = 0;
if (num_fields > 1) groupFields = num_fields - 1;
}
char *refIdBuffer = httpGetCmdsString(pContext, cmd->values);
char *aliasBuffer = httpGetCmdsString(pContext, cmd->table);
char *targetBuffer = httpGetCmdsString(pContext, cmd->timestamp);
if (groupFields == -1 && cmd->numOfRows == 0) {
gcWriteTargetStartJson(jsonBuf, refIdBuffer, aliasBuffer);
}
cmd->numOfRows += numOfRows;
for (int32_t k = 0; k < numOfRows; ++k) {
TAOS_ROW row = taos_fetch_row(result);
if (row == NULL) {
cmd->numOfRows--;
continue;
}
int32_t *length = taos_fetch_lengths(result);
// for group by
if (groupFields != -1) {
char target[HTTP_GC_TARGET_SIZE] = {0};
int32_t len;
len = snprintf(target, HTTP_GC_TARGET_SIZE, "%s{", aliasBuffer);
for (int32_t i = dataFields + 1; i < num_fields; i++) {
if (row[i] == NULL) {
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:nil", fields[i].name);
if (i < num_fields - 1) {
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, ", ");
}
continue;
}
switch (fields[i].type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%d", fields[i].name, *((int8_t *)row[i]));
break;
case TSDB_DATA_TYPE_SMALLINT:
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%d", fields[i].name, *((int16_t *)row[i]));
break;
case TSDB_DATA_TYPE_INT:
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%d,", fields[i].name, *((int32_t *)row[i]));
break;
case TSDB_DATA_TYPE_BIGINT:
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%" PRId64, fields[i].name, *((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_FLOAT:
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%.5f", fields[i].name, GET_FLOAT_VAL(row[i]));
break;
case TSDB_DATA_TYPE_DOUBLE:
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%.9f", fields[i].name, GET_DOUBLE_VAL(row[i]));
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (row[i] != NULL) {
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:", fields[i].name);
memcpy(target + len, (char *)row[i], length[i]);
len = (int32_t)strlen(target);
}
break;
default:
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%s", fields[i].name, "-");
break;
}
if (i < num_fields - 1) {
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, ", ");
}
}
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "}");
if (strcmp(target, targetBuffer) != 0) {
// first target not write this section
if (strlen(targetBuffer) != 0) {
gcWriteTargetEndJson(jsonBuf);
}
// start new target
gcWriteTargetStartJson(jsonBuf, refIdBuffer, target);
strncpy(targetBuffer, target, HTTP_GC_TARGET_SIZE);
}
} // end of group by
// data row array begin
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
for (int32_t i = dataFields; i >= 0; i--) {
httpJsonItemToken(jsonBuf);
if (row == NULL || i >= num_fields || row[i] == NULL) {
httpJsonOriginString(jsonBuf, "null", 4);
continue;
}
switch (fields[i].type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
httpJsonInt(jsonBuf, *((int8_t *)row[i]));
break;
case TSDB_DATA_TYPE_SMALLINT:
httpJsonInt(jsonBuf, *((int16_t *)row[i]));
break;
case TSDB_DATA_TYPE_INT:
httpJsonInt(jsonBuf, *((int32_t *)row[i]));
break;
case TSDB_DATA_TYPE_BIGINT:
httpJsonInt64(jsonBuf, *((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_FLOAT:
httpJsonFloat(jsonBuf, GET_FLOAT_VAL(row[i]));
break;
case TSDB_DATA_TYPE_DOUBLE:
httpJsonDouble(jsonBuf, GET_DOUBLE_VAL(row[i]));
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
httpJsonStringForTransMean(jsonBuf, (char *)row[i], fields[i].bytes);
break;
case TSDB_DATA_TYPE_TIMESTAMP: {
int64_t ts = convertTimePrecision(*((int64_t *)row[i]), precision, TSDB_TIME_PRECISION_MILLI);
httpJsonInt64(jsonBuf, ts);
break;
}
default:
httpJsonString(jsonBuf, "-", 1);
break;
}
}
if (dataFields == 0) {
httpJsonItemToken(jsonBuf);
httpJsonString(jsonBuf, "-", 1);
}
// data row array end
httpJsonToken(jsonBuf, JsonArrEnd);
}
return true;
}
void gcSendHeartBeatResp(HttpContext *pContext, HttpSqlCmd *cmd) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
char *desc = "Grafana server receive a quest from you!";
httpInitJsonBuf(jsonBuf, pContext);
httpJsonToken(jsonBuf, JsonObjStt);
httpJsonPair(jsonBuf, "message", (int32_t)strlen("message"), desc, (int32_t)strlen(desc));
httpJsonToken(jsonBuf, JsonObjEnd);
char head[1024];
int32_t hLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_GRAFANA], httpVersionStr[pContext->parser->httpVersion],
httpKeepAliveStr[pContext->parser->keepAlive], (jsonBuf->lst - jsonBuf->buf));
httpWriteBuf(pContext, head, hLen);
httpWriteBuf(pContext, jsonBuf->buf, (int32_t)(jsonBuf->lst - jsonBuf->buf));
}

165
src/plugins/http/src/httpGzip.c
View File

@ -1,165 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "zlib.h"
#include "httpGzip.h"
typedef enum {
EHTTP_GZIP_INITING,
EHTTP_GZIP_READY,
EHTTP_GZIP_CLOSED,
} EHTTP_GZIP_STATE;
struct ehttp_gzip_s {
ehttp_gzip_conf_t conf;
ehttp_gzip_callbacks_t callbacks;
void * arg;
z_stream * gzip;
gz_header * header;
char * chunk;
int32_t state;
};
static void dummy_on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) {}
static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) {
switch (gzip->state) {
case EHTTP_GZIP_READY: {
inflateEnd(gzip->gzip);
} break;
default:
break;
}
if (gzip->gzip) {
free(gzip->gzip);
gzip->gzip = NULL;
}
if (gzip->header) {
free(gzip->header);
gzip->header = NULL;
}
if (gzip->chunk) {
free(gzip->chunk);
gzip->chunk = NULL;
}
gzip->state = EHTTP_GZIP_CLOSED;
}
ehttp_gzip_t *ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) {
ehttp_gzip_t *gzip = (ehttp_gzip_t *)calloc(1, sizeof(*gzip));
if (!gzip) return NULL;
do {
gzip->conf = conf;
gzip->callbacks = callbacks;
gzip->arg = arg;
if (gzip->callbacks.on_data == NULL) gzip->callbacks.on_data = dummy_on_data;
gzip->gzip = (z_stream *)calloc(1, sizeof(*gzip->gzip));
if (gzip->conf.get_header) {
gzip->header = (gz_header *)calloc(1, sizeof(*gzip->header));
}
if (gzip->conf.chunk_size <= 0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT;
gzip->chunk = (char *)malloc(gzip->conf.chunk_size);
if (!gzip->gzip || (gzip->conf.get_header && !gzip->header) || !gzip->chunk) break;
gzip->gzip->zalloc = Z_NULL;
gzip->gzip->zfree = Z_NULL;
gzip->gzip->opaque = Z_NULL;
// 863 windowBits can also be greater than 15 for optional gzip decoding. Add
// 864 32 to windowBits to enable zlib and gzip decoding with automatic header
// 865 detection, or add 16 to decode only the gzip format (the zlib format will
// 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a
// 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see
// 868 below), inflate() will not automatically decode concatenated gzip streams.
// 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state
// 870 would need to be reset to continue decoding a subsequent gzip stream.
int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS
if (ret != Z_OK) break;
if (gzip->header) {
ret = inflateGetHeader(gzip->gzip, gzip->header);
}
if (ret != Z_OK) break;
gzip->gzip->next_out = (z_const Bytef *)gzip->chunk;
gzip->gzip->avail_out = gzip->conf.chunk_size;
gzip->state = EHTTP_GZIP_READY;
return gzip;
} while (0);
ehttp_gzip_destroy(gzip);
return NULL;
}
ehttp_gzip_t *ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg);
void ehttp_gzip_destroy(ehttp_gzip_t *gzip) {
ehttp_gzip_cleanup(gzip);
free(gzip);
}
int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) {
if (gzip->state != EHTTP_GZIP_READY) return -1;
if (len <= 0) return 0;
gzip->gzip->next_in = (z_const Bytef*)buf;
gzip->gzip->avail_in = len;
while (gzip->gzip->avail_in) {
int32_t ret;
if (gzip->header) {
ret = inflate(gzip->gzip, Z_BLOCK);
} else {
ret = inflate(gzip->gzip, Z_SYNC_FLUSH);
}
if (ret != Z_OK && ret != Z_STREAM_END) return -1;
if (gzip->gzip->avail_out > 0) {
if (ret != Z_STREAM_END) continue;
}
int32_t _len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk);
gzip->gzip->next_out[0] = '\0';
gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, _len);
gzip->gzip->next_out = (z_const Bytef *)gzip->chunk;
gzip->gzip->avail_out = gzip->conf.chunk_size;
}
return 0;
}
int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) {
if (gzip->state != EHTTP_GZIP_READY) return -1;
gzip->gzip->next_in = NULL;
gzip->gzip->avail_in = 0;
int32_t ret;
ret = inflate(gzip->gzip, Z_FINISH);
if (ret != Z_STREAM_END) return -1;
int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk);
gzip->gzip->next_out[0] = '\0';
gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len);
gzip->gzip->next_out = NULL;
gzip->gzip->avail_out = 0;
return 0;
}

60
src/plugins/http/src/httpHandle.c
View File

@ -1,60 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "httpInt.h"
#include "httpResp.h"
#include "httpContext.h"
#include "httpHandle.h"
bool httpDecodeRequest(HttpContext* pContext) {
if (pContext->decodeMethod->fpDecode == NULL) {
return false;
}
return (*pContext->decodeMethod->fpDecode)(pContext);
}
/**
* Process the request from http pServer
*/
bool httpProcessData(HttpContext* pContext) {
if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_HANDLING)) {
httpTrace("context:%p, fd:%d, state:%s not in ready state, stop process request", pContext, pContext->fd,
httpContextStateStr(pContext->state));
pContext->error = true;
httpCloseContextByApp(pContext);
return false;
}
// handle Cross-domain request
if (strcmp(pContext->parser->method, "OPTIONS") == 0) {
httpTrace("context:%p, fd:%d, process options request", pContext, pContext->fd);
httpSendOptionResp(pContext, "process options request success");
} else {
if (!httpDecodeRequest(pContext)) {
/*
* httpCloseContextByApp has been called when parsing the error
*/
// httpCloseContextByApp(pContext);
} else {
httpClearParser(pContext->parser);
httpProcessRequest(pContext);
}
}
return true;
}

534
src/plugins/http/src/httpJson.c
View File

@ -1,534 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosmsg.h"
#include "taoserror.h"
#include "tglobal.h"
#include "http.h"
#include "httpLog.h"
#include "httpJson.h"
#include "httpResp.h"
#include "httpUtil.h"
#define MAX_NUM_STR_SZ 25
char JsonItmTkn = ',';
char JsonObjStt = '{';
char JsonObjEnd = '}';
char JsonArrStt = '[';
char JsonArrEnd = ']';
char JsonStrStt = '\"';
char JsonStrEnd = '\"';
char JsonPairTkn = ':';
char JsonNulTkn[] = "null";
char JsonTrueTkn[] = "true";
char JsonFalseTkn[] = "false";
int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t sz) {
int32_t len;
int32_t countWait = 0;
int32_t writeLen = 0;
do {
if (pContext->fd > 2) {
len = (int32_t)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL);
} else {
return sz;
}
if (len < 0) {
httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno),
countWait);
if (++countWait > HTTP_WRITE_RETRY_TIMES) break;
taosMsleep(HTTP_WRITE_WAIT_TIME_MS);
continue;
} else if (len == 0) {
httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno,
strerror(errno));
break;
} else {
countWait = 0;
writeLen += len;
}
} while (writeLen < sz);
return writeLen;
}
int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz) {
int32_t writeSz = httpWriteBufByFd(pContext, buf, sz);
if (writeSz != sz) {
httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response:\n%s", pContext, pContext->fd, sz,
writeSz, buf);
} else {
httpTrace("context:%p, fd:%d, dataSize:%d, writeSize:%d, response:\n%s", pContext, pContext->fd, sz, writeSz, buf);
}
return writeSz;
}
int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz) {
int32_t writeSz = httpWriteBufByFd(pContext, buf, sz);
if (writeSz != sz) {
httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response", pContext, pContext->fd, sz,
writeSz);
}
return writeSz;
}
int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
int32_t remain = 0;
char sLen[24];
int32_t srcLen = (int32_t)(buf->lst - buf->buf);
if (buf->pContext->fd <= 0) {
httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd);
buf->pContext->fd = -1;
}
/*
* HTTP servers often use compression to optimize transmission, for example
* with Content-Encoding: gzip or Content-Encoding: deflate.
* If both compression and chunked encoding are enabled, then the content stream is first compressed, then chunked;
* so the chunk encoding itself is not compressed, and the data in each chunk is not compressed individually.
* The remote endpoint then decodes the stream by concatenating the chunks and uncompressing the result.
*/
if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) {
if (buf->lst == buf->buf) {
httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd);
return 0; // there is no data to dump.
} else {
int32_t len = sprintf(sLen, "%x\r\n", srcLen);
httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen,
buf->buf);
httpWriteBufNoTrace(buf->pContext, sLen, len);
remain = httpWriteBufNoTrace(buf->pContext, buf->buf, srcLen);
}
} else {
char compressBuf[JSON_BUFFER_SIZE] = {0};
int32_t compressBufLen = JSON_BUFFER_SIZE;
int32_t ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast);
if (ret == 0) {
if (compressBufLen > 0) {
int32_t len = sprintf(sLen, "%x\r\n", compressBufLen);
httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", buf->pContext,
buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf);
httpWriteBufNoTrace(buf->pContext, sLen, len);
remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen);
} else {
httpDebug("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext,
buf->pContext->fd, isTheLast, buf->buf);
remain = 0; // there is no data to dump.
}
} else {
httpError("context:%p, fd:%d, failed to compress data, chunkSize:%d, last:%d, error:%d, response:\n%s",
buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf);
remain = 0;
}
}
httpWriteBufNoTrace(buf->pContext, "\r\n", 2);
buf->total += (int32_t)(buf->lst - buf->buf);
buf->lst = buf->buf;
memset(buf->buf, 0, (size_t)buf->size);
return remain;
}
void httpWriteJsonBufHead(JsonBuf* buf) {
if (buf->pContext->fd <= 0) {
buf->pContext->fd = -1;
}
char msg[1024] = {0};
int32_t len = -1;
if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) {
len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion],
httpKeepAliveStr[buf->pContext->parser->keepAlive]);
} else {
len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion],
httpKeepAliveStr[buf->pContext->parser->keepAlive]);
}
httpWriteBuf(buf->pContext, (const char*)msg, len);
}
void httpWriteJsonBufEnd(JsonBuf* buf) {
if (buf->pContext->fd <= 0) {
httpTrace("context:%p, fd:%d, json buf fd is 0", buf->pContext, buf->pContext->fd);
buf->pContext->fd = -1;
}
httpWriteJsonBufBody(buf, true);
httpWriteBufNoTrace(buf->pContext, "0\r\n\r\n", 5); // end of chunked resp
}
void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext) {
buf->lst = buf->buf;
buf->total = 0;
buf->size = JSON_BUFFER_SIZE; // option setting
buf->pContext = pContext;
memset(buf->lst, 0, JSON_BUFFER_SIZE);
if (pContext->parser->acceptEncodingGzip == 1 && tsHttpEnableCompress) {
httpGzipCompressInit(buf->pContext);
}
httpTrace("context:%p, fd:%d, json buffer initialized", buf->pContext, buf->pContext->fd);
}
void httpJsonItemToken(JsonBuf* buf) {
char c = *(buf->lst - 1);
if (c == JsonArrStt || c == JsonObjStt || c == JsonPairTkn || c == JsonItmTkn) {
return;
}
if (buf->lst > buf->buf) httpJsonToken(buf, JsonItmTkn);
}
void httpJsonString(JsonBuf* buf, char* sVal, int32_t len) {
httpJsonItemToken(buf);
httpJsonToken(buf, JsonStrStt);
httpJsonPrint(buf, sVal, len);
httpJsonToken(buf, JsonStrEnd);
}
void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len) {
httpJsonItemToken(buf);
httpJsonPrint(buf, sVal, len);
}
void httpJsonStringForTransMean(JsonBuf* buf, char* sVal, int32_t maxLen) {
httpJsonItemToken(buf);
httpJsonToken(buf, JsonStrStt);
if (sVal != NULL) {
// dispose transferred meaning byte
char* lastPos = sVal;
char* curPos = sVal;
for (int32_t i = 0; i < maxLen; ++i) {
if (*curPos == 0) {
break;
}
if (*curPos == '\"') {
httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos));
curPos++;
lastPos = curPos;
httpJsonPrint(buf, "\\\"", 2);
} else if (*curPos == '\\') {
httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos));
curPos++;
lastPos = curPos;
httpJsonPrint(buf, "\\\\", 2);
} else {
curPos++;
}
}
if (*lastPos) {
httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos));
}
}
httpJsonToken(buf, JsonStrEnd);
}
void httpJsonInt64(JsonBuf* buf, int64_t num) {
httpJsonItemToken(buf);
httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%" PRId64, num);
}
void httpJsonUInt64(JsonBuf* buf, uint64_t num) {
httpJsonItemToken(buf);
httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%" PRIu64, num);
}
void httpJsonTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision) {
char ts[35] = {0};
int32_t fractionLen;
char* format = NULL;
time_t quot = 0;
int64_t mod = 0;
switch (timePrecision) {
case TSDB_TIME_PRECISION_MILLI: {
mod = ((t) % 1000 + 1000) % 1000;
if (t < 0 && mod != 0) {
t -= 1000;
}
quot = t / 1000;
fractionLen = 5;
format = ".%03" PRId64;
break;
}
case TSDB_TIME_PRECISION_MICRO: {
mod = ((t) % 1000000 + 1000000) % 1000000;
if (t < 0 && mod != 0) {
t -= 1000000;
}
quot = t / 1000000;
fractionLen = 8;
format = ".%06" PRId64;
break;
}
case TSDB_TIME_PRECISION_NANO: {
mod = ((t) % 1000000000 + 1000000000) % 1000000000;
if (t < 0 && mod != 0) {
t -= 1000000000;
}
quot = t / 1000000000;
fractionLen = 11;
format = ".%09" PRId64;
break;
}
default:
fractionLen = 0;
assert(false);
}
struct tm ptm = {0};
localtime_r(&quot, &ptm);
int32_t length = (int32_t)strftime(ts, 35, "%Y-%m-%d %H:%M:%S", &ptm);
length += snprintf(ts + length, fractionLen, format, mod);
httpJsonString(buf, ts, length);
}
void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision) {
char ts[40] = {0};
struct tm* ptm;
int32_t fractionLen;
char* format = NULL;
time_t quot = 0;
long mod = 0;
switch (timePrecision) {
case TSDB_TIME_PRECISION_MILLI: {
mod = ((t) % 1000 + 1000) % 1000;
if (t < 0 && mod != 0) {
t -= 1000;
}
quot = t / 1000;
fractionLen = 5;
format = ".%03" PRId64;
break;
}
case TSDB_TIME_PRECISION_MICRO: {
mod = ((t) % 1000000 + 1000000) % 1000000;
if (t < 0 && mod != 0) {
t -= 1000000;
}
quot = t / 1000000;
fractionLen = 8;
format = ".%06" PRId64;
break;
}
case TSDB_TIME_PRECISION_NANO: {
mod = ((t) % 1000000000 + 1000000000) % 1000000000;
if (t < 0 && mod != 0) {
t -= 1000000000;
}
quot = t / 1000000000;
fractionLen = 11;
format = ".%09" PRId64;
break;
}
default:
fractionLen = 0;
assert(false);
}
ptm = localtime(&quot);
int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm);
length += snprintf(ts + length, fractionLen, format, mod);
length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm);
httpJsonString(buf, ts, length);
}
void httpJsonInt(JsonBuf* buf, int32_t num) {
httpJsonItemToken(buf);
httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%d", num);
}
void httpJsonUInt(JsonBuf* buf, uint32_t num) {
httpJsonItemToken(buf);
httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%u", num);
}
void httpJsonFloat(JsonBuf* buf, float num) {
httpJsonItemToken(buf);
httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
if (isinf(num) || isnan(num)) {
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "null");
} else if (num > 1E10 || num < -1E10) {
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.5e", num);
} else {
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.5f", num);
}
}
void httpJsonDouble(JsonBuf* buf, double num) {
httpJsonItemToken(buf);
httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
if (isinf(num) || isnan(num)) {
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "null");
} else if (num > 1E10 || num < -1E10) {
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.9e", num);
} else {
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.9f", num);
}
}
void httpJsonNull(JsonBuf* buf) { httpJsonString(buf, "null", 4); }
void httpJsonBool(JsonBuf* buf, int32_t val) {
if (val == 0)
httpJsonPrint(buf, JsonFalseTkn, sizeof(JsonFalseTkn));
else
httpJsonPrint(buf, JsonTrueTkn, sizeof(JsonTrueTkn));
}
void httpJsonPairHead(JsonBuf* buf, char* name, int32_t len) {
httpJsonItemToken(buf);
httpJsonString(buf, name, len);
httpJsonToken(buf, JsonPairTkn);
}
void httpJsonPair(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen) {
httpJsonPairHead(buf, name, nameLen);
httpJsonString(buf, sVal, valLen);
}
void httpJsonPairOriginString(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen) {
httpJsonPairHead(buf, name, nameLen);
httpJsonOriginString(buf, sVal, valLen);
}
void httpJsonPairIntVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonInt(buf, num);
}
void httpJsonPairInt64Val(JsonBuf* buf, char* name, int32_t nNameLen, int64_t num) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonInt64(buf, num);
}
void httpJsonPairBoolVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonBool(buf, num);
}
void httpJsonPairFloatVal(JsonBuf* buf, char* name, int32_t nNameLen, float num) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonFloat(buf, num);
}
void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int32_t nNameLen, double num) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonDouble(buf, num);
}
void httpJsonPairNullVal(JsonBuf* buf, char* name, int32_t nNameLen) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonNull(buf);
}
void httpJsonPairArray(JsonBuf* buf, char* name, int32_t len, httpJsonBuilder fnBuilder, void* dsHandle) {
httpJsonPairHead(buf, name, len);
httpJsonArray(buf, fnBuilder, dsHandle);
}
void httpJsonPairObject(JsonBuf* buf, char* name, int32_t len, httpJsonBuilder fnBuilder, void* dsHandle) {
httpJsonPairHead(buf, name, len);
httpJsonObject(buf, fnBuilder, dsHandle);
}
void httpJsonObject(JsonBuf* buf, httpJsonBuilder fnBuilder, void* dsHandle) {
httpJsonItemToken(buf);
httpJsonToken(buf, JsonObjStt);
(*fnBuilder)(buf, dsHandle);
httpJsonToken(buf, JsonObjEnd);
}
void httpJsonArray(JsonBuf* buf, httpJsonBuilder fnBuilder, void* jsonHandle) {
httpJsonItemToken(buf);
httpJsonToken(buf, JsonArrStt);
(*fnBuilder)(buf, jsonHandle);
httpJsonToken(buf, JsonArrEnd);
}
void httpJsonTestBuf(JsonBuf* buf, int32_t safety) {
if ((buf->lst - buf->buf + safety) < buf->size) return;
// buf->slot = *buf->lst;
httpWriteJsonBufBody(buf, false);
}
void httpJsonToken(JsonBuf* buf, char c) {
httpJsonTestBuf(buf, MAX_NUM_STR_SZ); // maybe object stack
*buf->lst++ = c;
}
void httpJsonPrint(JsonBuf* buf, const char* json, int32_t len) {
if (len == 0 || len >= JSON_BUFFER_SIZE) {
return;
}
if (len > buf->size) {
httpWriteJsonBufBody(buf, false);
httpJsonPrint(buf, json, len);
// buf->slot = json[len - 1];
return;
}
httpJsonTestBuf(buf, len + 2);
memcpy(buf->lst, json, (size_t)len);
buf->lst += len;
}
void httpJsonPairStatus(JsonBuf* buf, int32_t code) {
if (code == 0) {
httpJsonPair(buf, "status", 6, "succ", 4);
} else {
httpJsonPair(buf, "status", 6, "error", 5);
httpJsonItemToken(buf);
httpJsonPairIntVal(buf, "code", 4, code & 0XFFFF);
httpJsonItemToken(buf);
if (code == TSDB_CODE_MND_DB_NOT_SELECTED) {
httpJsonPair(buf, "desc", 4, "failed to create database", 23);
} else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
httpJsonPair(buf, "desc", 4, "failed to create table", 22);
} else {
httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int32_t)strlen(tstrerror(code)));
}
}
}

184
src/plugins/http/src/httpMetricsHandle.c
View File

@ -1,184 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tfs.h"
#include "httpMetricsHandle.h"
#include "dnode.h"
#include "httpLog.h"
static HttpDecodeMethod metricsDecodeMethod = {"metrics", metricsProcessRequest};
void metricsInitHandle(HttpServer* pServer) {
httpAddMethod(pServer, &metricsDecodeMethod);
}
bool metricsProcessRequest(HttpContext* pContext) {
httpDebug("context:%p, fd:%d, user:%s, process admin grant msg", pContext, pContext->fd, pContext->user);
JsonBuf* jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) {
httpError("failed to allocate memory for metrics");
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
return false;
}
httpInitJsonBuf(jsonBuf, pContext);
httpWriteJsonBufHead(jsonBuf);
httpJsonToken(jsonBuf, JsonObjStt);
{
char* keyDisks = "tags";
httpJsonPairHead(jsonBuf, keyDisks, (int32_t)strlen(keyDisks));
httpJsonToken(jsonBuf, JsonArrStt);
{
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonObjStt);
char* keyTagName = "name";
char* keyTagValue = "value";
httpJsonPairOriginString(jsonBuf, keyTagName, (int32_t)strlen(keyTagName), "\"dnode_id\"",
(int32_t)strlen("\"dnode_id\""));
int32_t dnodeId = dnodeGetDnodeId();
httpJsonPairIntVal(jsonBuf, keyTagValue, (int32_t)strlen(keyTagValue), dnodeId);
httpJsonToken(jsonBuf, JsonObjEnd);
}
httpJsonToken(jsonBuf, JsonArrEnd);
}
{
if (tsDnodeStartTime != 0) {
int64_t now = taosGetTimestampMs();
int64_t upTime = now-tsDnodeStartTime;
char* keyUpTime = "up_time";
httpJsonPairInt64Val(jsonBuf, keyUpTime, (int32_t)strlen(keyUpTime), upTime);
}
}
{
int32_t cpuCores = taosGetCpuCores();
char* keyCpuCores = "cpu_cores";
httpJsonPairIntVal(jsonBuf, keyCpuCores, (int32_t)strlen(keyCpuCores), cpuCores);
float sysCpuUsage = 0;
float procCpuUsage = 0;
bool succeeded = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
if (!succeeded) {
httpError("failed to get cpu usage");
} else {
if (sysCpuUsage <= procCpuUsage) {
sysCpuUsage = procCpuUsage + 0.1f;
}
char* keyCpuSystem = "cpu_system";
char* keyCpuEngine = "cpu_engine";
httpJsonPairFloatVal(jsonBuf, keyCpuSystem, (int32_t)strlen(keyCpuSystem), sysCpuUsage);
httpJsonPairFloatVal(jsonBuf, keyCpuEngine, (int32_t)strlen(keyCpuEngine), procCpuUsage);
}
}
{
float sysMemoryUsedMB = 0;
bool succeeded = taosGetSysMemory(&sysMemoryUsedMB);
if (!succeeded) {
httpError("failed to get sys memory info");
} else {
char* keyMemSystem = "mem_system";
httpJsonPairFloatVal(jsonBuf, keyMemSystem, (int32_t)strlen(keyMemSystem), sysMemoryUsedMB);
}
float procMemoryUsedMB = 0;
succeeded = taosGetProcMemory(&procMemoryUsedMB);
if (!succeeded) {
httpError("failed to get proc memory info");
} else {
char* keyMemEngine = "mem_engine";
httpJsonPairFloatVal(jsonBuf, keyMemEngine, (int32_t)strlen(keyMemEngine), procMemoryUsedMB);
}
}
{
int64_t bytes = 0, rbytes = 0, tbytes = 0;
bool succeeded = taosGetCardInfo(&bytes, &rbytes, &tbytes);
if (!succeeded) {
httpError("failed to get network info");
} else {
char* keyNetIn = "net_in";
char* keyNetOut = "net_out";
httpJsonPairInt64Val(jsonBuf, keyNetIn, (int32_t)strlen(keyNetIn), rbytes);
httpJsonPairInt64Val(jsonBuf, keyNetOut, (int32_t)strlen(keyNetOut), tbytes);
}
}
{
int64_t rchars = 0;
int64_t wchars = 0;
bool succeeded = taosReadProcIO(&rchars, &wchars);
if (!succeeded) {
httpError("failed to get io info");
} else {
char* keyIORead = "io_read";
char* keyIOWrite = "io_write";
httpJsonPairInt64Val(jsonBuf, keyIORead, (int32_t)strlen(keyIORead), rchars);
httpJsonPairInt64Val(jsonBuf, keyIOWrite, (int32_t)strlen(keyIOWrite), wchars);
}
}
{
const int8_t numTiers = 3;
SFSMeta fsMeta;
STierMeta* tierMetas = calloc(numTiers, sizeof(STierMeta));
tfsUpdateInfo(&fsMeta, tierMetas, numTiers);
{
char* keyDiskUsed = "disk_used";
char* keyDiskTotal = "disk_total";
httpJsonPairInt64Val(jsonBuf, keyDiskTotal, (int32_t)strlen(keyDiskTotal), fsMeta.tsize);
httpJsonPairInt64Val(jsonBuf, keyDiskUsed, (int32_t)strlen(keyDiskUsed), fsMeta.used);
char* keyDisks = "disks";
httpJsonPairHead(jsonBuf, keyDisks, (int32_t)strlen(keyDisks));
httpJsonToken(jsonBuf, JsonArrStt);
for (int i = 0; i < numTiers; ++i) {
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonObjStt);
char* keyDataDirLevelUsed = "datadir_used";
char* keyDataDirLevelTotal = "datadir_total";
httpJsonPairInt64Val(jsonBuf, keyDataDirLevelUsed, (int32_t)strlen(keyDataDirLevelUsed), tierMetas[i].used);
httpJsonPairInt64Val(jsonBuf, keyDataDirLevelTotal, (int32_t)strlen(keyDataDirLevelTotal), tierMetas[i].size);
httpJsonToken(jsonBuf, JsonObjEnd);
}
httpJsonToken(jsonBuf, JsonArrEnd);
}
free(tierMetas);
}
{
SStatisInfo info = dnodeGetStatisInfo();
{
char* keyReqHttp = "req_http";
char* keyReqSelect = "req_select";
char* keyReqInsert = "req_insert";
httpJsonPairInt64Val(jsonBuf, keyReqHttp, (int32_t)strlen(keyReqHttp), info.httpReqNum);
httpJsonPairInt64Val(jsonBuf, keyReqSelect, (int32_t)strlen(keyReqSelect), info.queryReqNum);
httpJsonPairInt64Val(jsonBuf, keyReqInsert, (int32_t)strlen(keyReqInsert), info.submitReqNum);
}
}
httpJsonToken(jsonBuf, JsonObjEnd);
httpWriteJsonBufEnd(jsonBuf);
pContext->reqType = HTTP_REQTYPE_OTHERS;
httpFreeJsonBuf(pContext);
return false;
}

1182
src/plugins/http/src/httpParser.c
View File

File diff suppressed because it is too large Load Diff

157
src/plugins/http/src/httpQueue.c
View File

@ -1,157 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "httpQueue.h"
#include "../../../../include/client/taos.h"
#include "httpAuth.h"
#include "httpContext.h"
#include "httpInt.h"
#include "httpResp.h"
#include "httpSession.h"
#include "httpSql.h"
#include "os.h"
#include "tnote.h"
#include "tqueue.h"
#include "tsclient.h"
typedef struct {
pthread_t thread;
int32_t workerId;
} SHttpWorker;
typedef struct {
int32_t num;
SHttpWorker *httpWorker;
} SHttpWorkerPool;
typedef struct {
void * param;
void * result;
int32_t code;
int32_t rows;
FHttpResultFp fp;
} SHttpResult;
static SHttpWorkerPool tsHttpPool;
static taos_qset tsHttpQset;
static taos_queue tsHttpQueue;
void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) {
if (tsHttpQueue != NULL) {
SHttpResult *pMsg = taosAllocateQitem(sizeof(SHttpResult));
pMsg->param = param;
pMsg->result = result;
pMsg->code = code;
pMsg->rows = rows;
pMsg->fp = fp;
taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg);
} else {
taos_stop_query(result);
taos_free_result(result);
//(*fp)(param, result, code, rows);
}
}
static void *httpProcessResultQueue(void *param) {
SHttpResult *pMsg;
int32_t type;
void * unUsed;
setThreadName("httpResultQ");
while (1) {
if (taosReadQitemFromQset(tsHttpQset, &type, (void **)&pMsg, &unUsed) == 0) {
httpDebug("qset:%p, http queue got no message from qset, exiting", tsHttpQset);
break;
}
httpTrace("context:%p, res:%p will be processed in result queue, code:%d rows:%d", pMsg->param, pMsg->result,
pMsg->code, pMsg->rows);
(*pMsg->fp)(pMsg->param, pMsg->result, pMsg->code, pMsg->rows);
taosFreeQitem(pMsg);
}
return NULL;
}
static bool httpAllocateResultQueue() {
tsHttpQueue = taosOpenQueue();
if (tsHttpQueue == NULL) return false;
taosAddIntoQset(tsHttpQset, tsHttpQueue, NULL);
for (int32_t i = 0; i < tsHttpPool.num; ++i) {
SHttpWorker *pWorker = tsHttpPool.httpWorker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pWorker->thread, &thAttr, httpProcessResultQueue, pWorker) != 0) {
httpError("failed to create thread to process http result queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
httpDebug("http result worker:%d is launched, total:%d", pWorker->workerId, tsHttpPool.num);
}
httpInfo("http result queue is opened");
return true;
}
static void httpFreeResultQueue() {
taosCloseQueue(tsHttpQueue);
tsHttpQueue = NULL;
}
bool httpInitResultQueue() {
tsHttpQset = taosOpenQset();
tsHttpPool.num = tsHttpMaxThreads;
tsHttpPool.httpWorker = (SHttpWorker *)calloc(sizeof(SHttpWorker), tsHttpPool.num);
if (tsHttpPool.httpWorker == NULL) return -1;
for (int32_t i = 0; i < tsHttpPool.num; ++i) {
SHttpWorker *pWorker = tsHttpPool.httpWorker + i;
pWorker->workerId = i;
}
return httpAllocateResultQueue();
}
void httpCleanupResultQueue() {
httpFreeResultQueue();
for (int32_t i = 0; i < tsHttpPool.num; ++i) {
SHttpWorker *pWorker = tsHttpPool.httpWorker + i;
if (taosCheckPthreadValid(pWorker->thread)) {
taosQsetThreadResume(tsHttpQset);
}
}
for (int32_t i = 0; i < tsHttpPool.num; ++i) {
SHttpWorker *pWorker = tsHttpPool.httpWorker + i;
if (taosCheckPthreadValid(pWorker->thread)) {
pthread_join(pWorker->thread, NULL);
}
}
taosCloseQset(tsHttpQset);
free(tsHttpPool.httpWorker);
httpInfo("http result queue is closed");
}

211
src/plugins/http/src/httpResp.c
View File

@ -1,211 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "httpLog.h"
#include "httpResp.h"
#include "httpJson.h"
#include "httpContext.h"
const char *httpKeepAliveStr[] = {"", "Connection: Keep-Alive\r\n", "Connection: Close\r\n"};
const char *httpVersionStr[] = {"HTTP/1.0", "HTTP/1.1", "HTTP/1.2"};
const char *httpRespTemplate[] = {
// HTTP_RESPONSE_JSON_OK
// HTTP_RESPONSE_JSON_ERROR
"{\"status\":\"succ\",\"code\":%d,\"desc\":\"%s\"}",
"{\"status\":\"error\",\"code\":%d,\"desc\":\"%s\"}",
// HTTP_RESPONSE_OK
// HTTP_RESPONSE_ERROR
"%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n",
"%s %d %s\r\nAccess-Control-Allow-Origin:*\r\n%sContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n",
// HTTP_RESPONSE_CHUNKED_UN_COMPRESS, HTTP_RESPONSE_CHUNKED_COMPRESS
"%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sContent-Type: application/json;charset=utf-8\r\nTransfer-Encoding: chunked\r\n\r\n",
"%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sContent-Type: application/json;charset=utf-8\r\nContent-Encoding: gzip\r\nTransfer-Encoding: chunked\r\n\r\n",
// HTTP_RESPONSE_OPTIONS
"%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\nAccess-Control-Allow-Methods: *\r\nAccess-Control-Max-Age: 3600\r\nAccess-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, authorization\r\n\r\n",
// HTTP_RESPONSE_GRAFANA
"%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sAccess-Control-Allow-Methods:POST, GET, OPTIONS, DELETE, PUT\r\nAccess-Control-Allow-Headers:Accept, Content-Type\r\nContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n"
};
static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char *httpCodeStr, int32_t errNo, const char *desc) {
httpError("context:%p, fd:%d, code:%d, error:%s", pContext, pContext->fd, httpCode, desc);
char head[512] = {0};
char body[512] = {0};
int8_t httpVersion = 0;
int8_t keepAlive = 0;
if (pContext->parser != NULL) {
httpVersion = pContext->parser->httpVersion;
keepAlive = pContext->parser->keepAlive;
}
int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_ERROR], errNo, desc);
int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[httpVersion], httpCode,
httpCodeStr, httpKeepAliveStr[keepAlive], bodyLen);
httpWriteBuf(pContext, head, headLen);
httpWriteBuf(pContext, body, bodyLen);
httpCloseContextByApp(pContext);
}
void httpSendErrorResp(HttpContext *pContext, int32_t errNo) {
int32_t httpCode = HTTP_CODE_INTERNAL_SERVER_ERROR;
if (errNo == TSDB_CODE_SUCCESS)
httpCode = HTTP_CODE_OK;
else if (errNo == TSDB_CODE_HTTP_SERVER_OFFLINE)
httpCode = HTTP_CODE_NOT_FOUND;
else if (errNo == TSDB_CODE_HTTP_UNSUPPORT_URL)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_INVALID_URL)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_NO_ENOUGH_MEMORY)
httpCode = HTTP_CODE_INSUFFICIENT_STORAGE;
else if (errNo == TSDB_CODE_HTTP_REQUSET_TOO_BIG)
httpCode = HTTP_CODE_PAYLOAD_TOO_LARGE;
else if (errNo == TSDB_CODE_HTTP_NO_AUTH_INFO)
httpCode = HTTP_CODE_UNAUTHORIZED;
else if (errNo == TSDB_CODE_HTTP_NO_MSG_INPUT)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_NO_SQL_INPUT)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_NO_EXEC_USEDB)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_SESSION_FULL)
httpCode = HTTP_CODE_INTERNAL_SERVER_ERROR;
else if (errNo == TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR)
httpCode = HTTP_CODE_INTERNAL_SERVER_ERROR;
else if (errNo == TSDB_CODE_HTTP_INVALID_MULTI_REQUEST)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_CREATE_GZIP_FAILED)
httpCode = HTTP_CODE_INTERNAL_SERVER_ERROR;
else if (errNo == TSDB_CODE_HTTP_FINISH_GZIP_FAILED)
httpCode = HTTP_CODE_INTERNAL_SERVER_ERROR;
else if (errNo == TSDB_CODE_HTTP_INVALID_VERSION)
httpCode = HTTP_CODE_HTTP_VER_NOT_SUPPORTED;
else if (errNo == TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH)
httpCode = HTTP_CODE_LENGTH_REQUIRED;
else if (errNo == TSDB_CODE_HTTP_INVALID_AUTH_TYPE)
httpCode = HTTP_CODE_UNAUTHORIZED;
else if (errNo == TSDB_CODE_HTTP_INVALID_AUTH_FORMAT)
httpCode = HTTP_CODE_UNAUTHORIZED;
else if (errNo == TSDB_CODE_HTTP_INVALID_BASIC_AUTH)
httpCode = HTTP_CODE_UNAUTHORIZED;
else if (errNo == TSDB_CODE_HTTP_INVALID_TAOSD_AUTH)
httpCode = HTTP_CODE_UNAUTHORIZED;
else if (errNo == TSDB_CODE_HTTP_PARSE_METHOD_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_TARGET_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_VERSION_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_SP_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_STATUS_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_PHRASE_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_CRLF_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_CHUNK_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_END_FAILED)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_INVALID_STATE)
httpCode = HTTP_CODE_BAD_REQUEST;
else if (errNo == TSDB_CODE_HTTP_PARSE_ERROR_STATE)
httpCode = HTTP_CODE_BAD_REQUEST;
else
httpCode = HTTP_CODE_BAD_REQUEST;
if (pContext->parser && pContext->parser->httpCode != 0) {
httpCode = pContext->parser->httpCode;
}
pContext->error = true;
char *httpCodeStr = httpGetStatusDesc(httpCode);
httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo & 0XFFFF, tstrerror(errNo));
}
void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char *errMsg) {
int32_t httpCode = HTTP_CODE_BAD_REQUEST;
char temp[512] = {0};
int32_t len = sprintf(temp, "invalid SQL: %s", errMsg);
for (int32_t i = 0; i < len; ++i) {
if (temp[i] == '\"') {
temp[i] = '\'';
} else if (temp[i] == '\n') {
temp[i] = ' ';
} else {
}
}
httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_TSC_INVALID_OPERATION & 0XFFFF, temp);
}
void httpSendSuccResp(HttpContext *pContext, char *desc) {
char head[1024] = {0};
char body[1024] = {0};
int8_t httpVersion = 0;
int8_t keepAlive = 0;
if (pContext->parser != NULL) {
httpVersion = pContext->parser->httpVersion;
keepAlive = pContext->parser->keepAlive;
}
int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc);
int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[httpVersion],
httpKeepAliveStr[keepAlive], bodyLen);
httpWriteBuf(pContext, head, headLen);
httpWriteBuf(pContext, body, bodyLen);
httpCloseContextByApp(pContext);
}
void httpSendOptionResp(HttpContext *pContext, char *desc) {
char head[1024] = {0};
char body[1024] = {0};
int8_t httpVersion = 0;
int8_t keepAlive = 0;
if (pContext->parser != NULL) {
httpVersion = pContext->parser->httpVersion;
keepAlive = pContext->parser->keepAlive;
}
int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc);
int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[httpVersion],
httpKeepAliveStr[keepAlive], bodyLen);
httpWriteBuf(pContext, head, headLen);
httpWriteBuf(pContext, body, bodyLen);
httpCloseContextByApp(pContext);
}

251
src/plugins/http/src/httpRestHandle.c
View File

@ -1,251 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "httpLog.h"
#include "httpRestHandle.h"
#include "httpRestJson.h"
#include "tglobal.h"
static HttpDecodeMethod restDecodeMethod = {"rest", restProcessRequest};
static HttpDecodeMethod restDecodeMethod2 = {"restful", restProcessRequest};
static HttpEncodeMethod restEncodeSqlTimestampMethod = {
.startJsonFp = restStartSqlJson,
.stopJsonFp = restStopSqlJson,
.buildQueryJsonFp = restBuildSqlTimestampJson,
.buildAffectRowJsonFp = restBuildSqlAffectRowsJson,
.initJsonFp = NULL,
.cleanJsonFp = NULL,
.checkFinishedFp = NULL,
.setNextCmdFp = NULL
};
static HttpEncodeMethod restEncodeSqlLocalTimeStringMethod = {
.startJsonFp = restStartSqlJson,
.stopJsonFp = restStopSqlJson,
.buildQueryJsonFp = restBuildSqlLocalTimeStringJson,
.buildAffectRowJsonFp = restBuildSqlAffectRowsJson,
.initJsonFp = NULL,
.cleanJsonFp = NULL,
.checkFinishedFp = NULL,
.setNextCmdFp = NULL
};
static HttpEncodeMethod restEncodeSqlUtcTimeStringMethod = {
.startJsonFp = restStartSqlJson,
.stopJsonFp = restStopSqlJson,
.buildQueryJsonFp = restBuildSqlUtcTimeStringJson,
.buildAffectRowJsonFp = restBuildSqlAffectRowsJson,
.initJsonFp = NULL,
.cleanJsonFp = NULL,
.checkFinishedFp = NULL,
.setNextCmdFp = NULL
};
void restInitHandle(HttpServer* pServer) {
httpAddMethod(pServer, &restDecodeMethod);
httpAddMethod(pServer, &restDecodeMethod2);
}
bool restGetUserFromUrl(HttpContext* pContext) {
HttpParser* pParser = pContext->parser;
if (pParser->path[REST_USER_USEDB_URL_POS].pos >= TSDB_USER_LEN || pParser->path[REST_USER_USEDB_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->user, pParser->path[REST_USER_USEDB_URL_POS].str, TSDB_USER_LEN);
return true;
}
bool restGetPassFromUrl(HttpContext* pContext) {
HttpParser* pParser = pContext->parser;
if (pParser->path[REST_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].str, HTTP_PASSWORD_LEN);
return true;
}
bool restProcessLoginRequest(HttpContext* pContext) {
httpDebug("context:%p, fd:%d, user:%s, process restful login msg", pContext, pContext->fd, pContext->user);
pContext->reqType = HTTP_REQTYPE_LOGIN;
return true;
}
bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) {
httpDebug("context:%p, fd:%d, user:%s, process restful sql msg", pContext, pContext->fd, pContext->user);
char* sql = pContext->parser->body.str;
if (sql == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT);
return false;
}
/*
* for async test
*
if (httpCheckUsedbSql(sql)) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_EXEC_USEDB);
return false;
}
*/
HttpSqlCmd* cmd = &(pContext->singleCmd);
cmd->nativSql = sql;
/* find if there is db_name in url */
pContext->db[0] = '\0';
HttpString *path = &pContext->parser->path[REST_USER_USEDB_URL_POS];
if (tsHttpDbNameMandatory) {
if (path->pos == 0) {
httpError("context:%p, fd:%d, user:%s, database name is mandatory", pContext, pContext->fd, pContext->user);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_URL);
return false;
}
}
if (path->pos > 0 && !(strlen(sql) > 4 && (sql[0] == 'u' || sql[0] == 'U') &&
(sql[1] == 's' || sql[1] == 'S') && (sql[2] == 'e' || sql[2] == 'E') && sql[3] == ' '))
{
snprintf(pContext->db, /*TSDB_ACCT_ID_LEN + */TSDB_DB_NAME_LEN, "%s", path->str);
}
pContext->reqType = HTTP_REQTYPE_SINGLE_SQL;
if (timestampFmt == REST_TIMESTAMP_FMT_LOCAL_STRING) {
pContext->encodeMethod = &restEncodeSqlLocalTimeStringMethod;
} else if (timestampFmt == REST_TIMESTAMP_FMT_TIMESTAMP) {
pContext->encodeMethod = &restEncodeSqlTimestampMethod;
} else if (timestampFmt == REST_TIMESTAMP_FMT_UTC_STRING) {
pContext->encodeMethod = &restEncodeSqlUtcTimeStringMethod;
}
return true;
}
#define REST_FUNC_URL_POS 2
#define REST_OUTP_URL_POS 3
#define REST_AGGR_URL_POS 4
#define REST_BUFF_URL_POS 5
#define HTTP_FUNC_LEN 32
#define HTTP_OUTP_LEN 16
#define HTTP_AGGR_LEN 2
#define HTTP_BUFF_LEN 32
static int udfSaveFile(const char *fname, const char *content, long len) {
int fd = open(fname, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755);
if (fd < 0)
return -1;
if (taosWrite(fd, (void *)content, len) < 0)
return -1;
return 0;
}
static bool restProcessUdfRequest(HttpContext* pContext) {
HttpParser* pParser = pContext->parser;
if (pParser->path[REST_FUNC_URL_POS].pos >= HTTP_FUNC_LEN || pParser->path[REST_FUNC_URL_POS].pos <= 0) {
return false;
}
if (pParser->path[REST_OUTP_URL_POS].pos >= HTTP_OUTP_LEN || pParser->path[REST_OUTP_URL_POS].pos <= 0) {
return false;
}
if (pParser->path[REST_AGGR_URL_POS].pos >= HTTP_AGGR_LEN || pParser->path[REST_AGGR_URL_POS].pos <= 0) {
return false;
}
if (pParser->path[REST_BUFF_URL_POS].pos >= HTTP_BUFF_LEN || pParser->path[REST_BUFF_URL_POS].pos <= 0) {
return false;
}
char* sql = pContext->parser->body.str;
int len = pContext->parser->body.pos;
if (sql == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT);
return false;
}
char udfDir[256] = {0};
char buf[64] = "udf-";
char funcName[64] = {0};
int aggr = 0;
char outputType[16] = {0};
char buffSize[32] = {0};
tstrncpy(funcName, pParser->path[REST_FUNC_URL_POS].str, HTTP_FUNC_LEN);
tstrncpy(buf + 4, funcName, HTTP_FUNC_LEN);
if (pParser->path[REST_AGGR_URL_POS].str[0] != '0') {
aggr = 1;
}
tstrncpy(outputType, pParser->path[REST_OUTP_URL_POS].str, HTTP_OUTP_LEN);
tstrncpy(buffSize, pParser->path[REST_BUFF_URL_POS].str, HTTP_BUFF_LEN);
taosGetTmpfilePath(funcName, udfDir);
udfSaveFile(udfDir, sql, len);
tfree(sql);
pContext->parser->body.str = malloc(1024);
sql = pContext->parser->body.str;
int sqlLen = sprintf(sql, "create %s function %s as \"%s\" outputtype %s bufsize %s",
aggr == 1 ? "aggregate" : " ", funcName, udfDir, outputType, buffSize);
pContext->parser->body.pos = sqlLen;
pContext->parser->body.size = sqlLen + 1;
HttpSqlCmd* cmd = &(pContext->singleCmd);
cmd->nativSql = sql;
pContext->reqType = HTTP_REQTYPE_SINGLE_SQL;
pContext->encodeMethod = &restEncodeSqlLocalTimeStringMethod;
return true;
}
bool restProcessRequest(struct HttpContext* pContext) {
if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "login")) {
restGetUserFromUrl(pContext);
restGetPassFromUrl(pContext);
}
if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO);
return false;
}
if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "sql")) {
return restProcessSqlRequest(pContext, REST_TIMESTAMP_FMT_LOCAL_STRING);
} else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "sqlt")) {
return restProcessSqlRequest(pContext, REST_TIMESTAMP_FMT_TIMESTAMP);
} else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "sqlutc")) {
return restProcessSqlRequest(pContext, REST_TIMESTAMP_FMT_UTC_STRING);
} else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "login")) {
return restProcessLoginRequest(pContext);
} else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "udf")) {
return restProcessUdfRequest(pContext);
} else {
}
httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_URL);
return false;
}

264
src/plugins/http/src/httpRestJson.c
View File

@ -1,264 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tglobal.h"
#include "tsclient.h"
#include "httpLog.h"
#include "httpJson.h"
#include "httpRestHandle.h"
#include "httpRestJson.h"
void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
// data row array begin
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
httpJsonItemToken(jsonBuf);
httpJsonInt(jsonBuf, affect_rows);
// data row array end
httpJsonToken(jsonBuf, JsonArrEnd);
cmd->numOfRows = 1;
}
void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
TAOS_FIELD *fields = taos_fetch_fields(result);
int32_t num_fields = taos_num_fields(result);
httpInitJsonBuf(jsonBuf, pContext);
httpWriteJsonBufHead(jsonBuf);
// object begin
httpJsonToken(jsonBuf, JsonObjStt);
// status, and data
httpJsonItemToken(jsonBuf);
httpJsonPair(jsonBuf, REST_JSON_STATUS, REST_JSON_STATUS_LEN, REST_JSON_SUCCESS, REST_JSON_SUCCESS_LEN);
// head begin
httpJsonItemToken(jsonBuf);
httpJsonPairHead(jsonBuf, REST_JSON_HEAD, REST_JSON_HEAD_LEN);
// head array begin
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
SSqlObj *pObj = (SSqlObj *) result;
bool isAlterSql = (pObj->sqlstr == NULL) ? false : httpCheckAlterSql(pObj->sqlstr);
if (num_fields == 0) {
httpJsonItemToken(jsonBuf);
httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN);
} else {
if (isAlterSql == true) {
httpJsonItemToken(jsonBuf);
httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN);
} else {
for (int32_t i = 0; i < num_fields; ++i) {
httpJsonItemToken(jsonBuf);
httpJsonString(jsonBuf, fields[i].name, (int32_t)strlen(fields[i].name));
}
}
}
// head array end
httpJsonToken(jsonBuf, JsonArrEnd);
// column_meta begin
httpJsonItemToken(jsonBuf);
httpJsonPairHead(jsonBuf, REST_JSON_HEAD_INFO, REST_JSON_HEAD_INFO_LEN);
// column_meta array begin
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
if (num_fields == 0) {
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
httpJsonItemToken(jsonBuf);
httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN);
httpJsonItemToken(jsonBuf);
httpJsonInt(jsonBuf, TSDB_DATA_TYPE_INT);
httpJsonItemToken(jsonBuf);
httpJsonInt(jsonBuf, 4);
httpJsonToken(jsonBuf, JsonArrEnd);
} else {
for (int32_t i = 0; i < num_fields; ++i) {
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
if (isAlterSql == true) {
httpJsonItemToken(jsonBuf);
httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN);
} else {
httpJsonItemToken(jsonBuf);
httpJsonString(jsonBuf, fields[i].name, (int32_t)strlen(fields[i].name));
}
httpJsonItemToken(jsonBuf);
httpJsonInt(jsonBuf, fields[i].type);
httpJsonItemToken(jsonBuf);
httpJsonInt(jsonBuf, fields[i].bytes);
httpJsonToken(jsonBuf, JsonArrEnd);
}
}
// column_meta array end
httpJsonToken(jsonBuf, JsonArrEnd);
// data begin
httpJsonItemToken(jsonBuf);
httpJsonPairHead(jsonBuf, REST_JSON_DATA, REST_JSON_DATA_LEN);
// data array begin
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
}
bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows,
int32_t timestampFormat) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return false;
int32_t num_fields = taos_num_fields(result);
TAOS_FIELD *fields = taos_fetch_fields(result);
for (int32_t k = 0; k < numOfRows; ++k) {
TAOS_ROW row = taos_fetch_row(result);
if (row == NULL) {
continue;
}
int32_t *length = taos_fetch_lengths(result);
// data row array begin
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
for (int32_t i = 0; i < num_fields; i++) {
httpJsonItemToken(jsonBuf);
if (row[i] == NULL) {
httpJsonOriginString(jsonBuf, "null", 4);
continue;
}
switch (fields[i].type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
httpJsonInt(jsonBuf, *((int8_t *)row[i]));
break;
case TSDB_DATA_TYPE_SMALLINT:
httpJsonInt(jsonBuf, *((int16_t *)row[i]));
break;
case TSDB_DATA_TYPE_INT:
httpJsonInt(jsonBuf, *((int32_t *)row[i]));
break;
case TSDB_DATA_TYPE_BIGINT:
httpJsonInt64(jsonBuf, *((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_UTINYINT:
httpJsonUInt(jsonBuf, *((uint8_t *)row[i]));
break;
case TSDB_DATA_TYPE_USMALLINT:
httpJsonUInt(jsonBuf, *((uint16_t *)row[i]));
break;
case TSDB_DATA_TYPE_UINT:
httpJsonUInt(jsonBuf, *((uint32_t *)row[i]));
break;
case TSDB_DATA_TYPE_UBIGINT:
httpJsonUInt64(jsonBuf, *((uint64_t *)row[i]));
break;
case TSDB_DATA_TYPE_FLOAT:
httpJsonFloat(jsonBuf, GET_FLOAT_VAL(row[i]));
break;
case TSDB_DATA_TYPE_DOUBLE:
httpJsonDouble(jsonBuf, GET_DOUBLE_VAL(row[i]));
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
httpJsonStringForTransMean(jsonBuf, (char *)row[i], length[i]);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
if (timestampFormat == REST_TIMESTAMP_FMT_LOCAL_STRING) {
httpJsonTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result));
} else if (timestampFormat == REST_TIMESTAMP_FMT_TIMESTAMP) {
httpJsonInt64(jsonBuf, *((int64_t *)row[i]));
} else {
httpJsonUtcTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result));
}
break;
default:
break;
}
}
// data row array end
httpJsonToken(jsonBuf, JsonArrEnd);
cmd->numOfRows++;
if (pContext->fd <= 0) {
httpError("context:%p, fd:%d, user:%s, conn closed, abort retrieve", pContext, pContext->fd, pContext->user);
return false;
}
if (cmd->numOfRows >= tsRestRowLimit) {
httpDebug("context:%p, fd:%d, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext,
pContext->fd, pContext->user, cmd->numOfRows, tsRestRowLimit);
return false;
}
}
httpDebug("context:%p, fd:%d, user:%s, retrieved row:%d", pContext, pContext->fd, pContext->user, cmd->numOfRows);
return true;
}
bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) {
return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP);
}
bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) {
return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING);
}
bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) {
return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING);
}
void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
// data array end
httpJsonToken(jsonBuf, JsonArrEnd);
// rows
httpJsonItemToken(jsonBuf);
httpJsonPairHead(jsonBuf, REST_JSON_ROWS, REST_JSON_ROWS_LEN);
httpJsonInt64(jsonBuf, cmd->numOfRows);
// object end
httpJsonToken(jsonBuf, JsonObjEnd);
httpWriteJsonBufEnd(jsonBuf);
}

417
src/plugins/http/src/httpServer.c
View File

@ -1,417 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosmsg.h"
#include "tsocket.h"
#include "tutil.h"
#include "ttimer.h"
#include "tglobal.h"
#include "httpInt.h"
#include "httpContext.h"
#include "httpResp.h"
#include "httpUtil.h"
static bool httpReadData(HttpContext *pContext);
#ifdef __APPLE__
static int sv_dummy = 0;
#endif
static void httpStopThread(HttpThread *pThread) {
pThread->stop = true;
// signal the thread to stop, try graceful method first,
// and use pthread_cancel when failed
#ifdef __APPLE__
int sv[2];
sv[0] = sv[1] = -1;
int r = socketpair(PF_LOCAL, SOCK_STREAM, 0, sv);
do {
if (r) break;
struct epoll_event ev = {0};
ev.events = EPOLLIN;
ev.data.ptr = &sv_dummy;
pThread->stop = true;
r = epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, sv[0], &ev);
if (r) break;
if (1 != send(sv[1], "1", 1, 0)) {
r = -1;
break;
}
} while (0);
if (r) {
pthread_cancel(pThread->thread);
}
#else
struct epoll_event event = {.events = EPOLLIN};
eventfd_t fd = eventfd(1, 0);
if (fd == -1) {
httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s",
pThread->label, strerror(errno));
pThread->stop = true;
pthread_cancel(pThread->thread);
} else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s",
pThread->label, strerror(errno));
pthread_cancel(pThread->thread);
}
#endif // __APPLE__
pthread_join(pThread->thread, NULL);
#ifdef __APPLE__
if (sv[0] != -1) {
close(sv[0]);
sv[0] = -1;
}
if (sv[1] != -1) {
close(sv[1]);
sv[1] = -1;
}
#else // __APPLE__
if (fd != -1) {
taosCloseSocket(fd);
}
#endif // __APPLE__
EpollClose(pThread->pollFd);
pthread_mutex_destroy(&(pThread->threadMutex));
}
void httpCleanUpConnect() {
HttpServer *pServer = &tsHttpServer;
if (pServer->pThreads == NULL) return;
if (taosCheckPthreadValid(pServer->thread)) {
pthread_join(pServer->thread, NULL);
}
for (int32_t i = 0; i < pServer->numOfThreads; ++i) {
HttpThread *pThread = pServer->pThreads + i;
if (pThread != NULL) {
httpStopThread(pThread);
}
}
httpDebug("http server:%s is cleaned up", pServer->label);
}
static void httpProcessHttpData(void *param) {
HttpServer * pServer = &tsHttpServer;
HttpThread * pThread = (HttpThread *)param;
HttpContext *pContext;
int32_t fdNum;
taosSetMaskSIGPIPE();
setThreadName("httpData");
while (1) {
struct epoll_event events[HTTP_MAX_EVENTS];
//-1 means uncertainty, 0-nowait, 1-wait 1 ms, set it from -1 to 1
fdNum = epoll_wait(pThread->pollFd, events, HTTP_MAX_EVENTS, TAOS_EPOLL_WAIT_TIME);
if (pThread->stop) {
httpDebug("%p, http thread get stop event, exiting...", pThread);
break;
}
if (fdNum <= 0) continue;
for (int32_t i = 0; i < fdNum; ++i) {
#ifdef __APPLE__
if (events[i].data.ptr == &sv_dummy) {
// no need to drain the recv buffer of sv[0]
// since there's only one time to send at most 1 byte to sv[0]
// btw, pThread->stop shall be already set, thus never reached here
httpDebug("if you see this line, there's internal logic error");
continue;
}
#endif // __APPLE__
pContext = httpGetContext(events[i].data.ptr);
if (pContext == NULL) {
httpError("context:%p, is already released, close connect", events[i].data.ptr);
// epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL);
// taosClose(events[i].data.fd);
continue;
}
if (events[i].events & EPOLLPRI) {
httpDebug("context:%p, fd:%d, state:%s, EPOLLPRI events occured, accessed:%d, close connect", pContext,
pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes);
httpCloseContextByServer(pContext);
continue;
}
if (events[i].events & EPOLLRDHUP) {
httpDebug("context:%p, fd:%d, state:%s, EPOLLRDHUP events occured, accessed:%d, close connect", pContext,
pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes);
httpCloseContextByServer(pContext);
continue;
}
if (events[i].events & EPOLLERR) {
httpDebug("context:%p, fd:%d, state:%s, EPOLLERR events occured, accessed:%d, close connect", pContext,
pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes);
httpCloseContextByServer(pContext);
continue;
}
if (events[i].events & EPOLLHUP) {
httpDebug("context:%p, fd:%d, state:%s, EPOLLHUP events occured, accessed:%d, close connect", pContext,
pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes);
httpCloseContextByServer(pContext);
continue;
}
if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
httpDebug("context:%p, fd:%d, state:%s, not in ready state, ignore read events", pContext, pContext->fd,
httpContextStateStr(pContext->state));
httpReleaseContext(pContext/*, true*/);
continue;
}
if (pServer->status != HTTP_SERVER_RUNNING) {
httpDebug("context:%p, fd:%d, state:%s, server is not running, accessed:%d, close connect", pContext,
pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_SERVER_OFFLINE);
httpNotifyContextClose(pContext);
} else {
if (httpReadData(pContext)) {
(*(pThread->processData))(pContext);
atomic_fetch_add_32(&pServer->requestNum, 1);
}
}
}
}
}
static void *httpAcceptHttpConnection(void *arg) {
SOCKET connFd = -1;
struct sockaddr_in clientAddr;
int32_t threadId = 0;
HttpServer * pServer = &tsHttpServer;
HttpThread * pThread = NULL;
HttpContext * pContext = NULL;
int32_t totalFds = 0;
taosSetMaskSIGPIPE();
setThreadName("httpAcceptConn");
pServer->fd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort);
if (pServer->fd < 0) {
httpError("http server:%s, failed to open http socket, ip:%s:%u error:%s", pServer->label,
taosIpStr(pServer->serverIp), pServer->serverPort, strerror(errno));
return NULL;
} else {
httpInfo("http server init success at %u", pServer->serverPort);
pServer->status = HTTP_SERVER_RUNNING;
}
while (1) {
socklen_t addrlen = sizeof(clientAddr);
connFd = accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen);
if (pServer->stop) {
httpDebug("http server:%s socket stop, exiting...", pServer->label);
break;
}
if (connFd == -1) {
if (errno == EINVAL) {
httpDebug("http server:%s socket was shutdown, exiting...", pServer->label);
break;
}
httpError("http server:%s, accept connect failure, errno:%d reason:%s", pServer->label, errno, strerror(errno));
continue;
}
totalFds = 1;
for (int32_t i = 0; i < pServer->numOfThreads; ++i) {
totalFds += pServer->pThreads[i].numOfContexts;
}
#if 0
if (totalFds > tsHttpCacheSessions * 100) {
httpError("fd:%d, ip:%s:%u, totalFds:%d larger than httpCacheSessions:%d*100, refuse connection", connFd,
taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions);
taosCloseSocket(connFd);
continue;
}
#endif
taosKeepTcpAlive(connFd);
taosSetNonblocking(connFd, 1);
// pick up the thread to handle this connection
pThread = pServer->pThreads + threadId;
pContext = httpCreateContext(connFd);
if (pContext == NULL) {
httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd,
taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port));
taosCloseSocket(connFd);
continue;
}
pContext->pThread = pThread;
sprintf(pContext->ipstr, "%s:%u", taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port));
struct epoll_event event;
#ifndef _TD_NINGSI_60
event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP;
#else
event.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP | EPOLLRDHUP;
#endif
event.data.ptr = pContext;
if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd,
pContext->ipstr, pThread->label, strerror(errno));
taosCloseSocket(pContext->fd);
httpReleaseContext(pContext/*, true*/);
continue;
}
// notify the data process, add into the FdObj list
atomic_add_fetch_32(&pThread->numOfContexts, 1);
httpDebug("context:%p, fd:%d, ip:%s, thread:%s numOfContexts:%d totalContext:%d, accept a new connection", pContext,
connFd, pContext->ipstr, pThread->label, pThread->numOfContexts, totalFds);
// pick up next thread for next connection
threadId++;
threadId = threadId % pServer->numOfThreads;
}
taosCloseSocket(pServer->fd);
return NULL;
}
bool httpInitConnect() {
HttpServer *pServer = &tsHttpServer;
pServer->pThreads = calloc(pServer->numOfThreads, sizeof(HttpThread));
if (pServer->pThreads == NULL) {
httpError("init error no enough memory");
return false;
}
HttpThread *pThread = pServer->pThreads;
for (int32_t i = 0; i < pServer->numOfThreads; ++i) {
sprintf(pThread->label, "%s%d", pServer->label, i);
pThread->processData = pServer->processData;
pThread->threadId = i;
if (pthread_mutex_init(&(pThread->threadMutex), NULL) < 0) {
httpError("http thread:%s, failed to init HTTP process data mutex, reason:%s", pThread->label, strerror(errno));
return false;
}
pThread->pollFd = (EpollFd)epoll_create(HTTP_MAX_EVENTS); // size does not matter
if (pThread->pollFd <= 0) {
httpError("http thread:%s, failed to create HTTP epoll", pThread->label);
pthread_mutex_destroy(&(pThread->threadMutex));
return false;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) {
httpError("http thread:%s, failed to create HTTP process data thread, reason:%s", pThread->label,
strerror(errno));
pthread_mutex_destroy(&(pThread->threadMutex));
return false;
}
pthread_attr_destroy(&thattr);
httpDebug("http thread:%p:%s, initialized", pThread, pThread->label);
pThread++;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pServer->thread), &thattr, (void *)httpAcceptHttpConnection, (void *)(pServer)) != 0) {
httpError("http server:%s, failed to create Http accept thread, reason:%s", pServer->label, strerror(errno));
httpCleanUpConnect();
return false;
}
pthread_attr_destroy(&thattr);
httpDebug("http server:%s, initialized, ip:%s:%u, numOfThreads:%d", pServer->label, taosIpStr(pServer->serverIp),
pServer->serverPort, pServer->numOfThreads);
return true;
}
static bool httpReadData(HttpContext *pContext) {
HttpParser *pParser = pContext->parser;
if (!pParser->inited) {
httpInitParser(pParser);
}
if (pParser->parsed) {
httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed);
return false;
}
pContext->accessTimes++;
pContext->lastAccessTime = taosGetTimestampSec();
char buf[HTTP_STEP_SIZE + 1] = {0};
while (1) {
int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE);
if (nread > 0) {
buf[nread] = '\0';
httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf);
int32_t ok = httpParseBuf(pParser, buf, nread);
if (ok) {
httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok,
pParser->parseCode);
httpSendErrorResp(pContext, pParser->parseCode);
httpNotifyContextClose(pContext);
return false;
}
if (pParser->parseCode) {
httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode);
httpSendErrorResp(pContext, pParser->parseCode);
httpNotifyContextClose(pContext);
return false;
}
if (!pParser->parsed) {
httpTrace("context:%p, fd:%d, read not finished", pContext, pContext->fd);
continue;
} else {
httpDebug("context:%p, fd:%d, bodyLen:%d", pContext, pContext->fd, pParser->body.pos);
return true;
}
} else if (nread < 0) {
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno);
continue; // later again
} else {
httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno);
taosCloseSocket(pContext->fd);
httpReleaseContext(pContext/*, false */);
return false;
}
} else {
httpError("context:%p, fd:%d, nread:%d, wait another event", pContext, pContext->fd, nread);
taosCloseSocket(pContext->fd);
httpReleaseContext(pContext/*, false */);
return false;
}
}
}

125
src/plugins/http/src/httpSession.c
View File

@ -1,125 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "httpSession.h"
#include "../../../../include/client/taos.h"
#include "httpContext.h"
#include "httpInt.h"
#include "os.h"
#include "taoserror.h"
#include "tcache.h"
#include "tglobal.h"
void httpCreateSession(HttpContext *pContext, void *taos) {
HttpServer *server = &tsHttpServer;
httpReleaseSession(pContext);
pthread_mutex_lock(&server->serverMutex);
HttpSession session;
memset(&session, 0, sizeof(HttpSession));
session.taos = taos;
session.refCount = 1;
int32_t len = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
pContext->session =
taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire * 1000);
// void *temp = pContext->session;
// taosCacheRelease(server->sessionCache, (void **)&temp, false);
if (pContext->session == NULL) {
httpError("context:%p, fd:%d, user:%s, error:%s", pContext, pContext->fd, pContext->user,
tstrerror(TSDB_CODE_HTTP_SESSION_FULL));
taos_close(taos);
pthread_mutex_unlock(&server->serverMutex);
return;
}
httpDebug("context:%p, fd:%d, user:%s, create a new session:%p:%p sessionRef:%d", pContext, pContext->fd,
pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount);
pthread_mutex_unlock(&server->serverMutex);
}
static void httpFetchSessionImp(HttpContext *pContext) {
HttpServer *server = &tsHttpServer;
pthread_mutex_lock(&server->serverMutex);
char sessionId[HTTP_SESSION_ID_LEN];
int32_t len = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
pContext->session = taosCacheAcquireByKey(server->sessionCache, sessionId, len);
if (pContext->session != NULL) {
atomic_add_fetch_32(&pContext->session->refCount, 1);
httpDebug("context:%p, fd:%d, user:%s, find an exist session:%p:%p, sessionRef:%d", pContext, pContext->fd,
pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount);
} else {
httpDebug("context:%p, fd:%d, user:%s, session not found", pContext, pContext->fd, pContext->user);
}
pthread_mutex_unlock(&server->serverMutex);
}
void httpGetSession(HttpContext *pContext) {
if (pContext->session == NULL) {
httpFetchSessionImp(pContext);
} else {
char sessionId[HTTP_SESSION_ID_LEN];
snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
httpReleaseSession(pContext);
httpFetchSessionImp(pContext);
}
}
void httpReleaseSession(HttpContext *pContext) {
if (pContext == NULL || pContext->session == NULL) return;
int32_t refCount = atomic_sub_fetch_32(&pContext->session->refCount, 1);
assert(refCount >= 0);
httpDebug("context:%p, release session:%p:%p, sessionRef:%d", pContext, pContext->session, pContext->session->taos,
pContext->session->refCount);
taosCacheRelease(tsHttpServer.sessionCache, (void **)&pContext->session, false);
pContext->session = NULL;
}
static void httpDestroySession(void *data) {
HttpSession *session = data;
httpDebug("session:%p:%p, is destroyed, sessionRef:%d", session, session->taos, session->refCount);
if (session->taos != NULL) {
taos_close(session->taos);
session->taos = NULL;
}
}
void httpCleanUpSessions() {
if (tsHttpServer.sessionCache != NULL) {
SCacheObj *cache = tsHttpServer.sessionCache;
httpInfo("session cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable));
taosCacheCleanup(tsHttpServer.sessionCache);
tsHttpServer.sessionCache = NULL;
}
}
bool httpInitSessions() {
tsHttpServer.sessionCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, 5, false, httpDestroySession, "rests");
if (tsHttpServer.sessionCache == NULL) {
httpError("failed to init session cache");
return false;
}
return true;
}

491
src/plugins/http/src/httpSql.c
View File

@ -1,491 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "httpSql.h"
#include "../../../../include/client/taos.h"
#include "httpAuth.h"
#include "httpContext.h"
#include "httpInt.h"
#include "httpQueue.h"
#include "httpResp.h"
#include "httpSession.h"
#include "os.h"
#include "taoserror.h"
#include "tnote.h"
#include "tsclient.h"
void httpProcessMultiSql(HttpContext *pContext);
void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows);
void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
HttpSqlCmds * multiCmds = pContext->multiCmds;
HttpEncodeMethod *encode = pContext->encodeMethod;
HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos;
char * sql = httpGetCmdsString(pContext, singleCmd->sql);
bool isContinue = false;
if (code == TSDB_CODE_SUCCESS && numOfRows > 0) {
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->buildQueryJsonFp) {
isContinue = (encode->buildQueryJsonFp)(pContext, singleCmd, result, numOfRows);
}
}
if (isContinue) {
// retrieve next batch of rows
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s", pContext,
pContext->fd, pContext->user, multiCmds->pos, numOfRows, sql);
taos_fetch_rows_a(result, httpProcessMultiSqlRetrieveCallBack, param);
} else {
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, numOfRows, sql);
if (code < 0) {
httpError("context:%p, fd:%d, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, tstrerror(code), sql);
}
taos_free_result(result);
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) {
(encode->stopJsonFp)(pContext, singleCmd);
}
multiCmds->pos++;
httpProcessMultiSql(pContext);
}
}
void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) {
int32_t code = taos_errno(result);
httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessMultiSqlRetrieveCallBackImp);
}
void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
HttpSqlCmds * multiCmds = pContext->multiCmds;
HttpEncodeMethod *encode = pContext->encodeMethod;
HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos;
char * sql = httpGetCmdsString(pContext, singleCmd->sql);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
httpWarn("context:%p, fd:%d, user:%s, process pos:%d, code:%s:inprogress, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, tstrerror(code), sql);
return;
}
if (code != TSDB_CODE_SUCCESS) {
if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code)) {
singleCmd->code = code;
httpDebug("context:%p, fd:%d, user:%s, process pos jump to:%d, last code:%s, last sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos + 1, tstrerror(code), sql);
} else {
singleCmd->code = code;
httpError("context:%p, fd:%d, user:%s, process pos:%d, error code:%s, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, tstrerror(code), sql);
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN) {
if (encode->startJsonFp) (encode->startJsonFp)(pContext, singleCmd, result);
if (encode->stopJsonFp) (encode->stopJsonFp)(pContext, singleCmd);
}
}
multiCmds->pos++;
httpProcessMultiSql(pContext);
taos_free_result(result);
return;
}
bool isUpdate = tscIsUpdateQuery(result);
if (isUpdate) {
// not select or show commands
int32_t affectRows = taos_affected_rows(result);
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, affect rows:%d, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, affectRows, sql);
singleCmd->code = 0;
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->startJsonFp) {
(encode->startJsonFp)(pContext, singleCmd, result);
}
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->buildAffectRowJsonFp) {
(encode->buildAffectRowJsonFp)(pContext, singleCmd, affectRows);
}
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) {
(encode->stopJsonFp)(pContext, singleCmd);
}
if (encode->setNextCmdFp) {
(encode->setNextCmdFp)(pContext, singleCmd, code);
} else {
multiCmds->pos++;
}
taos_free_result(result);
httpProcessMultiSql(pContext);
} else {
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, start retrieve, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, sql);
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->startJsonFp) {
(encode->startJsonFp)(pContext, singleCmd, result);
}
taos_fetch_rows_a(result, httpProcessMultiSqlRetrieveCallBack, pContext);
}
}
void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) {
int32_t code = taos_errno(result);
int32_t affectRows = taos_affected_rows(result);
httpDispatchToResultQueue(param, result, code, affectRows, httpProcessMultiSqlCallBackImp);
}
void httpProcessMultiSql(HttpContext *pContext) {
HttpSqlCmds * multiCmds = pContext->multiCmds;
HttpEncodeMethod *encode = pContext->encodeMethod;
if (multiCmds->pos >= multiCmds->size) {
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, size:%d, stop mulit-querys", pContext, pContext->fd,
pContext->user, multiCmds->pos, multiCmds->size);
if (encode->cleanJsonFp) {
(encode->cleanJsonFp)(pContext);
}
httpCloseContextByApp(pContext);
return;
}
HttpSqlCmd *cmd = multiCmds->cmds + multiCmds->pos;
char *sql = httpGetCmdsString(pContext, cmd->sql);
httpTraceL("context:%p, fd:%d, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd, pContext->user,
multiCmds->pos, sql);
nPrintHttp("%s", sql);
taos_query_a(pContext->session->taos, sql, httpProcessMultiSqlCallBack, (void *)pContext);
}
void httpProcessMultiSqlCmd(HttpContext *pContext) {
if (pContext == NULL) return;
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (multiCmds == NULL || multiCmds->size <= 0 || multiCmds->pos >= multiCmds->size || multiCmds->pos < 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_MULTI_REQUEST);
return;
}
httpDebug("context:%p, fd:%d, user:%s, start multi-querys pos:%d, size:%d", pContext, pContext->fd, pContext->user,
multiCmds->pos, multiCmds->size);
HttpEncodeMethod *encode = pContext->encodeMethod;
if (encode->initJsonFp) {
(encode->initJsonFp)(pContext);
}
httpProcessMultiSql(pContext);
}
void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows);
void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
HttpEncodeMethod *encode = pContext->encodeMethod;
bool isContinue = false;
if (code == TSDB_CODE_SUCCESS && numOfRows > 0) {
if (encode->buildQueryJsonFp) {
isContinue = (encode->buildQueryJsonFp)(pContext, &pContext->singleCmd, result, numOfRows);
}
}
if (isContinue) {
// retrieve next batch of rows
httpDebug("context:%p, fd:%d, user:%s, continue retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user,
numOfRows);
taos_fetch_rows_a(result, httpProcessSingleSqlRetrieveCallBack, param);
} else {
httpDebug("context:%p, fd:%d, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user,
numOfRows);
if (code < 0) {
httpError("context:%p, fd:%d, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->user,
tstrerror(code));
}
taos_free_result(result);
if (encode->stopJsonFp) {
(encode->stopJsonFp)(pContext, &pContext->singleCmd);
}
httpCloseContextByApp(pContext);
}
}
void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) {
int32_t code = taos_errno(result);
httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessSingleSqlRetrieveCallBackImp);
}
void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
HttpEncodeMethod *encode = pContext->encodeMethod;
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
httpError("context:%p, fd:%d, user:%s, query error, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd,
pContext->user, tstrerror(code), result);
return;
}
if (code != TSDB_CODE_SUCCESS) {
SSqlObj *pObj = (SSqlObj *)result;
if (code == TSDB_CODE_TSC_INVALID_OPERATION) {
terrno = code;
httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, pContext->fd,
pContext->user, tstrerror(code), pObj, taos_errstr(pObj));
httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj));
} else {
httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, pContext->user,
tstrerror(code), pObj);
httpSendErrorResp(pContext, code);
}
taos_free_result(result);
return;
}
bool isUpdate = tscIsUpdateQuery(result);
if (isUpdate) {
// not select or show commands
int32_t affectRows = taos_affected_rows(result);
assert(affectRows == affectRowsInput);
httpDebug("context:%p, fd:%d, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd,
pContext->user, affectRows, result);
if (encode->startJsonFp) {
(encode->startJsonFp)(pContext, &pContext->singleCmd, result);
}
if (encode->buildAffectRowJsonFp) {
(encode->buildAffectRowJsonFp)(pContext, &pContext->singleCmd, affectRows);
}
if (encode->stopJsonFp) {
(encode->stopJsonFp)(pContext, &pContext->singleCmd);
}
taos_free_result(result);
httpCloseContextByApp(pContext);
} else {
httpDebug("context:%p, fd:%d, user:%s, start retrieve", pContext, pContext->fd, pContext->user);
if (encode->startJsonFp) {
(encode->startJsonFp)(pContext, &pContext->singleCmd, result);
}
taos_fetch_rows_a(result, httpProcessSingleSqlRetrieveCallBack, pContext);
}
}
void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) {
int32_t code = taos_errno(result);
int32_t affectRows = taos_affected_rows(result);
httpDispatchToResultQueue(param, result, code, affectRows, httpProcessSingleSqlCallBackImp);
}
void httpProcessSingleSqlCmd(HttpContext *pContext) {
HttpSqlCmd * cmd = &pContext->singleCmd;
char * sql = cmd->nativSql;
HttpSession *pSession = pContext->session;
if (sql == NULL || sql[0] == 0) {
httpError("context:%p, fd:%d, user:%s, error:no sql input", pContext, pContext->fd, pContext->user);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT);
return;
}
httpTraceL("context:%p, fd:%d, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->user, sql);
nPrintHttp("%s", sql);
taos_query_a(pSession->taos, sql, httpProcessSingleSqlCallBack, (void *)pContext);
}
void httpProcessLoginCmd(HttpContext *pContext) {
char token[128] = {0};
if (httpGenTaosdAuthToken(pContext, token, 128) != 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR);
} else {
httpDebug("context:%p, fd:%d, user:%s, login via http, return token:%s", pContext, pContext->fd, pContext->user,
token);
httpSendSuccResp(pContext, token);
}
}
void httpProcessHeartBeatCmd(HttpContext *pContext) {
HttpEncodeMethod *encode = pContext->encodeMethod;
if (encode->startJsonFp) {
(encode->startJsonFp)(pContext, &pContext->singleCmd, NULL);
}
if (encode->stopJsonFp) {
(encode->stopJsonFp)(pContext, &pContext->singleCmd);
}
httpCloseContextByApp(pContext);
}
void httpExecCmd(HttpContext *pContext) {
switch (pContext->reqType) {
case HTTP_REQTYPE_LOGIN:
httpProcessLoginCmd(pContext);
break;
case HTTP_REQTYPE_SINGLE_SQL:
httpProcessSingleSqlCmd(pContext);
break;
case HTTP_REQTYPE_MULTI_SQL:
httpProcessMultiSqlCmd(pContext);
break;
case HTTP_REQTYPE_HEARTBEAT:
httpProcessHeartBeatCmd(pContext);
break;
case HTTP_REQTYPE_OTHERS:
httpCloseContextByApp(pContext);
break;
default:
httpCloseContextByApp(pContext);
break;
}
memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd));
}
void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) {
HttpContext *pContext = param;
taos_free_result(result);
if (pContext == NULL) return;
if (code < 0) {
httpError("context:%p, fd:%d, user:%s, login error, code:%s", pContext, pContext->fd, pContext->user,
tstrerror(code));
httpSendErrorResp(pContext, code);
return;
}
httpDebug("context:%p, fd:%d, user:%s, connect tdengine success, taos:%p", pContext, pContext->fd, pContext->user,
pContext->taos);
if (pContext->taos == NULL) {
httpError("context:%p, fd:%d, user:%s, login error, taos is empty", pContext, pContext->fd, pContext->user);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_LOGIN_FAILED);
return;
}
httpCreateSession(pContext, pContext->taos);
if (pContext->session == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_SESSION_FULL);
} else {
httpExecCmd(pContext);
}
}
void httpProcessRequest(HttpContext *pContext) {
httpGetSession(pContext);
if (pContext->session == NULL || pContext->reqType == HTTP_REQTYPE_LOGIN) {
taos_connect_a(NULL, pContext->user, pContext->pass, "", 0, httpProcessRequestCb, (void *)pContext,
&(pContext->taos));
httpDebug("context:%p, fd:%d, user:%s, try connect tdengine, taos:%p", pContext, pContext->fd, pContext->user,
pContext->taos);
if (pContext->taos != NULL) {
STscObj *pObj = pContext->taos;
pObj->from = TAOS_REQ_FROM_HTTP;
}
} else {
httpExecCmd(pContext);
}
}
int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql)
{
char *pos;
if (oldSql == NULL || newSql == NULL) {
return TSDB_CODE_SUCCESS;
}
/* bad sql clause */
pos = strstr(oldSql, "%%");
if (pos) {
httpError("bad sql:%s", oldSql);
return TSDB_CODE_HTTP_REQUEST_JSON_ERROR;
}
pos = strchr(oldSql, '%');
if (pos == NULL) {
httpDebug("sql:%s", oldSql);
*newSql = oldSql;
return TSDB_CODE_SUCCESS;
}
*newSql = (char *) calloc(1, (strlen(oldSql) << 1) + 1);
if (newSql == NULL) {
httpError("failed to allocate for new sql, old sql:%s", oldSql);
return TSDB_CODE_HTTP_NO_ENOUGH_MEMORY;
}
char *src = oldSql;
char *dst = *newSql;
size_t sqlLen = strlen(src);
while (1) {
memcpy(dst, src, pos - src + 1);
dst += pos - src + 1;
*dst++ = '%';
if (pos + 1 >= oldSql + sqlLen) {
break;
}
src = ++pos;
pos = strchr(pos, '%');
if (pos == NULL) {
memcpy(dst, src, strlen(src));
break;
}
}
return TSDB_CODE_SUCCESS;
}
void httpCheckFreeEscapedSql(char *oldSql, char *newSql)
{
if (oldSql && newSql) {
if (oldSql != newSql) {
free(newSql);
}
}
}

123
src/plugins/http/src/httpSystem.c
View File

@ -1,123 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "../../../../include/client/taos.h"
#include "httpContext.h"
#include "httpGcHandle.h"
#include "httpHandle.h"
#include "httpInt.h"
#include "httpMetricsHandle.h"
#include "httpQueue.h"
#include "httpResp.h"
#include "httpRestHandle.h"
#include "httpServer.h"
#include "httpSession.h"
#include "httpTgHandle.h"
#include "os.h"
#include "tadmin.h"
#include "tglobal.h"
#include "tsocket.h"
#include "ttimer.h"
#ifndef _ADMIN
void adminInitHandle(HttpServer* pServer) {}
void opInitHandle(HttpServer* pServer) {}
#endif
HttpServer tsHttpServer;
int32_t httpInitSystem() {
strcpy(tsHttpServer.label, "rest");
tsHttpServer.serverIp = 0;
tsHttpServer.serverPort = tsHttpPort;
tsHttpServer.numOfThreads = tsHttpMaxThreads;
tsHttpServer.processData = httpProcessData;
pthread_mutex_init(&tsHttpServer.serverMutex, NULL);
restInitHandle(&tsHttpServer);
adminInitHandle(&tsHttpServer);
gcInitHandle(&tsHttpServer);
tgInitHandle(&tsHttpServer);
opInitHandle(&tsHttpServer);
metricsInitHandle(&tsHttpServer);
return 0;
}
int32_t httpStartSystem() {
httpInfo("start http server ...");
if (tsHttpServer.status != HTTP_SERVER_INIT) {
httpError("http server is already started");
return -1;
}
if (!httpInitResultQueue()) {
httpError("http init result queue failed");
return -1;
}
if (!httpInitContexts()) {
httpError("http init contexts failed");
return -1;
}
if (!httpInitSessions()) {
httpError("http init session failed");
return -1;
}
if (!httpInitConnect()) {
httpError("http init server failed");
return -1;
}
return 0;
}
void httpStopSystem() {
tsHttpServer.status = HTTP_SERVER_CLOSING;
tsHttpServer.stop = 1;
#ifdef WINDOWS
closesocket(tsHttpServer.fd);
#elif __APPLE__
if (tsHttpServer.fd!=-1) {
close(tsHttpServer.fd);
tsHttpServer.fd = -1;
}
#else
shutdown(tsHttpServer.fd, SHUT_RD);
#endif
tgCleanupHandle();
}
void httpCleanUpSystem() {
httpInfo("http server cleanup");
httpStopSystem();
httpCleanUpConnect();
httpCleanupContexts();
httpCleanUpSessions();
httpCleanupResultQueue();
pthread_mutex_destroy(&tsHttpServer.serverMutex);
tfree(tsHttpServer.pThreads);
tsHttpServer.pThreads = NULL;
tsHttpServer.status = HTTP_SERVER_CLOSED;
}
int32_t httpGetReqCount() { return atomic_exchange_32(&tsHttpServer.requestNum, 0); }

917
src/plugins/http/src/httpTgHandle.c
View File

@ -1,917 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tglobal.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "httpInt.h"
#include "httpTgHandle.h"
#include "httpTgJson.h"
#include "cJSON.h"
/*
* taos.telegraf.cfg formats like
{
"metrics": [
{
"name" : "system",
"tbname" : "system_uptime",
"fields": [
"uptime"
]
},
{
"name": "system",
"tbname" : "system_uptime_format",
"fields": [
"uptime_format"
]
},
{
"name": "swap",
"tbname" : "swap_in",
"fields": [
"in"
]
},
{
"name": "cpu",
"tbname" : "cpu_usage",
"fields": [
"usage_active",
"usage_guest"
]
}
]
}
*/
#define TG_MAX_SORT_TAG_SIZE 20
static HttpDecodeMethod tgDecodeMethod = {"telegraf", tgProcessRquest};
static HttpEncodeMethod tgQueryMethod = {
.startJsonFp = tgStartQueryJson,
.stopJsonFp = tgStopQueryJson,
.buildQueryJsonFp = NULL,
.buildAffectRowJsonFp = tgBuildSqlAffectRowsJson,
.initJsonFp = tgInitQueryJson,
.cleanJsonFp = tgCleanQueryJson,
.checkFinishedFp = tgCheckFinished,
.setNextCmdFp = tgSetNextCmd
};
static const char DEFAULT_TELEGRAF_CFG[] =
"{\"metrics\":["
"{\"name\":\"system\",\"tbname\":\"system_uptime\",\"fields\":[\"uptime\"]},"
"{\"name\":\"system\",\"tbname\":\"system_uptime_format\",\"fields\":[\"uptime_format\"]},"
"{\"name\":\"swap\",\"tbname\":\"swap_in\",\"fields\":[\"in\"]},"
"{\"name\":\"cpu\",\"tbname\":\"cpu_usage\",\"fields\":[\"usage_guest\"]}"
"]}";
typedef struct {
char * name;
char * tbName;
char ** fields;
int32_t fieldNum;
} STgSchema;
typedef struct {
STgSchema *schemas;
int32_t size;
int32_t pos;
} STgSchemas;
static STgSchemas tgSchemas = {0};
void tgFreeSchema(STgSchema *schema) {
if (schema->name != NULL) {
free(schema->name);
schema->name = NULL;
}
if (schema->tbName != NULL) {
free(schema->tbName);
schema->tbName = NULL;
}
if (schema->fields != NULL) {
for (int32_t f = 0; f < schema->fieldNum; ++f) {
if (schema->fields[f] != NULL) {
free(schema->fields[f]);
schema->fields[f] = NULL;
}
}
free(schema->fields);
schema->fields = NULL;
schema->fieldNum = 0;
}
}
void tgFreeSchemas() {
if (tgSchemas.schemas != NULL) {
for (int32_t s = 0; s < tgSchemas.size; ++s) {
tgFreeSchema(&tgSchemas.schemas[s]);
}
free(tgSchemas.schemas);
tgSchemas.size = 0;
tgSchemas.schemas = NULL;
}
}
void tgInitSchemas(int32_t size) {
tgFreeSchemas();
tgSchemas.schemas = calloc(sizeof(STgSchema), size);
tgSchemas.size = 0;
}
void tgParseSchemaMetric(cJSON *metric) {
STgSchema schema = {0};
bool parsedOk = true;
// name
cJSON *name = cJSON_GetObjectItem(metric, "name");
if (name == NULL) {
parsedOk = false;
goto ParseEnd;
}
if (name->type != cJSON_String) {
parsedOk = false;
goto ParseEnd;
}
if (name->valuestring == NULL) {
parsedOk = false;
goto ParseEnd;
}
int32_t nameLen = (int32_t)strlen(name->valuestring);
if (nameLen == 0) {
parsedOk = false;
goto ParseEnd;
}
schema.name = calloc(nameLen + 1, 1);
strcpy(schema.name, name->valuestring);
// tbname
cJSON *tbname = cJSON_GetObjectItem(metric, "tbname");
if (tbname == NULL) {
parsedOk = false;
goto ParseEnd;
}
if (tbname->type != cJSON_String) {
parsedOk = false;
goto ParseEnd;
}
if (tbname->valuestring == NULL) {
parsedOk = false;
goto ParseEnd;
}
int32_t tbnameLen = (int32_t)strlen(tbname->valuestring);
if (tbnameLen == 0) {
parsedOk = false;
goto ParseEnd;
}
schema.tbName = calloc(tbnameLen + 1, 1);
strcpy(schema.tbName, tbname->valuestring);
// fields
cJSON *fields = cJSON_GetObjectItem(metric, "fields");
if (fields == NULL) {
goto ParseEnd;
}
int32_t fieldSize = cJSON_GetArraySize(fields);
if (fieldSize <= 0 || fieldSize > TSDB_MAX_COLUMNS) {
goto ParseEnd;
}
if (fieldSize > 0) {
schema.fields = calloc(sizeof(STgSchema), (size_t)fieldSize);
schema.fieldNum = fieldSize;
for (int32_t i = 0; i < fieldSize; i++) {
cJSON *field = cJSON_GetArrayItem(fields, i);
if (field == NULL) {
parsedOk = false;
goto ParseEnd;
}
if (field->valuestring == NULL) {
parsedOk = false;
goto ParseEnd;
}
nameLen = (int32_t)strlen(field->valuestring);
if (nameLen == 0 || nameLen >= TSDB_TABLE_NAME_LEN) {
parsedOk = false;
goto ParseEnd;
}
schema.fields[i] = calloc(nameLen + 1, 1);
strcpy(schema.fields[i], field->valuestring);
}
}
ParseEnd:
if (parsedOk) {
tgSchemas.schemas[tgSchemas.size++] = schema;
} else {
tgFreeSchema(&schema);
}
}
int32_t tgParseSchema(const char *content, char *fileName) {
cJSON *root = cJSON_Parse(content);
if (root == NULL) {
httpError("failed to parse telegraf schema file:%s, invalid json format, content:%s", fileName, content);
return -1;
}
int32_t size = 0;
cJSON * metrics = cJSON_GetObjectItem(root, "metrics");
if (metrics != NULL) {
size = cJSON_GetArraySize(metrics);
if (size <= 0) {
httpError("failed to parse telegraf schema file:%s, metrics size is 0", fileName);
cJSON_Delete(root);
return -1;
}
tgInitSchemas(size);
for (int32_t i = 0; i < size; i++) {
cJSON *metric = cJSON_GetArrayItem(metrics, i);
if (metric != NULL) {
tgParseSchemaMetric(metric);
}
}
} else {
size = 1;
tgInitSchemas(size);
tgParseSchemaMetric(root);
}
cJSON_Delete(root);
return size;
}
int32_t tgReadSchema(char *fileName) {
FILE *fp = fopen(fileName, "r");
if (fp == NULL) {
return -1;
}
httpInfo("open telegraf schema file:%s success", fileName);
fseek(fp, 0, SEEK_END);
int32_t contentSize = (int32_t)ftell(fp);
if (contentSize <= 0) {
fclose(fp);
return 0;
}
rewind(fp);
char * content = (char *)calloc(contentSize + 1, 1);
int32_t result = (int32_t)fread(content, 1, contentSize, fp);
if (result != contentSize) {
httpError("failed to read telegraf schema file:%s", fileName);
fclose(fp);
free(content);
return 0;
}
content[contentSize] = 0;
int32_t schemaNum = tgParseSchema(content, fileName);
free(content);
fclose(fp);
httpInfo("parse telegraf schema file:%s, schema size:%d", fileName, schemaNum);
return schemaNum;
}
void tgInitHandle(HttpServer *pServer) {
char fileName[TSDB_FILENAME_LEN * 2] = {0};
sprintf(fileName, "%s/taos.telegraf.cfg", configDir);
if (tgReadSchema(fileName) <= 0) {
tgFreeSchemas();
if (tgParseSchema(DEFAULT_TELEGRAF_CFG, "default") <= 0) {
tgFreeSchemas();
}
}
httpAddMethod(pServer, &tgDecodeMethod);
}
void tgCleanupHandle() { tgFreeSchemas(); }
bool tgGetUserFromUrl(HttpContext *pContext) {
HttpParser *pParser = pContext->parser;
if (pParser->path[TG_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[TG_USER_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->user, pParser->path[TG_USER_URL_POS].str, sizeof(pContext->user));
return true;
}
bool tgGetPassFromUrl(HttpContext *pContext) {
HttpParser *pParser = pContext->parser;
if (pParser->path[TG_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->pass, pParser->path[TG_PASS_URL_POS].str, sizeof(pContext->pass));
return true;
}
char *tgGetDbFromUrl(HttpContext *pContext) {
HttpParser *pParser = pContext->parser;
if (pParser->path[TG_DB_URL_POS].pos <= 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_DB_NOT_INPUT);
return NULL;
}
if (pParser->path[TG_DB_URL_POS].pos >= TSDB_DB_NAME_LEN) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_DB_TOO_LONG);
return NULL;
}
return pParser->path[TG_DB_URL_POS].str;
}
char *tgGetStableName(char *stname, cJSON *fields, int32_t fieldsSize) {
for (int32_t s = 0; s < tgSchemas.size; ++s) {
STgSchema *schema = &tgSchemas.schemas[s];
if (strcasecmp(schema->name, stname) != 0) {
continue;
}
bool schemaMatched = true;
for (int32_t f = 0; f < schema->fieldNum; ++f) {
char *fieldName = schema->fields[f];
bool fieldMatched = false;
for (int32_t i = 0; i < fieldsSize; i++) {
cJSON *field = cJSON_GetArrayItem(fields, i);
if (strcasecmp(field->string, fieldName) == 0) {
fieldMatched = true;
break;
}
}
if (!fieldMatched) {
schemaMatched = false;
break;
}
}
if (schemaMatched) {
return schema->tbName;
}
}
return stname;
}
/*
* parse single metric
{
"fields": {
"field_1": 30,
"field_2": 4,
"field_N": 59,
"n_images": 660
},
"name": "docker",
"tags": {
"host": "raynor"
},
"timestamp": 1458229140
}
*/
bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
// metric name
cJSON *name = cJSON_GetObjectItem(metric, "name");
if (name == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NULL);
return false;
}
if (name->type != cJSON_String) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_TYPE);
return false;
}
if (name->valuestring == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_NULL);
return false;
}
int32_t nameLen = (int32_t)strlen(name->valuestring);
if (nameLen == 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_NULL);
return false;
}
if (nameLen >= TSDB_TABLE_NAME_LEN - 8) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_LONG);
return false;
}
// timestamp
cJSON *timestamp = cJSON_GetObjectItem(metric, "timestamp");
if (timestamp == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_NULL);
return false;
}
if (timestamp->type != cJSON_Number) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE);
return false;
}
if (timestamp->valueint <= 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL);
return false;
}
// tags
cJSON *tags = cJSON_GetObjectItem(metric, "tags");
if (tags == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_NULL);
return false;
}
int32_t tagsSize = cJSON_GetArraySize(tags);
if (tagsSize <= 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_SIZE_0);
return false;
}
if (tagsSize > TG_MAX_SORT_TAG_SIZE) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG);
return false;
}
cJSON *host = NULL;
for (int32_t i = 0; i < tagsSize; i++) {
cJSON *tag = cJSON_GetArrayItem(tags, i);
if (tag == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NULL);
return false;
}
if (tag->string == NULL || strlen(tag->string) == 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_NULL);
return false;
}
/*
* tag size may be larget than TSDB_COL_NAME_LEN
* we keep the first TSDB_COL_NAME_LEN bytes
*/
if (0) {
if (strlen(tag->string) >= TSDB_COL_NAME_LEN) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_SIZE);
return false;
}
}
if (tag->type != cJSON_Number && tag->type != cJSON_String) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE);
return false;
}
if (tag->type == cJSON_String) {
if (tag->valuestring == NULL || strlen(tag->valuestring) == 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_VALUE_NULL);
return false;
}
}
if (strcasecmp(tag->string, "host") == 0) {
host = tag;
}
}
if (host == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TABLE_NULL);
return false;
}
if (host->type != cJSON_String) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_HOST_NOT_STRING);
return false;
}
if (strlen(host->valuestring) >= TSDB_TABLE_NAME_LEN - 1) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TABLE_SIZE);
return false;
}
// fields
cJSON *fields = cJSON_GetObjectItem(metric, "fields");
if (fields == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_NULL);
return false;
}
int32_t fieldsSize = cJSON_GetArraySize(fields);
if (fieldsSize <= 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_SIZE_0);
return false;
}
if (fieldsSize > (TSDB_MAX_COLUMNS - TSDB_MAX_TAGS - 1)) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG);
return false;
}
for (int32_t i = 0; i < fieldsSize; i++) {
cJSON *field = cJSON_GetArrayItem(fields, i);
if (field == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NULL);
return false;
}
if (field->string == NULL || strlen(field->string) == 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_NULL);
return false;
}
/*
* tag size may be larget than TSDB_COL_NAME_LEN
* we keep the first TSDB_COL_NAME_LEN bytes
*/
if (0) {
if (strlen(field->string) >= TSDB_COL_NAME_LEN) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE);
return false;
}
}
if (field->type != cJSON_Number && field->type != cJSON_String) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE);
return false;
}
if (field->type == cJSON_String) {
if (field->valuestring == NULL || strlen(field->valuestring) == 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL);
return false;
}
}
}
// assembling cmds
HttpSqlCmd *stable_cmd = httpNewSqlCmd(pContext);
if (stable_cmd == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
return false;
}
stable_cmd->cmdType = HTTP_CMD_TYPE_CREATE_STBALE;
stable_cmd->cmdReturnType = HTTP_CMD_RETURN_TYPE_NO_RETURN;
HttpSqlCmd *table_cmd = httpNewSqlCmd(pContext);
if (table_cmd == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
return false;
}
table_cmd->cmdType = HTTP_CMD_TYPE_INSERT;
// order by tag name
cJSON * orderedTags[TG_MAX_SORT_TAG_SIZE] = {0};
int32_t orderTagsLen = 0;
for (int32_t i = 0; i < tagsSize; ++i) {
cJSON *tag = cJSON_GetArrayItem(tags, i);
orderedTags[orderTagsLen++] = tag;
for (int32_t j = orderTagsLen - 1; j >= 1; --j) {
cJSON *tag1 = orderedTags[j];
cJSON *tag2 = orderedTags[j - 1];
if (strcasecmp(tag1->string, "host") == 0 || strcmp(tag1->string, tag2->string) < 0) {
orderedTags[j] = tag2;
orderedTags[j - 1] = tag1;
}
}
}
orderTagsLen = orderTagsLen < TSDB_MAX_TAGS ? orderTagsLen : TSDB_MAX_TAGS;
table_cmd->tagNum = stable_cmd->tagNum = (int8_t)orderTagsLen;
table_cmd->timestamp = stable_cmd->timestamp = httpAddToSqlCmdBuffer(pContext, "%" PRId64, timestamp->valueint);
// stable name
char *stname = tgGetStableName(name->valuestring, fields, fieldsSize);
table_cmd->metric = stable_cmd->metric = httpAddToSqlCmdBuffer(pContext, "%s", stname);
if (tsTelegrafUseFieldNum == 0) {
table_cmd->stable = stable_cmd->stable = httpAddToSqlCmdBuffer(pContext, "%s", stname);
} else {
table_cmd->stable = stable_cmd->stable =
httpAddToSqlCmdBuffer(pContext, "%s_%d_%d", stname, fieldsSize, orderTagsLen);
}
table_cmd->stable = stable_cmd->stable =
httpShrinkTableName(pContext, table_cmd->stable, httpGetCmdsString(pContext, table_cmd->stable));
// stable tag for detail
for (int32_t i = 0; i < orderTagsLen; ++i) {
cJSON *tag = orderedTags[i];
char *tagStr = NULL;
int32_t retCode = httpCheckAllocEscapeSql(tag->string, &tagStr);
if (retCode != TSDB_CODE_SUCCESS) {
httpSendErrorResp(pContext, retCode);
return false;
}
stable_cmd->tagNames[i] = table_cmd->tagNames[i] = httpAddToSqlCmdBuffer(pContext, tagStr);
httpCheckFreeEscapedSql(tag->string, tagStr);
if (tag->type == cJSON_String)
stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "'%s'", tag->valuestring);
else if (tag->type == cJSON_Number)
stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "%" PRId64, tag->valueint);
else if (tag->type == cJSON_True)
stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "1");
else if (tag->type == cJSON_False)
stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "0");
else
stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "NULL");
}
// table name
if (tsTelegrafUseFieldNum == 0) {
table_cmd->table = stable_cmd->table =
httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%s", stname, host->valuestring);
} else {
table_cmd->table = stable_cmd->table =
httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring);
}
for (int32_t i = 0; i < orderTagsLen; ++i) {
cJSON *tag = orderedTags[i];
if (tag == host) continue;
if (tag->type == cJSON_String)
httpAddToSqlCmdBufferNoTerminal(pContext, "_%s", tag->valuestring);
else if (tag->type == cJSON_Number)
httpAddToSqlCmdBufferNoTerminal(pContext, "_%" PRId64, tag->valueint);
else if (tag->type == cJSON_False)
httpAddToSqlCmdBufferNoTerminal(pContext, "_0");
else if (tag->type == cJSON_True)
httpAddToSqlCmdBufferNoTerminal(pContext, "_1");
else
httpAddToSqlCmdBufferNoTerminal(pContext, "_n");
}
httpAddToSqlCmdBuffer(pContext, "");
table_cmd->table = stable_cmd->table =
httpShrinkTableName(pContext, table_cmd->table, httpGetCmdsString(pContext, table_cmd->table));
// assembling create stable sql
stable_cmd->sql = httpAddToSqlCmdBufferNoTerminal(pContext, "create table if not exists %s.%s(ts timestamp", db,
httpGetCmdsString(pContext, table_cmd->stable));
for (int32_t i = 0; i < fieldsSize; ++i) {
cJSON *field = cJSON_GetArrayItem(fields, i);
char * field_type = "double";
if (field->type == cJSON_String)
field_type = "binary(32)";
else if (field->type == cJSON_False || field->type == cJSON_True)
field_type = "tinyint";
else {
}
char *field_name = field->string;
httpAddToSqlCmdBufferNoTerminal(pContext, ",f_%s %s", field_name, field_type);
}
httpAddToSqlCmdBufferNoTerminal(pContext, ") tags(");
for (int32_t i = 0; i < orderTagsLen; ++i) {
cJSON *tag = orderedTags[i];
char * tag_type = "bigint";
if (tag->type == cJSON_String)
tag_type = "binary(32)";
else if (tag->type == cJSON_False || tag->type == cJSON_True)
tag_type = "tinyint";
else {
}
char *tag_name = tag->string;
if (i != orderTagsLen - 1)
httpAddToSqlCmdBufferNoTerminal(pContext, "t_%s %s,", tag_name, tag_type);
else
httpAddToSqlCmdBuffer(pContext, "t_%s %s)", tag_name, tag_type);
}
// assembling insert sql
table_cmd->sql = httpAddToSqlCmdBufferNoTerminal(pContext, "import into %s.%s using %s.%s tags(", db,
httpGetCmdsString(pContext, table_cmd->table), db,
httpGetCmdsString(pContext, table_cmd->stable));
for (int32_t i = 0; i < orderTagsLen; ++i) {
cJSON *tag = orderedTags[i];
if (i != orderTagsLen - 1) {
if (tag->type == cJSON_Number)
httpAddToSqlCmdBufferNoTerminal(pContext, "%" PRId64 ",", tag->valueint);
else if (tag->type == cJSON_String)
httpAddToSqlCmdBufferNoTerminal(pContext, "'%s',", tag->valuestring);
else if (tag->type == cJSON_False)
httpAddToSqlCmdBufferNoTerminal(pContext, "0,");
else if (tag->type == cJSON_True)
httpAddToSqlCmdBufferNoTerminal(pContext, "1,");
else {
httpAddToSqlCmdBufferNoTerminal(pContext, "NULL,");
}
} else {
if (tag->type == cJSON_Number)
httpAddToSqlCmdBufferNoTerminal(pContext, "%" PRId64 ")", tag->valueint);
else if (tag->type == cJSON_String)
httpAddToSqlCmdBufferNoTerminal(pContext, "'%s')", tag->valuestring);
else if (tag->type == cJSON_False)
httpAddToSqlCmdBufferNoTerminal(pContext, "0)");
else if (tag->type == cJSON_True)
httpAddToSqlCmdBufferNoTerminal(pContext, "1)");
else {
httpAddToSqlCmdBufferNoTerminal(pContext, "NULL)");
}
}
}
httpAddToSqlCmdBufferNoTerminal(pContext, " values(%" PRId64 ",", timestamp->valueint);
for (int32_t i = 0; i < fieldsSize; ++i) {
cJSON *field = cJSON_GetArrayItem(fields, i);
if (i != fieldsSize - 1) {
if (field->type == cJSON_Number)
httpAddToSqlCmdBufferNoTerminal(pContext, "%lf,", field->valuedouble);
else if (field->type == cJSON_String)
httpAddToSqlCmdBufferNoTerminal(pContext, "'%s',", field->valuestring);
else if (field->type == cJSON_False)
httpAddToSqlCmdBufferNoTerminal(pContext, "0,");
else if (field->type == cJSON_True)
httpAddToSqlCmdBufferNoTerminal(pContext, "1,");
else {
httpAddToSqlCmdBufferNoTerminal(pContext, "NULL,");
}
} else {
if (field->type == cJSON_Number)
httpAddToSqlCmdBuffer(pContext, "%lf)", field->valuedouble);
else if (field->type == cJSON_String)
httpAddToSqlCmdBuffer(pContext, "'%s')", field->valuestring);
else if (field->type == cJSON_False)
httpAddToSqlCmdBuffer(pContext, "0)");
else if (field->type == cJSON_True)
httpAddToSqlCmdBuffer(pContext, "1)");
else {
httpAddToSqlCmdBuffer(pContext, "NULL)");
}
}
}
return true;
}
/**
* request from telegraf 1.7.0
* single request:
{
"fields": {
"field_1": 30,
"field_2": 4,
"field_N": 59,
"n_images": 660
},
"name": "docker",
"tags": {
"host": "raynor"
},
"timestamp": 1458229140
}
* multiple request:
{
"metrics": [
{
"fields": {
"field_1": 30,
"field_2": 4,
"field_N": 59,
"n_images": 660
},
"name": "docker",
"tags": {
"host": "raynor"
},
"timestamp": 1458229140
},
{
"fields": {
"field_1": 30,
"field_2": 4,
"field_N": 59,
"n_images": 660
},
"name": "docker",
"tags": {
"host": "raynor"
},orderTagsLen
"timestamp": 1458229140
}
]
}
*/
bool tgProcessQueryRequest(HttpContext *pContext, char *db) {
httpDebug("context:%p, fd:%d, process telegraf query msg", pContext, pContext->fd);
char *filter = pContext->parser->body.str;
if (filter == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_MSG_INPUT);
return false;
}
cJSON *root = cJSON_Parse(filter);
if (root == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_INVALID_JSON);
return false;
}
cJSON *metrics = cJSON_GetObjectItem(root, "metrics");
if (metrics != NULL) {
int32_t size = cJSON_GetArraySize(metrics);
httpDebug("context:%p, fd:%d, multiple metrics:%d at one time", pContext, pContext->fd, size);
if (size <= 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRICS_NULL);
cJSON_Delete(root);
return false;
}
int32_t cmdSize = size * 2 + 1;
if (cmdSize > HTTP_MAX_CMD_SIZE) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRICS_SIZE);
cJSON_Delete(root);
return false;
}
if (!httpMallocMultiCmds(pContext, cmdSize, HTTP_BUFFER_SIZE)) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
HttpSqlCmd *cmd = httpNewSqlCmd(pContext);
if (cmd == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
cmd->cmdType = HTTP_CMD_TYPE_CREATE_DB;
cmd->cmdReturnType = HTTP_CMD_RETURN_TYPE_NO_RETURN;
cmd->sql = httpAddToSqlCmdBuffer(pContext, "create database if not exists %s", db);
for (int32_t i = 0; i < size; i++) {
cJSON *metric = cJSON_GetArrayItem(metrics, i);
if (metric != NULL) {
if (!tgProcessSingleMetric(pContext, metric, db)) {
cJSON_Delete(root);
return false;
}
}
}
} else {
httpDebug("context:%p, fd:%d, single metric", pContext, pContext->fd);
if (!httpMallocMultiCmds(pContext, 3, HTTP_BUFFER_SIZE)) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
HttpSqlCmd *cmd = httpNewSqlCmd(pContext);
if (cmd == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
cmd->cmdType = HTTP_CMD_TYPE_CREATE_DB;
cmd->cmdReturnType = HTTP_CMD_RETURN_TYPE_NO_RETURN;
cmd->sql = httpAddToSqlCmdBuffer(pContext, "create database if not exists %s", db);
if (!tgProcessSingleMetric(pContext, root, db)) {
cJSON_Delete(root);
return false;
}
}
cJSON_Delete(root);
pContext->reqType = HTTP_REQTYPE_MULTI_SQL;
pContext->encodeMethod = &tgQueryMethod;
pContext->multiCmds->pos = 2;
return true;
}
bool tgProcessRquest(struct HttpContext *pContext) {
if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO);
return false;
}
char *db = tgGetDbFromUrl(pContext);
if (db == NULL) {
return false;
}
return tgProcessQueryRequest(pContext, db);
}

151
src/plugins/http/src/httpTgJson.c
View File

@ -1,151 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosmsg.h"
#include "httpLog.h"
#include "httpJson.h"
#include "httpResp.h"
#include "httpTgHandle.h"
#include "httpTgJson.h"
void tgInitQueryJson(HttpContext *pContext) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
httpInitJsonBuf(jsonBuf, pContext);
httpWriteJsonBufHead(jsonBuf);
// array begin
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonObjStt);
httpJsonPairHead(jsonBuf, "metrics", 7);
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
}
void tgCleanQueryJson(HttpContext *pContext) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
// array end
httpJsonToken(jsonBuf, JsonArrEnd);
httpJsonToken(jsonBuf, JsonObjEnd);
httpWriteJsonBufEnd(jsonBuf);
}
void tgStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
// object begin
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonObjStt);
// data
httpJsonItemToken(jsonBuf);
httpJsonPair(jsonBuf, "metric", 6, httpGetCmdsString(pContext, cmd->stable),
(int32_t)strlen(httpGetCmdsString(pContext, cmd->metric)));
httpJsonItemToken(jsonBuf);
httpJsonPair(jsonBuf, "stable", 6, httpGetCmdsString(pContext, cmd->stable),
(int32_t)strlen(httpGetCmdsString(pContext, cmd->stable)));
httpJsonItemToken(jsonBuf);
httpJsonPair(jsonBuf, "table", 5, httpGetCmdsString(pContext, cmd->table),
(int32_t)strlen(httpGetCmdsString(pContext, cmd->table)));
httpJsonItemToken(jsonBuf);
httpJsonPair(jsonBuf, "timestamp", 9, httpGetCmdsString(pContext, cmd->timestamp),
(int32_t)strlen(httpGetCmdsString(pContext, cmd->timestamp))); // hack way
}
void tgStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
// data
httpJsonItemToken(jsonBuf);
httpJsonPairStatus(jsonBuf, cmd->code);
// object end
httpJsonToken(jsonBuf, JsonObjEnd);
}
void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
// data
httpJsonPairIntVal(jsonBuf, "affected_rows", 13, affect_rows);
}
bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
httpDebug("context:%p, fd:%d, check telegraf command, code:%s, state:%d, type:%d, rettype:%d, tags:%d", pContext,
pContext->fd, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum);
if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) {
if (cmd->cmdState == HTTP_CMD_STATE_NOT_RUN_YET) {
if (code == TSDB_CODE_MND_DB_NOT_SELECTED || code == TSDB_CODE_MND_INVALID_DB) {
cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED;
if (multiCmds->cmds[0].cmdState == HTTP_CMD_STATE_NOT_RUN_YET) {
multiCmds->pos = (int16_t)-1;
httpDebug("context:%p, fd:%d, import failed, try create database", pContext, pContext->fd);
return false;
}
} else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED;
if (multiCmds->cmds[multiCmds->pos - 1].cmdState == HTTP_CMD_STATE_NOT_RUN_YET) {
multiCmds->pos = (int16_t)(multiCmds->pos - 2);
httpDebug("context:%p, fd:%d, import failed, try create stable", pContext, pContext->fd);
return false;
}
} else {
}
} else {
}
} else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_DB) {
cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED;
httpDebug("context:%p, fd:%d, code:%s, create database failed", pContext, pContext->fd, tstrerror(code));
} else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_STBALE) {
cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED;
httpDebug("context:%p, fd:%d, code:%s, create stable failed", pContext, pContext->fd, tstrerror(code));
} else {
}
return true;
}
void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
httpDebug("context:%p, fd:%d, get telegraf next command, pos:%d, code:%s, state:%d, type:%d, rettype:%d, tags:%d",
pContext, pContext->fd, multiCmds->pos, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType,
cmd->tagNum);
if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) {
multiCmds->pos = (int16_t)(multiCmds->pos + 2);
} else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_DB) {
multiCmds->pos++;
} else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_STBALE) {
multiCmds->pos++;
} else {
multiCmds->pos++;
}
}

497
src/plugins/http/src/httpUtil.c
View File

@ -1,497 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "httpUtil.h"
#include "../../../../include/client/taos.h"
#include "httpInt.h"
#include "httpResp.h"
#include "httpSql.h"
#include "os.h"
#include "tmd5.h"
#include "ttoken.h"
bool httpCheckUsedbSql(char *sql) {
if (strstr(sql, "use ") != NULL) {
return true;
}
return false;
}
bool httpCheckAlterSql(char *sql) {
int32_t index = 0;
do {
SStrToken t0 = tStrGetToken(sql, &index, false);
if (t0.type != TK_LP) {
return t0.type == TK_ALTER;
}
} while (1);
}
void httpTimeToString(int32_t t, char *buf, int32_t buflen) {
memset(buf, 0, (size_t)buflen);
char ts[32] = {0};
struct tm *ptm;
time_t tt = t / 1000;
ptm = localtime(&tt);
strftime(ts, 31, "%Y-%m-%d %H:%M:%S", ptm);
sprintf(buf, "%s.%03d", ts, t % 1000);
}
int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...) {
HttpSqlCmds *cmd = pContext->multiCmds;
if (cmd->buffer == NULL) return -1;
int32_t remainLength = cmd->bufferSize - cmd->bufferPos;
if (remainLength < 4096) {
if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1;
}
char * buffer = cmd->buffer + cmd->bufferPos;
int32_t len = 0;
va_list argpointer;
va_start(argpointer, format);
len += vsnprintf(buffer, (size_t)remainLength, format, argpointer);
va_end(argpointer);
if (cmd->bufferPos + len + 1 >= cmd->bufferSize) {
return -1;
}
cmd->buffer[cmd->bufferPos + len] = 0;
cmd->bufferPos = cmd->bufferPos + len + 1;
remainLength = cmd->bufferSize - cmd->bufferPos;
if (remainLength < 4096) {
if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1;
}
return (int32_t)(buffer - cmd->buffer);
}
int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const format, ...) {
HttpSqlCmds *cmd = pContext->multiCmds;
if (cmd->buffer == NULL) return -1;
int32_t remainLength = cmd->bufferSize - cmd->bufferPos;
if (remainLength < 4096) {
if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1;
}
char * buffer = cmd->buffer + cmd->bufferPos;
int32_t len = 0;
va_list argpointer;
va_start(argpointer, format);
len += vsnprintf(buffer, (size_t)remainLength, format, argpointer);
va_end(argpointer);
if (cmd->bufferPos + len + 1 >= cmd->bufferSize) {
return -1;
}
cmd->bufferPos = cmd->bufferPos + len;
remainLength = cmd->bufferSize - cmd->bufferPos;
if (remainLength < 4096) {
if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1;
}
return (int32_t)(buffer - cmd->buffer);
}
int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext) {
HttpSqlCmds *cmd = pContext->multiCmds;
if (cmd->buffer == NULL) return -1;
int32_t remainLength = cmd->bufferSize - cmd->bufferPos;
if (remainLength < 4096) {
if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1;
}
char *buffer = cmd->buffer + cmd->bufferPos;
*buffer = 0;
cmd->bufferPos = cmd->bufferPos + 1;
remainLength = cmd->bufferSize - cmd->bufferPos;
if (remainLength < 4096) {
if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1;
}
return (int32_t)(buffer - cmd->buffer);
}
int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int32_t mallocSize) {
HttpSqlCmds *cmd = pContext->multiCmds;
if (cmd->buffer == NULL) return -1;
if (cmd->bufferPos + mallocSize >= cmd->bufferSize) {
if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1;
}
char *buffer = cmd->buffer + cmd->bufferPos;
memset(cmd->buffer + cmd->bufferPos, 0, (size_t)mallocSize);
cmd->bufferPos = cmd->bufferPos + mallocSize;
return (int32_t)(buffer - cmd->buffer);
}
bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferSize) {
if (cmdSize > HTTP_MAX_CMD_SIZE) {
httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user,
cmdSize, HTTP_MAX_CMD_SIZE);
return false;
}
if (pContext->multiCmds == NULL) {
pContext->multiCmds = (HttpSqlCmds *)malloc(sizeof(HttpSqlCmds));
if (pContext->multiCmds == NULL) {
httpError("context:%p, fd:%d, user:%s, malloc multiCmds error", pContext, pContext->fd, pContext->user);
return false;
}
memset(pContext->multiCmds, 0, sizeof(HttpSqlCmds));
}
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (multiCmds->cmds == NULL || cmdSize > multiCmds->maxSize) {
free(multiCmds->cmds);
multiCmds->cmds = (HttpSqlCmd *)malloc((size_t)cmdSize * sizeof(HttpSqlCmd));
if (multiCmds->cmds == NULL) {
httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize);
return false;
}
multiCmds->maxSize = (int16_t)cmdSize;
}
if (multiCmds->buffer == NULL || bufferSize > multiCmds->bufferSize) {
free(multiCmds->buffer);
multiCmds->buffer = (char *)malloc((size_t)bufferSize);
if (multiCmds->buffer == NULL) {
httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user,
bufferSize);
return false;
}
multiCmds->bufferSize = bufferSize;
}
multiCmds->pos = 0;
multiCmds->size = 0;
multiCmds->bufferPos = 0;
memset(multiCmds->cmds, 0, (size_t)multiCmds->maxSize * sizeof(HttpSqlCmd));
return true;
}
bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (cmdSize <= 0 || cmdSize > HTTP_MAX_CMD_SIZE) {
httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user,
cmdSize, HTTP_MAX_CMD_SIZE);
return false;
}
HttpSqlCmd *new_cmds = (HttpSqlCmd *)realloc(multiCmds->cmds, (size_t)cmdSize * sizeof(HttpSqlCmd));
if (new_cmds == NULL && multiCmds->cmds) {
free(multiCmds->cmds);
}
multiCmds->cmds = new_cmds;
if (multiCmds->cmds == NULL) {
httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize);
return false;
}
memset(multiCmds->cmds + multiCmds->maxSize, 0, (size_t)(cmdSize - multiCmds->maxSize) * sizeof(HttpSqlCmd));
multiCmds->maxSize = (int16_t)cmdSize;
return true;
}
bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (bufferSize <= 0 || bufferSize > HTTP_MAX_BUFFER_SIZE) {
httpError("context:%p, fd:%d, user:%s, mulitcmd buffer size:%d large then %d", pContext, pContext->fd,
pContext->user, bufferSize, HTTP_MAX_BUFFER_SIZE);
return false;
}
char *new_buffer = (char *)realloc(multiCmds->buffer, (size_t)bufferSize);
if (new_buffer == NULL && multiCmds->buffer) {
free(multiCmds->buffer);
}
multiCmds->buffer = new_buffer;
if (multiCmds->buffer == NULL) {
httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user, bufferSize);
return false;
}
memset(multiCmds->buffer + multiCmds->bufferSize, 0, (size_t)(bufferSize - multiCmds->bufferSize));
multiCmds->bufferSize = bufferSize;
return true;
}
void httpFreeMultiCmds(HttpContext *pContext) {
if (pContext->multiCmds != NULL) {
if (pContext->multiCmds->buffer != NULL) free(pContext->multiCmds->buffer);
if (pContext->multiCmds->cmds != NULL) free(pContext->multiCmds->cmds);
free(pContext->multiCmds);
pContext->multiCmds = NULL;
}
}
JsonBuf *httpMallocJsonBuf(HttpContext *pContext) {
if (pContext->jsonBuf == NULL) {
pContext->jsonBuf = (JsonBuf *)malloc(sizeof(JsonBuf));
if (pContext->jsonBuf == NULL) {
return NULL;
}
memset(pContext->jsonBuf, 0, sizeof(JsonBuf));
}
if (!pContext->jsonBuf->pContext) {
pContext->jsonBuf->pContext = pContext;
}
return pContext->jsonBuf;
}
void httpFreeJsonBuf(HttpContext *pContext) {
if (pContext->jsonBuf != NULL) {
free(pContext->jsonBuf);
pContext->jsonBuf = 0;
}
}
bool httpCompareMethod(HttpDecodeMethod *pSrc, HttpDecodeMethod *pCmp) {
if (strcmp(pSrc->module, pCmp->module) != 0) {
return false;
}
return true;
}
void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod) {
int32_t pos = 0;
for (pos = 0; pos < pServer->methodScannerLen; ++pos) {
if (httpCompareMethod(pServer->methodScanner[pos], pMethod)) {
break;
}
}
if (pos == pServer->methodScannerLen && pServer->methodScannerLen < HTTP_METHOD_SCANNER_SIZE) {
pServer->methodScanner[pos] = pMethod;
pServer->methodScannerLen++;
}
}
HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (multiCmds->size >= multiCmds->maxSize) {
if (!httpReMallocMultiCmdsSize(pContext, 2 * multiCmds->maxSize)) return NULL;
}
HttpSqlCmd *cmd = multiCmds->cmds + multiCmds->size++;
cmd->cmdType = HTTP_CMD_TYPE_UN_SPECIFIED;
cmd->cmdReturnType = HTTP_CMD_RETURN_TYPE_WITH_RETURN;
cmd->cmdState = HTTP_CMD_STATE_NOT_RUN_YET;
return cmd;
}
HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (multiCmds->size == 0) return NULL;
if (multiCmds->size > multiCmds->maxSize) return NULL;
return multiCmds->cmds + multiCmds->size - 1;
}
int32_t httpNextSqlCmdPos(HttpContext *pContext) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
return multiCmds->size;
}
void httpTrimTableName(char *name) {
for (int32_t i = 0; name[i] != 0; i++) {
if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'')
name[i] = '_';
if (i == TSDB_TABLE_NAME_LEN) {
name[i] = 0;
break;
}
}
}
int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name) {
int32_t len = 0;
for (int32_t i = 0; name[i] != 0; i++) {
if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'' ||
name[i] == '\"')
name[i] = '_';
len++;
}
if (len < TSDB_TABLE_NAME_LEN - 1) {
return pos;
}
MD5_CTX context;
MD5Init(&context);
MD5Update(&context, (uint8_t *)name, (uint32_t)len);
MD5Final(&context);
int32_t table_name = httpAddToSqlCmdBuffer(
pContext, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0],
context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6],
context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11],
context.digest[12], context.digest[13], context.digest[14], context.digest[15]);
if (table_name != -1) {
httpGetCmdsString(pContext, table_name)[0] = 't';
}
return table_name;
}
char *httpGetCmdsString(HttpContext *pContext, int32_t pos) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (pos < 0 || pos >= multiCmds->bufferSize) {
return "";
}
return multiCmds->buffer + pos;
}
int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData) {
int32_t err = 0;
z_stream gzipStream = {0};
static char dummyHead[2] = {
0x8 + 0x7 * 0x10,
(((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF,
};
gzipStream.zalloc = (alloc_func)0;
gzipStream.zfree = (free_func)0;
gzipStream.opaque = (voidpf)0;
gzipStream.next_in = (Bytef *)srcData;
gzipStream.avail_in = 0;
gzipStream.next_out = (Bytef *)destData;
if (inflateInit2(&gzipStream, 47) != Z_OK) {
return -1;
}
while (gzipStream.total_out < *nDestData && gzipStream.total_in < nSrcData) {
gzipStream.avail_in = gzipStream.avail_out = nSrcData; // 1
if ((err = inflate(&gzipStream, Z_NO_FLUSH)) == Z_STREAM_END) {
break;
}
if (err != Z_OK) {
if (err == Z_DATA_ERROR) {
gzipStream.next_in = (Bytef *)dummyHead;
gzipStream.avail_in = sizeof(dummyHead);
if ((err = inflate(&gzipStream, Z_NO_FLUSH)) != Z_OK) {
return -2;
}
} else {
return -3;
}
}
}
if (inflateEnd(&gzipStream) != Z_OK) {
return -4;
}
*nDestData = (int32_t)gzipStream.total_out;
return 0;
}
int32_t httpGzipCompressInit(HttpContext *pContext) {
pContext->gzipStream.zalloc = (alloc_func)0;
pContext->gzipStream.zfree = (free_func)0;
pContext->gzipStream.opaque = (voidpf)0;
if (deflateInit2(&pContext->gzipStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) !=
Z_OK) {
return -1;
}
return 0;
}
int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData,
bool isTheLast) {
int32_t err = 0;
int32_t lastTotalLen = (int32_t)(pContext->gzipStream.total_out);
pContext->gzipStream.next_in = (Bytef *)srcData;
pContext->gzipStream.avail_in = (uLong)nSrcData;
pContext->gzipStream.next_out = (Bytef *)destData;
pContext->gzipStream.avail_out = (uLong)(*nDestData);
while (pContext->gzipStream.avail_in != 0) {
if (deflate(&pContext->gzipStream, Z_FULL_FLUSH) != Z_OK) {
return -1;
}
int32_t cacheLen = (int32_t)(pContext->gzipStream.total_out - lastTotalLen);
if (cacheLen >= *nDestData) {
return -2;
}
}
if (pContext->gzipStream.avail_in != 0) {
return pContext->gzipStream.avail_in;
}
if (isTheLast) {
for (;;) {
if ((err = deflate(&pContext->gzipStream, Z_FINISH)) == Z_STREAM_END) {
break;
}
if (err != Z_OK) {
return -3;
}
}
if (deflateEnd(&pContext->gzipStream) != Z_OK) {
return -4;
}
}
*nDestData = (int32_t)(pContext->gzipStream.total_out) - lastTotalLen;
return 0;
}
bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp) {
HttpParser *pParser = pContext->parser;
if (pos < 0 || pos >= HTTP_MAX_URL) {
return false;
}
if (pParser->path[pos].pos <= 0) {
return false;
}
if (strcmp(pParser->path[pos].str, cmp) != 0) {
return false;
}
return true;
}

32
src/plugins/mqtt/CMakeLists.txt
View File

@ -1,32 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
INCLUDE_DIRECTORIES(inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/include)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/examples/templates)
AUX_SOURCE_DIRECTORY(src SRC)
IF (TD_LINUX)
ADD_LIBRARY(mqtt ${SRC})
TARGET_LINK_LIBRARIES(mqtt cJson mqttc)
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(mqtt taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(mqtt taos)
ENDIF ()
ENDIF ()
IF (TD_DARWIN)
ADD_LIBRARY(mqtt ${SRC})
TARGET_LINK_LIBRARIES(mqtt cJson mqttc)
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(mqtt taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(mqtt taos)
ENDIF ()
ENDIF ()

77
src/plugins/mqtt/inc/mqttInit.h
View File

@ -1,77 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MQTT_INIT_H
#define TDENGINE_MQTT_INIT_H
#ifdef __cplusplus
extern "C" {
#endif
/**
* @file
* A simple subscriber program that performs automatic reconnections.
*/
#include "mqtt.h"
#define QOS 1
#define TIMEOUT 10000L
#define MQTT_SEND_BUF_SIZE 102400
#define MQTT_RECV_BUF_SIZE 102400
/**
* @brief A structure that I will use to keep track of some data needed
* to setup the connection to the broker.
*
* An instance of this struct will be created in my \c main(). Then, whenever
* \ref mqttReconnectClient is called, this instance will be passed.
*/
typedef struct SMqttReconnectState {
uint8_t* sendbuf;
size_t sendbufsz;
uint8_t* recvbuf;
size_t recvbufsz;
} SMqttReconnectState;
/**
* @brief My reconnect callback. It will reestablish the connection whenever
* an error occurs.
*/
void mqttReconnectClient(struct mqtt_client* client, void** reconnect_state_vptr);
/**
* @brief The function will be called whenever a PUBLISH message is received.
*/
void mqttPublishCallback(void** unused, struct mqtt_response_publish* published);
/**
* @brief The client's refresher. This function triggers back-end routines to
* handle ingress/egress traffic to the broker.
*
* @note All this function needs to do is call \ref __mqtt_recv and
* \ref __mqtt_send every so often. I've picked 100 ms meaning that
* client ingress/egress traffic will be handled every 100 ms.
*/
void* mqttClientRefresher(void* client);
/**
* @brief Safelty closes the \p sockfd and cancels the \p client_daemon before \c exit.
*/
void mqttCleanupRes(int status, int sockfd, pthread_t* client_daemon);
#ifdef __cplusplus
}
#endif
#endif

30
src/plugins/mqtt/inc/mqttLog.h
View File

@ -1,30 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <mqtt://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MQTT_LOG_H
#define TDENGINE_MQTT_LOG_H
#include "tlog.h"
extern int32_t mqttDebugFlag;
#define mqttFatal(...) { if (mqttDebugFlag & DEBUG_FATAL) { taosPrintLog("MQT FATAL ", 255, __VA_ARGS__); }}
#define mqttError(...) { if (mqttDebugFlag & DEBUG_ERROR) { taosPrintLog("MQT ERROR ", 255, __VA_ARGS__); }}
#define mqttWarn(...) { if (mqttDebugFlag & DEBUG_WARN) { taosPrintLog("MQT WARN ", 255, __VA_ARGS__); }}
#define mqttInfo(...) { if (mqttDebugFlag & DEBUG_INFO) { taosPrintLog("MQT ", 255, __VA_ARGS__); }}
#define mqttDebug(...) { if (mqttDebugFlag & DEBUG_DEBUG) { taosPrintLog("MQT ", mqttDebugFlag, __VA_ARGS__); }}
#define mqttTrace(...) { if (mqttDebugFlag & DEBUG_TRACE) { taosPrintLog("MQT ", mqttDebugFlag, __VA_ARGS__); }}
#endif

29
src/plugins/mqtt/inc/mqttPayload.h
View File

@ -1,29 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MQTT_PLYLOAD_H
#define TDENGINE_MQTT_PLYLOAD_H
#ifdef __cplusplus
extern "C" {
#endif
char* mqttConverJsonToSql(char* json, int maxSize);
#ifdef __cplusplus
}
#endif
#endif

159
src/plugins/mqtt/src/mqttPayload.c
View File

@ -1,159 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "cJSON.h"
#include "mqttLog.h"
#include "mqttPayload.h"
// subscribe message like this
/*
/test {
"timestamp": 1599121290,
"gateway": {
"name": "AcuLink 810 Gateway",
"model": "AcuLink810-868",
"serial": "S8P20200207"
},
"device": {
"name": "Acuvim L V3 .221",
"model": "Acuvim-L-V3",
"serial": "221",
"online": true,
"readings": [
{
"param": "Freq_Hz",
"value": "59.977539",
"unit": "Hz"
},
{
"param": "Va_V",
"value": "122.002907",
"unit": "V"
},
{
"param": "DI4",
"value": "5.000000",
"unit": ""
}
]
}
}
*/
// send msg cmd
// mosquitto_pub -h test.mosquitto.org -t "/test" -m '{"timestamp": 1599121290,"gateway": {"name": "AcuLink 810 Gateway","model": "AcuLink810-868","serial": "S8P20200207"},"device": {"name": "Acuvim L V3 .221","model": "Acuvim-L-V3","serial": "221","online": true,"readings": [{"param": "Freq_Hz","value": "59.977539","unit": "Hz"},{"param": "Va_V","value": "122.002907","unit": "V"},{"param": "DI4","value": "5.000000","unit": ""}]}}'
/*
* This is an example, this function needs to be implemented in order to parse the json file into a sql statement
* Note that you need to create a super table and database before writing data
* In this case:
* create database mqttdb;
* create table mqttdb.devices(ts timestamp, value bigint) tags(name binary(32), model binary(32), serial binary(16), param binary(16), unit binary(16));
*/
char* mqttConverJsonToSql(char* json, int maxSize) {
// const int32_t maxSize = 10240;
maxSize *= 5;
char* sql = malloc(maxSize);
cJSON* root = cJSON_Parse(json);
if (root == NULL) {
mqttError("failed to parse msg, invalid json format");
goto MQTT_PARSE_OVER;
}
cJSON* timestamp = cJSON_GetObjectItem(root, "timestamp");
if (!timestamp || timestamp->type != cJSON_Number) {
mqttError("failed to parse msg, timestamp not found");
goto MQTT_PARSE_OVER;
}
cJSON* device = cJSON_GetObjectItem(root, "device");
if (!device) {
mqttError("failed to parse msg, device not found");
goto MQTT_PARSE_OVER;
}
cJSON* name = cJSON_GetObjectItem(device, "name");
if (!name || name->type != cJSON_String) {
mqttError("failed to parse msg, name not found");
goto MQTT_PARSE_OVER;
}
cJSON* model = cJSON_GetObjectItem(device, "model");
if (!model || model->type != cJSON_String) {
mqttError("failed to parse msg, model not found");
goto MQTT_PARSE_OVER;
}
cJSON* serial = cJSON_GetObjectItem(device, "serial");
if (!serial || serial->type != cJSON_String) {
mqttError("failed to parse msg, serial not found");
goto MQTT_PARSE_OVER;
}
cJSON* readings = cJSON_GetObjectItem(device, "readings");
if (!readings || readings->type != cJSON_Array) {
mqttError("failed to parse msg, readings not found");
goto MQTT_PARSE_OVER;
}
int count = cJSON_GetArraySize(readings);
if (count <= 0) {
mqttError("failed to parse msg, readings size smaller than 0");
goto MQTT_PARSE_OVER;
}
int len = snprintf(sql, maxSize, "insert into");
for (int i = 0; i < count; ++i) {
cJSON* reading = cJSON_GetArrayItem(readings, i);
if (reading == NULL) continue;
cJSON* param = cJSON_GetObjectItem(reading, "param");
if (!param || param->type != cJSON_String) {
mqttError("failed to parse msg, param not found");
goto MQTT_PARSE_OVER;
}
cJSON* value = cJSON_GetObjectItem(reading, "value");
if (!value || value->type != cJSON_String) {
mqttError("failed to parse msg, value not found");
goto MQTT_PARSE_OVER;
}
cJSON* unit = cJSON_GetObjectItem(reading, "unit");
if (!unit || unit->type != cJSON_String) {
mqttError("failed to parse msg, unit not found");
goto MQTT_PARSE_OVER;
}
len += snprintf(sql + len, maxSize - len,
" mqttdb.serial_%s_%s using mqttdb.devices tags('%s', '%s', '%s', '%s', '%s') values(%" PRId64 ", %s)",
serial->valuestring, param->valuestring, name->valuestring, model->valuestring, serial->valuestring,
param->valuestring, unit->valuestring, timestamp->valueint * 1000, value->valuestring);
}
cJSON_free(root);
return sql;
MQTT_PARSE_OVER:
cJSON_free(root);
free(sql);
return NULL;
}

146
src/plugins/mqtt/src/mqttSystem.c
View File

@ -1,146 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "../../../../include/client/taos.h"
#include "mqtt.h"
#include "mqttInit.h"
#include "mqttLog.h"
#include "mqttPayload.h"
#include "os.h"
#include "posix_sockets.h"
#include "taoserror.h"
#include "tglobal.h"
#include "tmqtt.h"
struct SMqttReconnectState tsMqttStatus = {0};
struct mqtt_client tsMqttClient = {0};
static pthread_t tsMqttClientDaemonThread = {0};
static void* tsMqttConnect = NULL;
static bool tsMqttIsRuning = false;
int32_t mqttInitSystem() { return 0; }
int32_t mqttStartSystem() {
tsMqttStatus.sendbufsz = MQTT_SEND_BUF_SIZE;
tsMqttStatus.recvbufsz = MQTT_RECV_BUF_SIZE;
tsMqttStatus.sendbuf = malloc(MQTT_SEND_BUF_SIZE);
tsMqttStatus.recvbuf = malloc(MQTT_RECV_BUF_SIZE);
tsMqttIsRuning = true;
mqtt_init_reconnect(&tsMqttClient, mqttReconnectClient, &tsMqttStatus, mqttPublishCallback);
if (pthread_create(&tsMqttClientDaemonThread, NULL, mqttClientRefresher, &tsMqttClient)) {
mqttError("mqtt failed to start daemon.");
mqttCleanupRes(EXIT_FAILURE, -1, NULL);
return -1;
}
mqttInfo("mqtt listening for topic:%s messages", tsMqttTopic);
return 0;
}
void mqttStopSystem() {
if (tsMqttIsRuning) {
tsMqttIsRuning = false;
tsMqttClient.error = MQTT_ERROR_SOCKET_ERROR;
taosMsleep(300);
mqttCleanupRes(EXIT_SUCCESS, tsMqttClient.socketfd, &tsMqttClientDaemonThread);
mqttInfo("mqtt is stopped");
}
}
void mqttCleanUpSystem() {
mqttStopSystem();
mqttInfo("mqtt is cleaned up");
}
void mqttPublishCallback(void** unused, struct mqtt_response_publish* published) {
const char* content = published->application_message;
mqttDebug("receive mqtt message, size:%d", (int)published->application_message_size);
if (tsMqttConnect == NULL) {
tsMqttConnect = taos_connect(NULL, "_root", tsInternalPass, "", 0);
if (tsMqttConnect == NULL) {
mqttError("failed to connect to tdengine, reason:%s", tstrerror(terrno));
return;
} else {
mqttInfo("successfully connected to the tdengine");
}
}
mqttTrace("receive mqtt message, content:%s", content);
char* sql = mqttConverJsonToSql((char*)content, (int)published->application_message_size);
if (sql != NULL) {
void* res = taos_query(tsMqttConnect, sql);
int code = taos_errno(res);
if (code != 0) {
mqttError("failed to exec sql, reason:%s sql:%s", tstrerror(code), sql);
} else {
mqttTrace("successfully to exec sql:%s", sql);
}
taos_free_result(res);
} else {
mqttError("failed to parse mqtt message");
}
}
void* mqttClientRefresher(void* client) {
setThreadName("mqttCliRefresh");
while (tsMqttIsRuning) {
mqtt_sync((struct mqtt_client*)client);
taosMsleep(100);
}
mqttDebug("mqtt quit refresher");
return NULL;
}
void mqttCleanupRes(int status, int sockfd, pthread_t* client_daemon) {
mqttInfo("clean up mqtt module");
if (sockfd != -1) {
close(sockfd);
}
if (client_daemon != NULL) {
pthread_cancel(*client_daemon);
}
}
void mqttReconnectClient(struct mqtt_client* client, void** unused) {
mqttInfo("mqtt tries to connect to the mqtt server");
if (client->error != MQTT_ERROR_INITIAL_RECONNECT) {
close(client->socketfd);
}
if (client->error != MQTT_ERROR_INITIAL_RECONNECT) {
mqttError("mqtt client was in error state %s", mqtt_error_str(client->error));
}
int sockfd = open_nb_socket(tsMqttHostName, tsMqttPort);
if (sockfd < 0) {
mqttError("mqtt client failed to open socket %s:%s", tsMqttHostName, tsMqttPort);
//mqttCleanupRes(EXIT_FAILURE, sockfd, NULL);
return;
}
mqtt_reinit(client, sockfd, tsMqttStatus.sendbuf, tsMqttStatus.sendbufsz, tsMqttStatus.recvbuf, tsMqttStatus.recvbufsz);
mqtt_connect(client, tsMqttClientId, NULL, NULL, 0, tsMqttUser, tsMqttPass, MQTT_CONNECT_CLEAN_SESSION, 400);
mqtt_subscribe(client, tsMqttTopic, 0);
}

16
src/sync/CMakeLists.txt
View File

@ -1,16 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
LIST(REMOVE_ITEM SRC src/syncArbitrator.c)
ADD_LIBRARY(sync ${SRC})
TARGET_LINK_LIBRARIES(sync tutil pthread common)
LIST(APPEND BIN_SRC src/syncArbitrator.c)
LIST(APPEND BIN_SRC src/syncTcp.c)
ADD_EXECUTABLE(tarbitrator ${BIN_SRC})
TARGET_LINK_LIBRARIES(tarbitrator sync common os tutil)
#ADD_SUBDIRECTORY(test)

143
src/sync/inc/syncInt.h
View File

@ -1,143 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_SYNC_INT_H
#define TDENGINE_SYNC_INT_H
#ifdef __cplusplus
extern "C" {
#endif
#include "syncMsg.h"
#include "twal.h"
#define sFatal(...) { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); }}
#define sError(...) { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); }}
#define sWarn(...) { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); }}
#define sInfo(...) { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
#define sDebug(...) { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
#define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
#define SYNC_TCP_THREADS 2
#define SYNC_MAX_NUM 2
#define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16)
#define SYNC_RECV_BUFFER_SIZE (5*1024*1024)
#define SYNC_MAX_FWDS 4096
#define SYNC_FWD_TIMER 300
#define SYNC_ROLE_TIMER 15000 // ms
#define SYNC_CHECK_INTERVAL 1000 // ms
#define SYNC_WAIT_AFTER_CHOOSE_MASTER 10 // ms
#define nodeRole pNode->peerInfo[pNode->selfIndex]->role
#define nodeVersion pNode->peerInfo[pNode->selfIndex]->version
#define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus
typedef struct {
char * buffer;
int32_t bufferSize;
char * offset;
int32_t forwards;
int32_t code;
} SRecvBuffer;
typedef struct {
uint64_t version;
void *mhandle;
int8_t acks;
int8_t nacks;
int8_t confirmed;
int32_t code;
int64_t time;
} SFwdInfo;
typedef struct {
int32_t first;
int32_t last;
int32_t fwds; // number of forwards
SFwdInfo fwdInfo[];
} SSyncFwds;
typedef struct SsyncPeer {
int32_t nodeId;
uint32_t ip;
uint16_t port;
int8_t role;
int8_t sstatus; // sync status
char fqdn[TSDB_FQDN_LEN]; // peer ip string
char id[TSDB_EP_LEN + 32]; // peer vgId + end point
uint64_t version;
uint64_t sversion; // track the peer version in retrieve process
uint64_t lastFileVer; // track the file version while retrieve
uint64_t lastWalVer; // track the wal version while retrieve
SOCKET syncFd;
SOCKET peerFd; // forward FD
int32_t numOfRetrieves; // number of retrieves tried
int32_t fileChanged; // a flag to indicate file is changed during retrieving process
int32_t refCount;
int8_t isArb;
int64_t rid;
void * timer;
void * pConn;
struct SSyncNode *pSyncNode;
} SSyncPeer;
typedef struct SSyncNode {
char path[TSDB_FILENAME_LEN];
int8_t replica;
int8_t quorum;
int8_t selfIndex;
uint32_t vgId;
int32_t refCount;
int64_t rid;
SSyncPeer * peerInfo[TAOS_SYNC_MAX_REPLICA + 1]; // extra one for arbitrator
SSyncPeer * pMaster;
SRecvBuffer *pRecv;
SSyncFwds * pSyncFwds; // saved forward info if quorum >1
void * pFwdTimer;
void * pRoleTimer;
void * pTsdb;
FGetWalInfo getWalInfoFp;
FWriteToCache writeToCacheFp;
FConfirmForward confirmForward;
FNotifyRole notifyRoleFp;
FNotifyFlowCtrl notifyFlowCtrlFp;
FStartSyncFile startSyncFileFp;
FStopSyncFile stopSyncFileFp;
FGetVersion getVersionFp;
FSendFile sendFileFp;
FRecvFile recvFileFp;
pthread_mutex_t mutex;
} SSyncNode;
// sync module global
extern int32_t tsSyncNum;
extern char tsNodeFqdn[TSDB_FQDN_LEN];
extern char * syncStatus[];
void * syncRetrieveData(void *param);
void * syncRestoreData(void *param);
int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
void syncRestartConnection(SSyncPeer *pPeer);
void syncBroadcastStatus(SSyncNode *pNode);
uint32_t syncResolvePeerFqdn(SSyncPeer *pPeer);
SSyncPeer *syncAcquirePeer(int64_t rid);
void syncReleasePeer(SSyncPeer *pPeer);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEPEER_H

33
src/sync/inc/syncMain.h
View File

@ -1,33 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_PLUGINS_SYNC_H
#define TDENGINE_PLUGINS_SYNC_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
int32_t syncTest1();
int32_t syncTest2();
#ifdef __cplusplus
}
#endif
#endif

141
src/sync/inc/syncMsg.h
View File

@ -1,141 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_SYNC_MSG_H
#define TDENGINE_SYNC_MSG_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tsync.h"
typedef enum {
TAOS_SMSG_START = 0,
TAOS_SMSG_SYNC_DATA = 1,
TAOS_SMSG_SYNC_DATA_RSP = 2,
TAOS_SMSG_SYNC_FWD = 3,
TAOS_SMSG_SYNC_FWD_RSP = 4,
TAOS_SMSG_SYNC_REQ = 5,
TAOS_SMSG_SYNC_REQ_RSP = 6,
TAOS_SMSG_SYNC_MUST = 7,
TAOS_SMSG_SYNC_MUST_RSP = 8,
TAOS_SMSG_STATUS = 9,
TAOS_SMSG_STATUS_RSP = 10,
TAOS_SMSG_SETUP = 11,
TAOS_SMSG_SETUP_RSP = 12,
TAOS_SMSG_SYNC_FILE = 13,
TAOS_SMSG_SYNC_FILE_RSP = 14,
TAOS_SMSG_TEST = 15,
TAOS_SMSG_END = 16
} ESyncMsgType;
typedef enum {
SYNC_STATUS_BROADCAST,
SYNC_STATUS_BROADCAST_RSP,
SYNC_STATUS_SETUP_CONN,
SYNC_STATUS_SETUP_CONN_RSP,
SYNC_STATUS_EXCHANGE_DATA,
SYNC_STATUS_EXCHANGE_DATA_RSP,
SYNC_STATUS_CHECK_ROLE,
SYNC_STATUS_CHECK_ROLE_RSP
} ESyncStatusType;
#pragma pack(push, 1)
typedef struct {
int8_t type; // msg type
int8_t protocol; // protocol version
uint16_t signature; // fixed value
int32_t code; //
int32_t cId; // cluster Id
int32_t vgId; // vg ID
int32_t len; // content length, does not include head
uint32_t cksum;
} SSyncHead;
typedef struct {
SSyncHead head;
uint16_t port;
uint16_t tranId;
int32_t sourceId; // only for arbitrator
char fqdn[TSDB_FQDN_LEN];
} SSyncMsg;
typedef struct {
SSyncHead head;
int8_t sync;
int8_t reserved;
uint16_t tranId;
int8_t reserverd[4];
} SSyncRsp;
typedef struct {
int8_t role;
uint64_t version;
} SPeerStatus;
typedef struct {
SSyncHead head;
int8_t role;
int8_t ack;
int8_t type;
int8_t reserved[3];
uint16_t tranId;
uint64_t version;
SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA];
} SPeersStatus;
typedef struct {
SSyncHead head;
uint64_t fversion;
} SFileVersion;
typedef struct {
SSyncHead head;
int8_t ack;
} SFileAck;
typedef struct {
SSyncHead head;
uint64_t version;
int32_t code;
} SFwdRsp;
#pragma pack(pop)
#define SYNC_PROTOCOL_VERSION 1
#define SYNC_SIGNATURE ((uint16_t)(0xCDEF))
extern char *statusType[];
uint16_t syncGenTranId();
int32_t syncCheckHead(SSyncHead *pHead);
void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len);
void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code);
void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId);
void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId);
void syncBuildFileAck(SFileAck *pMsg, int32_t vgId);
void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEPEER_H

43
src/sync/inc/syncTcp.h
View File

@ -1,43 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_SYNC_TCP_POOL_H
#define TDENGINE_SYNC_TCP_POOL_H
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
int32_t numOfThreads;
uint32_t serverIp;
int16_t port;
int32_t bufferSize;
void (*processBrokenLink)(int64_t handleId, int32_t closedByApp);
int32_t (*processIncomingMsg)(int64_t handleId, void *buffer);
void (*processIncomingConn)(SOCKET fd, uint32_t ip);
} SPoolInfo;
void *syncOpenTcpThreadPool(SPoolInfo *pInfo);
void syncCloseTcpThreadPool(void *);
void *syncAllocateTcpConn(void *, int64_t rid, SOCKET connFd);
void syncFreeTcpConn(void *);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TCP_POOL_H

189
src/sync/src/syncArbitrator.c
View File

@ -1,189 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
//#define _DEFAULT_SOURCE
#include "os.h"
#include "hash.h"
#include "tlog.h"
#include "tutil.h"
#include "ttimer.h"
#include "tsocket.h"
#include "tglobal.h"
#include "taoserror.h"
#include "twal.h"
#include "tsync.h"
#include "syncInt.h"
#include "syncTcp.h"
extern void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd);
static void arbSignalHandler(int32_t signum, void *sigInfo, void *context);
static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp);
static void arbProcessBrokenLink(int64_t rid, int32_t closedByApp);
static int32_t arbProcessPeerMsg(int64_t rid, void *buffer);
static tsem_t tsArbSem;
static void * tsArbTcpPool;
typedef struct {
char id[TSDB_EP_LEN + 24];
SOCKET nodeFd;
void * pConn;
} SNodeConn;
int32_t main(int32_t argc, char *argv[]) {
char arbLogPath[TSDB_FILENAME_LEN + 16] = {0};
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
tsArbitratorPort = atoi(argv[++i]);
} else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) {
debugFlag = atoi(argv[++i]);
} else if (strcmp(argv[i], "-g") == 0 && i < argc - 1) {
if (strlen(argv[++i]) > TSDB_FILENAME_LEN) continue;
tstrncpy(arbLogPath, argv[i], sizeof(arbLogPath));
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-p port]: arbitrator server port number, default is:%d\n", tsServerPort + TSDB_PORT_ARBITRATOR);
printf(" [-d debugFlag]: debug flag, option 131 | 135 | 143, default:0\n");
printf(" [-g logFilePath]: log file pathe, default:/arbitrator.log\n");
printf(" [-h help]: print out this help\n\n");
exit(0);
}
}
sDebugFlag = debugFlag;
if (tsem_init(&tsArbSem, 0, 0) != 0) {
printf("failed to create exit semphore\n");
exit(EXIT_FAILURE);
}
/* Set termination handler. */
taosSetSignal(SIGTERM, arbSignalHandler);
taosSetSignal(SIGINT, arbSignalHandler);
taosSetSignal(SIGHUP, arbSignalHandler);
taosSetSignal(SIGABRT, arbSignalHandler);
tsAsyncLog = 0;
strcat(arbLogPath, "/arbitrator.log");
taosInitLog(arbLogPath, 1000000, 10);
taosGetFqdn(tsNodeFqdn);
SPoolInfo info;
info.numOfThreads = 1;
info.serverIp = 0;
info.port = tsArbitratorPort;
info.bufferSize = SYNC_MAX_SIZE;
info.processBrokenLink = arbProcessBrokenLink;
info.processIncomingMsg = arbProcessPeerMsg;
info.processIncomingConn = arbProcessIncommingConnection;
tsArbTcpPool = syncOpenTcpThreadPool(&info);
if (tsArbTcpPool == NULL) {
sDebug("failed to open TCP thread pool, exit...");
return -1;
}
sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsArbitratorPort);
tsem_wait(&tsArbSem);
syncCloseTcpThreadPool(tsArbTcpPool);
sInfo("TAOS arbitrator is shut down");
closelog();
return 0;
}
static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) {
char ipstr[24];
tinet_ntoa(ipstr, sourceIp);
sDebug("peer TCP connection from ip:%s", ipstr);
SSyncMsg msg;
if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) {
sError("failed to read peer sync msg from ip:%s since %s", ipstr, strerror(errno));
taosCloseSocket(connFd);
return;
}
if (msg.head.type == TAOS_SMSG_TEST) {
syncProcessTestMsg(&msg, connFd);
return;
}
SNodeConn *pNode = calloc(sizeof(SNodeConn), 1);
if (pNode == NULL) {
sError("failed to allocate memory since %s", strerror(errno));
taosCloseSocket(connFd);
return;
}
msg.fqdn[TSDB_FQDN_LEN - 1] = 0;
snprintf(pNode->id, sizeof(pNode->id), "vgId:%d, peer:%s:%d", msg.sourceId, msg.fqdn, msg.port);
if (msg.head.vgId) {
sDebug("%s, vgId in head is not zero, close the connection", pNode->id);
tfree(pNode);
taosCloseSocket(connFd);
return;
}
sDebug("%s, arbitrator request is accepted", pNode->id);
pNode->nodeFd = connFd;
pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, (int64_t)pNode, connFd);
return;
}
static void arbProcessBrokenLink(int64_t rid, int32_t closedByApp) {
SNodeConn *pNode = (SNodeConn *)rid;
sDebug("%s, TCP link is broken since %s, closedByApp:%d", pNode->id, strerror(errno), closedByApp);
tfree(pNode);
}
static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) {
SNodeConn *pNode = (SNodeConn *)rid;
SSyncHead head;
int32_t bytes = 0;
char * cont = (char *)buffer;
int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(SSyncHead));
if (hlen != sizeof(SSyncHead)) {
sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen);
return -1;
}
bytes = taosReadMsg(pNode->nodeFd, cont, head.len);
if (bytes != head.len) {
sDebug("%s, failed to read, bytes:%d len:%d", pNode->id, bytes, head.len);
return -1;
}
sDebug("%s, msg is received, len:%d", pNode->id, head.len);
return 0;
}
static void arbSignalHandler(int32_t signum, void *sigInfo, void *context) {
taosIgnSignal(SIGTERM);
taosIgnSignal(SIGINT);
taosIgnSignal(SIGABRT);
taosIgnSignal(SIGHUP);
sInfo("shut down signal is %d", signum);
// inform main thread to exit
tsem_post(&tsArbSem);
}

1519
src/sync/src/syncMain.c
View File

File diff suppressed because it is too large Load Diff

110
src/sync/src/syncMsg.c
View File

@ -1,110 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tglobal.h"
#include "tchecksum.h"
#include "syncInt.h"
char *statusType[] = {
"broadcast",
"broadcast-rsp",
"setup-conn",
"setup-conn-rsp",
"exchange-data",
"exchange-data-rsp",
"check-role",
"check-role-rsp"
};
uint16_t syncGenTranId() {
return taosRand() & 0XFFFF;
}
static void syncBuildHead(SSyncHead *pHead) {
pHead->protocol = SYNC_PROTOCOL_VERSION;
pHead->signature = SYNC_SIGNATURE;
pHead->code = 0;
pHead->cId = 0;
taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead));
}
int32_t syncCheckHead(SSyncHead *pHead) {
if (pHead->protocol != SYNC_PROTOCOL_VERSION) return TSDB_CODE_SYN_MISMATCHED_PROTOCOL;
if (pHead->signature != SYNC_SIGNATURE) return TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
if (pHead->cId != 0) return TSDB_CODE_SYN_MISMATCHED_CLUSTERID;
if (pHead->len <= 0 || pHead->len > TSDB_MAX_WAL_SIZE) return TSDB_CODE_SYN_INVALID_MSGLEN;
if (pHead->type <= TAOS_SMSG_START || pHead->type >= TAOS_SMSG_END) return TSDB_CODE_SYN_INVALID_MSGTYPE;
if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SSyncHead))) return TSDB_CODE_SYN_INVALID_CHECKSUM;
return TSDB_CODE_SUCCESS;
}
void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len) {
pHead->type = TAOS_SMSG_SYNC_FWD;
pHead->vgId = vgId;
pHead->len = len;
syncBuildHead(pHead);
}
void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t _version, int32_t code) {
pMsg->head.type = TAOS_SMSG_SYNC_FWD_RSP;
pMsg->head.vgId = vgId;
pMsg->head.len = sizeof(SFwdRsp) - sizeof(SSyncHead);
syncBuildHead(&pMsg->head);
pMsg->version = _version;
pMsg->code = code;
}
static void syncBuildMsg(SSyncMsg *pMsg, int32_t vgId, ESyncMsgType type) {
pMsg->head.type = type;
pMsg->head.vgId = vgId;
pMsg->head.len = sizeof(SSyncMsg) - sizeof(SSyncHead);
syncBuildHead(&pMsg->head);
pMsg->port = tsSyncPort;
pMsg->tranId = syncGenTranId();
pMsg->sourceId = vgId;
tstrncpy(pMsg->fqdn, tsNodeFqdn, TSDB_FQDN_LEN);
}
void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_REQ); }
void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_DATA); }
void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SETUP); }
void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_TEST); }
void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId) {
pMsg->head.type = TAOS_SMSG_STATUS;
pMsg->head.vgId = vgId;
pMsg->head.len = sizeof(SPeersStatus) - sizeof(SSyncHead);
syncBuildHead(&pMsg->head);
}
void syncBuildFileAck(SFileAck *pMsg, int32_t vgId) {
pMsg->head.type = TAOS_SMSG_SYNC_FILE_RSP;
pMsg->head.vgId = vgId;
pMsg->head.len = sizeof(SFileAck) - sizeof(SSyncHead);
syncBuildHead(&pMsg->head);
}
void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId) {
pMsg->head.type = TAOS_SMSG_SYNC_FILE;
pMsg->head.vgId = vgId;
pMsg->head.len = sizeof(SFileVersion) - sizeof(SSyncHead);
syncBuildHead(&pMsg->head);
}

312
src/sync/src/syncRestore.c
View File

@ -1,312 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tlog.h"
#include "tutil.h"
#include "ttimer.h"
#include "tsocket.h"
#include "tqueue.h"
#include "twal.h"
#include "tsync.h"
#include "syncInt.h"
static int32_t syncRecvFileVersion(SSyncPeer *pPeer, uint64_t *fversion) {
SSyncNode *pNode = pPeer->pSyncNode;
SFileVersion fileVersion;
memset(&fileVersion, 0, sizeof(SFileVersion));
int32_t ret = taosReadMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion));
if (ret != sizeof(SFileVersion)) {
sError("%s, failed to read fver since %s", pPeer->id, strerror(errno));
return -1;
}
SFileAck fileVersionAck;
memset(&fileVersionAck, 0, sizeof(SFileAck));
syncBuildFileAck(&fileVersionAck, pNode->vgId);
ret = taosWriteMsg(pPeer->syncFd, &fileVersionAck, sizeof(SFileAck));
if (ret != sizeof(SFileAck)) {
sError("%s, failed to write fver ack since %s", pPeer->id, strerror(errno));
return -1;
}
*fversion = htobe64(fileVersion.fversion);
return 0;
}
static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
SSyncNode *pNode = pPeer->pSyncNode;
if (pNode->recvFileFp && (*pNode->recvFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) {
sError("%s, failed to restore file", pPeer->id);
return -1;
}
if (syncRecvFileVersion(pPeer, fversion) < 0) {
return -1;
}
sInfo("%s, all files are restored, fver:%" PRIu64, pPeer->id, *fversion);
return 0;
}
static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) {
SSyncNode *pNode = pPeer->pSyncNode;
int32_t ret, code = -1;
uint64_t lastVer = 0;
SWalHead *pHead = calloc(SYNC_MAX_SIZE, 1); // size for one record
if (pHead == NULL) return -1;
while (1) {
ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead));
if (ret != sizeof(SWalHead)) {
sError("%s, failed to read walhead while restore wal since %s", pPeer->id, strerror(errno));
break;
}
if (pHead->len == 0) {
sDebug("%s, wal is synced over, last wver:%" PRIu64, pPeer->id, lastVer);
code = 0;
break;
} // wal sync over
ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len);
if (ret != pHead->len) {
sError("%s, failed to read walcont, len:%d while restore wal since %s", pPeer->id, pHead->len, strerror(errno));
break;
}
sTrace("%s, restore a record, qtype:wal len:%d hver:%" PRIu64, pPeer->id, pHead->len, pHead->version);
if (lastVer == pHead->version) {
sError("%s, failed to restore record, same hver:%" PRIu64 ", wal sync failed" PRIu64, pPeer->id, lastVer);
break;
}
lastVer = pHead->version;
ret = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL);
if (ret != 0) {
sError("%s, failed to restore record since %s, hver:%" PRIu64, pPeer->id, tstrerror(ret), pHead->version);
break;
}
}
if (code < 0) {
sError("%s, failed to restore wal from syncFd:%d since %s", pPeer->id, pPeer->syncFd, strerror(errno));
}
free(pHead);
*wver = lastVer;
return code;
}
static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) {
SSyncNode *pNode = pPeer->pSyncNode;
SWalHead * pHead = (SWalHead *)offset;
(*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
offset += pHead->len + sizeof(SWalHead);
return offset;
}
static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) {
SSyncNode * pNode = pPeer->pSyncNode;
SRecvBuffer *pRecv = pNode->pRecv;
int32_t forwards = 0;
if (pRecv == NULL) {
sError("%s, recv buffer is null, restart connect", pPeer->id);
return -1;
}
sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards);
char *offset = pRecv->buffer;
while (forwards < pRecv->forwards) {
offset = syncProcessOneBufferedFwd(pPeer, offset);
forwards++;
}
pthread_mutex_lock(&pNode->mutex);
while (forwards < pRecv->forwards && pRecv->code == 0) {
offset = syncProcessOneBufferedFwd(pPeer, offset);
forwards++;
}
nodeRole = TAOS_SYNC_ROLE_SLAVE;
sDebug("%s, finish processing buffered fwds:%d", pPeer->id, forwards);
pthread_mutex_unlock(&pNode->mutex);
return pRecv->code;
}
int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
SSyncNode * pNode = pPeer->pSyncNode;
SRecvBuffer *pRecv = pNode->pRecv;
int32_t len = pHead->len + sizeof(SWalHead);
if (pRecv == NULL) {
sError("%s, recv buffer is not create yet", pPeer->id);
return -1;
}
if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) {
memcpy(pRecv->offset, pHead, len);
pRecv->offset += len;
pRecv->forwards++;
sTrace("%s, fwd is saved into queue, hver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards);
} else {
sError("%s, buffer size:%d is too small", pPeer->id, pRecv->bufferSize);
pRecv->code = -1; // set error code
}
return pRecv->code;
}
static void syncCloseRecvBuffer(SSyncNode *pNode) {
if (pNode->pRecv) {
sDebug("vgId:%d, recv buffer:%p is freed", pNode->vgId, pNode->pRecv);
tfree(pNode->pRecv->buffer);
}
tfree(pNode->pRecv);
}
static int32_t syncOpenRecvBuffer(SSyncNode *pNode) {
syncCloseRecvBuffer(pNode);
SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1);
if (pRecv == NULL) return -1;
pRecv->bufferSize = SYNC_RECV_BUFFER_SIZE;
pRecv->buffer = malloc(pRecv->bufferSize);
if (pRecv->buffer == NULL) {
free(pRecv);
return -1;
}
pRecv->offset = pRecv->buffer;
pRecv->forwards = 0;
pNode->pRecv = pRecv;
sDebug("vgId:%d, recv buffer:%p is created", pNode->vgId, pNode->pRecv);
return 0;
}
static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
nodeSStatus = TAOS_SYNC_STATUS_FILE;
uint64_t fversion = 0;
sInfo("%s, start to restore, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
SSyncRsp rsp = {.sync = 1, .tranId = syncGenTranId()};
if (taosWriteMsg(pPeer->syncFd, &rsp, sizeof(SSyncRsp)) != sizeof(SSyncRsp)) {
sError("%s, failed to send sync rsp since %s", pPeer->id, strerror(errno));
return -1;
}
sDebug("%s, send sync rsp to peer, tranId:%u", pPeer->id, rsp.tranId);
sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
(*pNode->startSyncFileFp)(pNode->vgId);
int32_t code = syncRestoreFile(pPeer, &fversion);
if (code < 0) {
(*pNode->stopSyncFileFp)(pNode->vgId, fversion);
sError("%s, failed to restore files", pPeer->id);
return -1;
}
(*pNode->stopSyncFileFp)(pNode->vgId, fversion);
nodeVersion = fversion;
sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion);
uint64_t wver = 0;
code = syncRestoreWal(pPeer, &wver); // lastwar
if (code < 0) {
sError("%s, failed to restore wal, code:%d", pPeer->id, code);
return -1;
}
if (wver != 0) {
nodeVersion = wver;
sDebug("%s, restore wal finished, set sver:%" PRIu64, pPeer->id, nodeVersion);
}
nodeSStatus = TAOS_SYNC_STATUS_CACHE;
sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
if (syncProcessBufferedFwd(pPeer) < 0) {
sError("%s, failed to insert buffered points", pPeer->id);
return -1;
}
return 0;
}
void *syncRestoreData(void *param) {
setThreadName("syncRestoreData");
int64_t rid = (int64_t)param;
SSyncPeer *pPeer = syncAcquirePeer(rid);
if (pPeer == NULL) {
sError("failed to restore data, invalid peer rid:%" PRId64, rid);
return NULL;
}
SSyncNode *pNode = pPeer->pSyncNode;
taosBlockSIGPIPE();
atomic_add_fetch_32(&tsSyncNum, 1);
sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
nodeRole = TAOS_SYNC_ROLE_SYNCING;
(*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
if (syncOpenRecvBuffer(pNode) < 0) {
sError("%s, failed to allocate recv buffer, restart connection", pPeer->id);
syncRestartConnection(pPeer);
} else {
if (syncRestoreDataStepByStep(pPeer) == 0) {
sInfo("%s, it is synced successfully", pPeer->id);
nodeRole = TAOS_SYNC_ROLE_SLAVE;
syncBroadcastStatus(pNode);
} else {
sError("%s, failed to restore data, restart connection", pPeer->id);
nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
syncRestartConnection(pPeer);
}
}
(*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
nodeSStatus = TAOS_SYNC_STATUS_INIT;
sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
taosCloseSocket(pPeer->syncFd);
syncCloseRecvBuffer(pNode);
atomic_sub_fetch_32(&tsSyncNum, 1);
// The ref is obtained in both the create thread and the current thread, so it is released twice
syncReleasePeer(pPeer);
syncReleasePeer(pPeer);
return NULL;
}

472
src/sync/src/syncRetrieve.c
View File

@ -1,472 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tlog.h"
#include "tutil.h"
#include "tglobal.h"
#include "ttimer.h"
#include "tsocket.h"
#include "twal.h"
#include "tsync.h"
#include "syncInt.h"
static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
uint64_t fver, wver;
int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
if (code != 0) {
sInfo("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
return -1;
}
pPeer->lastWalVer = wver;
return code;
}
static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) {
uint64_t fver, wver;
int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
if (code != 0) {
sInfo("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
return true;
}
if (wver != pPeer->lastWalVer) {
sInfo("%s, wal is modified while retrieve, wver:%" PRIu64 ", last:%" PRIu64, pPeer->id, wver, pPeer->lastWalVer);
return true;
}
return false;
}
static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
uint64_t fver, wver;
int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
if (code != 0) {
sInfo("%s, vnode is commiting while get fver for retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
return -1;
}
pPeer->lastFileVer = fver;
return code;
}
static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) {
uint64_t fver, wver;
int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
if (code != 0) {
sInfo("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
pPeer->fileChanged = 1;
return true;
}
if (fver != pPeer->lastFileVer) {
sInfo("%s, files are modified while retrieve, fver:%" PRIu64 ", last:%" PRIu64, pPeer->id, fver, pPeer->lastFileVer);
pPeer->fileChanged = 1;
return true;
}
pPeer->fileChanged = 0;
return false;
}
static int32_t syncSendFileVersion(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SFileVersion fileVersion;
memset(&fileVersion, 0, sizeof(SFileVersion));
syncBuildFileVersion(&fileVersion, pNode->vgId);
uint64_t fver = pPeer->lastFileVer;
fileVersion.fversion = htobe64(fver);
int32_t ret = taosWriteMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion));
if (ret != sizeof(SFileVersion)) {
sError("%s, failed to write fver:%" PRIu64 " since %s", pPeer->id, fver, strerror(errno));
return -1;
}
SFileAck fileAck;
memset(&fileAck, 0, sizeof(SFileAck));
ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
if (ret != sizeof(SFileAck)) {
sError("%s, failed to read fver ack since %s", pPeer->id, strerror(errno));
return -1;
}
// set the peer sync version
pPeer->sversion = fver;
return 0;
}
static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
if (syncGetFileVersion(pNode, pPeer) < 0) {
pPeer->fileChanged = 1;
return -1;
}
if (pNode->sendFileFp && (*pNode->sendFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) {
sError("%s, failed to retrieve file", pPeer->id);
return -1;
}
if (syncSendFileVersion(pPeer) < 0) {
return -1;
}
sInfo("%s, all files are retrieved", pPeer->id);
return 0;
}
// if only a partial record is read out, upper layer will reload the file to get a complete record
static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) {
int32_t ret = read(sfd, pHead, sizeof(SWalHead));
if (ret < 0) {
sError("sfd:%d, failed to read wal head since %s, ret:%d", sfd, strerror(errno), ret);
return -1;
}
if (ret == 0) {
sInfo("sfd:%d, read to the end of file, ret:%d", sfd, ret);
return 0;
}
if (ret != sizeof(SWalHead)) {
// file is not at end yet, it shall be reloaded
sInfo("sfd:%d, a partial wal head is read out, ret:%d", sfd, ret);
return 0;
}
assert(pHead->len <= TSDB_MAX_WAL_SIZE);
ret = read(sfd, pHead->cont, pHead->len);
if (ret < 0) {
sError("sfd:%d, failed to read wal content since %s, ret:%d", sfd, strerror(errno), ret);
return -1;
}
if (ret != pHead->len) {
// file is not at end yet, it shall be reloaded
sInfo("sfd:%d, a partial wal conetnt is read out, ret:%d", sfd, ret);
return 0;
}
return sizeof(SWalHead) + pHead->len;
}
static int64_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) {
int32_t sfd = open(name, O_RDONLY | O_BINARY);
if (sfd < 0) {
sError("%s, failed to open wal:%s for retrieve since:%s", pPeer->id, name, tstrerror(errno));
return -1;
}
int64_t code = taosLSeek(sfd, offset, SEEK_SET);
if (code < 0) {
sError("%s, failed to seek %" PRId64 " in wal:%s for retrieve since:%s", pPeer->id, offset, name, tstrerror(errno));
close(sfd);
return -1;
}
sInfo("%s, retrieve last wal:%s, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, name, offset, fversion);
SWalHead *pHead = malloc(SYNC_MAX_SIZE);
int64_t bytes = 0;
while (1) {
code = syncReadOneWalRecord(sfd, pHead);
if (code < 0) {
sError("%s, failed to read one record from wal:%s", pPeer->id, name);
break;
}
if (code == 0) {
code = bytes;
sInfo("%s, read to the end of wal, bytes:%" PRId64, pPeer->id, bytes);
break;
}
sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version);
int32_t wsize = (int32_t)code;
int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
if (ret != wsize) {
code = -1;
sError("%s, failed to forward wal since %s, hver:%" PRIu64, pPeer->id, strerror(errno), pHead->version);
break;
}
pPeer->sversion = pHead->version;
bytes += wsize;
if (pHead->version >= fversion && fversion > 0) {
code = 0;
sInfo("%s, retrieve wal finished, hver:%" PRIu64 " fver:%" PRIu64, pPeer->id, pHead->version, fversion);
break;
}
}
free(pHead);
close(sfd);
return code;
}
static int64_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) {
SSyncNode *pNode = pPeer->pSyncNode;
int32_t once = 0; // last WAL has once ever been processed
int64_t offset = 0;
uint64_t fversion = 0;
char fname[TSDB_FILENAME_LEN * 2] = {0}; // full path to wal file
// get full path to wal file
snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
sInfo("%s, start to retrieve last wal:%s", pPeer->id, fname);
while (1) {
if (syncAreFilesModified(pNode, pPeer)) return -1;
if (syncGetWalVersion(pNode, pPeer) < 0) return -1;
int64_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset);
if (bytes < 0) {
sInfo("%s, failed to retrieve last wal, bytes:%" PRId64, pPeer->id, bytes);
return bytes;
}
// check file changes
bool walModified = syncIsWalModified(pNode, pPeer);
// if file is not updated or updated once, set the fversion and sstatus
if (!walModified || once) {
if (fversion == 0) {
pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt
fversion = nodeVersion; // must read data to fversion
sInfo("%s, set sstatus:%s and fver:%" PRIu64, pPeer->id, syncStatus[pPeer->sstatus], fversion);
}
}
// if all data up to fversion is read out, it is over
if (pPeer->sversion >= fversion && fversion > 0) {
sInfo("%s, data up to fver:%" PRIu64 " has been read out, bytes:%" PRId64 " sver:%" PRIu64, pPeer->id, fversion, bytes,
pPeer->sversion);
return 0;
}
// if all data are read out, and no update
if (bytes == 0 && !walModified) {
// wal not closed, it means some data not flushed to disk, wait for a while
taosMsleep(10);
}
// if bytes > 0, file is updated, or fversion is not reached but file still open, read again
once = 1;
offset += bytes;
sInfo("%s, continue retrieve last wal, bytes:%" PRId64 " offset:%" PRId64 " sver:%" PRIu64 " fver:%" PRIu64, pPeer->id,
bytes, offset, pPeer->sversion, fversion);
}
return -1;
}
static int64_t syncRetrieveWal(SSyncPeer *pPeer) {
SSyncNode * pNode = pPeer->pSyncNode;
char fname[TSDB_FILENAME_LEN * 3];
char wname[TSDB_FILENAME_LEN * 2];
int32_t size;
int64_t code = -1;
int64_t index = 0;
while (1) {
// retrieve wal info
wname[0] = 0;
code = (*pNode->getWalInfoFp)(pNode->vgId, wname, &index);
if (code < 0) {
sError("%s, failed to get wal info since:%s, code:0x%" PRIx64, pPeer->id, strerror(errno), code);
break;
}
if (wname[0] == 0) { // no wal file
code = 0;
sInfo("%s, no wal file anymore", pPeer->id);
break;
}
if (code == 0) { // last wal
code = syncProcessLastWal(pPeer, wname, index);
sInfo("%s, last wal processed, code:%" PRId64, pPeer->id, code);
break;
}
// get the full path to wal file
snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
// send wal file, old wal file won't be modified, even remove is ok
struct stat fstat;
if (stat(fname, &fstat) < 0) {
code = -1;
sInfo("%s, failed to stat wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code);
break;
}
size = fstat.st_size;
sInfo("%s, retrieve wal:%s size:%d", pPeer->id, fname, size);
int32_t sfd = open(fname, O_RDONLY | O_BINARY);
if (sfd < 0) {
code = -1;
sError("%s, failed to open wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code);
break;
}
code = taosSendFile(pPeer->syncFd, sfd, NULL, size);
close(sfd);
if (code < 0) {
sError("%s, failed to send wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code);
break;
}
if (syncAreFilesModified(pNode, pPeer)) {
code = -1;
break;
}
}
if (code == 0) {
SWalHead walHead;
memset(&walHead, 0, sizeof(walHead));
if (taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)) == sizeof(walHead)) {
pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
} else {
sError("%s, failed to send last wal record since %s", pPeer->id, strerror(errno));
code = -1;
}
} else {
sError("%s, failed to send wal since %s, code:0x%" PRIx64, pPeer->id, strerror(errno), code);
}
return code;
}
static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SSyncMsg msg;
syncBuildSyncDataMsg(&msg, pNode->vgId);
if (taosWriteMsg(pPeer->syncFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) {
sError("%s, failed to send sync-data msg since %s, tranId:%u", pPeer->id, strerror(errno), msg.tranId);
return -1;
}
sInfo("%s, send sync-data msg to peer, tranId:%u", pPeer->id, msg.tranId);
SSyncRsp rsp;
if (taosReadMsg(pPeer->syncFd, &rsp, sizeof(SSyncRsp)) != sizeof(SSyncRsp)) {
sError("%s, failed to read sync-data rsp since %s, tranId:%u", pPeer->id, strerror(errno), msg.tranId);
return -1;
}
sInfo("%s, recv sync-data rsp from peer, tranId:%u rsp-tranId:%u", pPeer->id, msg.tranId, rsp.tranId);
return 0;
}
static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
sInfo("%s, start to retrieve, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
if (syncRetrieveFirstPkt(pPeer) < 0) {
sError("%s, failed to start retrieve", pPeer->id);
return -1;
}
pPeer->sversion = 0;
pPeer->sstatus = TAOS_SYNC_STATUS_FILE;
sInfo("%s, start to retrieve files, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
if (syncRetrieveFile(pPeer) != 0) {
sError("%s, failed to retrieve files", pPeer->id);
return -1;
}
// if no files are synced, there must be wal to sync, sversion must be larger than one
if (pPeer->sversion == 0) pPeer->sversion = 1;
sInfo("%s, start to retrieve wals", pPeer->id);
int64_t code = syncRetrieveWal(pPeer);
if (code < 0) {
sError("%s, failed to retrieve wals, code:0x%" PRIx64, pPeer->id, code);
return -1;
}
return 0;
}
void *syncRetrieveData(void *param) {
int64_t rid = (int64_t)param;
SSyncPeer *pPeer = syncAcquirePeer(rid);
if (pPeer == NULL) {
sError("failed to retrieve data, invalid peer rid:%" PRId64, rid);
return NULL;
}
uint32_t ip = syncResolvePeerFqdn(pPeer);
if (!ip) {
syncReleasePeer(pPeer);
return NULL;
}
SSyncNode *pNode = pPeer->pSyncNode;
taosBlockSIGPIPE();
sInfo("%s, start to retrieve data, sstatus:%s, numOfRetrieves:%d", pPeer->id, syncStatus[pPeer->sstatus],
pPeer->numOfRetrieves);
if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, pPeer->numOfRetrieves);
pPeer->syncFd = taosOpenTcpClientSocket(ip, pPeer->port, 0);
if (pPeer->syncFd < 0) {
sError("%s, failed to open socket to sync", pPeer->id);
} else {
sInfo("%s, sync tcp is setup", pPeer->id);
if (syncRetrieveDataStepByStep(pPeer) == 0) {
sInfo("%s, sync retrieve process is successful", pPeer->id);
} else {
sError("%s, failed to retrieve data, restart connection", pPeer->id);
syncRestartConnection(pPeer);
}
}
if (pPeer->fileChanged) {
pPeer->numOfRetrieves++;
} else {
pPeer->numOfRetrieves = 0;
// if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
}
if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
pPeer->fileChanged = 0;
taosCloseSocket(pPeer->syncFd);
// The ref is obtained in both the create thread and the current thread, so it is released twice
sInfo("%s, sync retrieve data over, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
syncReleasePeer(pPeer);
syncReleasePeer(pPeer);
return NULL;
}

338
src/sync/src/syncTcp.c
View File

@ -1,338 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tulog.h"
#include "tutil.h"
#include "tsocket.h"
#include "taoserror.h"
#include "twal.h"
#include "tsync.h"
#include "syncInt.h"
#include "syncTcp.h"
typedef struct SThreadObj {
pthread_t thread;
bool stop;
SOCKET pollFd;
int32_t numOfFds;
struct SPoolObj *pPool;
} SThreadObj;
typedef struct SPoolObj {
SPoolInfo info;
SThreadObj **pThread;
pthread_t thread;
int32_t nextId;
SOCKET acceptFd; // FD for accept new connection
int8_t stop;
} SPoolObj;
typedef struct {
SThreadObj *pThread;
int64_t handleId;
SOCKET fd;
int32_t closedByApp;
} SConnObj;
static void *syncAcceptPeerTcpConnection(void *argv);
static void *syncProcessTcpData(void *param);
static void syncStopPoolThread(SThreadObj *pThread);
static SThreadObj *syncGetTcpThread(SPoolObj *pPool);
void *syncOpenTcpThreadPool(SPoolInfo *pInfo) {
pthread_attr_t thattr;
SPoolObj *pPool = calloc(sizeof(SPoolObj), 1);
if (pPool == NULL) {
sError("failed to alloc pool for TCP server since no enough memory");
return NULL;
}
pPool->info = *pInfo;
pPool->pThread = calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
if (pPool->pThread == NULL) {
sError("failed to alloc pool thread for TCP server since no enough memory");
tfree(pPool);
return NULL;
}
pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port);
if (pPool->acceptFd < 0) {
tfree(pPool->pThread);
tfree(pPool);
sError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
return NULL;
}
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pPool->thread), &thattr, (void *)syncAcceptPeerTcpConnection, pPool) != 0) {
sError("failed to create accept thread for TCP server since %s", strerror(errno));
taosCloseSocket(pPool->acceptFd);
tfree(pPool->pThread);
tfree(pPool);
return NULL;
}
pthread_attr_destroy(&thattr);
sDebug("%p TCP pool is created", pPool);
return pPool;
}
void syncCloseTcpThreadPool(void *param) {
SPoolObj * pPool = param;
SThreadObj *pThread;
pPool->stop = 1;
#ifdef WINDOWS
closesocket(pPool->acceptFd);
#elif defined(__APPLE__)
if (pPool->acceptFd!=-1) {
close(pPool->acceptFd);
pPool->acceptFd = -1;
}
#else
shutdown(pPool->acceptFd, SHUT_RD);
#endif
pthread_join(pPool->thread, NULL);
for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) {
pThread = pPool->pThread[i];
if (pThread) syncStopPoolThread(pThread);
}
sDebug("%p TCP pool is closed", pPool);
tfree(pPool->pThread);
tfree(pPool);
}
void *syncAllocateTcpConn(void *param, int64_t rid, SOCKET connFd) {
struct epoll_event event;
SPoolObj *pPool = param;
SConnObj *pConn = calloc(sizeof(SConnObj), 1);
if (pConn == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
SThreadObj *pThread = syncGetTcpThread(pPool);
if (pThread == NULL) {
tfree(pConn);
return NULL;
}
pConn->fd = connFd;
pConn->pThread = pThread;
pConn->handleId = rid;
pConn->closedByApp = 0;
event.events = EPOLLIN | EPOLLRDHUP;
event.data.ptr = pConn;
if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
sError("failed to add fd:%d since %s", connFd, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
tfree(pConn);
pConn = NULL;
} else {
pThread->numOfFds++;
sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds);
}
return pConn;
}
void syncFreeTcpConn(void *param) {
SConnObj * pConn = param;
SThreadObj *pThread = pConn->pThread;
sDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
pConn->closedByApp = 1;
shutdown(pConn->fd, SHUT_WR);
}
static void taosProcessBrokenLink(SConnObj *pConn) {
SThreadObj *pThread = pConn->pThread;
SPoolObj * pPool = pThread->pPool;
SPoolInfo * pInfo = &pPool->info;
if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR);
(*pInfo->processBrokenLink)(pConn->handleId, pConn->closedByApp);
pThread->numOfFds--;
epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL);
sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds);
taosCloseSocket(pConn->fd);
tfree(pConn);
}
#define maxEvents 10
static void *syncProcessTcpData(void *param) {
SThreadObj *pThread = (SThreadObj *)param;
SPoolObj * pPool = pThread->pPool;
SPoolInfo * pInfo = &pPool->info;
SConnObj * pConn = NULL;
struct epoll_event events[maxEvents];
setThreadName("syncTcpData");
void *buffer = malloc(pInfo->bufferSize);
taosBlockSIGPIPE();
while (1) {
if (pThread->stop) break;
int32_t fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
if (pThread->stop) {
sDebug("%p TCP epoll thread is exiting...", pThread);
break;
}
if (fdNum < 0) {
sError("epoll_wait failed since %s", strerror(errno));
continue;
}
for (int32_t i = 0; i < fdNum; ++i) {
pConn = events[i].data.ptr;
assert(pConn);
if (events[i].events & EPOLLERR) {
sDebug("conn is broken since EPOLLERR");
taosProcessBrokenLink(pConn);
continue;
}
if (events[i].events & EPOLLHUP) {
sDebug("conn is broken since EPOLLHUP");
taosProcessBrokenLink(pConn);
continue;
}
if (events[i].events & EPOLLRDHUP) {
sDebug("conn is broken since EPOLLRDHUP");
taosProcessBrokenLink(pConn);
continue;
}
if (pConn->closedByApp == 0) {
if ((*pInfo->processIncomingMsg)(pConn->handleId, buffer) < 0) {
syncFreeTcpConn(pConn);
continue;
}
}
}
if (pThread->stop) break;
}
sDebug("%p TCP epoll thread exits", pThread);
EpollClose(pThread->pollFd);
tfree(pThread);
tfree(buffer);
return NULL;
}
static void *syncAcceptPeerTcpConnection(void *argv) {
SPoolObj * pPool = (SPoolObj *)argv;
SPoolInfo *pInfo = &pPool->info;
taosBlockSIGPIPE();
setThreadName("acceptTcpConn");
while (1) {
struct sockaddr_in clientAddr;
socklen_t addrlen = sizeof(clientAddr);
SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen);
if (pPool->stop) {
sDebug("%p TCP server accept is stopped", pPool);
break;
}
if (connFd < 0) {
if (errno == EINVAL) {
sDebug("%p TCP server accept is exiting...", pPool);
break;
} else {
sError("TCP accept failure since %s", strerror(errno));
continue;
}
}
// sDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port);
taosKeepTcpAlive(connFd);
(*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr);
}
taosCloseSocket(pPool->acceptFd);
return NULL;
}
static SThreadObj *syncGetTcpThread(SPoolObj *pPool) {
SThreadObj *pThread = pPool->pThread[pPool->nextId];
if (pThread) return pThread;
pThread = (SThreadObj *)calloc(1, sizeof(SThreadObj));
if (pThread == NULL) return NULL;
pThread->pPool = pPool;
pThread->pollFd = (EpollFd)epoll_create(10); // size does not matter
if (pThread->pollFd < 0) {
tfree(pThread);
return NULL;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)syncProcessTcpData, pThread);
pthread_attr_destroy(&thattr);
if (ret != 0) {
EpollClose(pThread->pollFd);
tfree(pThread);
return NULL;
}
sDebug("%p TCP epoll thread is created", pThread);
pPool->pThread[pPool->nextId] = pThread;
pPool->nextId++;
pPool->nextId = pPool->nextId % pPool->info.numOfThreads;
return pThread;
}
static void syncStopPoolThread(SThreadObj *pThread) {
pthread_t thread = pThread->thread;
if (!taosCheckPthreadValid(thread)) {
return;
}
pThread->stop = true;
if (taosComparePthread(thread, pthread_self())) {
pthread_detach(pthread_self());
return;
}
pthread_join(thread, NULL);
}

15
src/sync/test/CMakeLists.txt
View File

@ -1,15 +0,0 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
IF (TD_LINUX)
INCLUDE_DIRECTORIES(../inc)
LIST(APPEND CLIENT_SRC ./syncClient.c)
ADD_EXECUTABLE(syncClient ${CLIENT_SRC})
TARGET_LINK_LIBRARIES(syncClient sync trpc common)
LIST(APPEND SERVER_SRC ./syncServer.c)
ADD_EXECUTABLE(syncServer ${SERVER_SRC})
TARGET_LINK_LIBRARIES(syncServer sync trpc common)
ENDIF ()

194
src/sync/test/syncClient.c
View File

@ -1,194 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tglobal.h"
#include "tulog.h"
#include "trpc.h"
#include "taoserror.h"
typedef struct {
int index;
SRpcEpSet epSet;
int num;
int numOfReqs;
int msgSize;
tsem_t rspSem;
tsem_t * pOverSem;
pthread_t thread;
void * pRpc;
} SInfo;
void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SInfo *pInfo = (SInfo *)pMsg->ahandle;
uDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen,
pMsg->code);
if (pEpSet) pInfo->epSet = *pEpSet;
rpcFreeCont(pMsg->pCont);
tsem_post(&pInfo->rspSem);
}
int tcount = 0;
void *sendRequest(void *param) {
SInfo * pInfo = (SInfo *)param;
SRpcMsg rpcMsg = {0};
uDebug("thread:%d, start to send request", pInfo->index);
while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) {
pInfo->num++;
rpcMsg.pCont = rpcMallocCont(pInfo->msgSize);
rpcMsg.contLen = pInfo->msgSize;
rpcMsg.ahandle = pInfo;
rpcMsg.msgType = 1;
uDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
if (pInfo->num % 20000 == 0) {
uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
}
tsem_wait(&pInfo->rspSem);
}
uDebug("thread:%d, it is over", pInfo->index);
tcount++;
return NULL;
}
int main(int argc, char *argv[]) {
SRpcInit rpcInit;
SRpcEpSet epSet;
char secret[TSDB_KEY_LEN] = "mypassword";
int msgSize = 128;
int numOfReqs = 0;
int appThreads = 1;
char serverIp[40] = "127.0.0.1";
struct timeval systemTime;
int64_t startTime, endTime;
pthread_attr_t thattr;
// server info
epSet.numOfEps = 1;
epSet.inUse = 0;
epSet.port[0] = 7000;
epSet.port[1] = 7000;
strcpy(epSet.fqdn[0], serverIp);
strcpy(epSet.fqdn[1], "192.168.0.1");
// client info
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 0;
rpcInit.label = "APP";
rpcInit.numOfThreads = 1;
rpcInit.cfp = processResponse;
rpcInit.sessions = 100;
rpcInit.idleTime = tsShellActivityTimer*1000;
rpcInit.user = "michael";
rpcInit.secret = secret;
rpcInit.ckey = "key";
rpcInit.spi = 1;
rpcInit.connType = TAOS_CONN_CLIENT;
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
epSet.port[0] = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) {
tstrncpy(epSet.fqdn[0], argv[++i], TSDB_FQDN_LEN);
} else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) {
msgSize = atoi(argv[++i]);
} else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) {
rpcInit.sessions = atoi(argv[++i]);
} else if (strcmp(argv[i], "-n") == 0 && i < argc - 1) {
numOfReqs = atoi(argv[++i]);
} else if (strcmp(argv[i], "-a") == 0 && i < argc - 1) {
appThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) {
tsCompressMsgSize = atoi(argv[++i]);
} else if (strcmp(argv[i], "-u") == 0 && i < argc - 1) {
rpcInit.user = argv[++i];
} else if (strcmp(argv[i], "-k") == 0 && i < argc - 1) {
rpcInit.secret = argv[++i];
} else if (strcmp(argv[i], "-spi") == 0 && i < argc - 1) {
rpcInit.spi = atoi(argv[++i]);
} else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) {
rpcDebugFlag = atoi(argv[++i]);
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-i ip]: first server IP address, default is:%s\n", serverIp);
printf(" [-p port]: server port number, default is:%d\n", epSet.port[0]);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
printf(" [-m msgSize]: message body size, default is:%d\n", msgSize);
printf(" [-a threads]: number of app threads, default is:%d\n", appThreads);
printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs);
printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize);
printf(" [-u user]: user name for the connection, default is:%s\n", rpcInit.user);
printf(" [-k secret]: password for the connection, default is:%s\n", rpcInit.secret);
printf(" [-spi SPI]: security parameter index, default is:%d\n", rpcInit.spi);
printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag);
printf(" [-h help]: print out this help\n\n");
exit(0);
}
}
taosInitLog("client.log", 100000, 10);
void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) {
uError("failed to initialize RPC");
return -1;
}
uInfo("client is initialized");
gettimeofday(&systemTime, NULL);
startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads);
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
for (int i = 0; i < appThreads; ++i) {
pInfo->index = i;
pInfo->epSet = epSet;
pInfo->numOfReqs = numOfReqs;
pInfo->msgSize = msgSize;
tsem_init(&pInfo->rspSem, 0, 0);
pInfo->pRpc = pRpc;
pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo);
pInfo++;
}
do {
usleep(1);
} while (tcount < appThreads);
gettimeofday(&systemTime, NULL);
endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
float usedTime = (endTime - startTime) / 1000.0; // mseconds
uInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads);
uInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize);
taosCloseLog();
return 0;
}

472
src/sync/test/syncServer.c
View File

@ -1,472 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
//#define _DEFAULT_SOURCE
#include <stdint.h>
#include "os.h"
#include "tulog.h"
#include "tglobal.h"
#include "tsocket.h"
#include "trpc.h"
#include "tqueue.h"
#include "twal.h"
#include "tsync.h"
int msgSize = 128;
int commit = 0;
int dataFd = -1;
void * qhandle = NULL;
int walNum = 0;
uint64_t tversion = 0;
int64_t syncHandle;
int role;
int nodeId;
char path[256];
int numOfWrites;
SSyncInfo syncInfo;
SSyncCfg *pCfg;
int writeIntoWal(SWalHead *pHead) {
if (dataFd < 0) {
char walName[280];
snprintf(walName, sizeof(walName), "%s/wal/wal.%d", path, walNum);
(void)remove(walName);
dataFd = open(walName, O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (dataFd < 0) {
uInfo("failed to open wal file:%s(%s)", walName, strerror(errno));
return -1;
} else {
walNum++;
uInfo("file:%s is opened to write, walNum:%d", walName, walNum);
}
}
if (write(dataFd, pHead, sizeof(SWalHead) + pHead->len) < 0) {
uError("ver:%" PRIu64 ", failed to write wal file(%s)", pHead->version, strerror(errno));
} else {
uDebug("ver:%" PRIu64 ", written to wal", pHead->version);
}
numOfWrites++;
if (numOfWrites >= 10000) {
uInfo("%d request have been written into disk", numOfWrites);
close(dataFd);
dataFd = -1;
numOfWrites = 0;
}
return 0;
}
void confirmForward(int32_t vgId, void *mhandle, int32_t code) {
SRpcMsg * pMsg = (SRpcMsg *)mhandle;
SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
uDebug("ver:%" PRIu64 ", confirm is received", pHead->version);
rpcFreeCont(pMsg->pCont);
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = rpcMallocCont(msgSize);
rpcMsg.contLen = msgSize;
rpcMsg.handle = pMsg->handle;
rpcMsg.code = code;
rpcSendResponse(&rpcMsg);
taosFreeQitem(mhandle);
}
int processRpcMsg(void *item) {
SRpcMsg * pMsg = (SRpcMsg *)item;
SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
int code = -1;
if (role != TAOS_SYNC_ROLE_MASTER) {
uError("not master, write failed, role:%s", syncRole[role]);
} else {
pHead->version = ++tversion;
pHead->msgType = pMsg->msgType;
pHead->len = pMsg->contLen;
uDebug("ver:%" PRIu64 ", rsp from client processed", pHead->version);
writeIntoWal(pHead);
syncForwardToPeer(syncHandle, pHead, item, TAOS_QTYPE_RPC);
code = 0;
}
if (pCfg->quorum <= 1) {
rpcFreeCont(pMsg->pCont);
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = rpcMallocCont(msgSize);
rpcMsg.contLen = msgSize;
rpcMsg.handle = pMsg->handle;
rpcMsg.code = code;
rpcSendResponse(&rpcMsg);
taosFreeQitem(item);
}
return code;
}
int processFwdMsg(void *item) {
SWalHead *pHead = (SWalHead *)item;
if (pHead->version <= tversion) {
uError("ver:%" PRIu64 ", forward is even lower than local:%" PRIu64, pHead->version, tversion);
return -1;
}
uDebug("ver:%" PRIu64 ", forward from peer is received", pHead->version);
writeIntoWal(pHead);
tversion = pHead->version;
if (pCfg->quorum > 1) syncConfirmForward(syncHandle, pHead->version, 0);
// write into cache
/*
if (pHead->handle) {
syncSendFwdAck(syncHandle, pHead->handle, 0);
}
*/
taosFreeQitem(item);
return 0;
}
int processWalMsg(void *item) {
SWalHead *pHead = (SWalHead *)item;
if (pHead->version <= tversion) {
uError("ver:%" PRIu64 ", wal is even lower than local:%" PRIu64, pHead->version, tversion);
return -1;
};
uDebug("ver:%" PRIu64 ", wal from peer is received", pHead->version);
writeIntoWal(pHead);
tversion = pHead->version;
// write into cache
/*
if (pHead->handle) {
syncSendFwdAck(syncHandle, pHead->handle, 0);
}
*/
taosFreeQitem(item);
return 0;
}
void *processWriteQueue(void *param) {
int type;
void *item;
setThreadName("syncWrite");
while (1) {
int ret = taosReadQitem(qhandle, &type, &item);
if (ret <= 0) {
usleep(1000);
continue;
}
if (type == TAOS_QTYPE_RPC) {
processRpcMsg(item);
} else if (type == TAOS_QTYPE_WAL) {
processWalMsg(item);
} else if (type == TAOS_QTYPE_FWD) {
processFwdMsg(item);
}
}
return NULL;
}
int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char *ckey) {
// app shall retrieve the auth info based on meterID from DB or a data file
// demo code here only for simple demo
int ret = 0;
if (strcmp(meterId, "michael") == 0) {
*spi = 1;
*encrypt = 0;
strcpy(secret, "mypassword");
strcpy(ckey, "key");
} else if (strcmp(meterId, "jeff") == 0) {
*spi = 0;
*encrypt = 0;
} else {
ret = -1; // user not there
}
return ret;
}
void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SRpcMsg *pTemp;
pTemp = taosAllocateQitem(sizeof(SRpcMsg));
memcpy(pTemp, pMsg, sizeof(SRpcMsg));
uDebug("request is received, type:%d, len:%d", pMsg->msgType, pMsg->contLen);
taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp);
}
uint32_t getFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
uint32_t magic;
struct stat fstat;
char aname[280];
if (*index == 2) {
uInfo("wait for a while .....");
sleep(3);
}
if (name[0] == 0) {
// find the file
snprintf(aname, sizeof(aname), "%s/data/data.%d", path, *index);
sprintf(name, "data/data.%d", *index);
} else {
snprintf(aname, sizeof(aname), "%s/%s", path, name);
}
uInfo("get file info:%s", aname);
if (stat(aname, &fstat) < 0) return 0;
*size = fstat.st_size;
magic = fstat.st_size;
return magic;
}
int getWalInfo(int32_t vgId, char *name, int64_t *index) {
struct stat fstat;
char aname[280];
name[0] = 0;
if (*index + 1 > walNum) return 0;
snprintf(aname, sizeof(aname), "%s/wal/wal.%d", path, *index);
sprintf(name, "wal/wal.%d", *index);
uInfo("get wal info:%s", aname);
if (stat(aname, &fstat) < 0) return -1;
if (*index >= walNum - 1) return 0; // no more
return 1;
}
int writeToCache(int32_t vgId, void *data, int type) {
SWalHead *pHead = data;
uDebug("rsp from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type);
int msgSize = pHead->len + sizeof(SWalHead);
void *pMsg = taosAllocateQitem(msgSize);
memcpy(pMsg, pHead, msgSize);
taosWriteQitem(qhandle, type, pMsg);
return 0;
}
void confirmFwd(int32_t vgId, int64_t version) { return; }
void notifyRole(int32_t vgId, int8_t r) {
role = r;
printf("current role:%s\n", syncRole[role]);
}
void initSync() {
pCfg->replica = 1;
pCfg->quorum = 1;
syncInfo.vgId = 1;
syncInfo.getWalInfoFp = getWalInfo;
syncInfo.writeToCacheFp = writeToCache;
syncInfo.confirmForward = confirmForward;
syncInfo.notifyRoleFp = notifyRole;
pCfg->nodeInfo[0].nodeId = 1;
pCfg->nodeInfo[0].nodePort = 7010;
taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
pCfg->nodeInfo[1].nodeId = 2;
pCfg->nodeInfo[1].nodePort = 7110;
taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
pCfg->nodeInfo[2].nodeId = 3;
pCfg->nodeInfo[2].nodePort = 7210;
taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
pCfg->nodeInfo[3].nodeId = 4;
pCfg->nodeInfo[3].nodePort = 7310;
taosGetFqdn(pCfg->nodeInfo[3].nodeFqdn);
pCfg->nodeInfo[4].nodeId = 5;
pCfg->nodeInfo[4].nodePort = 7410;
taosGetFqdn(pCfg->nodeInfo[4].nodeFqdn);
}
void doSync() {
for (int i = 0; i < 5; ++i) {
if (tsSyncPort == pCfg->nodeInfo[i].nodePort) nodeId = pCfg->nodeInfo[i].nodeId;
}
snprintf(path, sizeof(path), "/root/test/d%d", nodeId);
tstrncpy(syncInfo.path, path, sizeof(syncInfo.path));
if (syncHandle == NULL) {
syncHandle = syncStart(&syncInfo);
} else {
if (syncReconfig(syncHandle, pCfg) < 0) syncHandle = NULL;
}
uInfo("nodeId:%d path:%s syncPort:%d", nodeId, path, tsSyncPort);
}
int main(int argc, char *argv[]) {
SRpcInit rpcInit;
char dataName[20] = "server.data";
pCfg = &syncInfo.syncCfg;
initSync();
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 7000;
rpcInit.label = "SER";
rpcInit.numOfThreads = 1;
rpcInit.cfp = processRequestMsg;
rpcInit.sessions = 1000;
rpcInit.idleTime = tsShellActivityTimer * 1500;
rpcInit.afp = retrieveAuthInfo;
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
rpcInit.localPort = atoi(argv[++i]);
} else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) {
msgSize = atoi(argv[++i]);
} else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) {
rpcInit.sessions = atoi(argv[++i]);
} else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) {
tsCompressMsgSize = atoi(argv[++i]);
} else if (strcmp(argv[i], "-w") == 0 && i < argc - 1) {
commit = atoi(argv[++i]);
} else if (strcmp(argv[i], "-v") == 0 && i < argc - 1) {
syncInfo.version = atoi(argv[++i]);
} else if (strcmp(argv[i], "-r") == 0 && i < argc - 1) {
pCfg->replica = atoi(argv[++i]);
} else if (strcmp(argv[i], "-q") == 0 && i < argc - 1) {
pCfg->quorum = atoi(argv[++i]);
} else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) {
rpcDebugFlag = atoi(argv[++i]);
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions);
printf(" [-m msgSize]: message body size, default is:%d\n", msgSize);
printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize);
printf(" [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit);
printf(" [-v version]: initial node version, default is:%" PRId64 "\n", syncInfo.version);
printf(" [-r replica]: replicacation number, default is:%d\n", pCfg->replica);
printf(" [-q quorum]: quorum, default is:%d\n", pCfg->quorum);
printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag);
printf(" [-h help]: print out this help\n\n");
exit(0);
}
}
uDebugFlag = rpcDebugFlag;
dDebugFlag = rpcDebugFlag;
// tmrDebugFlag = rpcDebugFlag;
tsAsyncLog = 0;
taosInitLog("server.log", 1000000, 10);
rpcInit.connType = TAOS_CONN_SERVER;
void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) {
uError("failed to start RPC server");
return -1;
}
tsSyncPort = rpcInit.localPort + 10;
qhandle = taosOpenQueue();
doSync();
pthread_attr_t thattr;
pthread_t thread;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&thread, &thattr, processWriteQueue, NULL) != 0) {
uError("failed to create thread, reason:%s", strerror(errno));
return -1;
}
printf("server is running, localPort:%d\n", rpcInit.localPort);
SNodesRole nroles;
while (1) {
int c = getchar();
switch (c) {
case '1':
pCfg->replica = 1;
doSync();
break;
case '2':
pCfg->replica = 2;
doSync();
break;
case '3':
pCfg->replica = 3;
doSync();
break;
case '4':
pCfg->replica = 4;
doSync();
break;
case '5':
pCfg->replica = 5;
doSync();
break;
case 's':
syncGetNodesRole(syncHandle, &nroles);
for (int i = 0; i < pCfg->replica; ++i)
printf("=== nodeId:%d role:%s\n", nroles.nodeId[i], syncRole[nroles.role[i]]);
break;
default:
break;
}
if (c == 'q') break;
}
syncStop(syncHandle);
if (dataFd >= 0) {
close(dataFd);
remove(dataName);
}
return 0;
}