From 521d7c1519ce5b8bee984189e392c5308dafd7a0 Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 1 Aug 2020 08:22:03 +0800 Subject: [PATCH 001/105] add ehttp_* --- src/common/inc/elog.h | 71 ++ src/common/src/elog.c | 95 +++ src/plugins/http/inc/ehttp_gzip.h | 30 + src/plugins/http/inc/ehttp_parser.h | 36 + src/plugins/http/inc/ehttp_util_string.h | 18 + src/plugins/http/inc/httpInt.h | 11 + src/plugins/http/src/ehttp_gzip.c | 154 ++++ src/plugins/http/src/ehttp_parser.c | 965 +++++++++++++++++++++++ src/plugins/http/src/ehttp_util_string.c | 30 + src/plugins/http/src/httpContext.c | 161 ++++ src/plugins/http/src/httpServer.c | 63 ++ 11 files changed, 1634 insertions(+) create mode 100644 src/common/inc/elog.h create mode 100644 src/common/src/elog.c create mode 100644 src/plugins/http/inc/ehttp_gzip.h create mode 100644 src/plugins/http/inc/ehttp_parser.h create mode 100644 src/plugins/http/inc/ehttp_util_string.h create mode 100644 src/plugins/http/src/ehttp_gzip.c create mode 100644 src/plugins/http/src/ehttp_parser.c create mode 100644 src/plugins/http/src/ehttp_util_string.c diff --git a/src/common/inc/elog.h b/src/common/inc/elog.h new file mode 100644 index 0000000000..8e11c3e761 --- /dev/null +++ b/src/common/inc/elog.h @@ -0,0 +1,71 @@ +#ifndef _elog_h_8897be44_dda8_45b6_9d37_8d8691cb05fb_ +#define _elog_h_8897be44_dda8_45b6_9d37_8d8691cb05fb_ + +#include + +typedef enum { + ELOG_DEBUG, + ELOG_INFO, + ELOG_WARN, + ELOG_ERROR, + ELOG_CRITICAL, + ELOG_VERBOSE, + ELOG_ABORT, +} ELOG_LEVEL; + +void elog_set_level(ELOG_LEVEL base); // only log those not less than base +void elog_set_thread_name(const char *name); + +void elog(ELOG_LEVEL level, int fd, const char *file, int line, const char *func, const char *fmt, ...) +#ifdef __GNUC__ + __attribute__((format(printf, 6, 7))) +#endif +; + +#define DLOG(fd, fmt, ...) elog(ELOG_DEBUG, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define ILOG(fd, fmt, ...) elog(ELOG_INFO, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define WLOG(fd, fmt, ...) elog(ELOG_WARN, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define ELOG(fd, fmt, ...) elog(ELOG_ERROR, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define CLOG(fd, fmt, ...) elog(ELOG_CRITICAL, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define VLOG(fd, fmt, ...) elog(ELOG_VERBOSE, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define ALOG(fd, fmt, ...) elog(ELOG_ABORT, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) + +#define D(fmt, ...) elog(ELOG_DEBUG, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define I(fmt, ...) elog(ELOG_INFO, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define W(fmt, ...) elog(ELOG_WARN, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define E(fmt, ...) elog(ELOG_ERROR, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define C(fmt, ...) elog(ELOG_CRITICAL, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define V(fmt, ...) elog(ELOG_VERBOSE, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) +#define A(fmt, ...) elog(ELOG_ABORT, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) + + + +// NOTE: https://en.wikipedia.org/wiki/Fail-fast +// for the sake of simplicity, both implementation and usage, +// we'll follow `fail-fast` or `let-it-crash` philosophy. + +// assertion in both debug/release build +#define EQ_ABORT(fmt, ...) A("Assertion failure: "fmt, ##__VA_ARGS__) + +#define EQ_ASSERT(statement) do { \ + if (statement) break; \ + A("Assertion failure: %s", #statement); \ +} while (0) + +#define EQ_ASSERT_EXT(statement, fmt, ...) do { \ + if (statement) break; \ + A("Assertion failure: %s: "fmt, #statement, ##__VA_ARGS__); \ +} while (0) + +#define EQ_ASSERT_API0(statement) do { \ + if (statement) break; \ + A("Assertion failure: %s failed: [%d]%s", #statement, errno, strerror(errno)); \ +} while (0) + +#define EQ_ASSERT_API(api) do { \ + A("Assertion failure: %s failed: [%d]%s", #api, errno, strerror(errno)); \ +} while (0) + + +#endif // _elog_h_8897be44_dda8_45b6_9d37_8d8691cb05fb_ + diff --git a/src/common/src/elog.c b/src/common/src/elog.c new file mode 100644 index 0000000000..95b580962c --- /dev/null +++ b/src/common/src/elog.c @@ -0,0 +1,95 @@ +#include "elog.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define gettid() syscall(__NR_gettid) + +static ELOG_LEVEL elog_level_base = ELOG_DEBUG; + +static __thread long elog_thread_id; +static __thread char elog_thread_name[24] = {0}; + +void elog_set_level(ELOG_LEVEL base) { + elog_level_base = base; +} + +void elog_set_thread_name(const char *name) { + elog_thread_id = gettid(); + snprintf(elog_thread_name, sizeof(elog_thread_name), "%s", name); +} + +void elog(ELOG_LEVEL level, int fd, const char *file, int line, const char *func, const char *fmt, ...) +{ + if (level < elog_level_base) return; + if (fd == -1) return; + + if (elog_thread_name[0]=='\0') { + elog_set_thread_name("unknown"); + } + + char *p; + int n; + size_t bytes; + + char buf[4096]; + snprintf(buf, sizeof(buf), "%s", file); + + char fn[1024]; + snprintf(fn, sizeof(fn), "%s", basename(buf)); + + char C; + switch (level) { + case ELOG_DEBUG: C = 'D'; break; + case ELOG_INFO: C = 'I'; break; + case ELOG_WARN: C = 'W'; break; + case ELOG_ERROR: C = 'E'; break; + case ELOG_CRITICAL: C = 'C'; break; + case ELOG_VERBOSE: C = 'V'; break; + case ELOG_ABORT: C = 'A'; break; + default: return; + } + + struct tm t; + struct timeval tv; + + if (gettimeofday(&tv, NULL)) return; + if (!localtime_r(&tv.tv_sec, &t)) return; + + p = buf; + bytes = sizeof(buf); + + n = snprintf(p, bytes, "%c[%02d/%02d %02d:%02d:%02d.%06ld][%06ld]: ==", + C, + t.tm_mon + 1, t.tm_mday, + t.tm_hour, t.tm_min, t.tm_sec, + tv.tv_usec, + elog_thread_id); + p += n; bytes -= n; + + va_list arg; + va_start(arg, fmt); + if (bytes>0) { + n = vsnprintf(p, bytes, fmt, arg); + p += n; bytes -= n; + } + va_end(arg); + + if (bytes>0) { + n = snprintf(p, bytes, "== t:%s#%s[%d]#%s()", + elog_thread_name, fn, line, func); + } + + dprintf(fd, "%s\n", buf); + + if (level == ELOG_ABORT) { + abort(); + } +} + diff --git a/src/plugins/http/inc/ehttp_gzip.h b/src/plugins/http/inc/ehttp_gzip.h new file mode 100644 index 0000000000..b2d6ace9b0 --- /dev/null +++ b/src/plugins/http/inc/ehttp_gzip.h @@ -0,0 +1,30 @@ +#ifndef _ehttp_gzip_h_9196791b_ac2a_4d73_9979_f4b41abbc4c0_ +#define _ehttp_gzip_h_9196791b_ac2a_4d73_9979_f4b41abbc4c0_ + +#include + +#define EHTTP_GZIP_CHUNK_SIZE_DEFAULT (1024*16) + +typedef struct ehttp_gzip_s ehttp_gzip_t; + +typedef struct ehttp_gzip_callbacks_s ehttp_gzip_callbacks_t; +typedef struct ehttp_gzip_conf_s ehttp_gzip_conf_t; + +struct ehttp_gzip_callbacks_s { + void (*on_data)(ehttp_gzip_t *gzip, void *arg, const char *buf, size_t len); +}; + +struct ehttp_gzip_conf_s { + int get_header:2; // 0: not fetching header info + size_t chunk_size; // 0: fallback to default: EHTTP_GZIP_CHUNK_SIZE_DEFAULT +}; + +ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); +ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); +void ehttp_gzip_destroy(ehttp_gzip_t *gzip); + +int ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, size_t len); +int ehttp_gzip_finish(ehttp_gzip_t *gzip); + +#endif // _ehttp_gzip_h_9196791b_ac2a_4d73_9979_f4b41abbc4c0_ + diff --git a/src/plugins/http/inc/ehttp_parser.h b/src/plugins/http/inc/ehttp_parser.h new file mode 100644 index 0000000000..be87650128 --- /dev/null +++ b/src/plugins/http/inc/ehttp_parser.h @@ -0,0 +1,36 @@ +#ifndef _ehttp_parser_fc7f9ac9_52da_4ee3_b556_deb2e1c3866e +#define _ehttp_parser_fc7f9ac9_52da_4ee3_b556_deb2e1c3866e + +#include + +typedef struct ehttp_parser_s ehttp_parser_t; +typedef struct ehttp_parser_callbacks_s ehttp_parser_callbacks_t; +typedef struct ehttp_parser_conf_s ehttp_parser_conf_t; +typedef struct ehttp_status_code_s ehttp_status_code_t; + +struct ehttp_parser_callbacks_s { + void (*on_request_line)(void *arg, const char *method, const char *target, const char *version, const char *target_raw); + void (*on_status_line)(void *arg, const char *version, int status_code, const char *reason_phrase); + void (*on_header_field)(void *arg, const char *key, const char *val); + void (*on_body)(void *arg, const char *chunk, size_t len); + void (*on_end)(void *arg); + void (*on_error)(void *arg, int status_code); +}; + +struct ehttp_parser_conf_s { + size_t flush_block_size; // <=0: immediately +}; + +ehttp_parser_t* ehttp_parser_create(ehttp_parser_callbacks_t callbacks, ehttp_parser_conf_t conf, void *arg); +void ehttp_parser_destroy(ehttp_parser_t *parser); +int ehttp_parser_parse(ehttp_parser_t *parser, const char *buf, size_t len); +int ehttp_parser_parse_string(ehttp_parser_t *parser, const char *str); +int ehttp_parser_parse_char(ehttp_parser_t *parser, const char c); +int ehttp_parser_parse_end(ehttp_parser_t *parser); + +char* ehttp_parser_urldecode(const char *enc); + +const char* ehttp_status_code_get_desc(const int status_code); + +#endif // _ehttp_parser_fc7f9ac9_52da_4ee3_b556_deb2e1c3866e + diff --git a/src/plugins/http/inc/ehttp_util_string.h b/src/plugins/http/inc/ehttp_util_string.h new file mode 100644 index 0000000000..46c5a42827 --- /dev/null +++ b/src/plugins/http/inc/ehttp_util_string.h @@ -0,0 +1,18 @@ +#ifndef _ehttp_util_string_h_99dacde5_2e7d_4662_97d6_04611fde683b_ +#define _ehttp_util_string_h_99dacde5_2e7d_4662_97d6_04611fde683b_ + +#include + +typedef struct ehttp_util_string_s ehttp_util_string_t; + +struct ehttp_util_string_s { + char *str; + size_t len; +}; + +void ehttp_util_string_cleanup(ehttp_util_string_t *str); +int ehttp_util_string_append(ehttp_util_string_t *str, const char *s, size_t len); +void ehttp_util_string_clear(ehttp_util_string_t *str); + +#endif // _ehttp_util_string_h_99dacde5_2e7d_4662_97d6_04611fde683b_ + diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index ffd621be7a..044b5cc4cc 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -28,6 +28,8 @@ #include "httpLog.h" #include "httpJson.h" +#include "ehttp_parser.h" + #define HTTP_MAX_CMD_SIZE 1024 #define HTTP_MAX_BUFFER_SIZE 1024*1024 @@ -162,6 +164,11 @@ typedef struct { int32_t len; } HttpBuf; +typedef enum { + EHTTP_CONTEXT_PROCESS_FAILED = 0x01, + EHTTP_CONTEXT_PARSER_FAILED = 0x02 +} EHTTP_CONTEXT_FAILED_CAUSE; + typedef struct { char buffer[HTTP_BUFFER_SIZE]; int bufsize; @@ -172,6 +179,10 @@ typedef struct { HttpBuf data; // body content HttpBuf token; // auth token HttpDecodeMethod *pMethod; + + ehttp_parser_t *parser; + int inited:2; + int failed:4; } HttpParser; typedef struct HttpContext { diff --git a/src/plugins/http/src/ehttp_gzip.c b/src/plugins/http/src/ehttp_gzip.c new file mode 100644 index 0000000000..ded344dfea --- /dev/null +++ b/src/plugins/http/src/ehttp_gzip.c @@ -0,0 +1,154 @@ +#include "ehttp_gzip.h" + +#include "os.h" +#include "zlib.h" + +#include + +typedef enum { + EHTTP_GZIP_INITING, + EHTTP_GZIP_READY, + EHTTP_GZIP_CLOSED, +} EHTTP_GZIP_STATE; + +struct ehttp_gzip_s { + ehttp_gzip_conf_t conf; + ehttp_gzip_callbacks_t callbacks; + void *arg; + z_stream *gzip; + gz_header *header; + char *chunk; + + int state; +}; + +static void dummy_on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, size_t len) { +} + +static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) { + switch(gzip->state) { + case EHTTP_GZIP_READY: { + inflateEnd(gzip->gzip); + } break; + default: break; + } + if (gzip->gzip) { + free(gzip->gzip); + gzip->gzip = NULL; + } + if (gzip->header) { + free(gzip->header); + gzip->header = NULL; + } + if (gzip->chunk) { + free(gzip->chunk); + gzip->chunk = NULL; + } + gzip->state = EHTTP_GZIP_CLOSED; +} + +ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) { + ehttp_gzip_t *gzip = (ehttp_gzip_t*)calloc(1, sizeof(*gzip)); + if (!gzip) return NULL; + + do { + gzip->conf = conf; + gzip->callbacks = callbacks; + gzip->arg = arg; + if (gzip->callbacks.on_data == NULL) gzip->callbacks.on_data = dummy_on_data; + gzip->gzip = (z_stream*)calloc(1, sizeof(*gzip->gzip)); + if (gzip->conf.get_header) { + gzip->header = (gz_header*)calloc(1, sizeof(*gzip->header)); + } + if (gzip->conf.chunk_size<=0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT; + gzip->chunk = (char*)malloc(gzip->conf.chunk_size); + if (!gzip->gzip || (gzip->conf.get_header && !gzip->header) || !gzip->chunk) break; + gzip->gzip->zalloc = Z_NULL; + gzip->gzip->zfree = Z_NULL; + gzip->gzip->opaque = Z_NULL; + + // 863 windowBits can also be greater than 15 for optional gzip decoding. Add + // 864 32 to windowBits to enable zlib and gzip decoding with automatic header + // 865 detection, or add 16 to decode only the gzip format (the zlib format will + // 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a + // 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + // 868 below), inflate() will not automatically decode concatenated gzip streams. + // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state + // 870 would need to be reset to continue decoding a subsequent gzip stream. + int ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS + if (ret != Z_OK) break; + if (gzip->header) { + ret = inflateGetHeader(gzip->gzip, gzip->header); + } + if (ret != Z_OK) break; + + gzip->gzip->next_out = (z_const Bytef*)gzip->chunk; + gzip->gzip->avail_out = gzip->conf.chunk_size; + gzip->state = EHTTP_GZIP_READY; + return gzip; + } while (0); + + ehttp_gzip_destroy(gzip); + return NULL; +} + +ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); + +void ehttp_gzip_destroy(ehttp_gzip_t *gzip) { + ehttp_gzip_cleanup(gzip); + + free(gzip); +} + +int ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, size_t len) { + if (gzip->state != EHTTP_GZIP_READY) return -1; + if (len <= 0) return 0; + + gzip->gzip->next_in = (z_const Bytef*)buf; + gzip->gzip->avail_in = len; + + while (gzip->gzip->avail_in) { + int ret; + if (gzip->header) { + ret = inflate(gzip->gzip, Z_BLOCK); + } else { + ret = inflate(gzip->gzip, Z_SYNC_FLUSH); + } + if (ret != Z_OK && ret != Z_STREAM_END) return -1; + + if (gzip->gzip->avail_out>0) { + if (ret!=Z_STREAM_END) continue; + } + + size_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + + gzip->gzip->next_out[0] = '\0'; + gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); + gzip->gzip->next_out = (z_const Bytef*)gzip->chunk; + gzip->gzip->avail_out = gzip->conf.chunk_size; + } + + return 0; +} + +int ehttp_gzip_finish(ehttp_gzip_t *gzip) { + if (gzip->state != EHTTP_GZIP_READY) return -1; + + gzip->gzip->next_in = NULL; + gzip->gzip->avail_in = 0; + + int ret; + ret = inflate(gzip->gzip, Z_FINISH); + + if (ret != Z_STREAM_END) return -1; + + size_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + + gzip->gzip->next_out[0] = '\0'; + gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); + gzip->gzip->next_out = NULL; + gzip->gzip->avail_out = 0; + + return 0; +} + diff --git a/src/plugins/http/src/ehttp_parser.c b/src/plugins/http/src/ehttp_parser.c new file mode 100644 index 0000000000..fbe15661b5 --- /dev/null +++ b/src/plugins/http/src/ehttp_parser.c @@ -0,0 +1,965 @@ +#include "ehttp_parser.h" + +#include "ehttp_gzip.h" +#include "ehttp_util_string.h" +#include "elog.h" + +#include +#include +#include + +struct ehttp_status_code_s { + int status_code; + const char *status_desc; +}; + +static ehttp_status_code_t status_codes[] = { + {100, "Continue"}, + {101, "Switching Protocol"}, + {102, "Processing (WebDAV)"}, + {103, "Early Hints"}, + {200, "OK"}, + {201, "Created"}, + {202, "Accepted"}, + {203, "Non-Authoritative Information"}, + {204, "No Content"}, + {205, "Reset Content"}, + {206, "Partial Content"}, + {207, "Multi-Status (WebDAV)"}, + {208, "Already Reported (WebDAV)"}, + {226, "IM Used (HTTP Delta encoding)"}, + {300, "Multiple Choice"}, + {301, "Moved Permanently"}, + {302, "Found"}, + {303, "See Other"}, + {304, "Not Modified"}, + {305, "Use Proxy"}, + {306, "unused"}, + {307, "Temporary Redirect"}, + {308, "Permanent Redirect"}, + {400, "Bad Request"}, + {401, "Unauthorized"}, + {402, "Payment Required"}, + {403, "Forbidden"}, + {404, "Not Found"}, + {405, "Method Not Allowed"}, + {406, "Not Acceptable"}, + {407, "Proxy Authentication Required"}, + {408, "Request Timeout"}, + {409, "Conflict"}, + {410, "Gone"}, + {411, "Length Required"}, + {412, "Precondition Failed"}, + {413, "Payload Too Large"}, + {414, "URI Too Long"}, + {415, "Unsupported Media Type"}, + {416, "Range Not Satisfiable"}, + {417, "Expectation Failed"}, + {418, "I'm a teapot"}, + {421, "Misdirected Request"}, + {422, "Unprocessable Entity (WebDAV)"}, + {423, "Locked (WebDAV)"}, + {424, "Failed Dependency (WebDAV)"}, + {425, "Too Early"}, + {426, "Upgrade Required"}, + {428, "Precondition Required"}, + {429, "Too Many Requests"}, + {431, "Request Header Fields Too Large"}, + {451, "Unavailable For Legal Reasons"}, + {500, "Internal Server Error"}, + {501, "Not Implemented"}, + {502, "Bad Gateway"}, + {503, "Service Unavailable"}, + {504, "Gateway Timeout"}, + {505, "HTTP Version Not Supported"}, + {506, "Variant Also Negotiates"}, + {507, "Insufficient Storage"}, + {508, "Loop Detected (WebDAV)"}, + {510, "Not Extended"}, + {511, "Network Authentication Required"}, + {0, NULL} +}; + +const char* ehttp_status_code_get_desc(const int status_code) { + ehttp_status_code_t *p = status_codes; + while (p->status_code!=0) { + if (p->status_code==status_code) return p->status_desc; + ++p; + } + return "Unknow status code"; +} + +typedef enum HTTP_PARSER_STATE { + HTTP_PARSER_BEGIN, + HTTP_PARSER_REQUEST_OR_RESPONSE, + HTTP_PARSER_METHOD, + HTTP_PARSER_TARGET, + HTTP_PARSER_HTTP_VERSION, + HTTP_PARSER_SP, + HTTP_PARSER_STATUS_CODE, + HTTP_PARSER_REASON_PHRASE, + HTTP_PARSER_CRLF, + HTTP_PARSER_HEADER, + HTTP_PARSER_HEADER_KEY, + HTTP_PARSER_HEADER_VAL, + HTTP_PARSER_CHUNK_SIZE, + HTTP_PARSER_CHUNK, + HTTP_PARSER_END, + HTTP_PARSER_ERROR, +} HTTP_PARSER_STATE; + +typedef struct ehttp_parser_kv_s ehttp_parser_kv_t; + +struct ehttp_parser_kv_s { + char *key; + char *val; +}; + +struct ehttp_parser_s { + ehttp_parser_callbacks_t callbacks; + void *arg; + ehttp_parser_conf_t conf; + + char *method; + char *target; + char *target_raw; + char *version; + + int http_10:2; + int http_11:2; + int accept_encoding_gzip:2; + int accept_encoding_chunked:2; + int transfer_gzip:2; + int transfer_chunked:2; + int content_length_specified:2; + int content_chunked:2; + + + int status_code; + char *reason_phrase; + + char *key; + char *val; + ehttp_parser_kv_t *kvs; + size_t kvs_count; + + char *auth_basic; + + size_t content_length; + + size_t chunk_size; + size_t received_chunk_size; + size_t received_size; + + ehttp_gzip_t *gzip; + ehttp_util_string_t str; + HTTP_PARSER_STATE *stacks; + size_t stacks_count; +}; + +static void dummy_on_request_line(void *arg, const char *method, const char *target, const char *version, const char *target_raw) { +} + +static void dummy_on_status_line(void *arg, const char *version, int status_code, const char *reason_phrase) { +} + +static void dummy_on_header_field(void *arg, const char *key, const char *val) { +} + +static void dummy_on_body(void *arg, const char *chunk, size_t len) { +} + +static void dummy_on_end(void *arg) { +} + +static void dummy_on_error(void *arg, int status_code) { +} + + +static HTTP_PARSER_STATE ehttp_parser_top(ehttp_parser_t *parser) { + EQ_ASSERT(parser->stacks_count >= 1); + EQ_ASSERT(parser->stacks); + + return parser->stacks[parser->stacks_count-1]; +} + +static int ehttp_parser_push(ehttp_parser_t *parser, HTTP_PARSER_STATE state) { + size_t n = parser->stacks_count + 1; + HTTP_PARSER_STATE *stacks = (HTTP_PARSER_STATE*)reallocarray(parser->stacks, n, sizeof(*stacks)); + if (!stacks) return -1; + + parser->stacks_count = n; + parser->stacks = stacks; + parser->stacks[n-1] = state; + + return 0; +} + +static int ehttp_parser_pop(ehttp_parser_t *parser) { + if (parser->stacks_count <= 0) return -1; + --parser->stacks_count; + + return 0; +} + +ehttp_parser_t *ehttp_parser_create(ehttp_parser_callbacks_t callbacks, ehttp_parser_conf_t conf, void *arg) { + ehttp_parser_t *parser = (ehttp_parser_t*)calloc(1, sizeof(*parser)); + if (!parser) return NULL; + + parser->callbacks = callbacks; + parser->arg = arg; + parser->conf = conf; + + if (parser->callbacks.on_request_line == NULL) { + parser->callbacks.on_request_line = dummy_on_request_line; + } + if (parser->callbacks.on_status_line == NULL) { + parser->callbacks.on_status_line = dummy_on_status_line; + } + if (parser->callbacks.on_header_field == NULL) { + parser->callbacks.on_header_field = dummy_on_header_field; + } + if (parser->callbacks.on_body == NULL) { + parser->callbacks.on_body = dummy_on_body; + } + if (parser->callbacks.on_end == NULL) { + parser->callbacks.on_end = dummy_on_end; + } + if (parser->callbacks.on_error == NULL) { + parser->callbacks.on_error = dummy_on_error; + } + + ehttp_parser_push(parser, HTTP_PARSER_BEGIN); + + return parser; +} + +static void ehttp_parser_kvs_destroy(ehttp_parser_t *parser) { + if (!parser->kvs) return; + + for (size_t i=0; ikvs_count; ++i) { + ehttp_parser_kv_t *p = &parser->kvs[i]; + free(p->key); p->key = NULL; + free(p->val); p->val = NULL; + } + free(parser->kvs); + parser->kvs = NULL; + parser->kvs_count = 0; + + free(parser->auth_basic); + parser->auth_basic = NULL; +} + +void ehttp_parser_destroy(ehttp_parser_t *parser) { + if (!parser) return; + + free(parser->method); parser->method = NULL; + free(parser->target); parser->target = NULL; + free(parser->target_raw); parser->target_raw = NULL; + free(parser->version); parser->version = NULL; + free(parser->reason_phrase); parser->reason_phrase = NULL; + free(parser->key); parser->key = NULL; + free(parser->val); parser->val = NULL; + free(parser->auth_basic); parser->auth_basic = NULL; + free(parser->stacks); parser->stacks = NULL; + + parser->stacks_count = 0; + + ehttp_parser_kvs_destroy(parser); + + ehttp_util_string_cleanup(&parser->str); + if (parser->gzip) { + ehttp_gzip_destroy(parser->gzip); + parser->gzip = NULL; + } + + free(parser); +} + +#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) + +char *ehttp_parser_urldecode(const char *enc) { + int ok = 1; + ehttp_util_string_t str = {0}; + while (*enc) { + char *p = strchr(enc, '%'); + if (!p) break; + int hex, cnt; + int n = sscanf(p+1, "%2x%n", &hex, &cnt); + if (n!=1 && cnt !=2) { ok = 0; break; } + if (ehttp_util_string_append(&str, enc, p-enc)) { ok = 0; break; } + char c = (char)hex; + if (ehttp_util_string_append(&str, &c, 1)) { ok = 0; break; } + enc = p+3; + } + char *dec = NULL; + if (ok && *enc) { + if (ehttp_util_string_append(&str, enc, strlen(enc))) { ok = 0; } + } + if (ok) { + dec = str.str; + str.str = NULL; + } + ehttp_util_string_cleanup(&str); + return dec; +} + +static void on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, size_t len) { + ehttp_parser_t *parser = (ehttp_parser_t*)arg; + parser->callbacks.on_body(parser->arg, buf, len); +} + +static int ehttp_parser_check_field(ehttp_parser_t *parser, const char *key, const char *val) { + int ok = 0; + do { + if (0==strcasecmp(key, "Content-Length")) { + size_t len = 0; + int bytes = 0; + int n = sscanf(val, "%ld%n", &len, &bytes); + if (n==1 && bytes==strlen(val)) { + parser->content_length = len; + parser->chunk_size = len; + parser->content_length_specified = 1; + break; + } + ok = -1; + break; + } + if (0==strcasecmp(key, "Accept-Encoding")) { + if (strstr(val, "gzip")) { + parser->accept_encoding_gzip = 1; + } + if (strstr(val, "chunked")) { + parser->accept_encoding_chunked = 1; + } + break; + } + if (0==strcasecmp(key, "Content-Encoding")) { + if (0==strcmp(val, "gzip")) { + parser->content_chunked = 1; + } + break; + } + if (0==strcasecmp(key, "Transfer-Encoding")) { + if (strstr(val, "gzip")) { + parser->transfer_gzip = 1; + ehttp_gzip_conf_t conf = {0}; + ehttp_gzip_callbacks_t callbacks = {0}; + + callbacks.on_data = on_data; + + parser->gzip = ehttp_gzip_create_decompressor(conf, callbacks, parser); + + if (!parser->gzip) { + E("failed to create gzip decompressor"); + ok = -1; + break; + } + } + if (strstr(val, "chunked")) { + parser->transfer_chunked = 1; + } + break; + } + if (0==strcasecmp(key, "Authorization")) { + char *t = NULL; + char *s = NULL; + int bytes = 0; + int n = sscanf(val, "%ms %ms%n", &t, &s, &bytes); + if (n==2 && t && s && bytes==strlen(val) && strcmp(t, "Basic")) { + free(parser->auth_basic); + parser->auth_basic = s; s = NULL; + } else { + ok = -1; + } + free(t); free(s); + break; + } + } while (0); + return ok; +} + +static int ehttp_parser_kvs_append_kv(ehttp_parser_t *parser, const char *key, const char *val) { + ehttp_parser_kv_t *kvs = (ehttp_parser_kv_t*)reallocarray(parser->kvs, parser->kvs_count + 1, sizeof(*kvs)); + if (!kvs) return -1; + + parser->kvs = kvs; + + kvs[parser->kvs_count].key = strdup(key); + kvs[parser->kvs_count].val = strdup(val); + + if (kvs[parser->kvs_count].key && kvs[parser->kvs_count].val) { + ++parser->kvs_count; + return 0; + } + + free(kvs[parser->kvs_count].key); + kvs[parser->kvs_count].key = NULL; + free(kvs[parser->kvs_count].val); + kvs[parser->kvs_count].val = NULL; + + return -1; +} + +static int on_begin(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (c=='G' || c=='P' || c=='H' || c=='D' || c=='C' || c=='O' || c=='T') { + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + ehttp_parser_pop(parser); + ehttp_parser_push(parser, HTTP_PARSER_REQUEST_OR_RESPONSE); + break; + } + E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 400); + } while (0); + return ok; +} + +static int on_request_or_response(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (parser->str.len==1) { + if (c=='T' && parser->str.str[0]=='H') { + ehttp_parser_pop(parser); + ehttp_parser_push(parser, HTTP_PARSER_END); + ehttp_parser_push(parser, HTTP_PARSER_HEADER); + ehttp_parser_push(parser, HTTP_PARSER_CRLF); + ehttp_parser_push(parser, HTTP_PARSER_REASON_PHRASE); + ehttp_parser_push(parser, HTTP_PARSER_SP); + ehttp_parser_push(parser, HTTP_PARSER_STATUS_CODE); + ehttp_parser_push(parser, HTTP_PARSER_SP); + ehttp_parser_push(parser, HTTP_PARSER_HTTP_VERSION); + *again = 1; + break; + } + ehttp_parser_pop(parser); + ehttp_parser_push(parser, HTTP_PARSER_END); + ehttp_parser_push(parser, HTTP_PARSER_HEADER); + ehttp_parser_push(parser, HTTP_PARSER_CRLF); + ehttp_parser_push(parser, HTTP_PARSER_HTTP_VERSION); + ehttp_parser_push(parser, HTTP_PARSER_SP); + ehttp_parser_push(parser, HTTP_PARSER_TARGET); + ehttp_parser_push(parser, HTTP_PARSER_SP); + ehttp_parser_push(parser, HTTP_PARSER_METHOD); + *again = 1; + break; + } + E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 400); + } while (0); + return ok; +} + +static int on_method(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + break; + } + parser->method = strdup(parser->str.str); + if (!parser->method) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + *again = 1; + } while (0); + return ok; +} + +static int on_target(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (!isspace(c) && c!='\r' && c!='\n') { + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + break; + } + parser->target_raw = strdup(parser->str.str); + parser->target = ehttp_parser_urldecode(parser->str.str); + if (!parser->target_raw || !parser->target) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + *again = 1; + } while (0); + return ok; +} + +static int on_version(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + const char *prefix = "HTTP/1."; + int len = strlen(prefix); + if (parser->str.len < len) { + if (prefix[parser->str.len]!=c) { + E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 400); + break; + } + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + break; + } + + if (c!='0' && c!='1') { + E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 400); + break; + } + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + if (c=='0') parser->http_10 = 1; + if (c=='1') parser->http_11 = 1; + + parser->version = strdup(parser->str.str); + if (!parser->version) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + + if (parser->method) { + parser->callbacks.on_request_line(parser->arg, parser->method, parser->target, parser->version, parser->target_raw); + } + + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + } while (0); + return ok; +} + +static int on_sp(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (c==' ') { + ehttp_parser_pop(parser); + break; + } + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + } while (0); + return ok; +} + +static int on_status_code(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (isdigit(c)) { + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + if (parser->str.len < 3) break; + + sscanf(parser->str.str, "%d", &parser->status_code); + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + break; + } + E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 400); + } while (0); + return ok; +} + +static int on_reason_phrase(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (c=='\r') { + parser->reason_phrase = strdup(parser->str.str); + if (!parser->reason_phrase) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + parser->callbacks.on_status_line(parser->arg, parser->version, parser->status_code, parser->reason_phrase); + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + *again = 1; + break; + } + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + } while (0); + return ok; +} + +static int post_process(ehttp_parser_t *parser) { + if (parser->gzip) { + if (ehttp_gzip_finish(parser->gzip)) { + E("gzip failed"); + parser->callbacks.on_error(parser->arg, 507); + return -1; + } + } + parser->callbacks.on_end(parser->arg); + return 0; +} + +static int on_crlf(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + const char *s = "\r\n"; + int len = strlen(s); + if (s[parser->str.len]!=c) { + E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 400); + break; + } + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + if (parser->str.len == len) { + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + if (ehttp_parser_top(parser) == HTTP_PARSER_END) { + ok = post_process(parser); + } + } + break; + } while (0); + return ok; +} + +static int on_header(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (c=='\r') { + ehttp_parser_pop(parser); + if (parser->transfer_chunked) { + ehttp_parser_push(parser, HTTP_PARSER_CHUNK_SIZE); + ehttp_parser_push(parser, HTTP_PARSER_CRLF); + } else { + if (parser->content_length > 0) { + ehttp_parser_push(parser, HTTP_PARSER_CHUNK); + } + ehttp_parser_push(parser, HTTP_PARSER_CRLF); + } + *again = 1; + break; + } + if (c!=' ' && c!='\t' && c!=':' ) { + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + ehttp_parser_push(parser, HTTP_PARSER_CRLF); + ehttp_parser_push(parser, HTTP_PARSER_HEADER_VAL); + ehttp_parser_push(parser, HTTP_PARSER_SP); + ehttp_parser_push(parser, HTTP_PARSER_HEADER_KEY); + break; + } + E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 400); + } while (0); + return ok; +} + +static int on_header_key(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + break; + } + if (c==':') { + parser->key = strdup(parser->str.str); + if (!parser->key) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + break; + } + E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 400); + } while (0); + return ok; +} + +static int on_header_val(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (c != '\r' && c != '\n' && (!isspace(c) || parser->str.len>0)) { + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + break; + } + const char *val = parser->str.str; + ok = ehttp_parser_check_field(parser, parser->key, val); + if (ehttp_parser_kvs_append_kv(parser, parser->key, val)) { + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + } else { + parser->callbacks.on_header_field(parser->arg, parser->key, val); + } + free(parser->key); parser->key = NULL; + val = NULL; + if (ok==-1) break; + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + *again = 1; + } while (0); + return ok; +} + +static int on_chunk_size(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + int bytes; + size_t len; + int n; + do { + if (isxdigit(c)) { + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + break; + } + if (c=='\r') { + n = sscanf(parser->str.str, "%lx%n", &len, &bytes); + if (n==1 && bytes==strlen(parser->str.str) && len>=0) { + if (len==0) { + if (parser->content_length_specified == 0 || parser->received_size == parser->content_length) { + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + ehttp_parser_push(parser, HTTP_PARSER_CRLF); + ehttp_parser_push(parser, HTTP_PARSER_CRLF); + *again = 1; + break; + } + } else { + if (parser->content_length_specified == 0 || parser->received_size + len <= parser->content_length) { + parser->chunk_size = len; + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + ehttp_parser_push(parser, HTTP_PARSER_CHUNK_SIZE); + ehttp_parser_push(parser, HTTP_PARSER_CRLF); + ehttp_parser_push(parser, HTTP_PARSER_CHUNK); + ehttp_parser_push(parser, HTTP_PARSER_CRLF); + *again = 1; + break; + } + } + } + } + E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 400); + } while (0); + return ok; +} + +static int on_chunk(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + if (ehttp_util_string_append(&parser->str, &c, 1)) { + E("parser state: %d, char: [%c]%02x, oom", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + ++parser->received_size; + ++parser->received_chunk_size; + if (parser->received_chunk_size < parser->chunk_size) break; + + if (parser->gzip) { + if (ehttp_gzip_write(parser->gzip, parser->str.str, parser->str.len)) { + E("gzip failed"); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + break; + } + } else { + parser->callbacks.on_body(parser->arg, parser->str.str, parser->str.len); + } + parser->received_chunk_size = 0; + ehttp_util_string_clear(&parser->str); + ehttp_parser_pop(parser); + if (ehttp_parser_top(parser) == HTTP_PARSER_END) { + ok = post_process(parser); + } + } while (0); + return ok; +} + +static int on_end(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int ok = 0; + do { + E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + ok = -1; + parser->callbacks.on_error(parser->arg, 507); + } while (0); + return ok; +} + +static int parse_char(ehttp_parser_t *parser, const char c, int *again) { + int ok = 0; + HTTP_PARSER_STATE state = ehttp_parser_top(parser); + do { + if (state == HTTP_PARSER_BEGIN) { + ok = on_begin(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_REQUEST_OR_RESPONSE) { + ok = on_request_or_response(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_METHOD) { + ok = on_method(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_TARGET) { + ok = on_target(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_HTTP_VERSION) { + ok = on_version(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_SP) { + ok = on_sp(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_STATUS_CODE) { + ok = on_status_code(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_REASON_PHRASE) { + ok = on_reason_phrase(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_CRLF) { + ok = on_crlf(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_HEADER) { + ok = on_header(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_HEADER_KEY) { + ok = on_header_key(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_HEADER_VAL) { + ok = on_header_val(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_CHUNK_SIZE) { + ok = on_chunk_size(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_CHUNK) { + ok = on_chunk(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_END) { + ok = on_end(parser, state, c, again); + break; + } + if (state == HTTP_PARSER_ERROR) { + ok = -2; + break; + } + E("unknown parser state: %d", state); + ok = -1; + parser->callbacks.on_error(parser->arg, 500); + } while (0); + if (ok==-1) { + ehttp_parser_push(parser, HTTP_PARSER_ERROR); + } + if (ok==-2) ok = -1; + return ok; +} + +int ehttp_parser_parse_string(ehttp_parser_t *parser, const char *str) { + return ehttp_parser_parse(parser, str, str?strlen(str):0); +} + +int ehttp_parser_parse_char(ehttp_parser_t *parser, const char c) { + return ehttp_parser_parse(parser, &c, 1); +} + +int ehttp_parser_parse(ehttp_parser_t *parser, const char *buf, size_t len) { + const char *p = buf; + int ret = 0; + size_t i = 0; + while (i < len) { + int again = 0; + ret = parse_char(parser, *p, &again); + if (ret) break; + if (again) continue; + ++p; + ++i; + } + return ret; +} + diff --git a/src/plugins/http/src/ehttp_util_string.c b/src/plugins/http/src/ehttp_util_string.c new file mode 100644 index 0000000000..94ebaaafa6 --- /dev/null +++ b/src/plugins/http/src/ehttp_util_string.c @@ -0,0 +1,30 @@ +#include "ehttp_util_string.h" + +#include +#include + +void ehttp_util_string_cleanup(ehttp_util_string_t *str) { + free(str->str); + str->str = NULL; + str->len = 0; +} + +int ehttp_util_string_append(ehttp_util_string_t *str, const char *s, size_t len) { + // int n = str->str?strlen(str->str):0; + int n = str->len; + char *p = (char*)realloc(str->str, n + len + 1); + if (!p) return -1; + strncpy(p+n, s, len); + p[n+len] = '\0'; + str->str = p; + str->len = n+len; + return 0; +} + +void ehttp_util_string_clear(ehttp_util_string_t *str) { + if (str->str) { + str->str[0] = '\0'; + str->len = 0; + } +} + diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index f46d3fb427..4e235e24e8 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -28,6 +28,22 @@ #include "httpSql.h" #include "httpSession.h" +#include "elog.h" + +// dirty tweak +extern bool httpGetHttpMethod(HttpContext* pContext); +extern bool httpParseURL(HttpContext* pContext); +extern bool httpParseHttpVersion(HttpContext* pContext); +extern bool httpGetDecodeMethod(HttpContext* pContext); +extern bool httpParseHead(HttpContext* pContext); + +static void on_request_line(void *arg, const char *method, const char *target, const char *version, const char *target_raw); +static void on_status_line(void *arg, const char *version, int status_code, const char *reason_phrase); +static void on_header_field(void *arg, const char *key, const char *val); +static void on_body(void *arg, const char *chunk, size_t len); +static void on_end(void *arg); +static void on_error(void *arg, int status_code); + static void httpRemoveContextFromEpoll(HttpContext *pContext) { HttpThread *pThread = pContext->pThread; if (pContext->fd >= 0) { @@ -149,6 +165,11 @@ void httpReleaseContext(HttpContext *pContext) { httpDebug("context:%p, won't be destroyed for cache is already released", pContext); // httpDestroyContext((void **)(&ppContext)); } + + if (pContext->parser.parser) { + ehttp_parser_destroy(pContext->parser.parser); + pContext->parser.parser = NULL; + } } bool httpInitContext(HttpContext *pContext) { @@ -168,6 +189,20 @@ bool httpInitContext(HttpContext *pContext) { memset(pParser, 0, sizeof(HttpParser)); pParser->pCur = pParser->pLast = pParser->buffer; + ehttp_parser_callbacks_t callbacks = { + on_request_line, + on_status_line, + on_header_field, + on_body, + on_end, + on_error + }; + ehttp_parser_conf_t conf = { + .flush_block_size = 0 + }; + pParser->parser = ehttp_parser_create(callbacks, conf, pContext); + pParser->inited = 1; + httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, parsed:%d", pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, pContext->parsed); return true; @@ -230,3 +265,129 @@ void httpCloseContextByServer(HttpContext *pContext) { httpRemoveContextFromEpoll(pContext); httpReleaseContext(pContext); } + + + + + +static void on_request_line(void *arg, const char *method, const char *target, const char *version, const char *target_raw) { + HttpContext *pContext = (HttpContext*)arg; + HttpParser *pParser = &pContext->parser; + + int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); + int n = snprintf(pParser->pLast, avail, + "%s %s %s\r\n", method, target_raw, version); + + char *last = pParser->pLast; + + do { + if (n>=avail) { + httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_request_line(%s,%s,%s,%s), exceeding buffer size", + pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, method, target, version, target_raw); + break; + } + pParser->bufsize += n; + + if (!httpGetHttpMethod(pContext)) { + httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_request_line(%s,%s,%s,%s), parse http method failed", + pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, method, target, version, target_raw); + break; + } + if (!httpParseURL(pContext)) { + httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_request_line(%s,%s,%s,%s), parse http url failed", + pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, method, target, version, target_raw); + break; + } + if (!httpParseHttpVersion(pContext)) { + httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_request_line(%s,%s,%s,%s), parse http version failed", + pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, method, target, version, target_raw); + break; + } + if (!httpGetDecodeMethod(pContext)) { + httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_request_line(%s,%s,%s,%s), get decode method failed", + pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, method, target, version, target_raw); + break; + } + + last += n; + pParser->pLast = last; + return; + } while (0); + + pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; +} + +static void on_status_line(void *arg, const char *version, int status_code, const char *reason_phrase) { + HttpContext *pContext = (HttpContext*)arg; + HttpParser *pParser = &pContext->parser; + + pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; +} + +static void on_header_field(void *arg, const char *key, const char *val) { + HttpContext *pContext = (HttpContext*)arg; + HttpParser *pParser = &pContext->parser; + + if (pParser->failed) return; + + int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); + int n = snprintf(pParser->pLast, avail, + "%s: %s\r\n", key, val); + + char *last = pParser->pLast; + + do { + if (n>=avail) { + httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_header_field(%s,%s), exceeding buffer size", + pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, key, val); + break; + } + pParser->bufsize += n; + pParser->pCur = pParser->pLast; + + if (!httpParseHead(pContext)) { + httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_header_field(%s,%s), parse head failed", + pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, key, val); + break; + } + + last += n; + pParser->pLast = last; + return; + } while (0); + + pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; +} + +static void on_body(void *arg, const char *chunk, size_t len) { + HttpContext *pContext = (HttpContext*)arg; + HttpParser *pParser = &pContext->parser; + + if (pParser->failed) return; + + if (!pContext->parsed) { + pContext->parsed = true; + } + + A("not implemented yet"); +} + +static void on_end(void *arg) { + HttpContext *pContext = (HttpContext*)arg; + HttpParser *pParser = &pContext->parser; + + if (pParser->failed) return; + + if (!pContext->parsed) { + pContext->parsed = true; + } +} + +static void on_error(void *arg, int status_code) { + HttpContext *pContext = (HttpContext*)arg; + HttpParser *pParser = &pContext->parser; + + D("=="); + pParser->failed |= EHTTP_CONTEXT_PARSER_FAILED; +} + diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index ec3d2c0d44..d0b01c628c 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -26,10 +26,14 @@ #include "httpResp.h" #include "httpUtil.h" +#include "elog.h" + #ifndef EPOLLWAKEUP #define EPOLLWAKEUP (1u << 29) #endif +static bool ehttpReadData(HttpContext *pContext); + static void httpStopThread(HttpThread* pThread) { pThread->stop = true; @@ -134,6 +138,8 @@ static bool httpDecompressData(HttpContext *pContext) { } static bool httpReadData(HttpContext *pContext) { + if (1) return ehttpReadData(pContext); + if (!pContext->parsed) { httpInitContext(pContext); } @@ -405,3 +411,60 @@ bool httpInitConnect() { pServer->serverPort, pServer->numOfThreads); return true; } + + + + +static bool ehttpReadData(HttpContext *pContext) { + HttpParser *pParser = &pContext->parser; + EQ_ASSERT(!pContext->parsed); + if (!pParser->parser) { + if (!pParser->inited) { + httpInitContext(pContext); + } + if (!pParser->parser) { + return false; + } + } + + pContext->accessTimes++; + pContext->lastAccessTime = taosGetTimestampSec(); + + char buf[HTTP_STEP_SIZE+1] = {0}; + int nread = (int)taosReadSocket(pContext->fd, buf, sizeof(buf)); + if (nread > 0) { + buf[nread] = '\0'; + if (strstr(buf, "GET ")==buf && !strchr(buf, '\r') && !strchr(buf, '\n')) { + D("==half of request line received:\n%s\n==", buf); + } + if (ehttp_parser_parse(pParser->parser, buf, nread)) { + D("==parsing failed=="); + httpCloseContextByServer(pContext); + return false; + } + if (pContext->parser.failed) { + D("==parsing failed: [0x%x]==", pContext->parser.failed); + httpNotifyContextClose(pContext); + return false; + } + return pContext->parsed; + } else if (nread < 0) { + if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { + httpDebug("context:%p, fd:%d, ip:%s, read from socket error:%d, wait another event", + pContext, pContext->fd, pContext->ipstr, errno); + return false; // later again + } else { + httpError("context:%p, fd:%d, ip:%s, read from socket error:%d, close connect", + pContext, pContext->fd, pContext->ipstr, errno); + D("==releasing because of reading failed=="); + httpReleaseContext(pContext); + return false; + } + } else { + // eof + D("==releasing because of remote close/reset=="); + httpReleaseContext(pContext); + return false; + } +} + From d9c04b18e9b96a888e363210b8b903e19735ec7d Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 1 Aug 2020 10:59:12 +0800 Subject: [PATCH 002/105] elog to util; parser.data.pos --- src/plugins/http/src/httpContext.c | 13 ++++++++++--- src/plugins/http/src/httpServer.c | 25 ++++++++++++++++++++++++- src/{common => util}/inc/elog.h | 0 src/{common => util}/src/elog.c | 0 4 files changed, 34 insertions(+), 4 deletions(-) rename src/{common => util}/inc/elog.h (100%) rename src/{common => util}/src/elog.c (100%) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 4e235e24e8..1e36fdef75 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -55,6 +55,7 @@ static void httpRemoveContextFromEpoll(HttpContext *pContext) { static void httpDestroyContext(void *data) { HttpContext *pContext = *(HttpContext **)data; + D("==context[%p] destroyed==", pContext); if (pContext->fd > 0) tclose(pContext->fd); HttpThread *pThread = pContext->pThread; @@ -80,6 +81,7 @@ bool httpInitContexts() { httpError("failed to init context cache"); return false; } + D("==cache [%p] created==", tsHttpServer.contextCache); return true; } @@ -120,6 +122,8 @@ HttpContext *httpCreateContext(int32_t fd) { HttpContext *pContext = calloc(1, sizeof(HttpContext)); if (pContext == NULL) return NULL; + D("==context[%p] created==", pContext); + pContext->fd = fd; pContext->httpVersion = HTTP_VERSION_10; pContext->lastAccessTime = taosGetTimestampSec(); @@ -209,6 +213,7 @@ bool httpInitContext(HttpContext *pContext) { } void httpCloseContextByApp(HttpContext *pContext) { + D("=="); pContext->parsed = false; bool keepAlive = true; @@ -220,6 +225,7 @@ void httpCloseContextByApp(HttpContext *pContext) { } if (keepAlive) { + D("==keepAlive=="); if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) { httpDebug("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd, pContext->ipstr); @@ -240,6 +246,7 @@ void httpCloseContextByApp(HttpContext *pContext) { pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); } } else { + D("==not keepAlive=="); httpRemoveContextFromEpoll(pContext); httpDebug("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close context", pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); @@ -365,9 +372,7 @@ static void on_body(void *arg, const char *chunk, size_t len) { if (pParser->failed) return; - if (!pContext->parsed) { - pContext->parsed = true; - } + if (pParser->data.pos == 0) pParser->data.pos = pParser->pLast; A("not implemented yet"); } @@ -378,6 +383,8 @@ static void on_end(void *arg) { if (pParser->failed) return; + if (pParser->data.pos == 0) pParser->data.pos = pParser->pLast; + if (!pContext->parsed) { pContext->parsed = true; } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index d0b01c628c..d21fa89d5c 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -138,7 +138,7 @@ static bool httpDecompressData(HttpContext *pContext) { } static bool httpReadData(HttpContext *pContext) { - if (1) return ehttpReadData(pContext); + if (0) return ehttpReadData(pContext); if (!pContext->parsed) { httpInitContext(pContext); @@ -447,6 +447,29 @@ static bool ehttpReadData(HttpContext *pContext) { httpNotifyContextClose(pContext); return false; } + if (pContext->parsed) { + int ret = httpCheckReadCompleted(pContext); + if (ret == HTTP_CHECK_BODY_CONTINUE) { + //httpDebug("context:%p, fd:%d, ip:%s, not finished yet, wait another event", pContext, pContext->fd, pContext->ipstr); + httpReleaseContext(pContext); + return false; + } else if (ret == HTTP_CHECK_BODY_SUCCESS){ + httpDebug("context:%p, fd:%d, ip:%s, thread:%s, read size:%d, dataLen:%d", + pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->parser.bufsize, pContext->parser.data.len); + if (httpDecompressData(pContext)) { + return true; + } else { + httpNotifyContextClose(pContext); + httpReleaseContext(pContext); + return false; + } + } else { + httpError("context:%p, fd:%d, ip:%s, failed to read http body, close connect", pContext, pContext->fd, pContext->ipstr); + httpNotifyContextClose(pContext); + httpReleaseContext(pContext); + return false; + } + } return pContext->parsed; } else if (nread < 0) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { diff --git a/src/common/inc/elog.h b/src/util/inc/elog.h similarity index 100% rename from src/common/inc/elog.h rename to src/util/inc/elog.h diff --git a/src/common/src/elog.c b/src/util/src/elog.c similarity index 100% rename from src/common/src/elog.c rename to src/util/src/elog.c From 93cb6386df27403bc04d957586b36140a477bf31 Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 1 Aug 2020 13:28:39 +0800 Subject: [PATCH 003/105] body process --- src/plugins/http/src/httpContext.c | 17 ++++++++++++++--- src/plugins/http/src/httpServer.c | 6 ++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 1e36fdef75..5012fd15f5 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -337,6 +337,7 @@ static void on_header_field(void *arg, const char *key, const char *val) { if (pParser->failed) return; + D("==key:[%s], val:[%s]==", key, val); int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); int n = snprintf(pParser->pLast, avail, "%s: %s\r\n", key, val); @@ -350,7 +351,7 @@ static void on_header_field(void *arg, const char *key, const char *val) { break; } pParser->bufsize += n; - pParser->pCur = pParser->pLast; + pParser->pCur = pParser->pLast + n; if (!httpParseHead(pContext)) { httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_header_field(%s,%s), parse head failed", @@ -372,9 +373,19 @@ static void on_body(void *arg, const char *chunk, size_t len) { if (pParser->failed) return; - if (pParser->data.pos == 0) pParser->data.pos = pParser->pLast; + if (pParser->data.pos == 0) { + pParser->data.pos = pParser->pLast; + pParser->data.len = 0; + } - A("not implemented yet"); + int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); + if (len+1>=avail) { + pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; + return; + } + memcpy(pParser->pLast, chunk, len); + pParser->pLast += len; + pParser->data.len += len; } static void on_end(void *arg) { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index d21fa89d5c..4f2ea63dc2 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -138,7 +138,7 @@ static bool httpDecompressData(HttpContext *pContext) { } static bool httpReadData(HttpContext *pContext) { - if (0) return ehttpReadData(pContext); + if (1) return ehttpReadData(pContext); if (!pContext->parsed) { httpInitContext(pContext); @@ -448,7 +448,9 @@ static bool ehttpReadData(HttpContext *pContext) { return false; } if (pContext->parsed) { - int ret = httpCheckReadCompleted(pContext); + // int ret = httpCheckReadCompleted(pContext); + // already done in ehttp_parser + int ret = HTTP_CHECK_BODY_SUCCESS; if (ret == HTTP_CHECK_BODY_CONTINUE) { //httpDebug("context:%p, fd:%d, ip:%s, not finished yet, wait another event", pContext, pContext->fd, pContext->ipstr); httpReleaseContext(pContext); From 7c434d6108e26a52995069978e1911efe40287b4 Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 1 Aug 2020 15:19:43 +0800 Subject: [PATCH 004/105] add env FALLBACK, for the sake of easy debug in different mode --- src/plugins/http/inc/httpInt.h | 2 ++ src/plugins/http/src/httpContext.c | 40 ++++++++++++++++-------------- src/plugins/http/src/httpServer.c | 6 +++-- src/plugins/http/src/httpSystem.c | 6 +++++ 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 044b5cc4cc..bde799d6d6 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -242,6 +242,8 @@ typedef struct HttpServer { pthread_mutex_t serverMutex; HttpDecodeMethod *methodScanner[HTTP_METHOD_SCANNER_SIZE]; bool (*processData)(HttpContext *pContext); + + int fallback:2; } HttpServer; extern const char *httpKeepAliveStr[]; diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 5012fd15f5..4440da6d45 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -72,6 +72,13 @@ static void httpDestroyContext(void *data) { httpFreeJsonBuf(pContext); httpFreeMultiCmds(pContext); + if (!tsHttpServer.fallback) { + if (pContext->parser.parser) { + ehttp_parser_destroy(pContext->parser.parser); + pContext->parser.parser = NULL; + } + } + tfree(pContext); } @@ -169,11 +176,6 @@ void httpReleaseContext(HttpContext *pContext) { httpDebug("context:%p, won't be destroyed for cache is already released", pContext); // httpDestroyContext((void **)(&ppContext)); } - - if (pContext->parser.parser) { - ehttp_parser_destroy(pContext->parser.parser); - pContext->parser.parser = NULL; - } } bool httpInitContext(HttpContext *pContext) { @@ -193,19 +195,21 @@ bool httpInitContext(HttpContext *pContext) { memset(pParser, 0, sizeof(HttpParser)); pParser->pCur = pParser->pLast = pParser->buffer; - ehttp_parser_callbacks_t callbacks = { - on_request_line, - on_status_line, - on_header_field, - on_body, - on_end, - on_error - }; - ehttp_parser_conf_t conf = { - .flush_block_size = 0 - }; - pParser->parser = ehttp_parser_create(callbacks, conf, pContext); - pParser->inited = 1; + if (!tsHttpServer.fallback) { + ehttp_parser_callbacks_t callbacks = { + on_request_line, + on_status_line, + on_header_field, + on_body, + on_end, + on_error + }; + ehttp_parser_conf_t conf = { + .flush_block_size = 0 + }; + pParser->parser = ehttp_parser_create(callbacks, conf, pContext); + pParser->inited = 1; + } httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, parsed:%d", pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, pContext->parsed); diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 4f2ea63dc2..819f7a5f4a 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -138,7 +138,7 @@ static bool httpDecompressData(HttpContext *pContext) { } static bool httpReadData(HttpContext *pContext) { - if (1) return ehttpReadData(pContext); + if (!tsHttpServer.fallback) return ehttpReadData(pContext); if (!pContext->parsed) { httpInitContext(pContext); @@ -437,11 +437,13 @@ static bool ehttpReadData(HttpContext *pContext) { if (strstr(buf, "GET ")==buf && !strchr(buf, '\r') && !strchr(buf, '\n')) { D("==half of request line received:\n%s\n==", buf); } + if (ehttp_parser_parse(pParser->parser, buf, nread)) { D("==parsing failed=="); httpCloseContextByServer(pContext); return false; } + if (pContext->parser.failed) { D("==parsing failed: [0x%x]==", pContext->parser.failed); httpNotifyContextClose(pContext); @@ -450,7 +452,7 @@ static bool ehttpReadData(HttpContext *pContext) { if (pContext->parsed) { // int ret = httpCheckReadCompleted(pContext); // already done in ehttp_parser - int ret = HTTP_CHECK_BODY_SUCCESS; + int ret = HTTP_CHECK_BODY_SUCCESS; if (ret == HTTP_CHECK_BODY_CONTINUE) { //httpDebug("context:%p, fd:%d, ip:%s, not finished yet, wait another event", pContext, pContext->fd, pContext->ipstr); httpReleaseContext(pContext); diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 3a0998f2e8..43466ee57e 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -39,6 +39,12 @@ HttpServer tsHttpServer; void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); int httpInitSystem() { + tsHttpServer.fallback = 0; + const char *v = getenv("FALLBACK"); + if (v) { + tsHttpServer.fallback = 1; + } + strcpy(tsHttpServer.label, "rest"); tsHttpServer.serverIp = 0; tsHttpServer.serverPort = tsHttpPort; From e55a55a6f82e523e65a81785db6d3c36fb875561 Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 1 Aug 2020 21:57:07 +0800 Subject: [PATCH 005/105] add ehttpInc/DecContextRef --- src/plugins/http/inc/httpContext.h | 3 ++ src/plugins/http/inc/httpInt.h | 2 + src/plugins/http/src/httpContext.c | 76 ++++++++++++++++++++++++++---- src/plugins/http/src/httpServer.c | 29 ++++++++---- 4 files changed, 92 insertions(+), 18 deletions(-) diff --git a/src/plugins/http/inc/httpContext.h b/src/plugins/http/inc/httpContext.h index a2d50d6b7f..594900d0cf 100644 --- a/src/plugins/http/inc/httpContext.h +++ b/src/plugins/http/inc/httpContext.h @@ -31,4 +31,7 @@ void httpCloseContextByApp(HttpContext *pContext); void httpNotifyContextClose(HttpContext *pContext); bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState); +void ehttpIncContextRef(HttpContext *pContext); +void ehttpDecContextRef(HttpContext **ppContext); + #endif diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index bde799d6d6..40f980f101 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -212,6 +212,8 @@ typedef struct HttpContext { void * timer; HttpEncodeMethod * encodeMethod; struct HttpThread *pThread; + + int closed:2; } HttpContext; typedef struct HttpThread { diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 4440da6d45..b229673df2 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -28,6 +28,7 @@ #include "httpSql.h" #include "httpSession.h" +#include "httpContext.h" #include "elog.h" // dirty tweak @@ -44,12 +45,20 @@ static void on_body(void *arg, const char *chunk, size_t len); static void on_end(void *arg); static void on_error(void *arg, int status_code); +static void httpDestroyContext(void *data); +static void httpMightDestroyContext(void *data); +static void ehttpReleaseContext(HttpContext *pContext); + static void httpRemoveContextFromEpoll(HttpContext *pContext) { HttpThread *pThread = pContext->pThread; if (pContext->fd >= 0) { epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); - taosCloseSocket(pContext->fd); + int32_t fd = pContext->fd; pContext->fd = -1; + taosCloseSocket(fd); + if (!tsHttpServer.fallback) { + ehttpDecContextRef(&pContext); + } } } @@ -83,12 +92,11 @@ static void httpDestroyContext(void *data) { } bool httpInitContexts() { - tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpDestroyContext, "restc"); + tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpMightDestroyContext, "restc"); if (tsHttpServer.contextCache == NULL) { httpError("failed to init context cache"); return false; } - D("==cache [%p] created==", tsHttpServer.contextCache); return true; } @@ -136,10 +144,12 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->lastAccessTime = taosGetTimestampSec(); pContext->state = HTTP_CONTEXT_STATE_READY; + ehttpIncContextRef(pContext); HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &pContext, sizeof(int64_t), &pContext, sizeof(int64_t), 3); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); + ehttpIncContextRef(pContext); // set the ref to 0 taosCacheRelease(tsHttpServer.contextCache, (void**)&ppContext, false); @@ -148,10 +158,13 @@ HttpContext *httpCreateContext(int32_t fd) { HttpContext *httpGetContext(void *ptr) { HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &ptr, sizeof(HttpContext *)); + EQ_ASSERT(ppContext); + EQ_ASSERT(*ppContext); if (ppContext) { HttpContext *pContext = *ppContext; if (pContext) { + if (!tsHttpServer.fallback) return pContext; int32_t refCount = atomic_add_fetch_32(&pContext->refCount, 1); httpDebug("context:%p, fd:%d, is accquired, data:%p refCount:%d", pContext, pContext->fd, ppContext, refCount); return pContext; @@ -161,6 +174,10 @@ HttpContext *httpGetContext(void *ptr) { } void httpReleaseContext(HttpContext *pContext) { + if (!tsHttpServer.fallback) { + ehttpReleaseContext(pContext); + return; + } int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); if (refCount < 0) { httpError("context:%p, is already released, refCount:%d", pContext, refCount); @@ -217,7 +234,9 @@ bool httpInitContext(HttpContext *pContext) { } void httpCloseContextByApp(HttpContext *pContext) { - D("=="); + if (!tsHttpServer.fallback) { + if (pContext->parsed == false) return; + } pContext->parsed = false; bool keepAlive = true; @@ -229,7 +248,6 @@ void httpCloseContextByApp(HttpContext *pContext) { } if (keepAlive) { - D("==keepAlive=="); if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) { httpDebug("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd, pContext->ipstr); @@ -250,16 +268,19 @@ void httpCloseContextByApp(HttpContext *pContext) { pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); } } else { - D("==not keepAlive=="); httpRemoveContextFromEpoll(pContext); httpDebug("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close context", pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); } - httpReleaseContext(pContext); + if (tsHttpServer.fallback) httpReleaseContext(pContext); } void httpCloseContextByServer(HttpContext *pContext) { + if (!tsHttpServer.fallback) { + if (pContext->closed) return; + pContext->closed = 1; + } if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) { httpDebug("context:%p, fd:%d, ip:%s, epoll finished, still used by app", pContext, pContext->fd, pContext->ipstr); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) { @@ -274,7 +295,7 @@ void httpCloseContextByServer(HttpContext *pContext) { pContext->parsed = false; httpRemoveContextFromEpoll(pContext); - httpReleaseContext(pContext); + if (tsHttpServer.fallback) httpReleaseContext(pContext); } @@ -409,7 +430,44 @@ static void on_error(void *arg, int status_code) { HttpContext *pContext = (HttpContext*)arg; HttpParser *pParser = &pContext->parser; - D("=="); pParser->failed |= EHTTP_CONTEXT_PARSER_FAILED; } +static void httpMightDestroyContext(void *data) { + HttpContext *pContext = *(HttpContext **)data; + if (!tsHttpServer.fallback) { + httpRemoveContextFromEpoll(pContext); + ehttpDecContextRef(&pContext); + return; + } + int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); + if (refCount>0) return; + EQ_ASSERT(refCount==0); + httpDestroyContext(data); +} + +static void ehttpReleaseContext(HttpContext *pContext) { + HttpContext **ppContext = pContext->ppContext; + + if (tsHttpServer.contextCache != NULL) { + taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); + } else { + httpDebug("context:%p, won't be destroyed for cache is already released", pContext); + // httpDestroyContext((void **)(&ppContext)); + } +} + +void ehttpIncContextRef(HttpContext *pContext) { + if (tsHttpServer.fallback) return; + atomic_add_fetch_32(&pContext->refCount, 1); +} + +void ehttpDecContextRef(HttpContext **ppContext) { + if (tsHttpServer.fallback) return; + HttpContext *pContext = *ppContext; + int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); + if (refCount>0) return; + EQ_ASSERT(refCount==0); + httpDestroyContext(ppContext); +} + diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 819f7a5f4a..5a785d2e55 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -194,6 +194,8 @@ static void httpProcessHttpData(void *param) { sigaddset(&set, SIGPIPE); pthread_sigmask(SIG_SETMASK, &set, NULL); + elog_set_thread_name("httpProcessHttpData"); + while (1) { struct epoll_event events[HTTP_MAX_EVENTS]; //-1 means uncertainty, 0-nowait, 1-wait 1 ms, set it from -1 to 1 @@ -209,14 +211,18 @@ static void httpProcessHttpData(void *param) { if (pContext == NULL) { httpError("context:%p, is already released, close connect", events[i].data.ptr); //epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL); - //tclose(events[i].data.fd); + //taosClose(events[i].data.fd); continue; } + ehttpIncContextRef(pContext); + if (events[i].events & EPOLLPRI) { httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLPRI events occured, accessed:%d, close connect", pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); httpCloseContextByServer(pContext); + if (!tsHttpServer.fallback) httpReleaseContext(pContext); + ehttpDecContextRef(&pContext); continue; } @@ -224,6 +230,8 @@ static void httpProcessHttpData(void *param) { httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLRDHUP events occured, accessed:%d, close connect", pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); httpCloseContextByServer(pContext); + httpReleaseContext(pContext); + ehttpDecContextRef(&pContext); continue; } @@ -231,6 +239,8 @@ static void httpProcessHttpData(void *param) { httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLERR events occured, accessed:%d, close connect", pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); httpCloseContextByServer(pContext); + if (!tsHttpServer.fallback) httpReleaseContext(pContext); + ehttpDecContextRef(&pContext); continue; } @@ -238,6 +248,8 @@ static void httpProcessHttpData(void *param) { httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLHUP events occured, accessed:%d, close connect", pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); httpCloseContextByServer(pContext); + if (!tsHttpServer.fallback) httpReleaseContext(pContext); + ehttpDecContextRef(&pContext); continue; } @@ -245,6 +257,7 @@ static void httpProcessHttpData(void *param) { httpDebug("context:%p, fd:%d, ip:%s, state:%s, not in ready state, ignore read events", pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state)); httpReleaseContext(pContext); + ehttpDecContextRef(&pContext); continue; } @@ -253,11 +266,15 @@ static void httpProcessHttpData(void *param) { pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); httpSendErrorResp(pContext, HTTP_SERVER_OFFLINE); httpNotifyContextClose(pContext); + if (!tsHttpServer.fallback) httpReleaseContext(pContext); + ehttpDecContextRef(&pContext); } else { if (httpReadData(pContext)) { (*(pThread->processData))(pContext); atomic_fetch_add_32(&pServer->requestNum, 1); } + if (!tsHttpServer.fallback) httpReleaseContext(pContext); + ehttpDecContextRef(&pContext); } } } @@ -338,7 +355,8 @@ static void *httpAcceptHttpConnection(void *arg) { httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd, pContext->ipstr, pThread->label, strerror(errno)); tclose(pContext->fd); - httpReleaseContext(pContext); + if (tsHttpServer.fallback) httpReleaseContext(pContext); + ehttpDecContextRef(&pContext); continue; } @@ -455,7 +473,6 @@ static bool ehttpReadData(HttpContext *pContext) { int ret = HTTP_CHECK_BODY_SUCCESS; if (ret == HTTP_CHECK_BODY_CONTINUE) { //httpDebug("context:%p, fd:%d, ip:%s, not finished yet, wait another event", pContext, pContext->fd, pContext->ipstr); - httpReleaseContext(pContext); return false; } else if (ret == HTTP_CHECK_BODY_SUCCESS){ httpDebug("context:%p, fd:%d, ip:%s, thread:%s, read size:%d, dataLen:%d", @@ -464,13 +481,11 @@ static bool ehttpReadData(HttpContext *pContext) { return true; } else { httpNotifyContextClose(pContext); - httpReleaseContext(pContext); return false; } } else { httpError("context:%p, fd:%d, ip:%s, failed to read http body, close connect", pContext, pContext->fd, pContext->ipstr); httpNotifyContextClose(pContext); - httpReleaseContext(pContext); return false; } } @@ -483,14 +498,10 @@ static bool ehttpReadData(HttpContext *pContext) { } else { httpError("context:%p, fd:%d, ip:%s, read from socket error:%d, close connect", pContext, pContext->fd, pContext->ipstr, errno); - D("==releasing because of reading failed=="); - httpReleaseContext(pContext); return false; } } else { // eof - D("==releasing because of remote close/reset=="); - httpReleaseContext(pContext); return false; } } From aac52f9691fcb4aa13eac068c22b1b287070e7e0 Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 1 Aug 2020 22:12:35 +0800 Subject: [PATCH 006/105] no need to dec ref in fallback mode --- src/plugins/http/src/httpContext.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 98028fdbb4..ab10234662 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -438,9 +438,6 @@ static void httpMightDestroyContext(void *data) { ehttpDecContextRef(&pContext); return; } - int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); - if (refCount>0) return; - EQ_ASSERT(refCount==0); httpDestroyContext(data); } From 46cf2db0a267994c8e1113fe1f4d750cfcdcaa26 Mon Sep 17 00:00:00 2001 From: freemine Date: Sat, 1 Aug 2020 22:41:29 +0800 Subject: [PATCH 007/105] reallocarray => realloc, because of configuration in travis --- src/plugins/http/src/ehttp_parser.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plugins/http/src/ehttp_parser.c b/src/plugins/http/src/ehttp_parser.c index fbe15661b5..30d37f8a0d 100644 --- a/src/plugins/http/src/ehttp_parser.c +++ b/src/plugins/http/src/ehttp_parser.c @@ -185,7 +185,8 @@ static HTTP_PARSER_STATE ehttp_parser_top(ehttp_parser_t *parser) { static int ehttp_parser_push(ehttp_parser_t *parser, HTTP_PARSER_STATE state) { size_t n = parser->stacks_count + 1; - HTTP_PARSER_STATE *stacks = (HTTP_PARSER_STATE*)reallocarray(parser->stacks, n, sizeof(*stacks)); + // HTTP_PARSER_STATE *stacks = (HTTP_PARSER_STATE*)reallocarray(parser->stacks, n, sizeof(*stacks)); + HTTP_PARSER_STATE *stacks = (HTTP_PARSER_STATE*)realloc(parser->stacks, n * sizeof(*stacks)); if (!stacks) return -1; parser->stacks_count = n; @@ -380,7 +381,8 @@ static int ehttp_parser_check_field(ehttp_parser_t *parser, const char *key, con } static int ehttp_parser_kvs_append_kv(ehttp_parser_t *parser, const char *key, const char *val) { - ehttp_parser_kv_t *kvs = (ehttp_parser_kv_t*)reallocarray(parser->kvs, parser->kvs_count + 1, sizeof(*kvs)); + // ehttp_parser_kv_t *kvs = (ehttp_parser_kv_t*)reallocarray(parser->kvs, parser->kvs_count + 1, sizeof(*kvs)); + ehttp_parser_kv_t *kvs = (ehttp_parser_kv_t*)realloc(parser->kvs, (parser->kvs_count + 1) * sizeof(*kvs)); if (!kvs) return -1; parser->kvs = kvs; From 0755ec1f12c9d02540bd7a548ea9bfe9b8b221ef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 7 Sep 2020 23:58:53 +0800 Subject: [PATCH 008/105] [td-225] fix bug while the content in binary column equals to the maximum allowed length --- src/client/src/TSDBJNIConnector.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index b25f620508..bd980b75a3 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -437,13 +437,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaData * @return */ jstring jniFromNCharToByteArray(JNIEnv *env, char *nchar, int32_t maxBytes) { - int len = (int)strlen(nchar); - if (len > maxBytes) { // no terminated symbol exists '\0' - len = maxBytes; - } - - jbyteArray bytes = (*env)->NewByteArray(env, len); - (*env)->SetByteArrayRegion(env, bytes, 0, len, (jbyte *)nchar); + jbyteArray bytes = (*env)->NewByteArray(env, maxBytes); + (*env)->SetByteArrayRegion(env, bytes, 0, maxBytes, (jbyte *)nchar); return bytes; } @@ -481,6 +476,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn } } + int32_t* length = taos_fetch_lengths(result); + char tmp[TSDB_MAX_BYTES_PER_ROW] = {0}; for (int i = 0; i < num_fields; i++) { @@ -515,15 +512,15 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv); } break; case TSDB_DATA_TYPE_BINARY: { - strncpy(tmp, row[i], (size_t)fields[i].bytes); // handle the case that terminated does not exist + memcpy(tmp, row[i], length[i]); // handle the case that terminated does not exist (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp)); - memset(tmp, 0, (size_t)fields[i].bytes); + memset(tmp, 0, length[i]); break; } case TSDB_DATA_TYPE_NCHAR: { (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteArrayFp, i, - jniFromNCharToByteArray(env, (char *)row[i], fields[i].bytes)); + jniFromNCharToByteArray(env, (char *)row[i], length[i])); break; } case TSDB_DATA_TYPE_TIMESTAMP: From ed7e81f1545c3f1a44c2282d6b3f0b579494503d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Sep 2020 13:17:24 +0800 Subject: [PATCH 009/105] [td-1393] opt last_row query with tags filter conditions. --- src/client/src/tscSQLParser.c | 97 +++++++++++++++++------------------ 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 48980fefb6..75d21fd9eb 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -92,7 +92,6 @@ static int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQueryS static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** exprString); static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type); static int32_t validateEp(char* ep); @@ -103,6 +102,7 @@ static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killTy static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); +static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); @@ -1330,7 +1330,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel /* * transfer sql functions that need secondary merge into another format - * in dealing with metric queries such as: count/first/last + * in dealing with super table queries such as: count/first/last */ if (isSTable) { tscTansformSQLFuncForSTableQuery(pQueryInfo); @@ -1515,16 +1515,16 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex) { + const char* msg1 = "not support column types"; + int16_t type = 0; int16_t bytes = 0; - char columnName[TSDB_COL_NAME_LEN] = {0}; - const char* msg1 = "not support column types"; + char columnName[TSDB_COL_NAME_LEN] = {0}; int32_t functionID = cvtFunc.execFuncId; if (functionID == TSDB_FUNC_SPREAD) { - if (pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BINARY || - pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_NCHAR || - pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BOOL) { + int32_t t1 = pSchema[pColIndex->columnIndex].type; + if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); return -1; } else { @@ -1541,8 +1541,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS } else { getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name); } - - + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName)); @@ -1783,10 +1782,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); } + + // NOTE: has time range condition or normal column filter condition, the last_row query will be transferred to last query SConvertFunc cvtFunc = {.originFuncId = functionID, .execFuncId = functionID}; - if (functionID == TSDB_FUNC_LAST_ROW && TSWINDOW_IS_EQUAL(pQueryInfo->window,TSWINDOW_INITIALIZER)) { + if (functionID == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || (hasNormalColumnFilter(pQueryInfo)))) { cvtFunc.execFuncId = TSDB_FUNC_LAST; } + if (!requireAllFields) { if (pItem->pNode->pParam->nExpr < 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -3204,19 +3206,6 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static UNUSED_FUNC int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { - char* start = *str; - - int32_t code = doArithmeticExprToString(pExpr, str); - if (code == TSDB_CODE_SUCCESS) { // remove out the parenthesis - int32_t len = (int32_t)strlen(start); - memmove(start, start + 1, len - 2); - start[len - 2] = 0; - } - - return code; -} - static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { if (pExpr->nSQLOptr == TK_ID) { if (*type == NON_ARITHMEIC_EXPR) { @@ -3615,7 +3604,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr } static void doCompactQueryExpr(tSQLExpr** pExpr) { - if (*pExpr == NULL || isExprDirectParentOfLeaftNode(*pExpr)) { + if (*pExpr == NULL || isExprDirectParentOfLeafNode(*pExpr)) { return; } @@ -3975,7 +3964,6 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql const char* msg2 = "invalid filter expression"; int32_t ret = TSDB_CODE_SUCCESS; - pQueryInfo->window = TSWINDOW_INITIALIZER; // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space SStringBuilder sb; memset(&sb, 0, sizeof(sb)); @@ -6109,11 +6097,27 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_TSC_INVALID_SQL; } + // set where info + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + if (pQuerySql->pWhere != NULL) { + if (parseWhereClause(pQueryInfo, &pQuerySql->pWhere, pSql) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + + pQuerySql->pWhere = NULL; + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { + pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; + pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; + } + } else { // set the time rang + if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed. + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); + } + } + int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2); - if (pQuerySql->pWhere) { - pQueryInfo->window = TSWINDOW_INITIALIZER; - } if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6133,26 +6137,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_TSC_INVALID_SQL; } - // set where info - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - if (pQuerySql->pWhere != NULL) { - if (parseWhereClause(pQueryInfo, &pQuerySql->pWhere, pSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - - pQuerySql->pWhere = NULL; - if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { - pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; - pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; - } - } else { // set the time rang - pQueryInfo->window = TSWINDOW_INITIALIZER; - if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed. - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); - } - } - // user does not specified the query time window, twa is not allowed in such case. if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX || (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { @@ -6303,7 +6287,8 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS (*pExpr)->_node.optr = getBinaryExprOptr(&t); assert((*pExpr)->_node.optr != 0); - + + // check for dividing by 0 if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) { if (pRight->nodeType == TSQL_NODE_VALUE) { if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i64Key == 0) { @@ -6316,7 +6301,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS // NOTE: binary|nchar data allows the >|< type filter if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { - if (pRight->nodeType == TSQL_NODE_VALUE) { + if (pRight != NULL && pRight->nodeType == TSQL_NODE_VALUE) { if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6326,3 +6311,15 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS return TSDB_CODE_SUCCESS; } + +bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) { + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); + if (pCol->numOfFilters > 0) { + return true; + } + } + + return false; +} \ No newline at end of file From fb25b058aecf496801d4fddd181bb6b9446ce618 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Sep 2020 13:22:45 +0800 Subject: [PATCH 010/105] [td-1394] add the returning value check after invoking the load data block from file function. --- src/query/src/qExecutor.c | 65 +++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d48d7d5ea1..37fe302d43 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2185,43 +2185,43 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { return false; } -int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock) { +int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { SQuery *pQuery = pRuntimeEnv->pQuery; - uint32_t status = 0; + *status = 0; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf > 0) { - status = BLK_DATA_ALL_NEEDED; + *status = BLK_DATA_ALL_NEEDED; } else { // check if this data block is required to load // Calculate all time windows that are overlapping or contain current data block. // If current data block is contained by all possible time window, do not load current data block. if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) { - status = BLK_DATA_ALL_NEEDED; + *status = BLK_DATA_ALL_NEEDED; } - if (status != BLK_DATA_ALL_NEEDED) { + if ((*status) != BLK_DATA_ALL_NEEDED) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg* pSqlFunc = &pQuery->pSelectExpr[i].base; int32_t functionId = pSqlFunc->functionId; int32_t colId = pSqlFunc->colInfo.colId; - status |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId); - if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { + (*status) |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId); + if (((*status) & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { break; } } } } - if (status == BLK_DATA_NO_NEEDED) { + if ((*status) == BLK_DATA_NO_NEEDED) { qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pRuntimeEnv->summary.discardBlocks += 1; - } else if (status == BLK_DATA_STATIS_NEEDED) { - if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { - // return DISK_DATA_LOAD_FAILED; - } + } else if ((*status) == BLK_DATA_STATIS_NEEDED) { + + // this function never returns error? + tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); pRuntimeEnv->summary.loadBlockStatis += 1; @@ -2230,24 +2230,26 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; } } else { - assert(status == BLK_DATA_ALL_NEEDED); + assert((*status) == BLK_DATA_ALL_NEEDED); // load the data block statistics to perform further filter pRuntimeEnv->summary.loadBlockStatis += 1; - if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { - } + tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { // current block has been discard due to filter applied pRuntimeEnv->summary.discardBlocks += 1; qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - return BLK_DATA_DISCARD; + (*status) = BLK_DATA_DISCARD; } pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; pRuntimeEnv->summary.loadBlocks += 1; *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); + if (*pDataBlock == NULL) { + return terrno; + } } return TSDB_CODE_SUCCESS; @@ -2431,15 +2433,18 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { ensureOutputBuffer(pRuntimeEnv, &blockInfo); SDataStatis *pStatis = NULL; - SArray *pDataBlock = NULL; - if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) { - pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step; - continue; + SArray * pDataBlock = NULL; + uint32_t status = 0; + + int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); + if (ret != TSDB_CODE_SUCCESS) { + break; } - if (terrno != TSDB_CODE_SUCCESS) { // load data block failed, abort query - longjmp(pRuntimeEnv->env, terrno); - break; + if (status == BLK_DATA_DISCARD) { + pQuery->current->lastKey = + QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; + continue; } // query start position can not move into tableApplyFunctionsOnBlock due to limit/offset condition @@ -4651,9 +4656,17 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { } SDataStatis *pStatis = NULL; - SArray *pDataBlock = NULL; - if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) { - pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step; + SArray * pDataBlock = NULL; + uint32_t status = 0; + + int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); + if (ret != TSDB_CODE_SUCCESS) { + break; + } + + if (status == BLK_DATA_DISCARD) { + pQuery->current->lastKey = + QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; continue; } From 2a395670bde9ac81cf96e00f8d07cd486a3eec4d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Sep 2020 13:24:07 +0800 Subject: [PATCH 011/105] [td-1369] --- src/client/src/tscSQLParser.c | 37 ++- src/client/src/tscUtil.c | 5 +- src/inc/taoserror.h | 2 +- src/query/inc/qAst.h | 1 + src/query/inc/sql.y | 72 ++--- src/query/src/qAst.c | 36 ++- src/query/src/qExecutor.c | 5 +- src/query/src/qFilterfunc.c | 81 +++++ src/query/src/qParserImpl.c | 5 + src/query/src/sql.c | 548 +++++++++++++++++----------------- src/tsdb/src/tsdbRead.c | 14 +- 11 files changed, 471 insertions(+), 335 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 75d21fd9eb..adb6bfccbc 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2784,6 +2784,12 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, case TK_LIKE: pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE; break; + case TK_ISNULL: + pColumnFilter->lowerRelOptr = TSDB_RELATION_ISNULL; + break; + case TK_NOTNULL: + pColumnFilter->lowerRelOptr = TSDB_RELATION_NOTNULL; + break; default: return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -2829,19 +2835,19 @@ static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) { return TSDB_CODE_SUCCESS; } +// pExpr->nSQLOptr == 0 while handling "is null" query static bool isExprLeafNode(tSQLExpr* pExpr) { return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && - (pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) || - pExpr->nSQLOptr == TK_SET); + (pExpr->nSQLOptr == 0 || pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) || pExpr->nSQLOptr == TK_SET); } -static bool isExprDirectParentOfLeaftNode(tSQLExpr* pExpr) { +static bool isExprDirectParentOfLeafNode(tSQLExpr* pExpr) { return (pExpr->pLeft != NULL && pExpr->pRight != NULL) && (isExprLeafNode(pExpr->pLeft) && isExprLeafNode(pExpr->pRight)); } static int32_t tSQLExprLeafToString(tSQLExpr* pExpr, bool addParentheses, char** output) { - if (!isExprDirectParentOfLeaftNode(pExpr)) { + if (!isExprDirectParentOfLeafNode(pExpr)) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -3052,7 +3058,7 @@ static int32_t getTagCondString(tSQLExpr* pExpr, char** str) { return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeaftNode(pExpr)) { + if (!isExprDirectParentOfLeafNode(pExpr)) { *(*str) = '('; *str += 1; @@ -3108,7 +3114,7 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeaftNode(pExpr)) { // internal node + if (!isExprDirectParentOfLeafNode(pExpr)) { // internal node int32_t ret = getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->nSQLOptr); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -3134,7 +3140,7 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeaftNode(pExpr)) { + if (!isExprDirectParentOfLeafNode(pExpr)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -3453,7 +3459,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - assert(isExprDirectParentOfLeaftNode(*pExpr)); + assert(isExprDirectParentOfLeafNode(*pExpr)); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -3499,7 +3505,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL } } - // in case of in operator, keep it in a seperate attribute + // in case of in operator, keep it in a seprate attribute if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { if (!validTableNameOptr(*pExpr)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); @@ -3520,7 +3526,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL *type = TSQL_EXPR_TBNAME; *pExpr = NULL; } else { - if (pRight->nSQLOptr == TK_ID) { // join on tag columns for stable query + if (pRight != NULL && pRight->nSQLOptr == TK_ID) { // join on tag columns for stable query if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -3573,7 +3579,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr int32_t leftType = -1; int32_t rightType = -1; - if (!isExprDirectParentOfLeaftNode(*pExpr)) { + if (!isExprDirectParentOfLeafNode(*pExpr)) { int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->nSQLOptr); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -3635,7 +3641,7 @@ static void doCompactQueryExpr(tSQLExpr** pExpr) { } static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* pQueryInfo, tSQLExpr** pOut, int32_t tableIndex) { - if (isExprDirectParentOfLeaftNode(*pExpr)) { + if (isExprDirectParentOfLeafNode(*pExpr)) { tSQLExpr* pLeft = (*pExpr)->pLeft; SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -3794,7 +3800,7 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLE return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeaftNode(pExpr)) { + if (!isExprDirectParentOfLeafNode(pExpr)) { if (pExpr->nSQLOptr == TK_OR) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -6215,6 +6221,11 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS return ret; } } + + if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->nSQLOptr == 0) { + *pExpr = calloc(1, sizeof(tExprNode)); + return TSDB_CODE_SUCCESS; + } if (pSqlExpr->pLeft == NULL) { if (pSqlExpr->nSQLOptr >= TK_BOOL && pSqlExpr->nSQLOptr <= TK_STRING) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 80e3828c9d..ad87825e34 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1531,9 +1531,10 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); assert(pQueryInfo->exprList == NULL); - pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; + pQueryInfo->window = TSWINDOW_INITIALIZER; } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index d8e5c8f1d7..3198d6d97e 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -98,7 +98,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, 0, 0x0216, "Syntax errr in SQL") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, 0, 0x0216, "Syntax error in SQL") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed") diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index 547616dee6..d3e60c21dc 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -32,6 +32,7 @@ struct tExprNode; struct SSchema; enum { + TSQL_NODE_DUMMY = 0x0, TSQL_NODE_EXPR = 0x1, TSQL_NODE_COL = 0x2, TSQL_NODE_VALUE = 0x4, diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 79aec2f349..e0dd9d98f0 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -567,53 +567,53 @@ where_opt(A) ::= WHERE expr(X). {A = X;} %type expr {tSQLExpr*} %destructor expr {tSQLExprDestroy($$);} -expr(A) ::= LP expr(X) RP. {A = X; } +expr(A) ::= LP expr(X) RP. {A = X; } -expr(A) ::= ID(X). {A = tSQLExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);} +expr(A) ::= ID(X). {A = tSQLExprIdValueCreate(&X, TK_ID);} +expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);} +expr(A) ::= ID(X) DOT STAR(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);} -expr(A) ::= INTEGER(X). {A = tSQLExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= INTEGER(X). {A = tSQLExprIdValueCreate(&X, TK_INTEGER);} expr(A) ::= MINUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);} expr(A) ::= PLUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). {A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= STRING(X). {A = tSQLExprIdValueCreate(&X, TK_STRING);} -expr(A) ::= NOW(X). {A = tSQLExprIdValueCreate(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). {A = tSQLExprIdValueCreate(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). {A = tSQLExprIdValueCreate(&X, TK_BOOL);} -// normal functions: min(x) -expr(A) ::= ID(X) LP exprlist(Y) RP(E). { - A = tSQLExprCreateFunction(Y, &X, &E, X.type); -} +expr(A) ::= FLOAT(X). {A = tSQLExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= MINUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= PLUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= STRING(X). {A = tSQLExprIdValueCreate(&X, TK_STRING);} +expr(A) ::= NOW(X). {A = tSQLExprIdValueCreate(&X, TK_NOW); } +expr(A) ::= VARIABLE(X). {A = tSQLExprIdValueCreate(&X, TK_VARIABLE);} +expr(A) ::= BOOL(X). {A = tSQLExprIdValueCreate(&X, TK_BOOL);} -// this is for: count(*)/first(*)/last(*) operation -expr(A) ::= ID(X) LP STAR RP(Y). { - A = tSQLExprCreateFunction(NULL, &X, &Y, X.type); -} +// ordinary functions: min(x), max(x), top(k, 20) +expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSQLExprCreateFunction(Y, &X, &E, X.type); } -//binary expression: a+2, b+3 -expr(A) ::= expr(X) AND expr(Y). {A = tSQLExprCreate(X, Y, TK_AND);} -expr(A) ::= expr(X) OR expr(Y). {A = tSQLExprCreate(X, Y, TK_OR); } +// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation +expr(A) ::= ID(X) LP STAR RP(Y). { A = tSQLExprCreateFunction(NULL, &X, &Y, X.type); } -//binary relational expression -expr(A) ::= expr(X) LT expr(Y). {A = tSQLExprCreate(X, Y, TK_LT);} -expr(A) ::= expr(X) GT expr(Y). {A = tSQLExprCreate(X, Y, TK_GT);} -expr(A) ::= expr(X) LE expr(Y). {A = tSQLExprCreate(X, Y, TK_LE);} -expr(A) ::= expr(X) GE expr(Y). {A = tSQLExprCreate(X, Y, TK_GE);} -expr(A) ::= expr(X) NE expr(Y). {A = tSQLExprCreate(X, Y, TK_NE);} -expr(A) ::= expr(X) EQ expr(Y). {A = tSQLExprCreate(X, Y, TK_EQ);} +// is (not) null expression +expr(A) ::= expr(X) IS NULL. {A = tSQLExprCreate(X, NULL, TK_ISNULL);} +expr(A) ::= expr(X) IS NOT NULL. {A = tSQLExprCreate(X, NULL, TK_NOTNULL);} -//binary arithmetic expression +// relational expression +expr(A) ::= expr(X) LT expr(Y). {A = tSQLExprCreate(X, Y, TK_LT);} +expr(A) ::= expr(X) GT expr(Y). {A = tSQLExprCreate(X, Y, TK_GT);} +expr(A) ::= expr(X) LE expr(Y). {A = tSQLExprCreate(X, Y, TK_LE);} +expr(A) ::= expr(X) GE expr(Y). {A = tSQLExprCreate(X, Y, TK_GE);} +expr(A) ::= expr(X) NE expr(Y). {A = tSQLExprCreate(X, Y, TK_NE);} +expr(A) ::= expr(X) EQ expr(Y). {A = tSQLExprCreate(X, Y, TK_EQ);} + +expr(A) ::= expr(X) AND expr(Y). {A = tSQLExprCreate(X, Y, TK_AND);} +expr(A) ::= expr(X) OR expr(Y). {A = tSQLExprCreate(X, Y, TK_OR); } + +// binary arithmetic expression expr(A) ::= expr(X) PLUS expr(Y). {A = tSQLExprCreate(X, Y, TK_PLUS); } expr(A) ::= expr(X) MINUS expr(Y). {A = tSQLExprCreate(X, Y, TK_MINUS); } expr(A) ::= expr(X) STAR expr(Y). {A = tSQLExprCreate(X, Y, TK_STAR); } expr(A) ::= expr(X) SLASH expr(Y). {A = tSQLExprCreate(X, Y, TK_DIVIDE);} expr(A) ::= expr(X) REM expr(Y). {A = tSQLExprCreate(X, Y, TK_REM); } -//like expression -expr(A) ::= expr(X) LIKE expr(Y). {A = tSQLExprCreate(X, Y, TK_LIKE); } +// like expression +expr(A) ::= expr(X) LIKE expr(Y). {A = tSQLExprCreate(X, Y, TK_LIKE); } //in expression expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSQLExprCreate(X, (tSQLExpr*)Y, TK_IN); } @@ -625,9 +625,9 @@ expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSQLExprCreate(X, (tSQLExpr*)Y, %destructor expritem {tSQLExprDestroy($$);} exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSQLExprListAppend(X,Y,0);} -exprlist(A) ::= expritem(X). {A = tSQLExprListAppend(0,X,0);} -expritem(A) ::= expr(X). {A = X;} -expritem(A) ::= . {A = 0;} +exprlist(A) ::= expritem(X). {A = tSQLExprListAppend(0,X,0);} +expritem(A) ::= expr(X). {A = X;} +expritem(A) ::= . {A = 0;} ///////////////////////////////////reset query cache////////////////////////////////////// cmd ::= RESET QUERY CACHE. { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 634f014d97..63411aaf3f 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -188,6 +188,10 @@ uint8_t getBinaryExprOptr(SStrToken *pToken) { return TSDB_BINARY_OP_REMAINDER; case TK_LIKE: return TSDB_RELATION_LIKE; + case TK_ISNULL: + return TSDB_RELATION_ISNULL; + case TK_NOTNULL: + return TSDB_RELATION_NOTNULL; default: { return 0; } } } @@ -486,29 +490,42 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr } else { int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID; if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - bool comp = true; + bool comp = true; int32_t ret = 0; - - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + if (comp) { ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); assert(ret <= 0); } - + if (ret == 0 && optr == TSDB_RELATION_LESS) { continue; } else { - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); comp = false; // no need to compare anymore } } + } else { + assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL); + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); + if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || + (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { + STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + } } } - free(cond.start); + free(cond.start); free(cond.end); tSkipListDestroyIter(iter); } @@ -683,6 +700,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, char * pData = SL_GET_NODE_DATA(pNode); tstr *name = (tstr*) tsdbGetTableName(*(void**) pData); + // todo speed up by using hash if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->optr == TSDB_RELATION_IN) { @@ -714,7 +732,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S // column project if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { - assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); + assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); param->setupInfoFn(pExpr, param->pExtInfo); if (pSkipList == NULL) { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 37fe302d43..2ca2545818 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -200,9 +200,6 @@ bool doFilterData(SQuery *pQuery, int32_t elemPos) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; char *pElem = (char*)pFilterInfo->pData + pFilterInfo->info.bytes * elemPos; - if (isNull(pElem, pFilterInfo->info.type)) { - return false; - } bool qualified = false; for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { @@ -1002,7 +999,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - if (QUERY_IS_INTERVAL_QUERY(pQuery)/* && tsCols != NULL*/) { + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { TSKEY ts = TSKEY_INITIAL_VAL; if (tsCols == NULL) { diff --git a/src/query/src/qFilterfunc.c b/src/query/src/qFilterfunc.c index 7e9f5c7da5..6b88171e71 100644 --- a/src/query/src/qFilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -284,6 +284,71 @@ bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0; } +//////////////////////////////////////////////////////////////// +bool isNull_i8(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return isNull(minval, TSDB_DATA_TYPE_TINYINT); +} + +bool isNull_i16(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return isNull(minval, TSDB_DATA_TYPE_SMALLINT); +} + +bool isNull_i32(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return isNull(minval, TSDB_DATA_TYPE_INT); +} + +bool isNull_i64(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return isNull(minval, TSDB_DATA_TYPE_BIGINT); +} + +bool isNull_ds(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return isNull(minval, TSDB_DATA_TYPE_FLOAT); +} + +bool isNull_dd(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return isNull(minval, TSDB_DATA_TYPE_DOUBLE); +} + +bool isNull_binary(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return isNull(minval, TSDB_DATA_TYPE_BINARY); +} + +bool isNull_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return isNull(minval, TSDB_DATA_TYPE_NCHAR); +} + +//////////////////////////////////////////////////////////////// +bool notNull_i8(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return !isNull(minval, TSDB_DATA_TYPE_TINYINT); +} + +bool notNull_i16(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return !isNull(minval, TSDB_DATA_TYPE_SMALLINT); +} + +bool notNull_i32(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return !isNull(minval, TSDB_DATA_TYPE_INT); +} + +bool notNull_i64(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return !isNull(minval, TSDB_DATA_TYPE_BIGINT); +} + +bool notNull_ds(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return !isNull(minval, TSDB_DATA_TYPE_FLOAT); +} + +bool notNull_dd(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return isNull(minval, TSDB_DATA_TYPE_DOUBLE); +} + +bool notNull_binary(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return !isNull(minval, TSDB_DATA_TYPE_BINARY); +} + +bool notNull_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { + return !isNull(minval, TSDB_DATA_TYPE_NCHAR); +} //////////////////////////////////////////////////////////////// @@ -398,6 +463,8 @@ bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i8, nequal_i8, NULL, + isNull_i8, + notNull_i8, }; bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -409,6 +476,8 @@ bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i16, nequal_i16, NULL, + isNull_i16, + notNull_i16, }; bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -420,6 +489,8 @@ bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i32, nequal_i32, NULL, + isNull_i32, + notNull_i32, }; bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -431,6 +502,8 @@ bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i64, nequal_i64, NULL, + isNull_i64, + notNull_i64, }; bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -442,6 +515,8 @@ bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_ds, nequal_ds, NULL, + isNull_ds, + notNull_ds, }; bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -453,6 +528,8 @@ bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_dd, nequal_dd, NULL, + isNull_dd, + notNull_dd, }; bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = { @@ -464,6 +541,8 @@ bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) NULL, nequal_str, like_str, + isNull_binary, + notNull_binary, }; bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = { @@ -475,6 +554,8 @@ bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxva NULL, nequal_nchar, like_nchar, + isNull_nchar, + notNull_nchar, }; bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 9629f24cc2..1e8ceffae6 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -275,6 +275,11 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { } else { pExpr->nSQLOptr = optrType; pExpr->pLeft = pLeft; + + if (pRight == NULL) { + pRight = calloc(1, sizeof(tSQLExpr)); + } + pExpr->pRight = pRight; } diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 373e57963c..35e36032d4 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -126,17 +126,17 @@ typedef union { #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 -#define YYNSTATE 244 -#define YYNRULE 225 +#define YYNSTATE 246 +#define YYNRULE 227 #define YYNTOKEN 206 -#define YY_MAX_SHIFT 243 -#define YY_MIN_SHIFTREDUCE 403 -#define YY_MAX_SHIFTREDUCE 627 -#define YY_ERROR_ACTION 628 -#define YY_ACCEPT_ACTION 629 -#define YY_NO_ACTION 630 -#define YY_MIN_REDUCE 631 -#define YY_MAX_REDUCE 855 +#define YY_MAX_SHIFT 245 +#define YY_MIN_SHIFTREDUCE 407 +#define YY_MAX_SHIFTREDUCE 633 +#define YY_ERROR_ACTION 634 +#define YY_ACCEPT_ACTION 635 +#define YY_NO_ACTION 636 +#define YY_MIN_REDUCE 637 +#define YY_MAX_REDUCE 863 /************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined @@ -202,121 +202,122 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (549) +#define YY_ACTTAB_COUNT (556) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 731, 444, 221, 729, 730, 629, 243, 510, 732, 445, - /* 10 */ 734, 735, 733, 41, 43, 526, 35, 36, 523, 11, - /* 20 */ 524, 29, 525, 444, 199, 39, 37, 40, 38, 155, - /* 30 */ 241, 445, 748, 34, 33, 219, 218, 32, 31, 30, - /* 40 */ 41, 43, 761, 35, 36, 136, 172, 173, 29, 137, - /* 50 */ 21, 199, 39, 37, 40, 38, 184, 141, 160, 843, - /* 60 */ 34, 33, 839, 772, 32, 31, 30, 404, 405, 406, - /* 70 */ 407, 408, 409, 410, 411, 412, 413, 414, 415, 242, - /* 80 */ 41, 43, 230, 35, 36, 746, 62, 137, 29, 137, - /* 90 */ 21, 199, 39, 37, 40, 38, 159, 843, 27, 842, - /* 100 */ 34, 33, 56, 838, 32, 31, 30, 105, 43, 8, - /* 110 */ 35, 36, 63, 115, 769, 29, 761, 527, 199, 39, - /* 120 */ 37, 40, 38, 168, 539, 747, 583, 34, 33, 18, - /* 130 */ 156, 32, 31, 30, 16, 210, 236, 235, 209, 208, - /* 140 */ 207, 234, 206, 233, 232, 231, 205, 727, 105, 715, - /* 150 */ 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, - /* 160 */ 726, 35, 36, 798, 837, 194, 29, 177, 157, 199, - /* 170 */ 39, 37, 40, 38, 181, 180, 21, 21, 34, 33, - /* 180 */ 444, 12, 32, 31, 30, 164, 596, 750, 445, 587, - /* 190 */ 153, 590, 154, 593, 105, 164, 596, 21, 17, 587, - /* 200 */ 150, 590, 196, 593, 60, 26, 90, 89, 144, 169, - /* 210 */ 217, 747, 747, 16, 149, 236, 235, 161, 162, 167, - /* 220 */ 234, 198, 233, 232, 231, 142, 670, 161, 162, 128, - /* 230 */ 222, 542, 747, 164, 596, 17, 143, 587, 750, 590, - /* 240 */ 105, 593, 26, 39, 37, 40, 38, 100, 170, 145, - /* 250 */ 797, 34, 33, 101, 26, 32, 31, 30, 32, 31, - /* 260 */ 30, 78, 183, 564, 565, 161, 162, 230, 589, 152, - /* 270 */ 592, 76, 80, 85, 88, 79, 240, 239, 97, 34, - /* 280 */ 33, 82, 42, 32, 31, 30, 118, 119, 70, 66, - /* 290 */ 69, 237, 42, 595, 679, 163, 61, 128, 132, 130, - /* 300 */ 93, 92, 91, 595, 671, 187, 585, 128, 594, 588, - /* 310 */ 750, 591, 171, 534, 47, 216, 215, 146, 594, 555, - /* 320 */ 186, 147, 556, 46, 613, 148, 14, 597, 13, 139, - /* 330 */ 42, 13, 50, 48, 3, 135, 75, 74, 140, 516, - /* 340 */ 515, 595, 586, 46, 22, 138, 203, 10, 9, 51, - /* 350 */ 22, 852, 530, 528, 531, 529, 594, 87, 86, 749, - /* 360 */ 808, 807, 165, 804, 803, 166, 771, 741, 220, 776, - /* 370 */ 763, 778, 102, 790, 789, 116, 117, 114, 681, 204, - /* 380 */ 133, 24, 213, 678, 214, 851, 72, 850, 848, 26, - /* 390 */ 120, 699, 25, 23, 185, 95, 134, 668, 81, 551, - /* 400 */ 666, 83, 84, 664, 188, 663, 174, 129, 661, 660, - /* 410 */ 659, 658, 657, 649, 131, 655, 653, 192, 52, 651, - /* 420 */ 760, 57, 49, 58, 791, 44, 197, 195, 193, 191, - /* 430 */ 189, 28, 212, 77, 223, 224, 225, 226, 227, 228, - /* 440 */ 229, 238, 627, 176, 175, 626, 201, 178, 179, 53, - /* 450 */ 625, 618, 182, 536, 64, 151, 186, 67, 552, 55, - /* 460 */ 103, 158, 662, 59, 200, 94, 96, 123, 700, 121, - /* 470 */ 126, 106, 107, 122, 124, 125, 127, 112, 108, 109, - /* 480 */ 113, 745, 110, 656, 111, 1, 2, 190, 5, 557, - /* 490 */ 104, 19, 6, 598, 20, 4, 15, 7, 65, 485, - /* 500 */ 202, 481, 479, 478, 477, 474, 448, 211, 68, 45, - /* 510 */ 71, 73, 22, 512, 511, 509, 54, 469, 467, 459, - /* 520 */ 465, 461, 463, 457, 455, 484, 483, 482, 480, 476, - /* 530 */ 475, 46, 446, 419, 417, 631, 630, 630, 630, 630, - /* 540 */ 630, 630, 630, 630, 630, 630, 630, 98, 99, + /* 0 */ 737, 448, 11, 735, 736, 635, 245, 448, 738, 449, + /* 10 */ 740, 741, 739, 35, 36, 449, 37, 38, 155, 243, + /* 20 */ 165, 29, 137, 136, 200, 41, 39, 43, 40, 105, + /* 30 */ 514, 160, 851, 34, 33, 778, 137, 32, 31, 30, + /* 40 */ 35, 36, 767, 37, 38, 159, 851, 165, 29, 767, + /* 50 */ 105, 200, 41, 39, 43, 40, 185, 157, 221, 220, + /* 60 */ 34, 33, 137, 156, 32, 31, 30, 35, 36, 448, + /* 70 */ 37, 38, 850, 141, 165, 29, 756, 449, 200, 41, + /* 80 */ 39, 43, 40, 197, 78, 60, 775, 34, 33, 232, + /* 90 */ 232, 32, 31, 30, 21, 41, 39, 43, 40, 32, + /* 100 */ 31, 30, 56, 34, 33, 847, 803, 32, 31, 30, + /* 110 */ 21, 21, 105, 408, 409, 410, 411, 412, 413, 414, + /* 120 */ 415, 416, 417, 418, 419, 244, 587, 169, 36, 753, + /* 130 */ 37, 38, 223, 50, 165, 29, 21, 62, 200, 41, + /* 140 */ 39, 43, 40, 170, 219, 753, 753, 34, 33, 27, + /* 150 */ 51, 32, 31, 30, 8, 37, 38, 63, 115, 165, + /* 160 */ 29, 101, 754, 200, 41, 39, 43, 40, 804, 224, + /* 170 */ 195, 753, 34, 33, 168, 846, 32, 31, 30, 16, + /* 180 */ 212, 238, 237, 211, 210, 209, 236, 208, 235, 234, + /* 190 */ 233, 207, 733, 756, 721, 722, 723, 724, 725, 726, + /* 200 */ 727, 728, 729, 730, 731, 732, 164, 600, 12, 239, + /* 210 */ 591, 17, 594, 188, 597, 105, 164, 600, 26, 559, + /* 220 */ 591, 845, 594, 46, 597, 34, 33, 150, 756, 32, + /* 230 */ 31, 30, 21, 90, 89, 144, 568, 569, 161, 162, + /* 240 */ 171, 149, 199, 76, 80, 85, 88, 79, 161, 162, + /* 250 */ 164, 600, 546, 82, 591, 589, 594, 100, 597, 242, + /* 260 */ 241, 97, 17, 16, 26, 238, 237, 752, 201, 26, + /* 270 */ 236, 61, 235, 234, 233, 118, 119, 70, 66, 69, + /* 280 */ 538, 676, 161, 162, 128, 530, 178, 187, 527, 184, + /* 290 */ 528, 590, 529, 182, 181, 593, 152, 596, 132, 130, + /* 300 */ 93, 92, 91, 42, 172, 543, 685, 218, 217, 128, + /* 310 */ 18, 163, 677, 42, 599, 128, 173, 174, 560, 619, + /* 320 */ 153, 47, 14, 13, 599, 592, 601, 595, 520, 598, + /* 330 */ 13, 519, 46, 154, 205, 22, 75, 74, 22, 598, + /* 340 */ 48, 10, 9, 534, 532, 535, 533, 42, 87, 86, + /* 350 */ 3, 139, 860, 140, 755, 142, 143, 603, 599, 147, + /* 360 */ 148, 146, 135, 145, 138, 814, 813, 166, 810, 809, + /* 370 */ 167, 777, 747, 598, 222, 769, 782, 784, 102, 796, + /* 380 */ 114, 116, 795, 117, 687, 206, 133, 531, 186, 26, + /* 390 */ 24, 95, 215, 684, 216, 859, 72, 858, 856, 120, + /* 400 */ 705, 25, 555, 23, 134, 674, 81, 672, 83, 189, + /* 410 */ 84, 670, 193, 669, 175, 52, 129, 667, 49, 666, + /* 420 */ 665, 106, 664, 44, 663, 107, 655, 131, 198, 766, + /* 430 */ 196, 661, 659, 657, 194, 57, 58, 797, 192, 190, + /* 440 */ 28, 214, 77, 225, 226, 227, 228, 229, 230, 203, + /* 450 */ 53, 231, 240, 633, 151, 177, 64, 67, 176, 668, + /* 460 */ 632, 179, 180, 631, 624, 187, 123, 183, 122, 706, + /* 470 */ 121, 125, 124, 94, 127, 662, 126, 96, 1, 2, + /* 480 */ 540, 112, 108, 109, 751, 110, 113, 111, 59, 55, + /* 490 */ 556, 103, 158, 19, 191, 20, 561, 104, 5, 602, + /* 500 */ 6, 4, 604, 15, 202, 7, 204, 65, 489, 485, + /* 510 */ 483, 482, 481, 478, 452, 213, 68, 45, 71, 22, + /* 520 */ 516, 515, 513, 54, 473, 471, 463, 469, 465, 467, + /* 530 */ 73, 461, 459, 488, 487, 486, 484, 480, 479, 46, + /* 540 */ 450, 423, 421, 637, 636, 636, 98, 636, 636, 636, + /* 550 */ 636, 636, 636, 636, 636, 99, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 226, 1, 210, 229, 230, 207, 208, 5, 234, 9, - /* 10 */ 236, 237, 238, 13, 14, 2, 16, 17, 5, 260, - /* 20 */ 7, 21, 9, 1, 24, 25, 26, 27, 28, 209, - /* 30 */ 210, 9, 240, 33, 34, 33, 34, 37, 38, 39, - /* 40 */ 13, 14, 244, 16, 17, 260, 33, 34, 21, 260, - /* 50 */ 210, 24, 25, 26, 27, 28, 258, 260, 269, 270, - /* 60 */ 33, 34, 260, 210, 37, 38, 39, 45, 46, 47, - /* 70 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 80 */ 13, 14, 78, 16, 17, 245, 247, 260, 21, 260, - /* 90 */ 210, 24, 25, 26, 27, 28, 269, 270, 259, 270, - /* 100 */ 33, 34, 102, 260, 37, 38, 39, 210, 14, 98, - /* 110 */ 16, 17, 101, 102, 261, 21, 244, 104, 24, 25, - /* 120 */ 26, 27, 28, 243, 103, 245, 99, 33, 34, 108, - /* 130 */ 258, 37, 38, 39, 85, 86, 87, 88, 89, 90, - /* 140 */ 91, 92, 93, 94, 95, 96, 97, 226, 210, 228, - /* 150 */ 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, - /* 160 */ 239, 16, 17, 266, 260, 268, 21, 126, 227, 24, - /* 170 */ 25, 26, 27, 28, 133, 134, 210, 210, 33, 34, - /* 180 */ 1, 44, 37, 38, 39, 1, 2, 246, 9, 5, - /* 190 */ 260, 7, 260, 9, 210, 1, 2, 210, 98, 5, - /* 200 */ 63, 7, 264, 9, 266, 105, 69, 70, 71, 243, - /* 210 */ 243, 245, 245, 85, 77, 87, 88, 33, 34, 227, - /* 220 */ 92, 37, 94, 95, 96, 260, 214, 33, 34, 217, - /* 230 */ 243, 37, 245, 1, 2, 98, 260, 5, 246, 7, - /* 240 */ 210, 9, 105, 25, 26, 27, 28, 98, 63, 260, - /* 250 */ 266, 33, 34, 210, 105, 37, 38, 39, 37, 38, - /* 260 */ 39, 72, 125, 115, 116, 33, 34, 78, 5, 132, - /* 270 */ 7, 64, 65, 66, 67, 68, 60, 61, 62, 33, - /* 280 */ 34, 74, 98, 37, 38, 39, 64, 65, 66, 67, - /* 290 */ 68, 227, 98, 109, 214, 59, 266, 217, 64, 65, - /* 300 */ 66, 67, 68, 109, 214, 262, 1, 217, 124, 5, - /* 310 */ 246, 7, 127, 99, 103, 130, 131, 260, 124, 99, - /* 320 */ 106, 260, 99, 103, 99, 260, 103, 99, 103, 260, - /* 330 */ 98, 103, 103, 122, 98, 260, 128, 129, 260, 99, - /* 340 */ 99, 109, 37, 103, 103, 260, 99, 128, 129, 120, - /* 350 */ 103, 246, 5, 5, 7, 7, 124, 72, 73, 246, - /* 360 */ 241, 241, 241, 241, 241, 241, 210, 242, 241, 210, - /* 370 */ 244, 210, 210, 267, 267, 210, 210, 248, 210, 210, - /* 380 */ 210, 210, 210, 210, 210, 210, 210, 210, 210, 105, - /* 390 */ 210, 210, 210, 210, 244, 59, 210, 210, 210, 109, - /* 400 */ 210, 210, 210, 210, 263, 210, 210, 210, 210, 210, - /* 410 */ 210, 210, 210, 210, 210, 210, 210, 263, 119, 210, - /* 420 */ 257, 211, 121, 211, 211, 118, 113, 117, 112, 111, - /* 430 */ 110, 123, 75, 84, 83, 49, 80, 82, 53, 81, - /* 440 */ 79, 75, 5, 5, 135, 5, 211, 135, 5, 211, - /* 450 */ 5, 86, 126, 99, 215, 211, 106, 215, 99, 107, - /* 460 */ 98, 1, 211, 103, 100, 212, 212, 219, 225, 224, - /* 470 */ 221, 256, 255, 223, 222, 220, 218, 250, 254, 253, - /* 480 */ 249, 244, 252, 211, 251, 216, 213, 98, 114, 99, - /* 490 */ 98, 103, 114, 99, 103, 98, 98, 98, 72, 9, - /* 500 */ 100, 5, 5, 5, 5, 5, 76, 15, 72, 16, - /* 510 */ 129, 129, 103, 5, 5, 99, 98, 5, 5, 5, - /* 520 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 530 */ 5, 103, 76, 59, 58, 0, 271, 271, 271, 271, - /* 540 */ 271, 271, 271, 271, 271, 271, 271, 21, 21, 271, - /* 550 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + /* 0 */ 226, 1, 260, 229, 230, 207, 208, 1, 234, 9, + /* 10 */ 236, 237, 238, 13, 14, 9, 16, 17, 209, 210, + /* 20 */ 20, 21, 260, 260, 24, 25, 26, 27, 28, 210, + /* 30 */ 5, 269, 270, 33, 34, 210, 260, 37, 38, 39, + /* 40 */ 13, 14, 244, 16, 17, 269, 270, 20, 21, 244, + /* 50 */ 210, 24, 25, 26, 27, 28, 258, 227, 33, 34, + /* 60 */ 33, 34, 260, 258, 37, 38, 39, 13, 14, 1, + /* 70 */ 16, 17, 270, 260, 20, 21, 246, 9, 24, 25, + /* 80 */ 26, 27, 28, 264, 72, 266, 261, 33, 34, 78, + /* 90 */ 78, 37, 38, 39, 210, 25, 26, 27, 28, 37, + /* 100 */ 38, 39, 102, 33, 34, 260, 266, 37, 38, 39, + /* 110 */ 210, 210, 210, 45, 46, 47, 48, 49, 50, 51, + /* 120 */ 52, 53, 54, 55, 56, 57, 99, 243, 14, 245, + /* 130 */ 16, 17, 210, 103, 20, 21, 210, 247, 24, 25, + /* 140 */ 26, 27, 28, 243, 243, 245, 245, 33, 34, 259, + /* 150 */ 120, 37, 38, 39, 98, 16, 17, 101, 102, 20, + /* 160 */ 21, 210, 240, 24, 25, 26, 27, 28, 266, 243, + /* 170 */ 268, 245, 33, 34, 227, 260, 37, 38, 39, 85, + /* 180 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 190 */ 96, 97, 226, 246, 228, 229, 230, 231, 232, 233, + /* 200 */ 234, 235, 236, 237, 238, 239, 1, 2, 44, 227, + /* 210 */ 5, 98, 7, 262, 9, 210, 1, 2, 105, 99, + /* 220 */ 5, 260, 7, 103, 9, 33, 34, 63, 246, 37, + /* 230 */ 38, 39, 210, 69, 70, 71, 115, 116, 33, 34, + /* 240 */ 63, 77, 37, 64, 65, 66, 67, 68, 33, 34, + /* 250 */ 1, 2, 37, 74, 5, 1, 7, 98, 9, 60, + /* 260 */ 61, 62, 98, 85, 105, 87, 88, 245, 15, 105, + /* 270 */ 92, 266, 94, 95, 96, 64, 65, 66, 67, 68, + /* 280 */ 99, 214, 33, 34, 217, 2, 126, 106, 5, 125, + /* 290 */ 7, 37, 9, 133, 134, 5, 132, 7, 64, 65, + /* 300 */ 66, 67, 68, 98, 127, 103, 214, 130, 131, 217, + /* 310 */ 108, 59, 214, 98, 109, 217, 33, 34, 99, 99, + /* 320 */ 260, 103, 103, 103, 109, 5, 99, 7, 99, 124, + /* 330 */ 103, 99, 103, 260, 99, 103, 128, 129, 103, 124, + /* 340 */ 122, 128, 129, 5, 5, 7, 7, 98, 72, 73, + /* 350 */ 98, 260, 246, 260, 246, 260, 260, 104, 109, 260, + /* 360 */ 260, 260, 260, 260, 260, 241, 241, 241, 241, 241, + /* 370 */ 241, 210, 242, 124, 241, 244, 210, 210, 210, 267, + /* 380 */ 248, 210, 267, 210, 210, 210, 210, 104, 244, 105, + /* 390 */ 210, 59, 210, 210, 210, 210, 210, 210, 210, 210, + /* 400 */ 210, 210, 109, 210, 210, 210, 210, 210, 210, 263, + /* 410 */ 210, 210, 263, 210, 210, 119, 210, 210, 121, 210, + /* 420 */ 210, 256, 210, 118, 210, 255, 210, 210, 113, 257, + /* 430 */ 117, 210, 210, 210, 112, 211, 211, 211, 111, 110, + /* 440 */ 123, 75, 84, 83, 49, 80, 82, 53, 81, 211, + /* 450 */ 211, 79, 75, 5, 211, 5, 215, 215, 135, 211, + /* 460 */ 5, 135, 5, 5, 86, 106, 219, 126, 223, 225, + /* 470 */ 224, 220, 222, 212, 218, 211, 221, 212, 216, 213, + /* 480 */ 99, 250, 254, 253, 244, 252, 249, 251, 103, 107, + /* 490 */ 99, 98, 1, 103, 98, 103, 99, 98, 114, 99, + /* 500 */ 114, 98, 104, 98, 100, 98, 100, 72, 9, 5, + /* 510 */ 5, 5, 5, 5, 76, 15, 72, 16, 129, 103, + /* 520 */ 5, 5, 99, 98, 5, 5, 5, 5, 5, 5, + /* 530 */ 129, 5, 5, 5, 5, 5, 5, 5, 5, 103, + /* 540 */ 76, 59, 58, 0, 271, 271, 21, 271, 271, 271, + /* 550 */ 271, 271, 271, 271, 271, 21, 271, 271, 271, 271, /* 560 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, /* 570 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, /* 580 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, @@ -336,83 +337,84 @@ static const YYCODETYPE yy_lookahead[] = { /* 720 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, /* 730 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, /* 740 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 750 */ 271, 271, 271, 271, 271, + /* 750 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + /* 760 */ 271, 271, }; -#define YY_SHIFT_COUNT (243) +#define YY_SHIFT_COUNT (245) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (535) +#define YY_SHIFT_MAX (543) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 137, 49, 128, 184, 232, 179, 179, 179, 179, 179, - /* 10 */ 179, 0, 22, 232, 13, 13, 13, 100, 179, 179, - /* 20 */ 179, 179, 179, 189, 4, 4, 549, 194, 232, 232, - /* 30 */ 232, 232, 232, 232, 232, 232, 232, 232, 232, 232, - /* 40 */ 232, 232, 232, 232, 232, 13, 13, 2, 2, 2, - /* 50 */ 2, 2, 2, 11, 2, 149, 179, 179, 179, 179, - /* 60 */ 148, 148, 21, 179, 179, 179, 179, 179, 179, 179, - /* 70 */ 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, - /* 80 */ 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, - /* 90 */ 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, - /* 100 */ 284, 336, 336, 290, 290, 336, 299, 301, 307, 313, - /* 110 */ 310, 316, 318, 320, 308, 284, 336, 336, 357, 357, - /* 120 */ 336, 349, 351, 386, 356, 355, 385, 358, 361, 336, - /* 130 */ 366, 336, 366, 549, 549, 27, 67, 67, 67, 94, - /* 140 */ 145, 218, 218, 218, 207, 246, 246, 246, 246, 222, - /* 150 */ 234, 185, 41, 221, 221, 216, 214, 220, 223, 225, - /* 160 */ 228, 263, 304, 305, 236, 211, 229, 240, 241, 247, - /* 170 */ 208, 219, 347, 348, 285, 437, 309, 438, 440, 312, - /* 180 */ 443, 445, 365, 326, 350, 354, 352, 360, 359, 362, - /* 190 */ 460, 389, 390, 392, 388, 374, 391, 378, 394, 397, - /* 200 */ 398, 364, 399, 400, 426, 490, 496, 497, 498, 499, - /* 210 */ 500, 430, 492, 436, 493, 381, 382, 409, 508, 509, - /* 220 */ 416, 418, 409, 512, 513, 514, 515, 516, 517, 518, - /* 230 */ 519, 520, 521, 522, 523, 524, 525, 428, 456, 526, - /* 240 */ 527, 474, 476, 535, + /* 0 */ 164, 94, 178, 205, 249, 6, 6, 6, 6, 6, + /* 10 */ 6, 0, 68, 249, 283, 283, 283, 113, 6, 6, + /* 20 */ 6, 6, 6, 12, 11, 11, 556, 215, 249, 249, + /* 30 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 40 */ 249, 249, 249, 249, 249, 283, 283, 25, 25, 25, + /* 50 */ 25, 25, 25, 56, 25, 159, 6, 6, 6, 6, + /* 60 */ 121, 121, 202, 6, 6, 6, 6, 6, 6, 6, + /* 70 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + /* 80 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + /* 90 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + /* 100 */ 284, 332, 332, 293, 293, 332, 296, 297, 305, 315, + /* 110 */ 313, 322, 327, 329, 317, 284, 332, 332, 366, 366, + /* 120 */ 332, 358, 360, 395, 365, 364, 394, 367, 372, 332, + /* 130 */ 377, 332, 377, 556, 556, 27, 54, 54, 54, 114, + /* 140 */ 139, 70, 70, 70, 179, 192, 192, 192, 192, 211, + /* 150 */ 234, 177, 160, 62, 62, 199, 181, 120, 219, 220, + /* 160 */ 227, 290, 320, 254, 252, 253, 218, 30, 229, 232, + /* 170 */ 235, 208, 213, 338, 339, 276, 448, 323, 450, 455, + /* 180 */ 326, 457, 458, 378, 341, 359, 381, 382, 385, 391, + /* 190 */ 393, 491, 396, 397, 399, 390, 384, 392, 386, 400, + /* 200 */ 403, 398, 405, 404, 407, 406, 435, 499, 504, 505, + /* 210 */ 506, 507, 508, 438, 500, 444, 501, 389, 401, 416, + /* 220 */ 515, 516, 423, 425, 416, 519, 520, 521, 522, 523, + /* 230 */ 524, 526, 527, 528, 529, 530, 531, 532, 533, 436, + /* 240 */ 464, 525, 534, 482, 484, 543, }; #define YY_REDUCE_COUNT (134) -#define YY_REDUCE_MIN (-241) -#define YY_REDUCE_MAX (273) +#define YY_REDUCE_MIN (-258) +#define YY_REDUCE_MAX (266) static const short yy_reduce_ofst[] = { - /* 0 */ -202, -79, -226, -211, -173, -103, -62, -120, -34, -33, - /* 10 */ -13, -147, -180, -171, -59, -8, 64, -128, 43, -16, - /* 20 */ 30, -208, -160, 12, 80, 90, -161, -241, -215, -203, - /* 30 */ -198, -157, -96, -70, -68, -35, -24, -11, 57, 61, - /* 40 */ 65, 69, 75, 78, 85, 105, 113, 119, 120, 121, - /* 50 */ 122, 123, 124, 125, 127, 126, 156, 159, 161, 162, - /* 60 */ 106, 107, 129, 165, 166, 168, 169, 170, 171, 172, - /* 70 */ 173, 174, 175, 176, 177, 178, 180, 181, 182, 183, - /* 80 */ 186, 187, 188, 190, 191, 192, 193, 195, 196, 197, - /* 90 */ 198, 199, 200, 201, 202, 203, 204, 205, 206, 209, - /* 100 */ 150, 210, 212, 141, 154, 213, 163, 215, 217, 224, - /* 110 */ 226, 230, 233, 227, 231, 237, 235, 238, 239, 242, - /* 120 */ 244, 243, 245, 250, 248, 252, 255, 249, 258, 251, - /* 130 */ 253, 272, 254, 269, 273, + /* 0 */ -202, -34, -226, -238, -224, -98, -181, -116, -100, -99, + /* 10 */ -74, -175, -191, -198, -170, -53, -18, -195, -49, -160, + /* 20 */ 5, -78, 22, 67, 92, 98, -110, -258, -237, -187, + /* 30 */ -155, -85, -39, 60, 73, 91, 93, 95, 96, 99, + /* 40 */ 100, 101, 102, 103, 104, 106, 108, 124, 125, 126, + /* 50 */ 127, 128, 129, 130, 133, 131, 161, 166, 167, 168, + /* 60 */ 112, 115, 132, 171, 173, 174, 175, 176, 180, 182, + /* 70 */ 183, 184, 185, 186, 187, 188, 189, 190, 191, 193, + /* 80 */ 194, 195, 196, 197, 198, 200, 201, 203, 204, 206, + /* 90 */ 207, 209, 210, 212, 214, 216, 217, 221, 222, 223, + /* 100 */ 144, 224, 225, 146, 149, 226, 172, 165, 170, 228, + /* 110 */ 230, 233, 236, 231, 237, 240, 238, 239, 241, 242, + /* 120 */ 243, 244, 246, 245, 247, 250, 251, 255, 256, 248, + /* 130 */ 261, 264, 265, 262, 266, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 628, 680, 669, 845, 845, 628, 628, 628, 628, 628, - /* 10 */ 628, 773, 646, 845, 628, 628, 628, 628, 628, 628, - /* 20 */ 628, 628, 628, 682, 682, 682, 768, 628, 628, 628, - /* 30 */ 628, 628, 628, 628, 628, 628, 628, 628, 628, 628, - /* 40 */ 628, 628, 628, 628, 628, 628, 628, 628, 628, 628, - /* 50 */ 628, 628, 628, 628, 628, 628, 628, 775, 777, 628, - /* 60 */ 794, 794, 766, 628, 628, 628, 628, 628, 628, 628, - /* 70 */ 628, 628, 628, 628, 628, 628, 628, 628, 628, 628, - /* 80 */ 628, 667, 628, 665, 628, 628, 628, 628, 628, 628, - /* 90 */ 628, 628, 628, 628, 628, 628, 628, 654, 628, 628, - /* 100 */ 628, 648, 648, 628, 628, 648, 801, 805, 799, 787, - /* 110 */ 795, 786, 782, 781, 809, 628, 648, 648, 677, 677, - /* 120 */ 648, 698, 696, 694, 686, 692, 688, 690, 684, 648, - /* 130 */ 675, 648, 675, 714, 728, 628, 810, 844, 800, 828, - /* 140 */ 827, 840, 834, 833, 628, 832, 831, 830, 829, 628, - /* 150 */ 628, 628, 628, 836, 835, 628, 628, 628, 628, 628, - /* 160 */ 628, 628, 628, 628, 812, 806, 802, 628, 628, 628, - /* 170 */ 628, 628, 628, 628, 628, 628, 628, 628, 628, 628, - /* 180 */ 628, 628, 628, 628, 765, 628, 628, 774, 628, 628, - /* 190 */ 628, 628, 628, 628, 796, 628, 788, 628, 628, 628, - /* 200 */ 628, 628, 628, 742, 628, 628, 628, 628, 628, 628, - /* 210 */ 628, 628, 628, 628, 628, 628, 628, 849, 628, 628, - /* 220 */ 628, 736, 847, 628, 628, 628, 628, 628, 628, 628, - /* 230 */ 628, 628, 628, 628, 628, 628, 628, 701, 628, 652, - /* 240 */ 650, 628, 644, 628, + /* 0 */ 634, 686, 675, 853, 853, 634, 634, 634, 634, 634, + /* 10 */ 634, 779, 652, 853, 634, 634, 634, 634, 634, 634, + /* 20 */ 634, 634, 634, 688, 688, 688, 774, 634, 634, 634, + /* 30 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, + /* 40 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, + /* 50 */ 634, 634, 634, 634, 634, 634, 634, 781, 783, 634, + /* 60 */ 800, 800, 772, 634, 634, 634, 634, 634, 634, 634, + /* 70 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, + /* 80 */ 634, 673, 634, 671, 634, 634, 634, 634, 634, 634, + /* 90 */ 634, 634, 634, 634, 634, 634, 634, 660, 634, 634, + /* 100 */ 634, 654, 654, 634, 634, 654, 807, 811, 805, 793, + /* 110 */ 801, 792, 788, 787, 815, 634, 654, 654, 683, 683, + /* 120 */ 654, 704, 702, 700, 692, 698, 694, 696, 690, 654, + /* 130 */ 681, 654, 681, 720, 734, 634, 816, 852, 806, 842, + /* 140 */ 841, 848, 840, 839, 634, 835, 836, 838, 837, 634, + /* 150 */ 634, 634, 634, 844, 843, 634, 634, 634, 634, 634, + /* 160 */ 634, 634, 634, 634, 818, 634, 812, 808, 634, 634, + /* 170 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, + /* 180 */ 634, 634, 634, 634, 634, 771, 634, 634, 780, 634, + /* 190 */ 634, 634, 634, 634, 634, 802, 634, 794, 634, 634, + /* 200 */ 634, 634, 634, 634, 634, 748, 634, 634, 634, 634, + /* 210 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 857, + /* 220 */ 634, 634, 634, 742, 855, 634, 634, 634, 634, 634, + /* 230 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 707, + /* 240 */ 634, 658, 656, 634, 650, 634, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1198,35 +1200,37 @@ static const char *const yyRuleName[] = { /* 193 */ "expr ::= BOOL", /* 194 */ "expr ::= ID LP exprlist RP", /* 195 */ "expr ::= ID LP STAR RP", - /* 196 */ "expr ::= expr AND expr", - /* 197 */ "expr ::= expr OR expr", + /* 196 */ "expr ::= expr IS NULL", + /* 197 */ "expr ::= expr IS NOT NULL", /* 198 */ "expr ::= expr LT expr", /* 199 */ "expr ::= expr GT expr", /* 200 */ "expr ::= expr LE expr", /* 201 */ "expr ::= expr GE expr", /* 202 */ "expr ::= expr NE expr", /* 203 */ "expr ::= expr EQ expr", - /* 204 */ "expr ::= expr PLUS expr", - /* 205 */ "expr ::= expr MINUS expr", - /* 206 */ "expr ::= expr STAR expr", - /* 207 */ "expr ::= expr SLASH expr", - /* 208 */ "expr ::= expr REM expr", - /* 209 */ "expr ::= expr LIKE expr", - /* 210 */ "expr ::= expr IN LP exprlist RP", - /* 211 */ "exprlist ::= exprlist COMMA expritem", - /* 212 */ "exprlist ::= expritem", - /* 213 */ "expritem ::= expr", - /* 214 */ "expritem ::=", - /* 215 */ "cmd ::= RESET QUERY CACHE", - /* 216 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 217 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 218 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 219 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 220 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 221 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 222 */ "cmd ::= KILL CONNECTION INTEGER", - /* 223 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 224 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 204 */ "expr ::= expr AND expr", + /* 205 */ "expr ::= expr OR expr", + /* 206 */ "expr ::= expr PLUS expr", + /* 207 */ "expr ::= expr MINUS expr", + /* 208 */ "expr ::= expr STAR expr", + /* 209 */ "expr ::= expr SLASH expr", + /* 210 */ "expr ::= expr REM expr", + /* 211 */ "expr ::= expr LIKE expr", + /* 212 */ "expr ::= expr IN LP exprlist RP", + /* 213 */ "exprlist ::= exprlist COMMA expritem", + /* 214 */ "exprlist ::= expritem", + /* 215 */ "expritem ::= expr", + /* 216 */ "expritem ::=", + /* 217 */ "cmd ::= RESET QUERY CACHE", + /* 218 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 219 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 220 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 221 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 222 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 223 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 224 */ "cmd ::= KILL CONNECTION INTEGER", + /* 225 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 226 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1880,35 +1884,37 @@ static const struct { { 260, -1 }, /* (193) expr ::= BOOL */ { 260, -4 }, /* (194) expr ::= ID LP exprlist RP */ { 260, -4 }, /* (195) expr ::= ID LP STAR RP */ - { 260, -3 }, /* (196) expr ::= expr AND expr */ - { 260, -3 }, /* (197) expr ::= expr OR expr */ + { 260, -3 }, /* (196) expr ::= expr IS NULL */ + { 260, -4 }, /* (197) expr ::= expr IS NOT NULL */ { 260, -3 }, /* (198) expr ::= expr LT expr */ { 260, -3 }, /* (199) expr ::= expr GT expr */ { 260, -3 }, /* (200) expr ::= expr LE expr */ { 260, -3 }, /* (201) expr ::= expr GE expr */ { 260, -3 }, /* (202) expr ::= expr NE expr */ { 260, -3 }, /* (203) expr ::= expr EQ expr */ - { 260, -3 }, /* (204) expr ::= expr PLUS expr */ - { 260, -3 }, /* (205) expr ::= expr MINUS expr */ - { 260, -3 }, /* (206) expr ::= expr STAR expr */ - { 260, -3 }, /* (207) expr ::= expr SLASH expr */ - { 260, -3 }, /* (208) expr ::= expr REM expr */ - { 260, -3 }, /* (209) expr ::= expr LIKE expr */ - { 260, -5 }, /* (210) expr ::= expr IN LP exprlist RP */ - { 269, -3 }, /* (211) exprlist ::= exprlist COMMA expritem */ - { 269, -1 }, /* (212) exprlist ::= expritem */ - { 270, -1 }, /* (213) expritem ::= expr */ - { 270, 0 }, /* (214) expritem ::= */ - { 208, -3 }, /* (215) cmd ::= RESET QUERY CACHE */ - { 208, -7 }, /* (216) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 208, -7 }, /* (217) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 208, -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 208, -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 208, -8 }, /* (220) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 208, -9 }, /* (221) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 208, -3 }, /* (222) cmd ::= KILL CONNECTION INTEGER */ - { 208, -5 }, /* (223) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 208, -5 }, /* (224) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 260, -3 }, /* (204) expr ::= expr AND expr */ + { 260, -3 }, /* (205) expr ::= expr OR expr */ + { 260, -3 }, /* (206) expr ::= expr PLUS expr */ + { 260, -3 }, /* (207) expr ::= expr MINUS expr */ + { 260, -3 }, /* (208) expr ::= expr STAR expr */ + { 260, -3 }, /* (209) expr ::= expr SLASH expr */ + { 260, -3 }, /* (210) expr ::= expr REM expr */ + { 260, -3 }, /* (211) expr ::= expr LIKE expr */ + { 260, -5 }, /* (212) expr ::= expr IN LP exprlist RP */ + { 269, -3 }, /* (213) exprlist ::= exprlist COMMA expritem */ + { 269, -1 }, /* (214) exprlist ::= expritem */ + { 270, -1 }, /* (215) expritem ::= expr */ + { 270, 0 }, /* (216) expritem ::= */ + { 208, -3 }, /* (217) cmd ::= RESET QUERY CACHE */ + { 208, -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 208, -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 208, -7 }, /* (220) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 208, -7 }, /* (221) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 208, -8 }, /* (222) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 208, -9 }, /* (223) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 208, -3 }, /* (224) cmd ::= KILL CONNECTION INTEGER */ + { 208, -5 }, /* (225) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 208, -5 }, /* (226) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2570,7 +2576,7 @@ static void yy_reduce( break; case 168: /* having_opt ::= */ case 178: /* where_opt ::= */ yytestcase(yyruleno==178); - case 214: /* expritem ::= */ yytestcase(yyruleno==214); + case 216: /* expritem ::= */ yytestcase(yyruleno==216); {yymsp[1].minor.yy66 = 0;} break; case 169: /* having_opt ::= HAVING expr */ @@ -2643,24 +2649,20 @@ static void yy_reduce( yymsp[0].minor.yy66 = yylhsminor.yy66; break; case 194: /* expr ::= ID LP exprlist RP */ -{ - yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); -} +{ yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy66 = yylhsminor.yy66; break; case 195: /* expr ::= ID LP STAR RP */ -{ - yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); -} +{ yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy66 = yylhsminor.yy66; break; - case 196: /* expr ::= expr AND expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);} + case 196: /* expr ::= expr IS NULL */ +{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, NULL, TK_ISNULL);} yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 197: /* expr ::= expr OR expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 197: /* expr ::= expr IS NOT NULL */ +{yylhsminor.yy66 = tSQLExprCreate(yymsp[-3].minor.yy66, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy66 = yylhsminor.yy66; break; case 198: /* expr ::= expr LT expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LT);} @@ -2686,57 +2688,65 @@ static void yy_reduce( {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_EQ);} yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 204: /* expr ::= expr PLUS expr */ + case 204: /* expr ::= expr AND expr */ +{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);} + yymsp[-2].minor.yy66 = yylhsminor.yy66; + break; + case 205: /* expr ::= expr OR expr */ +{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); } + yymsp[-2].minor.yy66 = yylhsminor.yy66; + break; + case 206: /* expr ::= expr PLUS expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_PLUS); } yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 205: /* expr ::= expr MINUS expr */ + case 207: /* expr ::= expr MINUS expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_MINUS); } yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 206: /* expr ::= expr STAR expr */ + case 208: /* expr ::= expr STAR expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_STAR); } yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 207: /* expr ::= expr SLASH expr */ + case 209: /* expr ::= expr SLASH expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_DIVIDE);} yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 208: /* expr ::= expr REM expr */ + case 210: /* expr ::= expr REM expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_REM); } yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 209: /* expr ::= expr LIKE expr */ + case 211: /* expr ::= expr LIKE expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LIKE); } yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 210: /* expr ::= expr IN LP exprlist RP */ + case 212: /* expr ::= expr IN LP exprlist RP */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-4].minor.yy66, (tSQLExpr*)yymsp[-1].minor.yy224, TK_IN); } yymsp[-4].minor.yy66 = yylhsminor.yy66; break; - case 211: /* exprlist ::= exprlist COMMA expritem */ + case 213: /* exprlist ::= exprlist COMMA expritem */ {yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224,yymsp[0].minor.yy66,0);} yymsp[-2].minor.yy224 = yylhsminor.yy224; break; - case 212: /* exprlist ::= expritem */ + case 214: /* exprlist ::= expritem */ {yylhsminor.yy224 = tSQLExprListAppend(0,yymsp[0].minor.yy66,0);} yymsp[0].minor.yy224 = yylhsminor.yy224; break; - case 213: /* expritem ::= expr */ + case 215: /* expritem ::= expr */ {yylhsminor.yy66 = yymsp[0].minor.yy66;} yymsp[0].minor.yy66 = yylhsminor.yy66; break; - case 215: /* cmd ::= RESET QUERY CACHE */ + case 217: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 216: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 218: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 217: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 219: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2747,14 +2757,14 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 218: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 220: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 219: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 221: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2765,7 +2775,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 220: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 222: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2779,7 +2789,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 221: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 223: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; @@ -2791,13 +2801,13 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 222: /* cmd ::= KILL CONNECTION INTEGER */ + case 224: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 223: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 225: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 224: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 226: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d829a85754..5fece58ef7 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2071,13 +2071,17 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { if (keyInfo.pTable != NULL) { totalNumOfTable++; taosArrayPush(pGroup, &keyInfo); + } else { + taosArrayRemove(groupList->pGroupList, j); + numOfGroups -= 1; + j -= 1; } } // window does not being updated, so set the original if (window.skey == INT64_MAX && window.ekey == INT64_MIN) { window = TSWINDOW_INITIALIZER; - assert(totalNumOfTable == 0); + assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == 0); } groupList->numOfTables = totalNumOfTable; @@ -2398,6 +2402,14 @@ static bool indexedNodeFilterFp(const void* pNode, void* param) { val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId); } + if (pInfo->optr == TSDB_RELATION_ISNULL || pInfo->optr == TSDB_RELATION_NOTNULL) { + if (pInfo->optr == TSDB_RELATION_ISNULL) { + return (val == NULL) || isNull(val, pInfo->sch.type); + } else if (pInfo->optr == TSDB_RELATION_NOTNULL) { + return (val != NULL) && (!isNull(val, pInfo->sch.type)); + } + } + int32_t ret = 0; if (val == NULL) { //the val is possible to be null, so check it out carefully ret = -1; // val is missing in table tags value pairs From 25b2cec2f5aee6db59745c96c58812c2c96e6f7c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Sep 2020 13:28:08 +0800 Subject: [PATCH 012/105] [td-1369] --- src/inc/taosdef.h | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 1a40f3b56d..7437eceecb 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -209,21 +209,24 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_RELATION_GREATER_EQUAL 5 #define TSDB_RELATION_NOT_EQUAL 6 #define TSDB_RELATION_LIKE 7 -#define TSDB_RELATION_IN 8 +#define TSDB_RELATION_ISNULL 8 +#define TSDB_RELATION_NOTNULL 9 +#define TSDB_RELATION_IN 10 -#define TSDB_RELATION_AND 9 -#define TSDB_RELATION_OR 10 -#define TSDB_RELATION_NOT 11 +#define TSDB_RELATION_AND 11 +#define TSDB_RELATION_OR 12 +#define TSDB_RELATION_NOT 13 -#define TSDB_BINARY_OP_ADD 12 -#define TSDB_BINARY_OP_SUBTRACT 13 -#define TSDB_BINARY_OP_MULTIPLY 14 -#define TSDB_BINARY_OP_DIVIDE 15 -#define TSDB_BINARY_OP_REMAINDER 16 +#define TSDB_BINARY_OP_ADD 30 +#define TSDB_BINARY_OP_SUBTRACT 31 +#define TSDB_BINARY_OP_MULTIPLY 32 +#define TSDB_BINARY_OP_DIVIDE 33 +#define TSDB_BINARY_OP_REMAINDER 34 #define TS_PATH_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 #define TSDB_USER_LEN TSDB_UNI_LEN + // ACCOUNT is a 32 bit positive integer // this is the length of its string representation // including the terminator zero From 5b6ca20f0bffd9a6e34c7af177e8aad77d2af7dd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Sep 2020 18:15:59 +0800 Subject: [PATCH 013/105] [td-225] refactor codes. --- src/client/inc/tscUtil.h | 7 +- src/client/src/tscSubquery.c | 13 +- src/client/src/tscUtil.c | 3 +- src/query/src/qExecutor.c | 2 +- tests/script/general/parser/join.sim | 2 + tests/script/general/parser/testSuite.sim | 168 +++++++++++----------- 6 files changed, 91 insertions(+), 104 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 9b31b8fc6a..748b19069b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -69,17 +69,12 @@ typedef struct SJoinSupporter { SSubqueryState* pState; SSqlObj* pObj; // parent SqlObj int32_t subqueryIndex; // index of sub query - char intervalTimeUnit; - char slidingTimeUnit; - int64_t intervalTime; // interval time - int64_t slidingTime; // sliding time SLimitVal limit; // limit info - uint64_t uid; // query meter uid + uint64_t uid; // query table uid SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution SArray* exprList; SFieldInfo fieldsInfo; STagCond tagCond; - SSqlGroupbyExpr groupbyExpr; struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array FILE* f; // temporary file in order to create TSBuf char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e264fa9b33..c4b07f7813 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -178,10 +178,6 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in pSupporter->subqueryIndex = index; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); - pSupporter->intervalTimeUnit = pQueryInfo->intervalTimeUnit; - pSupporter->slidingTime = pQueryInfo->slidingTimeUnit; - pSupporter->intervalTime = pQueryInfo->intervalTime; - pSupporter->slidingTime = pQueryInfo->slidingTime; pSupporter->limit = pQueryInfo->limit; STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index); @@ -311,18 +307,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { // set the second stage sub query for join process TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE); - pQueryInfo->intervalTimeUnit = pSupporter->intervalTimeUnit; - pQueryInfo->slidingTimeUnit = pSupporter->slidingTimeUnit; - pQueryInfo->intervalTime = pSupporter->intervalTime; - pQueryInfo->slidingTime = pSupporter->slidingTime; - pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; - tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); pQueryInfo->colList = pSupporter->colList; pQueryInfo->exprList = pSupporter->exprList; pQueryInfo->fieldsInfo = pSupporter->fieldsInfo; - + pSupporter->exprList = NULL; pSupporter->colList = NULL; memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo)); @@ -1221,7 +1211,6 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter pNewQueryInfo->limit.offset = 0; // backup the data and clear it in the sqlcmd object - pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr; memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); tscInitQueryInfo(pNewQueryInfo); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ad87825e34..5d50d7791a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1534,7 +1534,6 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; - pQueryInfo->window = TSWINDOW_INITIALIZER; } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { @@ -1555,6 +1554,8 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { } tscInitQueryInfo(pQueryInfo); + + pQueryInfo->window = TSWINDOW_INITIALIZER; pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 2ca2545818..fc932b8999 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1177,7 +1177,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { #if defined(_DEBUG_VIEW) printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, + elem.ts, key, elem.tag.i64Key, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex); #endif diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index ef3245ccaf..254571bda1 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -205,10 +205,12 @@ if $rows != 9 then endi if $data00 != @70-01-01 08:01:40.100@ then + print $data00 return -1 endi if $data10 != @70-01-01 08:01:40.200@ then + print $data10 return -1 endi diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 6790564cc7..721eb0bfa1 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -1,87 +1,87 @@ -sleep 2000 -run general/parser/alter.sim -sleep 2000 -run general/parser/alter1.sim -sleep 2000 -run general/parser/alter_stable.sim -sleep 2000 -run general/parser/auto_create_tb.sim -sleep 2000 -run general/parser/auto_create_tb_drop_tb.sim -sleep 2000 -run general/parser/col_arithmetic_operation.sim -sleep 2000 -run general/parser/columnValue.sim -sleep 2000 -run general/parser/commit.sim -sleep 2000 -run general/parser/create_db.sim -sleep 2000 -run general/parser/create_mt.sim -sleep 2000 -run general/parser/create_tb.sim -sleep 2000 -run general/parser/dbtbnameValidate.sim -sleep 2000 -run general/parser/fill.sim -sleep 2000 -run general/parser/fill_stb.sim -sleep 2000 -#run general/parser/fill_us.sim # -sleep 2000 -run general/parser/first_last.sim -sleep 2000 -run general/parser/import_commit1.sim -sleep 2000 -run general/parser/import_commit2.sim -sleep 2000 -run general/parser/import_commit3.sim -sleep 2000 -#run general/parser/import_file.sim -sleep 2000 -run general/parser/insert_tb.sim -sleep 2000 -run general/parser/tags_dynamically_specifiy.sim -sleep 2000 -run general/parser/interp.sim -sleep 2000 -run general/parser/lastrow.sim -sleep 2000 -run general/parser/limit.sim -sleep 2000 -run general/parser/limit1.sim -sleep 2000 -run general/parser/limit1_tblocks100.sim -sleep 2000 -run general/parser/limit2.sim -sleep 2000 -run general/parser/mixed_blocks.sim -sleep 2000 -run general/parser/nchar.sim -sleep 2000 -run general/parser/null_char.sim -sleep 2000 -run general/parser/selectResNum.sim -sleep 2000 -run general/parser/select_across_vnodes.sim -sleep 2000 -run general/parser/select_from_cache_disk.sim -sleep 2000 -run general/parser/set_tag_vals.sim -sleep 2000 -run general/parser/single_row_in_tb.sim -sleep 2000 -run general/parser/slimit.sim -sleep 2000 -run general/parser/slimit1.sim -sleep 2000 -run general/parser/slimit_alter_tags.sim -sleep 2000 -run general/parser/tbnameIn.sim -sleep 2000 -run general/parser/slimit_alter_tags.sim # persistent failed -sleep 2000 -run general/parser/join.sim +#sleep 2000 +#run general/parser/alter.sim +#sleep 2000 +#run general/parser/alter1.sim +#sleep 2000 +#run general/parser/alter_stable.sim +#sleep 2000 +#run general/parser/auto_create_tb.sim +#sleep 2000 +#run general/parser/auto_create_tb_drop_tb.sim +#sleep 2000 +#run general/parser/col_arithmetic_operation.sim +#sleep 2000 +#run general/parser/columnValue.sim +#sleep 2000 +#run general/parser/commit.sim +#sleep 2000 +#run general/parser/create_db.sim +#sleep 2000 +#run general/parser/create_mt.sim +#sleep 2000 +#run general/parser/create_tb.sim +#sleep 2000 +#run general/parser/dbtbnameValidate.sim +#sleep 2000 +#run general/parser/fill.sim +#sleep 2000 +#run general/parser/fill_stb.sim +#sleep 2000 +##run general/parser/fill_us.sim # +#sleep 2000 +#run general/parser/first_last.sim +#sleep 2000 +#run general/parser/import_commit1.sim +#sleep 2000 +#run general/parser/import_commit2.sim +#sleep 2000 +#run general/parser/import_commit3.sim +#sleep 2000 +##run general/parser/import_file.sim +#sleep 2000 +#run general/parser/insert_tb.sim +#sleep 2000 +#run general/parser/tags_dynamically_specifiy.sim +#sleep 2000 +#run general/parser/interp.sim +#sleep 2000 +#run general/parser/lastrow.sim +#sleep 2000 +#run general/parser/limit.sim +#sleep 2000 +#run general/parser/limit1.sim +#sleep 2000 +#run general/parser/limit1_tblocks100.sim +#sleep 2000 +#run general/parser/limit2.sim +#sleep 2000 +#run general/parser/mixed_blocks.sim +#sleep 2000 +#run general/parser/nchar.sim +#sleep 2000 +#run general/parser/null_char.sim +#sleep 2000 +#run general/parser/selectResNum.sim +#sleep 2000 +#run general/parser/select_across_vnodes.sim +#sleep 2000 +#run general/parser/select_from_cache_disk.sim +#sleep 2000 +#run general/parser/set_tag_vals.sim +#sleep 2000 +#run general/parser/single_row_in_tb.sim +#sleep 2000 +#run general/parser/slimit.sim +#sleep 2000 +#run general/parser/slimit1.sim +#sleep 2000 +#run general/parser/slimit_alter_tags.sim +#sleep 2000 +#run general/parser/tbnameIn.sim +#sleep 2000 +#run general/parser/slimit_alter_tags.sim # persistent failed +#sleep 2000 +#run general/parser/join.sim sleep 2000 run general/parser/join_multivnode.sim sleep 2000 From 9768c30be1f946830361873a091af1bf71993ff1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Sep 2020 22:15:12 +0800 Subject: [PATCH 014/105] [td-1372] --- src/client/src/tscFunctionImpl.c | 138 ++++++++++++++++++++++++++++--- src/query/inc/qPercentile.h | 4 +- src/query/src/qPercentile.c | 89 +++++++++++++++++--- 3 files changed, 206 insertions(+), 25 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 4b31a8001f..1356b98c67 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -117,6 +117,10 @@ typedef struct SFirstLastInfo { typedef struct SFirstLastInfo SLastrowInfo; typedef struct SPercentileInfo { tMemBucket *pMemBucket; + int32_t stage; + double minval; + double maxval; + int64_t numOfElems; } SPercentileInfo; typedef struct STopBotInfo { @@ -302,7 +306,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_PERCT) { *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; *bytes = (int16_t)sizeof(double); - *interBytes = (int16_t)sizeof(double); + *interBytes = (int16_t)sizeof(SPercentileInfo); } else if (functionId == TSDB_FUNC_LEASTSQR) { *type = TSDB_DATA_TYPE_BINARY; *bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE; // string @@ -2428,12 +2432,14 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - ((SPercentileInfo *)(pResInfo->interResultBuf))->pMemBucket = - tMemBucketCreate(pCtx->inputBytes, pCtx->inputType); - + // in the first round, get the min-max value of all involved data + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = pResInfo->interResultBuf; + pInfo->minval = DBL_MAX; + pInfo->maxval = -DBL_MAX; + pInfo->numOfElems = 0; + return true; } @@ -2442,7 +2448,65 @@ static void percentile_function(SQLFunctionCtx *pCtx) { SResultInfo * pResInfo = GET_RES_INFO(pCtx); SPercentileInfo *pInfo = pResInfo->interResultBuf; - + + // the first stage, only acquire the min/max value + if (pInfo->stage == 0) { + if (pCtx->preAggVals.isSet) { + if (pInfo->minval > pCtx->preAggVals.statis.min) { + pInfo->minval = pCtx->preAggVals.statis.min; + } + + if (pInfo->maxval < pCtx->preAggVals.statis.max) { + pInfo->maxval = pCtx->preAggVals.statis.max; + } + + pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); + } else { + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; + } + + // TODO extract functions + double v = 0; + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: + v = GET_INT8_VAL(data); + break; + case TSDB_DATA_TYPE_SMALLINT: + v = GET_INT16_VAL(data); + break; + case TSDB_DATA_TYPE_BIGINT: + v = (double)(GET_INT64_VAL(data)); + break; + case TSDB_DATA_TYPE_FLOAT: + v = GET_FLOAT_VAL(data); + break; + case TSDB_DATA_TYPE_DOUBLE: + v = GET_DOUBLE_VAL(data); + break; + default: + v = GET_INT32_VAL(data); + break; + } + + if (v < pInfo->minval) { + pInfo->minval = v; + } + + if (v > pInfo->maxval) { + pInfo->maxval = v; + } + + pInfo->numOfElems += 1; + } + } + + return; + } + + // the second stage, calculate the true percentile value for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { @@ -2462,10 +2526,47 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf; + + if (pInfo->stage == 0) { + // TODO extract functions + double v = 0; + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: + v = GET_INT8_VAL(pData); + break; + case TSDB_DATA_TYPE_SMALLINT: + v = GET_INT16_VAL(pData); + break; + case TSDB_DATA_TYPE_BIGINT: + v = (double)(GET_INT64_VAL(pData)); + break; + case TSDB_DATA_TYPE_FLOAT: + v = GET_FLOAT_VAL(pData); + break; + case TSDB_DATA_TYPE_DOUBLE: + v = GET_DOUBLE_VAL(pData); + break; + default: + v = GET_INT32_VAL(pData); + break; + } + + if (v < pInfo->minval) { + pInfo->minval = v; + } + + if (v > pInfo->maxval) { + pInfo->maxval = v; + } + + pInfo->numOfElems += 1; + return; + } + tMemBucketPut(pInfo->pMemBucket, pData, 1); SET_VAL(pCtx, 1, 1); @@ -2488,6 +2589,23 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } +static void percentile_next_step(SQLFunctionCtx *pCtx) { + SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = pResInfo->interResultBuf; + + if (pInfo->stage == 0) { + // all data are null, set it completed + if (pInfo->numOfElems == 0) { + pResInfo->complete = true; + } + + pInfo->stage += 1; + pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); + } else { + pResInfo->complete = true; + } +} + ////////////////////////////////////////////////////////////////////////////////// static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); @@ -4513,7 +4631,7 @@ SQLAggFuncElem aAggs[] = {{ percentile_function_setup, percentile_function, percentile_function_f, - no_next_step, + percentile_next_step, percentile_finalizer, noop1, noop1, diff --git a/src/query/inc/qPercentile.h b/src/query/inc/qPercentile.h index 0a52d4f205..c34c24c5b2 100644 --- a/src/query/inc/qPercentile.h +++ b/src/query/inc/qPercentile.h @@ -64,11 +64,11 @@ typedef struct tMemBucket { __perc_hash_func_t hashFunc; } tMemBucket; -tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType); +tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval); void tMemBucketDestroy(tMemBucket *pBucket); -void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); +int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); double getPercentile(tMemBucket *pMemBucket, double percent); diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index 1ce5861e52..ccc451108e 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -70,6 +70,33 @@ static void resetBoundingBox(MinMaxEntry* range, int32_t type) { } } +static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, double maxval) { + if (minval > maxval) { + return -1; + } + + switch(type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + range->iMinVal = minval; + range->iMaxVal = maxval; + break; + + case TSDB_DATA_TYPE_BIGINT: + range->i64MinVal = minval; + range->i64MaxVal = maxval; + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + range->dMinVal = minval; + range->dMaxVal = maxval; + break; + } + + return 0; +} + static void resetPosInfo(SSlotInfo* pInfo) { pInfo->size = 0; pInfo->pageId = -1; @@ -135,6 +162,11 @@ int32_t tBucketBigIntHash(tMemBucket *pBucket, const void *value) { return index; } else { + // out of range + if (v < pBucket->range.i64MinVal || v > pBucket->range.i64MaxVal) { + return -1; + } + // todo hash for bigint and float and double int64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal; if (span < pBucket->numOfSlots) { @@ -179,6 +211,11 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { return index; } else { + // out of range + if (v < pBucket->range.iMinVal || v > pBucket->range.iMaxVal) { + return -1; + } + // divide a range of [iMinVal, iMaxVal] into 1024 buckets int32_t span = pBucket->range.iMaxVal - pBucket->range.iMinVal; if (span < pBucket->numOfSlots) { @@ -209,6 +246,12 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { double posx = (v + DBL_MAX) / x; return ((int32_t)posx) % pBucket->numOfSlots; } else { + + // out of range + if (v < pBucket->range.dMinVal || v > pBucket->range.dMaxVal) { + return -1; + } + // divide a range of [dMinVal, dMaxVal] into 1024 buckets double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; if (span < pBucket->numOfSlots) { @@ -262,7 +305,7 @@ static void resetSlotInfo(tMemBucket* pBucket) { } } -tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType) { +tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) { tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket)); if (pBucket == NULL) { return NULL; @@ -278,9 +321,14 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType) { pBucket->maxCapacity = 200000; + if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { + uError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); + free(pBucket); + return NULL; + } + pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(tFilePage))/pBucket->bytes; pBucket->comparFn = getKeyComparFunc(pBucket->type); - resetBoundingBox(&pBucket->range, pBucket->type); pBucket->hashFunc = getHashFunc(pBucket->type); if (pBucket->hashFunc == NULL) { @@ -395,23 +443,25 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, char *data, int32_t dataType) { /* * in memory bucket, we only accept data array list */ -void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { +int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { assert(pBucket != NULL && data != NULL && size > 0); + pBucket->total += (int32_t)size; int32_t bytes = pBucket->bytes; - for (int32_t i = 0; i < size; ++i) { char *d = (char *) data + i * bytes; - int32_t slotIdx = (pBucket->hashFunc)(pBucket, d); - assert(slotIdx >= 0); + int32_t index = (pBucket->hashFunc)(pBucket, d); + if (index == -1) { // the value is out of range, do not add it into bucket + return -1; + } - tMemBucketSlot *pSlot = &pBucket->pSlots[slotIdx]; + tMemBucketSlot *pSlot = &pBucket->pSlots[index]; tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); // ensure available memory pages to allocate - int32_t groupId = getGroupId(pBucket->numOfSlots, slotIdx, pBucket->times); + int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times); int32_t pageId = -1; if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) { @@ -432,10 +482,12 @@ void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { pSlot->info.data->num += 1; pSlot->info.size += 1; } + + return 0; } //////////////////////////////////////////////////////////////////////////////////////////// -static void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minVal) { +static UNUSED_FUNC void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minVal) { *minVal = DBL_MAX; *maxVal = -DBL_MAX; @@ -681,16 +733,27 @@ double getPercentile(tMemBucket *pMemBucket, double percent) { // find the min/max value, no need to scan all data in bucket if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) { - double minx = 0, maxx = 0; - findMaxMinValue(pMemBucket, &maxx, &minx); + MinMaxEntry* pRange = &pMemBucket->range; - return fabs(percent - 100) < DBL_EPSILON ? maxx : minx; + switch(pMemBucket->type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + return fabs(percent - 100) < DBL_EPSILON? pRange->iMaxVal:pRange->iMinVal; + case TSDB_DATA_TYPE_BIGINT: + return fabs(percent - 100) < DBL_EPSILON? pRange->i64MaxVal:pRange->i64MinVal; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal; + default: + return -1; + } } double percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0); - int32_t orderIdx = (int32_t)percentVal; // do put data by using buckets + int32_t orderIdx = (int32_t)percentVal; return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx); } From 82714623852c0163f0f9c83d8b3aaa779a661a03 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Sep 2020 13:05:59 +0800 Subject: [PATCH 015/105] [td-1369] --- src/query/inc/qUtil.h | 6 +- src/query/src/qExecutor.c | 17 +++ src/query/src/qFilterfunc.c | 97 +++--------- tests/script/general/parser/testSuite.sim | 171 +++++++++++----------- 4 files changed, 128 insertions(+), 163 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 6de3c7c0e5..314159484d 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -50,14 +50,16 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3 tFilePage* page) { assert(pResult != NULL && pRuntimeEnv != NULL); - SQuery *pQuery = pRuntimeEnv->pQuery; -// tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + SQuery *pQuery = pRuntimeEnv->pQuery; int32_t realRowId = (int32_t)(pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + pQuery->pSelectExpr[columnIndex].bytes * realRowId; } +bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval); +bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval); + __filter_func_t *getRangeFilterFuncArray(int32_t type); __filter_func_t *getValueFilterFuncArray(int32_t type); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fc932b8999..d00e616841 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -205,6 +205,23 @@ bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j]; + bool isnull = isNull(pElem, pFilterInfo->info.type); + if (isnull) { + if (pFilterElem->fp == isNull_filter) { + qualified = true; + break; + } else { + continue; + } + } else { + if (pFilterElem->fp == notNull_filter) { + qualified = true; + break; + } else if (pFilterElem->fp == isNull_filter) { + continue; + } + } + if (pFilterElem->fp(pFilterElem, pElem, pElem)) { qualified = true; break; diff --git a/src/query/src/qFilterfunc.c b/src/query/src/qFilterfunc.c index 6b88171e71..b6050dddd8 100644 --- a/src/query/src/qFilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -285,69 +285,12 @@ bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0; } //////////////////////////////////////////////////////////////// -bool isNull_i8(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return isNull(minval, TSDB_DATA_TYPE_TINYINT); +bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval) { + return true; } -bool isNull_i16(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return isNull(minval, TSDB_DATA_TYPE_SMALLINT); -} - -bool isNull_i32(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return isNull(minval, TSDB_DATA_TYPE_INT); -} - -bool isNull_i64(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return isNull(minval, TSDB_DATA_TYPE_BIGINT); -} - -bool isNull_ds(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return isNull(minval, TSDB_DATA_TYPE_FLOAT); -} - -bool isNull_dd(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return isNull(minval, TSDB_DATA_TYPE_DOUBLE); -} - -bool isNull_binary(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return isNull(minval, TSDB_DATA_TYPE_BINARY); -} - -bool isNull_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return isNull(minval, TSDB_DATA_TYPE_NCHAR); -} - -//////////////////////////////////////////////////////////////// -bool notNull_i8(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return !isNull(minval, TSDB_DATA_TYPE_TINYINT); -} - -bool notNull_i16(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return !isNull(minval, TSDB_DATA_TYPE_SMALLINT); -} - -bool notNull_i32(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return !isNull(minval, TSDB_DATA_TYPE_INT); -} - -bool notNull_i64(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return !isNull(minval, TSDB_DATA_TYPE_BIGINT); -} - -bool notNull_ds(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return !isNull(minval, TSDB_DATA_TYPE_FLOAT); -} - -bool notNull_dd(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return isNull(minval, TSDB_DATA_TYPE_DOUBLE); -} - -bool notNull_binary(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return !isNull(minval, TSDB_DATA_TYPE_BINARY); -} - -bool notNull_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { - return !isNull(minval, TSDB_DATA_TYPE_NCHAR); +bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval) { + return true; } //////////////////////////////////////////////////////////////// @@ -463,8 +406,8 @@ bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i8, nequal_i8, NULL, - isNull_i8, - notNull_i8, + isNull_filter, + notNull_filter, }; bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -476,8 +419,8 @@ bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i16, nequal_i16, NULL, - isNull_i16, - notNull_i16, + isNull_filter, + notNull_filter, }; bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -489,8 +432,8 @@ bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i32, nequal_i32, NULL, - isNull_i32, - notNull_i32, + isNull_filter, + notNull_filter, }; bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -502,8 +445,8 @@ bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i64, nequal_i64, NULL, - isNull_i64, - notNull_i64, + isNull_filter, + notNull_filter, }; bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -515,8 +458,8 @@ bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_ds, nequal_ds, NULL, - isNull_ds, - notNull_ds, + isNull_filter, + notNull_filter, }; bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -528,8 +471,8 @@ bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_dd, nequal_dd, NULL, - isNull_dd, - notNull_dd, + isNull_filter, + notNull_filter, }; bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = { @@ -541,8 +484,8 @@ bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) NULL, nequal_str, like_str, - isNull_binary, - notNull_binary, + isNull_filter, + notNull_filter, }; bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = { @@ -554,8 +497,8 @@ bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxva NULL, nequal_nchar, like_nchar, - isNull_nchar, - notNull_nchar, + isNull_filter, + notNull_filter, }; bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 721eb0bfa1..f42254981c 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -1,87 +1,87 @@ -#sleep 2000 -#run general/parser/alter.sim -#sleep 2000 -#run general/parser/alter1.sim -#sleep 2000 -#run general/parser/alter_stable.sim -#sleep 2000 -#run general/parser/auto_create_tb.sim -#sleep 2000 -#run general/parser/auto_create_tb_drop_tb.sim -#sleep 2000 -#run general/parser/col_arithmetic_operation.sim -#sleep 2000 -#run general/parser/columnValue.sim -#sleep 2000 -#run general/parser/commit.sim -#sleep 2000 -#run general/parser/create_db.sim -#sleep 2000 -#run general/parser/create_mt.sim -#sleep 2000 -#run general/parser/create_tb.sim -#sleep 2000 -#run general/parser/dbtbnameValidate.sim -#sleep 2000 -#run general/parser/fill.sim -#sleep 2000 -#run general/parser/fill_stb.sim -#sleep 2000 -##run general/parser/fill_us.sim # -#sleep 2000 -#run general/parser/first_last.sim -#sleep 2000 -#run general/parser/import_commit1.sim -#sleep 2000 -#run general/parser/import_commit2.sim -#sleep 2000 -#run general/parser/import_commit3.sim -#sleep 2000 -##run general/parser/import_file.sim -#sleep 2000 -#run general/parser/insert_tb.sim -#sleep 2000 -#run general/parser/tags_dynamically_specifiy.sim -#sleep 2000 -#run general/parser/interp.sim -#sleep 2000 -#run general/parser/lastrow.sim -#sleep 2000 -#run general/parser/limit.sim -#sleep 2000 -#run general/parser/limit1.sim -#sleep 2000 -#run general/parser/limit1_tblocks100.sim -#sleep 2000 -#run general/parser/limit2.sim -#sleep 2000 -#run general/parser/mixed_blocks.sim -#sleep 2000 -#run general/parser/nchar.sim -#sleep 2000 -#run general/parser/null_char.sim -#sleep 2000 -#run general/parser/selectResNum.sim -#sleep 2000 -#run general/parser/select_across_vnodes.sim -#sleep 2000 -#run general/parser/select_from_cache_disk.sim -#sleep 2000 -#run general/parser/set_tag_vals.sim -#sleep 2000 -#run general/parser/single_row_in_tb.sim -#sleep 2000 -#run general/parser/slimit.sim -#sleep 2000 -#run general/parser/slimit1.sim -#sleep 2000 -#run general/parser/slimit_alter_tags.sim -#sleep 2000 -#run general/parser/tbnameIn.sim -#sleep 2000 -#run general/parser/slimit_alter_tags.sim # persistent failed -#sleep 2000 -#run general/parser/join.sim +sleep 2000 +run general/parser/alter.sim +sleep 2000 +run general/parser/alter1.sim +sleep 2000 +run general/parser/alter_stable.sim +sleep 2000 +run general/parser/auto_create_tb.sim +sleep 2000 +run general/parser/auto_create_tb_drop_tb.sim +sleep 2000 +run general/parser/col_arithmetic_operation.sim +sleep 2000 +run general/parser/columnValue.sim +sleep 2000 +run general/parser/commit.sim +sleep 2000 +run general/parser/create_db.sim +sleep 2000 +run general/parser/create_mt.sim +sleep 2000 +run general/parser/create_tb.sim +sleep 2000 +run general/parser/dbtbnameValidate.sim +sleep 2000 +run general/parser/fill.sim +sleep 2000 +run general/parser/fill_stb.sim +sleep 2000 +#run general/parser/fill_us.sim # +sleep 2000 +run general/parser/first_last.sim +sleep 2000 +run general/parser/import_commit1.sim +sleep 2000 +run general/parser/import_commit2.sim +sleep 2000 +run general/parser/import_commit3.sim +sleep 2000 +#run general/parser/import_file.sim +sleep 2000 +run general/parser/insert_tb.sim +sleep 2000 +run general/parser/tags_dynamically_specifiy.sim +sleep 2000 +run general/parser/interp.sim +sleep 2000 +run general/parser/lastrow.sim +sleep 2000 +run general/parser/limit.sim +sleep 2000 +run general/parser/limit1.sim +sleep 2000 +run general/parser/limit1_tblocks100.sim +sleep 2000 +run general/parser/limit2.sim +sleep 2000 +run general/parser/mixed_blocks.sim +sleep 2000 +run general/parser/nchar.sim +sleep 2000 +run general/parser/null_char.sim +sleep 2000 +run general/parser/selectResNum.sim +sleep 2000 +run general/parser/select_across_vnodes.sim +sleep 2000 +run general/parser/select_from_cache_disk.sim +sleep 2000 +run general/parser/set_tag_vals.sim +sleep 2000 +run general/parser/single_row_in_tb.sim +sleep 2000 +run general/parser/slimit.sim +sleep 2000 +run general/parser/slimit1.sim +sleep 2000 +run general/parser/slimit_alter_tags.sim +sleep 2000 +run general/parser/tbnameIn.sim +sleep 2000 +run general/parser/slimit_alter_tags.sim # persistent failed +sleep 2000 +run general/parser/join.sim sleep 2000 run general/parser/join_multivnode.sim sleep 2000 @@ -99,10 +99,13 @@ run general/parser/union.sim sleep 2000 run general/parser/constCol.sim sleep 2000 +run general/parser/where.sim +sleep 2000 run general/parser/timestamp.sim sleep 2000 run general/parser/sliding.sim + #sleep 2000 #run general/parser/repeatStream.sim #sleep 2000 From 1d35b18642017987f46ef859686dae41e76ef7b2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 10 Sep 2020 14:43:49 +0800 Subject: [PATCH 016/105] fix TD-1387 --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 632d299542..bcfa768341 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1314,7 +1314,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa int dcol = 0; // loop iter for SDataCols object while (dcol < pDataCols->numOfCols) { SDataCol *pDataCol = &(pDataCols->cols[dcol]); - if (ccol >= pCompData->numOfCols) { + if (dcol != 0 && ccol >= pCompData->numOfCols) { // Set current column as NULL and forward dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); dcol++; From b60e66bb78dfb97f05db3b7b529175ee0cc2a261 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Sep 2020 14:53:32 +0800 Subject: [PATCH 017/105] [td-1409] remove the dependency of JDBC driver on common-util package. #3459 --- src/connector/jdbc/pom.xml | 5 ----- .../src/main/java/com/taosdata/jdbc/TSDBDriver.java | 12 +++++------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index da0c5b12d4..0fed63fc80 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -48,11 +48,6 @@ - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - junit junit diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 02d642d643..e25bd64c73 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -14,8 +14,6 @@ *****************************************************************************/ package com.taosdata.jdbc; -import org.apache.commons.lang3.StringUtils; - import java.sql.*; import java.util.Properties; import java.util.logging.Logger; @@ -42,9 +40,8 @@ import java.util.logging.Logger; public class TSDBDriver implements java.sql.Driver { @Deprecated - private static final String URL_PREFIX1 = "jdbc:TSDB://"; - - private static final String URL_PREFIX = "jdbc:TAOS://"; + private static final String URL_PREFIX1 = "jdbc:tsdb://"; + private static final String URL_PREFIX = "jdbc:taos://"; /** * Key used to retrieve the database value from the properties instance passed @@ -188,7 +185,7 @@ public class TSDBDriver implements java.sql.Driver { } public boolean acceptsURL(String url) throws SQLException { - return StringUtils.isNotBlank(url) && url.startsWith(URL_PREFIX); + return (url != null && url.length() > 0 && url.trim().length() > 0) && url.toLowerCase().startsWith(URL_PREFIX); } public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { @@ -238,7 +235,8 @@ public class TSDBDriver implements java.sql.Driver { return null; } - if (!StringUtils.startsWithIgnoreCase(url, URL_PREFIX) && !StringUtils.startsWithIgnoreCase(url, URL_PREFIX1)) { + String lowerUrl = url.toLowerCase(); + if (!lowerUrl.startsWith(URL_PREFIX) && !lowerUrl.startsWith(URL_PREFIX1)) { return null; } From a30dd56278fc02055215894ff157f3dcde48a702 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Sep 2020 15:03:59 +0800 Subject: [PATCH 018/105] [td-225] --- src/client/src/tscLocal.c | 1 - tests/script/general/parser/where.sim | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index b240d357a8..4b6174e13d 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -16,7 +16,6 @@ #include "os.h" #include "taosmsg.h" -#include "qExtbuffer.h" #include "taosdef.h" #include "tcache.h" #include "tname.h" diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index dd3b11c2dc..fb15fb6dbe 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -2,6 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start sleep 3000 From d74478425bda4c2266c60374d75702aed95bb471 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Sep 2020 17:16:28 +0800 Subject: [PATCH 019/105] [td-1411] --- src/common/src/ttypes.c | 9 ++-- src/inc/taosdef.h | 2 +- src/query/src/qExtbuffer.c | 93 ++++++++++++++++++++++---------------- 3 files changed, 58 insertions(+), 46 deletions(-) diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 3910d80ecf..ff417b6cde 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -540,9 +540,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { } } -void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size) { - char tmpBuf[4096] = {0}; - +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { switch (type) { case TSDB_DATA_TYPE_INT: { SWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); @@ -575,10 +573,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size) { } default: { - assert(size <= 4096); - memcpy(tmpBuf, pLeft, size); + memcpy(buf, pLeft, size); memcpy(pLeft, pRight, size); - memcpy(pRight, tmpBuf, size); + memcpy(pRight, buf, size); break; } } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 7437eceecb..17cbbca834 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -198,7 +198,7 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); void* getNullValue(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); -void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); // TODO: check if below is necessary #define TSDB_RELATION_INVALID 0 diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index 192a31ebf5..1d3120ead4 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -502,22 +502,22 @@ FORCE_INLINE int32_t compare_sd(tOrderDescriptor *pDescriptor, int32_t numOfRows return compare_d(pDescriptor, numOfRows, idx1, data, numOfRows, idx2, data); } -static void swap(SColumnModel *pColumnModel, int32_t count, int32_t s1, char *data1, int32_t s2) { +static void swap(SColumnModel *pColumnModel, int32_t count, int32_t s1, char *data1, int32_t s2, void* buf) { for (int32_t i = 0; i < pColumnModel->numOfCols; ++i) { void *first = COLMODEL_GET_VAL(data1, pColumnModel, count, s1, i); void *second = COLMODEL_GET_VAL(data1, pColumnModel, count, s2, i); SSchema* pSchema = &pColumnModel->pFields[i].field; - tsDataSwap(first, second, pSchema->type, pSchema->bytes); + tsDataSwap(first, second, pSchema->type, pSchema->bytes, buf); } } static void tColDataInsertSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - __col_compar_fn_t compareFn) { + __col_compar_fn_t compareFn, void* buf) { for (int32_t i = start + 1; i <= end; ++i) { for (int32_t j = i; j > start; --j) { if (compareFn(pDescriptor, numOfRows, j, j - 1, data) == -1) { - swap(pDescriptor->pColumnModel, numOfRows, j - 1, data, j); + swap(pDescriptor->pColumnModel, numOfRows, j - 1, data, j, buf); } else { break; } @@ -553,7 +553,7 @@ static void UNUSED_FUNC tSortDataPrint(int32_t type, char *prefix, char *startx, } static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - __col_compar_fn_t compareFn) { + __col_compar_fn_t compareFn, void* buf) { int32_t midIdx = ((end - start) >> 1) + start; #if defined(_DEBUG_VIEW) @@ -567,15 +567,16 @@ static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta tSortDataPrint(pDescriptor->pColumnModel->pFields[colIdx].field.type, "before", startx, midx, endx); #endif + SColumnModel* pModel = pDescriptor->pColumnModel; if (compareFn(pDescriptor, numOfRows, midIdx, start, data) == 1) { - swap(pDescriptor->pColumnModel, numOfRows, start, data, midIdx); + swap(pModel, numOfRows, start, data, midIdx, buf); } if (compareFn(pDescriptor, numOfRows, midIdx, end, data) == 1) { - swap(pDescriptor->pColumnModel, numOfRows, midIdx, data, start); - swap(pDescriptor->pColumnModel, numOfRows, midIdx, data, end); + swap(pModel, numOfRows, midIdx, data, start, buf); + swap(pModel, numOfRows, midIdx, data, end, buf); } else if (compareFn(pDescriptor, numOfRows, start, end, data) == 1) { - swap(pDescriptor->pColumnModel, numOfRows, start, data, end); + swap(pModel, numOfRows, start, data, end, buf); } assert(compareFn(pDescriptor, numOfRows, midIdx, start, data) <= 0 && @@ -626,32 +627,20 @@ static UNUSED_FUNC void tRowModelDisplay(tOrderDescriptor *pDescriptor, int32_t printf("\n"); } -static int32_t qsort_call = 0; - -void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - int32_t orderType) { - // short array sort, incur another sort procedure instead of quick sort process - __col_compar_fn_t compareFn = (orderType == TSDB_ORDER_ASC) ? compare_sa : compare_sd; - - if (end - start + 1 <= 8) { - tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn); - return; - } - +static void columnwiseQSortImpl(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, + int32_t orderType, __col_compar_fn_t compareFn, void* buf) { #ifdef _DEBUG_VIEW -// printf("before sort:\n"); -// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); + printf("before sort:\n"); + tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); #endif int32_t s = start, e = end; - median(pDescriptor, numOfRows, start, end, data, compareFn); + median(pDescriptor, numOfRows, start, end, data, compareFn, buf); #ifdef _DEBUG_VIEW -// printf("%s called: %d\n", __FUNCTION__, qsort_call++); + // printf("%s called: %d\n", __FUNCTION__, qsort_call++); #endif - UNUSED(qsort_call); - int32_t end_same = end; int32_t start_same = start; @@ -663,17 +652,17 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta } if (ret == 0 && e != end_same) { - swap(pDescriptor->pColumnModel, numOfRows, e, data, end_same--); + swap(pDescriptor->pColumnModel, numOfRows, e, data, end_same--, buf); } e--; } if (e != s) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, e); + swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf); } #ifdef _DEBUG_VIEW -// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); + // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); #endif while (s < e) { @@ -683,16 +672,16 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta } if (ret == 0 && s != start_same) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, start_same++); + swap(pDescriptor->pColumnModel, numOfRows, s, data, start_same++, buf); } s++; } if (s != e) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, e); + swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf); } #ifdef _DEBUG_VIEW -// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); + // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); #endif } @@ -702,14 +691,14 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta int32_t right = end; while (right > end_same && left <= end_same) { - swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--); + swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf); } // (pivotal+1) + steps of number that are identical pivotal rightx += (end - end_same); #ifdef _DEBUG_VIEW -// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); + // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); #endif } @@ -719,26 +708,52 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta int32_t right = e - 1; while (left < start_same && right >= start_same) { - swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--); + swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf); } // (pivotal-1) - steps of number that are identical pivotal leftx -= (start_same - start); #ifdef _DEBUG_VIEW -// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); + // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); #endif } if (leftx > start) { - tColDataQSort(pDescriptor, numOfRows, start, leftx, data, orderType); + columnwiseQSortImpl(pDescriptor, numOfRows, start, leftx, data, orderType, compareFn, buf); } if (rightx < end) { - tColDataQSort(pDescriptor, numOfRows, rightx, end, data, orderType); + columnwiseQSortImpl(pDescriptor, numOfRows, rightx, end, data, orderType, compareFn, buf); } } +void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t order) { + // short array sort, incur another sort procedure instead of quick sort process + __col_compar_fn_t compareFn = (order == TSDB_ORDER_ASC) ? compare_sa : compare_sd; + + SColumnModel* pModel = pDescriptor->pColumnModel; + + size_t width = 0; + for(int32_t i = 0; i < pModel->numOfCols; ++i) { + SSchema* pSchema = &pModel->pFields[i].field; + if (width < pSchema->bytes) { + width = pSchema->bytes; + } + } + + char* buf = malloc(width); + assert(width > 0 && buf != NULL); + + if (end - start + 1 <= 8) { + tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn, buf); + } else { + columnwiseQSortImpl(pDescriptor, numOfRows, start, end, data, order, compareFn, buf); + } + + free(buf); +} + /* * deep copy of sschema */ From dd368d91af23935ea7d16e44b264b59dbab2c8b0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Sep 2020 22:12:17 +0800 Subject: [PATCH 020/105] [td-1412] --- src/query/src/qExecutor.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d00e616841..75614578fe 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -793,7 +793,7 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - pCtx[k].ptsList = &tsBuf[offset]; + pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset]; } // not a whole block involved in query processing, statistics data can not be used @@ -1475,12 +1475,14 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pCtx->preAggVals.dataBlockLoaded = (inputData != NULL); // limit/offset query will affect this value - pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos:0; pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1; + // minimum value no matter ascending/descending order query + pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size - 1); + uint32_t status = aAggs[functionId].nStatus; if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { - pCtx->ptsList = tsCol; + pCtx->ptsList = &tsCol[pCtx->startOffset]; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { From b9566424bae3182796bc676a69724245431283d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 11 Sep 2020 10:22:42 +0800 Subject: [PATCH 021/105] [td-1412] --- src/query/inc/tsqlfunction.h | 2 +- src/query/src/qExecutor.c | 28 +++++++++------------------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index c314087179..a78bb7ec51 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -177,7 +177,7 @@ typedef struct SQLFunctionCtx { int16_t outputType; int16_t outputBytes; // size of results, determined by function and input column data type bool hasNull; // null value exist in current block - bool requireNull; // require null in some function + bool requireNull; // require null in some function int16_t functionId; // function id void * aInputElemBuf; char * aOutputBuf; // final result output buffer, point to sdata->data diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 75614578fe..511d6c36ef 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -743,9 +743,9 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - int32_t num = -1; + int32_t num = -1; int32_t order = pQuery->order.order; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); STableQueryInfo* item = pQuery->current; @@ -779,27 +779,24 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, - int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) { +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, + int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].base.functionId; - pCtx[k].nStartQueryTimestamp = pWin->skey; pCtx[k].size = forwardStep; pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset]; + pCtx[k].ptsList = &tsCol[pCtx[k].startOffset]; } // not a whole block involved in query processing, statistics data can not be used - if (forwardStep != numOfTotal) { - pCtx[k].preAggVals.isSet = false; - } + pCtx[k].preAggVals.isSet = (forwardStep == numOfTotal); if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunction(&pCtx[k]); @@ -924,19 +921,11 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas char *dataBlock = NULL; SQuery *pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; int32_t functionId = pQuery->pSelectExpr[col].base.functionId; if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pSelectExpr[col]; - // set the start offset to be the lowest start position, no matter asc/desc query order - if (QUERY_IS_ASC_QUERY(pQuery)) { - pCtx->startOffset = pQuery->pos; - } else { - pCtx->startOffset = pQuery->pos - (size - 1); - } - sas->offset = 0; sas->colList = pQuery->colList; sas->numOfCols = pQuery->numOfCols; @@ -1478,7 +1467,8 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1; // minimum value no matter ascending/descending order query - pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size - 1); + pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size + 1); + assert(pCtx->startOffset >= 0); uint32_t status = aAggs[functionId].nStatus; if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { From bd0522b1a922f09019c93b0859d0e0e00e04b0aa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 11 Sep 2020 10:35:03 +0800 Subject: [PATCH 022/105] [td-225] add test cases. --- tests/script/general/parser/timestamp_query.sim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/script/general/parser/timestamp_query.sim b/tests/script/general/parser/timestamp_query.sim index 6994b2d295..783c03602b 100644 --- a/tests/script/general/parser/timestamp_query.sim +++ b/tests/script/general/parser/timestamp_query.sim @@ -21,6 +21,10 @@ $tsu = $rowNum * $delta $tsu = $tsu - $delta $tsu = $tsu + $ts0 +print ==================>issue #3481, normal column not allowed, +sql_error select ts,c1,min(c2) from ts_stb0 + + ##### select from supertable $tb = $tbPrefix . 0 sql select first(c1), last(c1), (1537325400 - 1537146000)/(5*60) v from $tb where ts >= $ts0 and ts < $tsu interval(5m) fill(value, -1) From 9a87f3ccd36abcdd44878d9aaee9647b80c0a803 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 11 Sep 2020 11:05:10 +0800 Subject: [PATCH 023/105] [td-1416] --- src/client/src/tscSQLParser.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index adb6bfccbc..eaf2692e7e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5226,7 +5226,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { } } -static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { +static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { @@ -5244,9 +5244,14 @@ static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { } } - assert(qualifiedCol); + // it is not a tag column/tbname column/user-defined column, return error + if (!qualifiedCol) { + return TSDB_CODE_TSC_INVALID_SQL; + } } } + + return TSDB_CODE_SUCCESS; } static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) { @@ -5329,7 +5334,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ || (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { - tagColExists = true; + tagColExists = true; // selectivity + ts/tag column break; } } @@ -5362,7 +5367,11 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) */ if (numOfSelectivity == 1) { doUpdateSqlFunctionForTagPrj(pQueryInfo); - doUpdateSqlFunctionForColPrj(pQueryInfo); + int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else if (numOfSelectivity > 1) { /* * If more than one selectivity functions exist, all the selectivity functions must be last_row. @@ -5381,7 +5390,10 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) } doUpdateSqlFunctionForTagPrj(pQueryInfo); - doUpdateSqlFunctionForColPrj(pQueryInfo); + int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } else { if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) { @@ -5392,7 +5404,10 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) if (numOfAggregation > 0 || numOfSelectivity > 0) { // clear the projection type flag pQueryInfo->type &= (~TSDB_QUERY_TYPE_PROJECTION_QUERY); - doUpdateSqlFunctionForColPrj(pQueryInfo); + int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } From 562f6ce245a524635bc6ce6fde251792bed921a2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 11 Sep 2020 11:22:45 +0800 Subject: [PATCH 024/105] [td-1416] --- src/client/src/tscSQLParser.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index eaf2692e7e..e4654efe1c 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5231,7 +5231,8 @@ static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - if (pExpr->functionId == TSDB_FUNC_PRJ) { + + if (pExpr->functionId == TSDB_FUNC_PRJ && (!TSDB_COL_IS_UD_COL(pExpr->colInfo.flag))) { bool qualifiedCol = false; for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); From eb4659d8d7aa9d63e25289bc8211044e80c477d0 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 11 Sep 2020 15:01:25 +0800 Subject: [PATCH 025/105] [TD-1389] : Update null value test --- tests/pytest/query/queryNormal.py | 2 ++ tests/pytest/query/queryNullValueTest.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/tests/pytest/query/queryNormal.py b/tests/pytest/query/queryNormal.py index 1ab285bbad..208ac54ecd 100644 --- a/tests/pytest/query/queryNormal.py +++ b/tests/pytest/query/queryNormal.py @@ -42,6 +42,8 @@ class TDTestCase: # join 3 tables -- bug exists tdSql.error("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id") + tdSql.error("select * from stb1 whern c1 > 'test' limit 100") + # query show stable tdSql.query("show stables") tdSql.checkRows(1) diff --git a/tests/pytest/query/queryNullValueTest.py b/tests/pytest/query/queryNullValueTest.py index f521f2e5e9..bc0b11827e 100644 --- a/tests/pytest/query/queryNullValueTest.py +++ b/tests/pytest/query/queryNullValueTest.py @@ -42,6 +42,9 @@ class TDTestCase: tdSql.prepare() for i in range(len(self.types)): + tdSql.execute("drop table if exists t0") + tdSql.execute("drop table if exists t1") + print("======== checking type %s ==========" % self.types[i]) tdSql.execute("create table t0 (ts timestamp, col %s)" % self.types[i]) tdSql.execute("insert into t0 values (%d, NULL)" % (self.ts)) From a28d714d98dce759db3ed4a86198d3b9da04fa19 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 11 Sep 2020 18:35:10 +0800 Subject: [PATCH 026/105] td-1423] fix bugs in group by nchar/binary columns. #3476 --- src/query/inc/qExecutor.h | 2 +- src/query/src/qExecutor.c | 43 +++++++++++++++++++++++++---------- src/query/src/qUtil.c | 47 +++++++++++++++++++++++++++++---------- 3 files changed, 67 insertions(+), 25 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 25fb04fb9a..169bf907c6 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -57,7 +57,7 @@ typedef struct SWindowResult { uint16_t numOfRows; // number of rows of current time window bool closed; // this result status: closed or opened SResultInfo* resultInfo; // For each result column, there is a resultInfo - TSKEY skey; // start key of current time window + union {STimeWindow win; char* key;}; // start key of current time window } SWindowResult; /** diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 511d6c36ef..f4927ecb78 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -518,7 +518,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t } else { int32_t slot = curTimeWindowIndex(pWindowResInfo); SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot); - w = GET_TIMEWINDOW(pWindowResInfo, pWindowRes); + w = pWindowRes->win; } if (w.skey > ts || w.ekey < ts) { @@ -624,7 +624,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes } // set time window for current result - pWindowRes->skey = win->skey; + pWindowRes->win = (*win); setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); return TSDB_CODE_SUCCESS; @@ -697,12 +697,12 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe continue; } - TSKEY ekey = pResult->skey + pWindowResInfo->interval; + TSKEY ekey = pResult->win.ekey; if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (pResult->skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { + (pResult->win.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { closeTimeWindow(pWindowResInfo, i); } else { - skey = pResult->skey; + skey = pResult->win.skey; break; } } @@ -715,7 +715,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->curIndex = i; } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].skey; + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].win.skey; // the number of completed slots are larger than the threshold, return current generated results to client. if (numOfClosed > pWindowResInfo->threshold) { @@ -1097,8 +1097,25 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - int64_t v = -1; // not assign result buffer yet, add new result buffer + char* d = pData; + int16_t len = bytes; + if (type == TSDB_DATA_TYPE_BINARY||type == TSDB_DATA_TYPE_NCHAR) { + d = varDataVal(pData); + len = varDataLen(pData); + } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); + qError("QInfo:%p group by not supported on double/float/binary/nchar columns, abort", pQInfo); + + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true); + if (pWindowRes == NULL) { + return -1; + } + + int64_t v = -1; switch(type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: v = GET_INT8_VAL(pData); break; @@ -1107,12 +1124,14 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat case TSDB_DATA_TYPE_BIGINT: v = GET_INT64_VAL(pData); break; } - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes, true); - if (pWindowRes == NULL) { - return -1; + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + pWindowRes->key = malloc(varDataTLen(pData)); + varDataCopy(pWindowRes->key, pData); + } else { + pWindowRes->win.skey = v; + pWindowRes->win.ekey = v; } - pWindowRes->skey = v; assert(pRuntimeEnv->windowResInfo.interval == 0); if (pWindowRes->pos.pageId == -1) { @@ -3004,7 +3023,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); - assert(ts == pWindowRes->skey); + assert(ts == pWindowRes->win.skey); int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); if (num <= 0) { cs.position[pos] += 1; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 509362863c..c195a0b76c 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -126,11 +126,26 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); assert(num >= 0 && num <= numOfClosed); - + + int16_t type = pWindowResInfo->type; + + char *key = NULL; + int16_t bytes = -1; + for (int32_t i = 0; i < num; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; if (pResult->closed) { // remove the window slot from hash table - taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->skey, pWindowResInfo->type); + + // todo refactor + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + key = varDataVal(pResult->key); + bytes = varDataLen(pResult->key); + } else { + key = (char*) &pResult->win.skey; + bytes = tDataTypeDesc[pWindowResInfo->type].nSize; + } + + taosHashRemove(pWindowResInfo->hashList, (const char *)key, bytes); } else { break; } @@ -150,15 +165,24 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { } pWindowResInfo->size = remain; + for (int32_t k = 0; k < pWindowResInfo->size; ++k) { SWindowResult *pResult = &pWindowResInfo->pResult[k]; - int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->skey, - tDataTypeDesc[pWindowResInfo->type].nSize); + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + key = varDataVal(pResult->key); + bytes = varDataLen(pResult->key); + } else { + key = (char*) &pResult->win.skey; + bytes = tDataTypeDesc[pWindowResInfo->type].nSize; + } + + int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)key, bytes); assert(p != NULL); + int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - taosHashPut(pWindowResInfo->hashList, (char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize, - (char *)&v, sizeof(int32_t)); + taosHashPut(pWindowResInfo->hashList, (char *)key, bytes, (char *)&v, sizeof(int32_t)); } pWindowResInfo->curIndex = -1; @@ -207,20 +231,19 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0].skey < pWindowResInfo->pResult[1].skey)? 1:-1; - + int32_t resultOrder = (pWindowResInfo->pResult[0].win.skey < pWindowResInfo->pResult[1].win.skey)? 1:-1; if (order != resultOrder) { return; } int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - TSKEY ekey = pWindowResInfo->pResult[i].skey + pWindowResInfo->interval; + TSKEY ekey = pWindowResInfo->pResult[i].win.ekey; while (i < pWindowResInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].skey > lastKey)) { + while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].win.skey > lastKey)) { ++i; } } @@ -258,7 +281,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow pWindowRes->numOfRows = 0; pWindowRes->pos = (SPosInfo){-1, -1}; pWindowRes->closed = false; - pWindowRes->skey = TSKEY_INITIAL_VAL; + pWindowRes->win = TSWINDOW_INITIALIZER; } /** @@ -268,7 +291,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow */ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { dst->numOfRows = src->numOfRows; - dst->skey = src->skey; + dst->win = src->win; dst->closed = src->closed; int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; From 2b1519c167beaa5b2a098a3b01dac46060891d71 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 12 Sep 2020 15:50:04 +0800 Subject: [PATCH 027/105] [td-1319] --- src/client/inc/tsclient.h | 20 ++-- src/client/src/tscAsync.c | 10 +- src/client/src/tscLocalMerge.c | 2 - src/client/src/tscServer.c | 45 +++++--- src/client/src/tscSql.c | 133 +++++++++++++----------- src/client/src/tscSubquery.c | 17 +-- src/client/src/tscSystem.c | 7 +- src/client/src/tscUtil.c | 58 +++++++++-- src/plugins/http/src/httpSql.c | 13 --- src/tsdb/src/tsdbRead.c | 1 + src/util/src/tcache.c | 9 +- tests/script/general/parser/groupby.sim | 2 + 12 files changed, 188 insertions(+), 129 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 5f4a46ddad..11b7815586 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -29,6 +29,7 @@ extern "C" { #include "tglobal.h" #include "tsqlfunction.h" #include "tutil.h" +#include "tcache.h" #include "qExecutor.h" #include "qSqlparser.h" @@ -359,6 +360,8 @@ typedef struct SSqlObj { uint16_t numOfSubs; struct SSqlObj **pSubs; struct SSqlObj * prev, *next; + + struct SSqlObj **self; } SSqlObj; typedef struct SSqlStream { @@ -413,7 +416,6 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); -void tscDestroyResPointerInfo(SSqlRes *pRes); void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache); @@ -425,17 +427,19 @@ void tscFreeSqlResult(SSqlObj *pSql); /** * only free part of resources allocated during query. + * TODO remove it later * Note: this function is multi-thread safe. * @param pObj */ -void tscPartiallyFreeSqlObj(SSqlObj *pObj); +void tscPartiallyFreeSqlObj(SSqlObj *pSql); /** * free sql object, release allocated resource - * @param pObj Free metric/meta information, dynamically allocated payload, and - * response buffer, object itself + * @param pObj */ -void tscFreeSqlObj(SSqlObj *pObj); +void tscFreeSqlObj(SSqlObj *pSql); + +void tscFreeSqlObjInCache(void *pSql); void tscCloseTscObj(STscObj *pObj); @@ -451,9 +455,6 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen) bool tscIsUpdateQuery(SSqlObj* pSql); bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); -// todo remove this function. -bool tscResultsetFetchCompleted(TAOS_RES *result); - char *tscGetErrorMsgPayload(SSqlCmd *pCmd); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); @@ -502,7 +503,8 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField } } -extern void * tscCacheHandle; +extern SCacheObj* tscCacheHandle; +extern SCacheObj* tscObjCache; extern void * tscTmr; extern void * tscQhandle; extern int tscKeepConn[]; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index d07089539a..5e9aa1b1f8 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -18,6 +18,7 @@ #include "tnote.h" #include "trpc.h" +#include "tcache.h" #include "tscLog.h" #include "tscSubquery.h" #include "tscLocalMerge.h" @@ -40,6 +41,8 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows); void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) { + SSqlCmd* pCmd = &pSql->cmd; + pSql->signature = pSql; pSql->param = param; pSql->pTscObj = pObj; @@ -59,7 +62,10 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const strntolower(pSql->sqlstr, sqlstr, (int32_t)sqlLen); tscDebugL("%p SQL: %s", pSql, pSql->sqlstr); - pSql->cmd.curSql = pSql->sqlstr; + pCmd->curSql = pSql->sqlstr; + + uint64_t handle = (uint64_t) pSql; + pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); int32_t code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; @@ -69,7 +75,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const tscQueueAsyncRes(pSql); return; } - + tscDoQuery(pSql); } diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 39a757795e..af6a546ff4 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -472,10 +472,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { return; } - tscDebug("%p start to free local reducer", pSql); SSqlRes *pRes = &(pSql->res); if (pRes->pLocalReducer == NULL) { - tscDebug("%p local reducer has been freed, abort", pSql); return; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index fbc02cc40e..0733690e3f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -27,10 +27,7 @@ #include "tutil.h" #include "tlockfree.h" -#define TSC_MGMT_VNODE 999 - SRpcCorEpSet tscMgmtEpSet; -SRpcEpSet tscDnodeEpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; @@ -236,12 +233,17 @@ int tscSendMsgToServer(SSqlObj *pSql) { } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { - SSqlObj *pSql = (SSqlObj *)rpcMsg->ahandle; - if (pSql == NULL || pSql->signature != pSql) { - tscError("%p sql is already released", pSql); + uint64_t handle = (uint64_t) rpcMsg->ahandle; + + void** p = taosCacheAcquireByKey(tscObjCache, &handle, sizeof(uint64_t)); + if (p == NULL) { + rpcFreeCont(rpcMsg->pCont); return; } + SSqlObj* pSql = *p; + assert(pSql != NULL); + STscObj *pObj = pSql->pTscObj; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -249,7 +251,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pObj->signature != pObj) { tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature); - tscFreeSqlObj(pSql); + taosCacheRelease(tscObjCache, (void**) &p, true); rpcFreeCont(rpcMsg->pCont); return; } @@ -261,18 +263,18 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature); - tscFreeSqlObj(pSql); + taosCacheRelease(tscObjCache, (void**) &p, true); rpcFreeCont(rpcMsg->pCont); return; } - if (pEpSet) { + if (pEpSet) { if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { - if(pCmd->command < TSDB_SQL_MGMT) { - tscUpdateVgroupInfo(pSql, pEpSet); + if (pCmd->command < TSDB_SQL_MGMT) { + tscUpdateVgroupInfo(pSql, pEpSet); } else { tscUpdateMgmtEpSet(pEpSet); - } + } } } @@ -294,7 +296,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pSql->retry > pSql->maxRetry) { tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); } else { - // wait for a little bit moment and then retry + // wait for a little bit moment and then retry, todo do not sleep in rpc callback thread if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { int32_t duration = getWaitingTimeInterval(pSql->retry); taosMsleep(duration); @@ -304,6 +306,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { // if there is an error occurring, proceed to the following error handling procedure. if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosCacheRelease(tscObjCache, (void**) &p, false); rpcFreeCont(rpcMsg->pCont); return; } @@ -365,18 +368,21 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); } + bool shouldFree = false; if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; - bool shouldFree = tscShouldBeFreed(pSql); + shouldFree = tscShouldBeFreed(pSql); (*pSql->fp)(pSql->param, pSql, rpcMsg->code); if (shouldFree) { + void** p1 = p; + taosCacheRelease(tscObjCache, (void **)&p1, true); tscDebug("%p sqlObj is automatically freed", pSql); - tscFreeSqlObj(pSql); } } + taosCacheRelease(tscObjCache, (void**) &p, false); rpcFreeCont(rpcMsg->pCont); } @@ -2000,7 +2006,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { createHBObj(pObj); - taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); +// taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); return 0; } @@ -2164,6 +2170,10 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->fp = tscTableMetaCallBack; pNew->param = pSql; + // TODO add test case on x86 platform + uint64_t adr = (uint64_t) pNew; + pNew->self = taosCachePut(tscObjCache, &adr, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2*60*1000); + int32_t code = tscProcessSql(pNew); if (code == TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated @@ -2265,6 +2275,9 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { } pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; + + uint64_t p = (uint64_t) pNew; + pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000); tscDebug("%p new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql, pNew, pNewQueryInfo->numOfTables); pNew->fp = tscTableMetaCallBack; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 9fa4db999f..33996307ad 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -100,6 +100,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con } pObj->signature = pObj; + pObj->pDnodeConn = pDnodeConn; tstrncpy(pObj->user, user, sizeof(pObj->user)); secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); @@ -132,20 +133,15 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con return NULL; } - pSql->pTscObj = pObj; + pSql->pTscObj = pObj; pSql->signature = pSql; - pSql->maxRetry = TSDB_MAX_REPLICA; - tsem_init(&pSql->rspSem, 0, 0); - - pObj->pDnodeConn = pDnodeConn; - - pSql->fp = fp; - pSql->param = param; - if (taos != NULL) { - *taos = pObj; - } - + pSql->maxRetry = TSDB_MAX_REPLICA; + pSql->fp = fp; + pSql->param = param; pSql->cmd.command = TSDB_SQL_CONNECT; + + tsem_init(&pSql->rspSem, 0, 0); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; rpcClose(pDnodeConn); @@ -154,7 +150,14 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con return NULL; } + if (taos != NULL) { + *taos = pObj; + } + + uint64_t key = (uint64_t) pSql; + pSql->self = taosCachePut(tscObjCache, &key, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); + return pSql; } @@ -533,60 +536,72 @@ int taos_select_db(TAOS *taos, const char *db) { } // send free message to vnode to free qhandle and corresponding resources in vnode -static bool tscKillQueryInVnode(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && !tscIsTwoStageSTableQuery(pQueryInfo, 0) && - (pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_SHOW || - pCmd->command == TSDB_SQL_RETRIEVE || - pCmd->command == TSDB_SQL_FETCH) && - (pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) { - - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]); - tscProcessSql(pSql); - return true; - } - - return false; -} +//static bool tscKillQueryInVnode(SSqlObj* pSql) { +// SSqlCmd* pCmd = &pSql->cmd; +// SSqlRes* pRes = &pSql->res; +// +// SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); +// STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); +// +// if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { +// return false; +// } +// +// if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && (pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL) && +// (pCmd->command == TSDB_SQL_SELECT || +// pCmd->command == TSDB_SQL_SHOW || +// pCmd->command == TSDB_SQL_RETRIEVE || +// pCmd->command == TSDB_SQL_FETCH)) { +// +// pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; +// tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]); +// tscProcessSql(pSql); +// return true; +// } +// +// return false; +//} void taos_free_result(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - - if (pSql == NULL || pSql->signature != pSql) { - tscDebug("%p sqlObj has been freed", pSql); - return; - } - - // The semaphore can not be changed while freeing async sub query objects. - SSqlRes *pRes = &pSql->res; - if (pRes == NULL || pRes->qhandle == 0) { - tscFreeSqlObj(pSql); - tscDebug("%p SqlObj is freed by app, qhandle is null", pSql); + if (res == NULL) { return; } - // set freeFlag to 1 in retrieve message if there are un-retrieved results data in node - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - if (pQueryInfo == NULL) { - tscFreeSqlObj(pSql); - tscDebug("%p SqlObj is freed by app", pSql); - return; - } - - pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - if (!tscKillQueryInVnode(pSql)) { - tscFreeSqlObj(pSql); - tscDebug("%p sqlObj is freed by app", pSql); - } + SSqlObj* pSql = (SSqlObj*) res; + taosCacheRelease(tscObjCache, (void**) &pSql->self, true); } +//static void doFreeResult(TAOS_RES *res) { +// SSqlObj *pSql = (SSqlObj *)res; +// +// if (pSql == NULL || pSql->signature != pSql) { +// tscDebug("%p sqlObj has been freed", pSql); +// return; +// } +// +// // The semaphore can not be changed while freeing async sub query objects. +// SSqlRes *pRes = &pSql->res; +// if (pRes == NULL || pRes->qhandle == 0) { +// tscFreeSqlObj(pSql); +// tscDebug("%p SqlObj is freed by app, qhandle is null", pSql); +// return; +// } +// +// // set freeFlag to 1 in retrieve message if there are un-retrieved results data in node +// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); +// if (pQueryInfo == NULL) { +// tscFreeSqlObj(pSql); +// tscDebug("%p SqlObj is freed by app", pSql); +// return; +// } +// +// pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; +// if (!tscKillQueryInVnode(pSql)) { +// tscFreeSqlObj(pSql); +// tscDebug("%p sqlObj is freed by app", pSql); +// } +//} + int taos_errno(TAOS_RES *tres) { SSqlObj *pSql = (SSqlObj *) tres; if (pSql == NULL || pSql->signature != pSql) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index c4b07f7813..e9ec272ea4 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1512,9 +1512,9 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { SSqlObj *pParentSql = trsupport->pParentSql; assert(pSql == pParentSql->pSubs[index]); - pParentSql->pSubs[index] = NULL; - - taos_free_result(pSql); +// pParentSql->pSubs[index] = NULL; +// +// taos_free_result(pSql); taosTFree(trsupport->localBuffer); taosTFree(trsupport); } @@ -1728,10 +1728,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(tres != NULL); SSqlObj *pSql = (SSqlObj *)tres; -// if (pSql == NULL) { // sql object has been released in error process, return immediately -// tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); -// return; -// } SSubqueryState* pState = trsupport->pState; assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); @@ -1907,9 +1903,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) pParentObj->res.code = pSql->res.code; } - taos_free_result(tres); taosTFree(pSupporter); - if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { return; } @@ -2029,11 +2023,6 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; _error: - for(int32_t j = 0; j < numOfSub; ++j) { - taosTFree(pSql->pSubs[j]->param); - taos_free_result(pSql->pSubs[j]); - } - taosTFree(pState); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 72f23881d2..2c7fcf05c9 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -30,7 +30,8 @@ #include "tlocale.h" // global, not configurable -void * tscCacheHandle; +SCacheObj* tscCacheHandle; +SCacheObj* tscObjCache; void * tscTmr; void * tscQhandle; void * tscCheckDiskUsageTmr; @@ -146,6 +147,7 @@ void taos_init_imp(void) { if (tscCacheHandle == NULL) { tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta"); + tscObjCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, refreshTime, false, tscFreeSqlObjInCache, "sqlObjHandle"); } tscDebug("client is initialized successfully"); @@ -157,6 +159,9 @@ void taos_cleanup() { if (tscCacheHandle != NULL) { taosCacheCleanup(tscCacheHandle); tscCacheHandle = NULL; + + taosCacheCleanup(tscObjCache); + tscObjCache = NULL; } if (tscQhandle != NULL) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 5d50d7791a..47abe60ddd 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -252,11 +252,11 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { if (pRes->tsrow == NULL) { int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; pRes->numOfCols = numOfOutput; - + pRes->tsrow = calloc(numOfOutput, POINTER_BYTES); pRes->length = calloc(numOfOutput, sizeof(int32_t)); pRes->buffer = calloc(numOfOutput, POINTER_BYTES); - + // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { taosTFree(pRes->tsrow); @@ -268,7 +268,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { return TSDB_CODE_SUCCESS; } -void tscDestroyResPointerInfo(SSqlRes* pRes) { +static void tscDestroyResPointerInfo(SSqlRes* pRes) { if (pRes->buffer != NULL) { // free all buffers containing the multibyte string for (int i = 0; i < pRes->numOfCols; i++) { taosTFree(pRes->buffer[i]); @@ -367,12 +367,36 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { tscResetSqlCmdObj(pCmd, false); } +static void tscFreeSubobj(SSqlObj* pSql) { + if (pSql->numOfSubs == 0) { + return; + } + + tscDebug("%p start to free sub SqlObj, numOfSub:%d", pSql, pSql->numOfSubs); + + for(int32_t i = 0; i < pSql->numOfSubs; ++i) { + tscDebug("%p free sub SqlObj:%p, index:%d", pSql, pSql->pSubs[i], i); + taos_free_result(pSql->pSubs[i]); + pSql->pSubs[i] = NULL; + } + + pSql->numOfSubs = 0; +} + +void tscFreeSqlObjInCache(void *pSql) { + assert(pSql != NULL); + SSqlObj** p = (SSqlObj**) pSql; + + tscFreeSqlObj(*p); +} + void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; } tscDebug("%p start to free sql object", pSql); + tscFreeSubobj(pSql); tscPartiallyFreeSqlObj(pSql); pSql->signature = NULL; @@ -724,13 +748,25 @@ void tscCloseTscObj(STscObj* pObj) { pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); - pthread_mutex_destroy(&pObj->mutex); - + + // wait for all sqlObjs created according to this connect closed + while(1) { + pthread_mutex_lock(&pObj->mutex); + void* p = pObj->sqlList; + pthread_mutex_unlock(&pObj->mutex); + + if (p == NULL) { + break; + } + } + if (pObj->pDnodeConn != NULL) { rpcClose(pObj->pDnodeConn); pObj->pDnodeConn = NULL; } - + + pthread_mutex_destroy(&pObj->mutex); + tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn); taosTFree(pObj); } @@ -1721,6 +1757,9 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL); + + uint64_t p = (uint64_t) pNew; + pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000); return pNew; } @@ -1960,6 +1999,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void tscDebug("%p new sub insertion: %p, vnodeIdx:%d", pSql, pNew, pTableMetaInfo->vgroupIndex); } + uint64_t p = (uint64_t) pNew; + pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 10); return pNew; _error: @@ -2101,11 +2142,6 @@ bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) { return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit); } -bool tscResultsetFetchCompleted(TAOS_RES *result) { - SSqlRes* pRes = result; - return pRes->completed; -} - char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; } /** diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 07cdea1380..e86db2021a 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -50,10 +50,6 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n } } - // if (tscResultsetFetchCompleted(result)) { - // isContinue = false; - // } - if (isContinue) { // retrieve next batch of rows httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s", @@ -223,15 +219,6 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int } } -#if 0 - // todo refactor - if (tscResultsetFetchCompleted(result)) { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, resultset fetch completed", pContext, pContext->fd, pContext->ipstr, - pContext->user); - isContinue = false; - } -#endif - if (isContinue) { // retrieve next batch of rows httpDebug("context:%p, fd:%d, ip:%s, user:%s, continue retrieve, numOfRows:%d", pContext, pContext->fd, diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 5fece58ef7..24a19e83d0 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -248,6 +248,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL && sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0); + // todo apply the lastkey of table check to avoid to load header file for (int32_t i = 0; i < sizeOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i); diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index ab489e2e46..7fda057483 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -165,7 +165,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - // set free cache node callback function for hash table + // set free cache node callback function pCacheObj->freeFp = fn; pCacheObj->refreshTime = refreshTimeInSeconds * 1000; pCacheObj->extendLifespan = extendLifespan; @@ -322,7 +322,12 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { } void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { - if (pCacheObj == NULL || (*data) == NULL || (taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0)) { + if (taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) { + return; + } + + if (pCacheObj == NULL || (*data) == NULL) { + uError("cache:%s, NULL data to release", pCacheObj->name); return; } diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 255e00ca41..bd0d3c1a12 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -423,6 +423,8 @@ if $data97 != @group_tb0@ then return -1 endi +print ---------------------------------> group by binary|nchar data add cases + #=========================== group by multi tags ====================== sql create table st (ts timestamp, c int) tags (t1 int, t2 int, t3 int, t4 int); From 6fb53b30523f824832a251f52374143b96980ce2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 12 Sep 2020 08:13:02 +0000 Subject: [PATCH 028/105] TD-1311 --- src/plugins/http/inc/httpContext.h | 3 - src/plugins/http/inc/httpInt.h | 4 - src/plugins/http/src/gcHandle.c | 26 ++-- src/plugins/http/src/httpAuth.c | 27 ++-- src/plugins/http/src/httpContext.c | 224 ++++++++++------------------- src/plugins/http/src/httpHandle.c | 162 +++------------------ src/plugins/http/src/httpJson.c | 45 +++--- src/plugins/http/src/httpResp.c | 4 +- src/plugins/http/src/httpServer.c | 189 +++++------------------- src/plugins/http/src/httpSession.c | 13 +- src/plugins/http/src/httpSql.c | 107 +++++++------- src/plugins/http/src/httpSystem.c | 6 - src/plugins/http/src/httpUtil.c | 27 ++-- src/plugins/http/src/restHandle.c | 6 +- src/plugins/http/src/restJson.c | 18 ++- src/plugins/http/src/tgHandle.c | 6 +- src/plugins/http/src/tgJson.c | 21 ++- 17 files changed, 265 insertions(+), 623 deletions(-) diff --git a/src/plugins/http/inc/httpContext.h b/src/plugins/http/inc/httpContext.h index 594900d0cf..a2d50d6b7f 100644 --- a/src/plugins/http/inc/httpContext.h +++ b/src/plugins/http/inc/httpContext.h @@ -31,7 +31,4 @@ void httpCloseContextByApp(HttpContext *pContext); void httpNotifyContextClose(HttpContext *pContext); bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState); -void ehttpIncContextRef(HttpContext *pContext); -void ehttpDecContextRef(HttpContext **ppContext); - #endif diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 40f980f101..044b5cc4cc 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -212,8 +212,6 @@ typedef struct HttpContext { void * timer; HttpEncodeMethod * encodeMethod; struct HttpThread *pThread; - - int closed:2; } HttpContext; typedef struct HttpThread { @@ -244,8 +242,6 @@ typedef struct HttpServer { pthread_mutex_t serverMutex; HttpDecodeMethod *methodScanner[HTTP_METHOD_SCANNER_SIZE]; bool (*processData)(HttpContext *pContext); - - int fallback:2; } HttpServer; extern const char *httpKeepAliveStr[]; diff --git a/src/plugins/http/src/gcHandle.c b/src/plugins/http/src/gcHandle.c index 72b73b4bad..4aed6eb5cc 100644 --- a/src/plugins/http/src/gcHandle.c +++ b/src/plugins/http/src/gcHandle.c @@ -67,8 +67,7 @@ bool gcGetPassFromUrl(HttpContext* pContext) { } bool gcProcessLoginRequest(HttpContext* pContext) { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, process grafana login msg", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpDebug("context:%p, fd:%d, user:%s, process grafana login msg", pContext, pContext->fd, pContext->user); pContext->reqType = HTTP_REQTYPE_LOGIN; return true; } @@ -143,7 +142,7 @@ bool gcProcessLoginRequest(HttpContext* pContext) { //}] bool gcProcessQueryRequest(HttpContext* pContext) { - httpDebug("context:%p, fd:%d, ip:%s, process grafana query msg", pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, process grafana query msg", pContext, pContext->fd); HttpParser* pParser = &pContext->parser; char* filter = pParser->data.pos; @@ -183,15 +182,13 @@ bool gcProcessQueryRequest(HttpContext* pContext) { cJSON* refId = cJSON_GetObjectItem(query, "refId"); if (refId == NULL || refId->valuestring == NULL || strlen(refId->valuestring) == 0) { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, refId is null", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpDebug("context:%p, fd:%d, user:%s, refId is null", pContext, pContext->fd, pContext->user); continue; } int refIdBuffer = httpAddToSqlCmdBuffer(pContext, refId->valuestring); if (refIdBuffer == -1) { - httpWarn("context:%p, fd:%d, ip:%s, user:%s, refId buffer is full", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpWarn("context:%p, fd:%d, user:%s, refId buffer is full", pContext, pContext->fd, pContext->user); break; } @@ -200,8 +197,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) { aliasBuffer = httpAddToSqlCmdBuffer(pContext, alias->valuestring); if (aliasBuffer == -1) { - httpWarn("context:%p, fd:%d, ip:%s, user:%s, alias buffer is full", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpWarn("context:%p, fd:%d, user:%s, alias buffer is full", pContext, pContext->fd, pContext->user); break; } } @@ -211,15 +207,13 @@ bool gcProcessQueryRequest(HttpContext* pContext) { cJSON* sql = cJSON_GetObjectItem(query, "sql"); if (sql == NULL || sql->valuestring == NULL || strlen(sql->valuestring) == 0) { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, sql is null", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpDebug("context:%p, fd:%d, user:%s, sql is null", pContext, pContext->fd, pContext->user); continue; } int sqlBuffer = httpAddToSqlCmdBuffer(pContext, sql->valuestring); if (sqlBuffer == -1) { - httpWarn("context:%p, fd:%d, ip:%s, user:%s, sql buffer is full", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpWarn("context:%p, fd:%d, user:%s, sql buffer is full", pContext, pContext->fd, pContext->user); break; } @@ -237,8 +231,8 @@ bool gcProcessQueryRequest(HttpContext* pContext) { cmd->timestamp = httpAddToSqlCmdBufferWithSize(pContext, HTTP_GC_TARGET_SIZE + 1); // hack way if (cmd->timestamp == -1) { - httpWarn("context:%p, fd:%d, ip:%s, user:%s, cant't malloc target size, sql buffer is full", - pContext, pContext->fd, pContext->ipstr, pContext->user); + httpWarn("context:%p, fd:%d, user:%s, cant't malloc target size, sql buffer is full", pContext, pContext->fd, + pContext->user); break; } } @@ -251,7 +245,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { } bool gcProcessHeartbeatRequest(HttpContext* pContext) { - httpDebug("context:%p, fd:%d, ip:%s, process grafana heartbeat msg", pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, process grafana heartbeat msg", pContext, pContext->fd); pContext->reqType = HTTP_REQTYPE_HEARTBEAT; pContext->encodeMethod = &gcHeartBeatMethod; return true; diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c index ea7024fad6..dd4d14c709 100644 --- a/src/plugins/http/src/httpAuth.c +++ b/src/plugins/http/src/httpAuth.c @@ -28,23 +28,21 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) { int outlen = 0; char *base64 = (char *)base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { - httpError("context:%p, fd:%d, ip:%s, basic token:%s parsed error", pContext, pContext->fd, pContext->ipstr, token); + httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token); free(base64); return false; } char *user = strstr(base64, ":"); if (user == NULL) { - httpError("context:%p, fd:%d, ip:%s, basic token:%s invalid format", pContext, pContext->fd, pContext->ipstr, - token); + httpError("context:%p, fd:%d, basic token:%s invalid format", pContext, pContext->fd, token); free(base64); return false; } int user_len = (int)(user - base64); if (user_len < 1 || user_len >= TSDB_USER_LEN) { - httpError("context:%p, fd:%d, ip:%s, basic token:%s parse user error", pContext, pContext->fd, pContext->ipstr, - token); + httpError("context:%p, fd:%d, basic token:%s parse user error", pContext, pContext->fd, token); free(base64); return false; } @@ -54,8 +52,7 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) { char *password = user + 1; int pass_len = (int)((base64 + outlen) - password); if (pass_len < 1 || pass_len >= TSDB_PASSWORD_LEN) { - httpError("context:%p, fd:%d, ip:%s, basic token:%s parse password error", pContext, pContext->fd, pContext->ipstr, - token); + httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token); free(base64); return false; } @@ -63,8 +60,7 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) { pContext->pass[pass_len] = 0; free(base64); - httpDebug("context:%p, fd:%d, ip:%s, basic token parsed success, user:%s", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpDebug("context:%p, fd:%d, basic token parsed success, user:%s", pContext, pContext->fd, pContext->user); return true; } @@ -73,28 +69,27 @@ bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) { int outlen = 0; unsigned char *base64 = base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { - httpError("context:%p, fd:%d, ip:%s, taosd token:%s parsed error", pContext, pContext->fd, pContext->ipstr, token); + httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token); if (base64) free(base64); return false; } if (outlen != (TSDB_USER_LEN + TSDB_PASSWORD_LEN)) { - httpError("context:%p, fd:%d, ip:%s, taosd token:%s length error", pContext, pContext->fd, pContext->ipstr, token); + httpError("context:%p, fd:%d, taosd token:%s length error", pContext, pContext->fd, token); free(base64); return false; } char *descrypt = taosDesDecode(KEY_DES_4, (char *)base64, outlen); if (descrypt == NULL) { - httpError("context:%p, fd:%d, ip:%s, taosd token:%s descrypt error", pContext, pContext->fd, pContext->ipstr, - token); + httpError("context:%p, fd:%d, taosd token:%s descrypt error", pContext, pContext->fd, token); free(base64); return false; } else { tstrncpy(pContext->user, descrypt, sizeof(pContext->user)); tstrncpy(pContext->pass, descrypt + TSDB_USER_LEN, sizeof(pContext->pass)); - httpDebug("context:%p, fd:%d, ip:%s, taosd token:%s parsed success, user:%s", pContext, pContext->fd, - pContext->ipstr, token, pContext->user); + httpDebug("context:%p, fd:%d, taosd token:%s parsed success, user:%s", pContext, pContext->fd, token, + pContext->user); free(base64); free(descrypt); return true; @@ -116,7 +111,7 @@ bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen) { free(encrypt); free(base64); - httpDebug("context:%p, fd:%d, ip:%s, gen taosd token:%s", pContext, pContext->fd, pContext->ipstr, token); + httpDebug("context:%p, fd:%d, gen taosd token:%s", pContext, pContext->fd, token); return true; } diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 16d8e91899..59c81f5960 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -37,16 +37,14 @@ extern bool httpParseHttpVersion(HttpContext* pContext); extern bool httpGetDecodeMethod(HttpContext* pContext); extern bool httpParseHead(HttpContext* pContext); -static void on_request_line(void *arg, const char *method, const char *target, const char *version, const char *target_raw); -static void on_status_line(void *arg, const char *version, int status_code, const char *reason_phrase); -static void on_header_field(void *arg, const char *key, const char *val); -static void on_body(void *arg, const char *chunk, size_t len); -static void on_end(void *arg); -static void on_error(void *arg, int status_code); +static void httpParseOnRequestLine(void *arg, const char *method, const char *target, const char *version, const char *target_raw); +static void httpParseOnStatusLine(void *arg, const char *version, int status_code, const char *reason_phrase); +static void httpParseOnHeaderField(void *arg, const char *key, const char *val); +static void httpParseOnBody(void *arg, const char *chunk, size_t len); +static void httpParseOnEnd(void *arg); +static void httpParseOnError(void *arg, int status_code); static void httpDestroyContext(void *data); -static void httpMightDestroyContext(void *data); -static void ehttpReleaseContext(HttpContext *pContext); static void httpRemoveContextFromEpoll(HttpContext *pContext) { HttpThread *pThread = pContext->pThread; @@ -54,15 +52,11 @@ static void httpRemoveContextFromEpoll(HttpContext *pContext) { epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); int32_t fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); taosCloseSocket(fd); - if (!tsHttpServer.fallback) { - ehttpDecContextRef(&pContext); - } } } static void httpDestroyContext(void *data) { HttpContext *pContext = *(HttpContext **)data; - D("==context[%p] destroyed==", pContext); if (pContext->fd > 0) taosClose(pContext->fd); HttpThread *pThread = pContext->pThread; @@ -79,18 +73,16 @@ static void httpDestroyContext(void *data) { httpFreeJsonBuf(pContext); httpFreeMultiCmds(pContext); - if (!tsHttpServer.fallback) { - if (pContext->parser.parser) { - ehttp_parser_destroy(pContext->parser.parser); - pContext->parser.parser = NULL; - } + if (pContext->parser.parser) { + ehttp_parser_destroy(pContext->parser.parser); + pContext->parser.parser = NULL; } taosTFree(pContext); } bool httpInitContexts() { - tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpMightDestroyContext, "restc"); + tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpDestroyContext, "restc"); if (tsHttpServer.contextCache == NULL) { httpError("failed to init context cache"); return false; @@ -135,20 +127,16 @@ HttpContext *httpCreateContext(int32_t fd) { HttpContext *pContext = calloc(1, sizeof(HttpContext)); if (pContext == NULL) return NULL; - D("==context[%p] created==", pContext); - pContext->fd = fd; pContext->httpVersion = HTTP_VERSION_10; pContext->lastAccessTime = taosGetTimestampSec(); pContext->state = HTTP_CONTEXT_STATE_READY; - ehttpIncContextRef(pContext); uint64_t handleVal = (uint64_t)pContext; HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, sizeof(int64_t), &pContext, sizeof(int64_t), 3000); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); - ehttpIncContextRef(pContext); // set the ref to 0 taosCacheRelease(tsHttpServer.contextCache, (void**)&ppContext, false); @@ -164,7 +152,6 @@ HttpContext *httpGetContext(void *ptr) { if (ppContext) { HttpContext *pContext = *ppContext; if (pContext) { - if (!tsHttpServer.fallback) return pContext; int32_t refCount = atomic_add_fetch_32(&pContext->refCount, 1); httpDebug("context:%p, fd:%d, is accquired, data:%p refCount:%d", pContext, pContext->fd, ppContext, refCount); return pContext; @@ -174,10 +161,6 @@ HttpContext *httpGetContext(void *ptr) { } void httpReleaseContext(HttpContext *pContext) { - if (!tsHttpServer.fallback) { - ehttpReleaseContext(pContext); - return; - } int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); if (refCount < 0) { httpError("context:%p, is already released, refCount:%d", pContext, refCount); @@ -212,31 +195,25 @@ bool httpInitContext(HttpContext *pContext) { memset(pParser, 0, sizeof(HttpParser)); pParser->pCur = pParser->pLast = pParser->buffer; - if (!tsHttpServer.fallback) { - ehttp_parser_callbacks_t callbacks = { - on_request_line, - on_status_line, - on_header_field, - on_body, - on_end, - on_error - }; - ehttp_parser_conf_t conf = { - .flush_block_size = 0 - }; - pParser->parser = ehttp_parser_create(callbacks, conf, pContext); - pParser->inited = 1; - } + ehttp_parser_callbacks_t callbacks = { + httpParseOnRequestLine, + httpParseOnStatusLine, + httpParseOnHeaderField, + httpParseOnBody, + httpParseOnEnd, + httpParseOnError + }; + ehttp_parser_conf_t conf = { + .flush_block_size = 0 + }; + pParser->parser = ehttp_parser_create(callbacks, conf, pContext); + pParser->inited = 1; - httpDebug("context:%p, fd:%d, ip:%s, accessTimes:%d, parsed:%d", pContext, pContext->fd, pContext->ipstr, - pContext->accessTimes, pContext->parsed); + httpDebug("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed); return true; } void httpCloseContextByApp(HttpContext *pContext) { - if (!tsHttpServer.fallback) { - if (pContext->parsed == false) return; - } pContext->parsed = false; bool keepAlive = true; @@ -249,150 +226,132 @@ void httpCloseContextByApp(HttpContext *pContext) { if (keepAlive) { if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) { - httpDebug("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd, - pContext->ipstr); + httpDebug("context:%p, fd:%d, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) { httpRemoveContextFromEpoll(pContext); - httpDebug("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect", pContext, pContext->fd, - pContext->ipstr); + httpDebug("context:%p, fd:%d, ast state:dropping, keepAlive:true, close connect", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) { - httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd, - pContext->ipstr); + httpDebug("context:%p, fd:%d, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) { httpRemoveContextFromEpoll(pContext); - httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect", pContext, pContext->fd, - pContext->ipstr); + httpDebug("context:%p, fd:%d, last state:ready, keepAlive:true, close connect", pContext, pContext->fd); } else { httpRemoveContextFromEpoll(pContext); - httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect", pContext, pContext->fd, - pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); + httpError("context:%p, fd:%d, last state:%s:%d, keepAlive:true, close connect", pContext, pContext->fd, + httpContextStateStr(pContext->state), pContext->state); } } else { httpRemoveContextFromEpoll(pContext); - httpDebug("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close context", pContext, pContext->fd, - pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); + httpDebug("context:%p, fd:%d, ilast state:%s:%d, keepAlive:false, close context", pContext, pContext->fd, + httpContextStateStr(pContext->state), pContext->state); } - if (tsHttpServer.fallback) httpReleaseContext(pContext); + httpReleaseContext(pContext); } void httpCloseContextByServer(HttpContext *pContext) { - if (!tsHttpServer.fallback) { - if (pContext->closed) return; - pContext->closed = 1; - } if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) { - httpDebug("context:%p, fd:%d, ip:%s, epoll finished, still used by app", pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, epoll finished, still used by app", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) { - httpDebug("context:%p, fd:%d, ip:%s, epoll already finished, wait app finished", pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, epoll already finished, wait app finished", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) { - httpDebug("context:%p, fd:%d, ip:%s, epoll finished, close connect", pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, epoll finished, close connect", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) { - httpDebug("context:%p, fd:%d, ip:%s, epoll finished, will be closed soon", pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, epoll finished, will be closed soon", pContext, pContext->fd); } else { - httpError("context:%p, fd:%d, ip:%s, unknown state:%d", pContext, pContext->fd, pContext->ipstr, pContext->state); + httpError("context:%p, fd:%d, unknown state:%d", pContext, pContext->fd, pContext->state); } pContext->parsed = false; httpRemoveContextFromEpoll(pContext); - if (tsHttpServer.fallback) httpReleaseContext(pContext); } - - - - -static void on_request_line(void *arg, const char *method, const char *target, const char *version, const char *target_raw) { +static void httpParseOnRequestLine(void *arg, const char *method, const char *target, const char *version, const char *target_raw) { HttpContext *pContext = (HttpContext*)arg; HttpParser *pParser = &pContext->parser; int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); - int n = snprintf(pParser->pLast, avail, - "%s %s %s\r\n", method, target_raw, version); - + int n = snprintf(pParser->pLast, avail, "%s %s %s\r\n", method, target_raw, version); char *last = pParser->pLast; do { - if (n>=avail) { - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_request_line(%s,%s,%s,%s), exceeding buffer size", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, method, target, version, target_raw); + if (n >= avail) { + httpDebug("context:%p, fd:%d, request line(%s,%s,%s,%s), exceeding buffer size", pContext, pContext->fd, method, + target, version, target_raw); break; } pParser->bufsize += n; if (!httpGetHttpMethod(pContext)) { - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_request_line(%s,%s,%s,%s), parse http method failed", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, method, target, version, target_raw); + httpDebug("context:%p, fd:%d, request line(%s,%s,%s,%s), parse http method failed", pContext, pContext->fd, + method, target, version, target_raw); break; } if (!httpParseURL(pContext)) { - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_request_line(%s,%s,%s,%s), parse http url failed", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, method, target, version, target_raw); + httpDebug("context:%p, fd:%d, request line(%s,%s,%s,%s), parse http url failed", pContext, pContext->fd, method, + target, version, target_raw); break; } if (!httpParseHttpVersion(pContext)) { - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_request_line(%s,%s,%s,%s), parse http version failed", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, method, target, version, target_raw); + httpDebug("context:%p, fd:%d, request line(%s,%s,%s,%s), parse http version failed", pContext, pContext->fd, + method, target, version, target_raw); break; } if (!httpGetDecodeMethod(pContext)) { - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_request_line(%s,%s,%s,%s), get decode method failed", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, method, target, version, target_raw); + httpDebug("context:%p, fd:%d, request line(%s,%s,%s,%s), get decode method failed", pContext, pContext->fd, + method, target, version, target_raw); break; } - last += n; - pParser->pLast = last; + last += n; + pParser->pLast = last; return; } while (0); pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; } -static void on_status_line(void *arg, const char *version, int status_code, const char *reason_phrase) { +static void httpParseOnStatusLine(void *arg, const char *version, int status_code, const char *reason_phrase) { HttpContext *pContext = (HttpContext*)arg; HttpParser *pParser = &pContext->parser; + httpDebug("context:%p, fd:%d, failed to parse status line ", pContext, pContext->fd); pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; } -static void on_header_field(void *arg, const char *key, const char *val) { +static void httpParseOnHeaderField(void *arg, const char *key, const char *val) { HttpContext *pContext = (HttpContext*)arg; HttpParser *pParser = &pContext->parser; if (pParser->failed) return; - D("==key:[%s], val:[%s]==", key, val); - int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); - int n = snprintf(pParser->pLast, avail, - "%s: %s\r\n", key, val); - + httpDebug("context:%p, fd:%d, key:%s val:%s", pContext, pContext->fd, key, val); + int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); + int n = snprintf(pParser->pLast, avail, "%s: %s\r\n", key, val); char *last = pParser->pLast; do { - if (n>=avail) { - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_header_field(%s,%s), exceeding buffer size", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, key, val); + if (n >= avail) { + httpDebug("context:%p, fd:%d, header field(%s,%s), exceeding buffer size", pContext, pContext->fd, key, val); break; } pParser->bufsize += n; - pParser->pCur = pParser->pLast + n; + pParser->pCur = pParser->pLast + n; if (!httpParseHead(pContext)) { - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, on_header_field(%s,%s), parse head failed", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, key, val); + httpDebug("context:%p, fd:%d, header field(%s,%s), parse failed", pContext, pContext->fd, key, val); break; } - last += n; - pParser->pLast = last; + last += n; + pParser->pLast = last; return; } while (0); pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; } -static void on_body(void *arg, const char *chunk, size_t len) { +static void httpParseOnBody(void *arg, const char *chunk, size_t len) { HttpContext *pContext = (HttpContext*)arg; HttpParser *pParser = &pContext->parser; @@ -404,16 +363,18 @@ static void on_body(void *arg, const char *chunk, size_t len) { } int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); - if (len+1>=avail) { + if (len + 1 >= avail) { + httpError("context:%p, fd:%d, failed parse body, exceeding buffer size", pContext, pContext->fd); pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; return; } + memcpy(pParser->pLast, chunk, len); pParser->pLast += len; pParser->data.len += len; } -static void on_end(void *arg) { +static void httpParseOnEnd(void *arg) { HttpContext *pContext = (HttpContext*)arg; HttpParser *pParser = &pContext->parser; @@ -424,47 +385,14 @@ static void on_end(void *arg) { if (!pContext->parsed) { pContext->parsed = true; } + + httpDebug("context:%p, fd:%d, parse success", pContext, pContext->fd); } -static void on_error(void *arg, int status_code) { - HttpContext *pContext = (HttpContext*)arg; - HttpParser *pParser = &pContext->parser; +static void httpParseOnError(void *arg, int status_code) { + HttpContext *pContext = (HttpContext *)arg; + HttpParser * pParser = &pContext->parser; + httpError("context:%p, fd:%d, failed to parse, status_code:%d", pContext, pContext->fd, status_code); pParser->failed |= EHTTP_CONTEXT_PARSER_FAILED; } - -static void httpMightDestroyContext(void *data) { - HttpContext *pContext = *(HttpContext **)data; - if (!tsHttpServer.fallback) { - httpRemoveContextFromEpoll(pContext); - ehttpDecContextRef(&pContext); - return; - } - httpDestroyContext(data); -} - -static void ehttpReleaseContext(HttpContext *pContext) { - HttpContext **ppContext = pContext->ppContext; - - if (tsHttpServer.contextCache != NULL) { - taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); - } else { - httpDebug("context:%p, won't be destroyed for cache is already released", pContext); - // httpDestroyContext((void **)(&ppContext)); - } -} - -void ehttpIncContextRef(HttpContext *pContext) { - if (tsHttpServer.fallback) return; - atomic_add_fetch_32(&pContext->refCount, 1); -} - -void ehttpDecContextRef(HttpContext **ppContext) { - if (tsHttpServer.fallback) return; - HttpContext *pContext = *ppContext; - int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); - if (refCount>0) return; - EQ_ASSERT(refCount==0); - httpDestroyContext(ppContext); -} - diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 407d19b307..59b3268392 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -103,15 +103,13 @@ bool httpParseHttpVersion(HttpContext* pContext) { HttpParser* pParser = &pContext->parser; char* pEnd = strchr(pParser->pLast, '1'); if (pEnd == NULL) { - httpError("context:%p, fd:%d, ip:%s, can't find http version at position:%s", pContext, pContext->fd, - pContext->ipstr, pParser->pLast); + httpError("context:%p, fd:%d, can't find http version at position:%s", pContext, pContext->fd, pParser->pLast); httpSendErrorResp(pContext, HTTP_PARSE_HTTP_VERSION_ERROR); return false; } if (*(pEnd + 1) != '.') { - httpError("context:%p, fd:%d, ip:%s, can't find http version at position:%s", pContext, pContext->fd, - pContext->ipstr, pParser->pLast); + httpError("context:%p, fd:%d, can't find http version at position:%s", pContext, pContext->fd, pParser->pLast); httpSendErrorResp(pContext, HTTP_PARSE_HTTP_VERSION_ERROR); return false; } @@ -125,8 +123,7 @@ bool httpParseHttpVersion(HttpContext* pContext) { else pContext->httpVersion = HTTP_VERSION_10; - httpDebug("context:%p, fd:%d, ip:%s, httpVersion:1.%d", pContext, pContext->fd, pContext->ipstr, - pContext->httpVersion); + httpDebug("context:%p, fd:%d, httpVersion:1.%d", pContext, pContext->fd, pContext->httpVersion); return true; } @@ -147,18 +144,20 @@ bool httpGetNextLine(HttpContext* pContext) { bool httpGetHttpMethod(HttpContext* pContext) { HttpParser* pParser = &pContext->parser; - char* pSeek = strchr(pParser->pLast, ' '); + if (pSeek == NULL) { + httpError("context:%p, fd:%d, failed to parse httpMethod", pContext, pContext->fd); httpSendErrorResp(pContext, HTTP_PARSE_HTTP_METHOD_ERROR); return false; } + pParser->method.pos = pParser->pLast; pParser->method.len = (int16_t)(pSeek - pParser->pLast); pParser->method.pos[pParser->method.len] = 0; pParser->pLast = pSeek + 1; - httpTrace("context:%p, fd:%d, ip:%s, httpMethod:%s", pContext, pContext->fd, pContext->ipstr, pParser->method.pos); + httpTrace("context:%p, fd:%d, httpMethod:%s", pContext, pContext->fd, pParser->method.pos); return true; } @@ -176,8 +175,8 @@ bool httpGetDecodeMethod(HttpContext* pContext) { return true; } - httpError("context:%p, fd:%d, ip:%s, error:the url is not support, method:%s, path:%s", - pContext, pContext->fd, pContext->ipstr, pParser->method.pos, pParser->path[0].pos); + httpError("context:%p, fd:%d, error:the url is not support, method:%s, path:%s", + pContext, pContext->fd, pParser->method.pos, pParser->path[0].pos); httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL); return false; @@ -187,23 +186,23 @@ bool httpParseHead(HttpContext* pContext) { HttpParser* pParser = &pContext->parser; if (strncasecmp(pParser->pLast, "Content-Length: ", 16) == 0) { pParser->data.len = (int32_t)atoi(pParser->pLast + 16); - httpTrace("context:%p, fd:%d, ip:%s, Content-Length:%d", pContext, pContext->fd, pContext->ipstr, + httpTrace("context:%p, fd:%d, Content-Length:%d", pContext, pContext->fd, pParser->data.len); } else if (strncasecmp(pParser->pLast, "Accept-Encoding: ", 17) == 0) { if (tsHttpEnableCompress && strstr(pParser->pLast + 17, "gzip") != NULL) { pContext->acceptEncoding = HTTP_COMPRESS_GZIP; - httpTrace("context:%p, fd:%d, ip:%s, Accept-Encoding:gzip", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, Accept-Encoding:gzip", pContext, pContext->fd); } else { pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY; - httpTrace("context:%p, fd:%d, ip:%s, Accept-Encoding:identity", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, Accept-Encoding:identity", pContext, pContext->fd); } } else if (strncasecmp(pParser->pLast, "Content-Encoding: ", 18) == 0) { if (strstr(pParser->pLast + 18, "gzip") != NULL) { pContext->contentEncoding = HTTP_COMPRESS_GZIP; - httpTrace("context:%p, fd:%d, ip:%s, Content-Encoding:gzip", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, Content-Encoding:gzip", pContext, pContext->fd); } else { pContext->contentEncoding = HTTP_COMPRESS_IDENTITY; - httpTrace("context:%p, fd:%d, ip:%s, Content-Encoding:identity", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, Content-Encoding:identity", pContext, pContext->fd); } } else if (strncasecmp(pParser->pLast, "Connection: ", 12) == 0) { if (strncasecmp(pParser->pLast + 12, "Keep-Alive", 10) == 0) { @@ -211,8 +210,7 @@ bool httpParseHead(HttpContext* pContext) { } else { pContext->httpKeepAlive = HTTP_KEEPALIVE_DISABLE; } - httpTrace("context:%p, fd:%d, ip:%s, keepAlive:%d", pContext, pContext->fd, pContext->ipstr, - pContext->httpKeepAlive); + httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->httpKeepAlive); } else if (strncasecmp(pParser->pLast, "Transfer-Encoding: ", 19) == 0) { if (strncasecmp(pParser->pLast + 19, "chunked", 7) == 0) { pContext->httpChunked = HTTP_CHUNKED; @@ -244,129 +242,6 @@ bool httpParseHead(HttpContext* pContext) { return true; } -bool httpParseChunkedBody(HttpContext* pContext, HttpParser* pParser, bool test) { - char* pEnd = pParser->buffer + pParser->bufsize; - char* pRet = pParser->data.pos; - char* pSize = pParser->data.pos; - size_t size = strtoul(pSize, NULL, 16); - if (size <= 0) return false; - - while (size > 0) { - char* pData = strstr(pSize, "\r\n"); - if (pData == NULL || pData >= pEnd) return false; - pData += 2; - - pSize = strstr(pData, "\r\n"); - if (pSize == NULL || pSize >= pEnd) return false; - if ((size_t)(pSize - pData) != size) return false; - pSize += 2; - - if (!test) { - memmove(pRet, pData, size); - pRet += size; - } - - size = strtoul(pSize, NULL, 16); - } - - if (!test) { - *pRet = '\0'; - } - - return true; -} - -int httpReadChunkedBody(HttpContext* pContext, HttpParser* pParser) { - bool parsedOk = httpParseChunkedBody(pContext, pParser, true); - if (parsedOk) { - httpParseChunkedBody(pContext, pParser, false); - return HTTP_CHECK_BODY_SUCCESS; - } else { - httpTrace("context:%p, fd:%d, ip:%s, chunked body not finished, continue read", pContext, pContext->fd, pContext->ipstr); - if (httpReadDataImp(pContext) != HTTP_READ_DATA_SUCCESS) { - httpError("context:%p, fd:%d, ip:%s, read chunked request error", pContext, pContext->fd, pContext->ipstr); - return HTTP_CHECK_BODY_ERROR; - } else { - return HTTP_CHECK_BODY_CONTINUE; - } - } -} - -int httpReadUnChunkedBody(HttpContext* pContext, HttpParser* pParser) { - int dataReadLen = pParser->bufsize - (int)(pParser->data.pos - pParser->buffer); - if (dataReadLen > pParser->data.len) { - httpError("context:%p, fd:%d, ip:%s, un-chunked body length invalid, read size:%d dataReadLen:%d > pContext->data.len:%d", - pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len); - return HTTP_CHECK_BODY_ERROR; - } else if (dataReadLen < pParser->data.len) { - httpTrace("context:%p, fd:%d, ip:%s, un-chunked body not finished, read size:%d dataReadLen:%d < pContext->data.len:%d, continue read", - pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len); - return HTTP_CHECK_BODY_CONTINUE; - } else { - return HTTP_CHECK_BODY_SUCCESS; - } -} - -bool httpParseRequest(HttpContext* pContext) { - HttpParser *pParser = &pContext->parser; - if (pContext->parsed) { - return true; - } - - httpTraceL("context:%p, fd:%d, ip:%s, thread:%s, numOfContexts:%d, read size:%d, raw data:\n%s", pContext, - pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfContexts, - pContext->parser.bufsize, pContext->parser.buffer); - - if (!httpGetHttpMethod(pContext)) { - return false; - } - - if (!httpParseURL(pContext)) { - return false; - } - - if (!httpParseHttpVersion(pContext)) { - return false; - } - - if (!httpGetDecodeMethod(pContext)) { - return false; - } - - do { - if (!httpGetNextLine(pContext)) { - return false; - } - - // Empty line, end of the HTTP HEAD - if (pParser->pCur - pParser->pLast == 1) { - pParser->data.pos = ++pParser->pCur; - break; - } - - if (!httpParseHead(pContext)) { - return false; - } - - pParser->pLast = ++pParser->pCur; - } while (1); - - httpDebug("context:%p, fd:%d, ip:%s, parse http head ok", pContext, pContext->fd, pContext->ipstr); - - pContext->parsed = true; - return true; -} - -int httpCheckReadCompleted(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - - if (pContext->httpChunked == HTTP_UNCUNKED) { - return httpReadUnChunkedBody(pContext, pParser); - } else { - return httpReadChunkedBody(pContext, pParser); - } -} - bool httpDecodeRequest(HttpContext* pContext) { HttpParser* pParser = &pContext->parser; if (pParser->pMethod->decodeFp == NULL) { @@ -380,17 +255,16 @@ bool httpDecodeRequest(HttpContext* pContext) { * Process the request from http pServer */ bool httpProcessData(HttpContext* pContext) { - if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_HANDLING)) { - httpDebug("context:%p, fd:%d, ip:%s, state:%s not in ready state, stop process request", - pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state)); + httpDebug("context:%p, fd:%d, state:%s not in ready state, stop process request", pContext, pContext->fd, + httpContextStateStr(pContext->state)); httpCloseContextByApp(pContext); return false; } // handle Cross-domain request if (strcmp(pContext->parser.method.pos, "OPTIONS") == 0) { - httpDebug("context:%p, fd:%d, ip:%s, process options request", pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, process options request", pContext, pContext->fd); httpSendOptionResp(pContext, "process options request success"); } else { if (!httpDecodeRequest(pContext)) { diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 4748f03b66..e200efbcef 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -52,14 +52,12 @@ int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz) { } if (len < 0) { - httpDebug("context:%p, fd:%d, ip:%s, socket write errno:%d, times:%d", - pContext, pContext->fd, pContext->ipstr, errno, countWait); + httpDebug("context:%p, fd:%d, socket write errno:%d, times:%d", pContext, pContext->fd, errno, countWait); if (++countWait > HTTP_WRITE_RETRY_TIMES) break; taosMsleep(HTTP_WRITE_WAIT_TIME_MS); continue; } else if (len == 0) { - httpDebug("context:%p, fd:%d, ip:%s, socket write errno:%d, connect already closed", - pContext, pContext->fd, pContext->ipstr, errno); + httpDebug("context:%p, fd:%d, socket write errno:%d, connect already closed", pContext, pContext->fd, errno); break; } else { countWait = 0; @@ -70,14 +68,13 @@ int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz) { return writeLen; } -int httpWriteBuf(struct HttpContext *pContext, const char *buf, int sz) { +int httpWriteBuf(struct HttpContext* pContext, const char* buf, int sz) { int writeSz = httpWriteBufByFd(pContext, buf, sz); if (writeSz != sz) { - httpError("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, failed to send response:\n%s", - pContext, pContext->fd, pContext->ipstr, sz, writeSz, buf); + httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response:\n%s", pContext, pContext->fd, sz, + writeSz, buf); } else { - httpTrace("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, response:\n%s", pContext, pContext->fd, - pContext->ipstr, sz, writeSz, buf); + httpTrace("context:%p, fd:%d, dataSize:%d, writeSize:%d, response:\n%s", pContext, pContext->fd, sz, writeSz, buf); } return writeSz; @@ -86,8 +83,8 @@ int httpWriteBuf(struct HttpContext *pContext, const char *buf, int sz) { int httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int sz) { int writeSz = httpWriteBufByFd(pContext, buf, sz); if (writeSz != sz) { - httpError("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, failed to send response", - pContext, pContext->fd, pContext->ipstr, sz, writeSz); + httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response", pContext, pContext->fd, sz, + writeSz); } return writeSz; @@ -99,7 +96,7 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { uint64_t srcLen = (uint64_t) (buf->lst - buf->buf); if (buf->pContext->fd <= 0) { - httpTrace("context:%p, fd:%d, ip:%s, write json body error", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); + httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd); buf->pContext->fd = -1; } @@ -113,12 +110,12 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { if (buf->pContext->acceptEncoding == HTTP_COMPRESS_IDENTITY) { if (buf->lst == buf->buf) { - httpTrace("context:%p, fd:%d, ip:%s, no data need dump", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); + httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd); return 0; // there is no data to dump. } else { int len = sprintf(sLen, "%lx\r\n", srcLen); - httpTrace("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", response:\n%s", - buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, buf->buf); + httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd, + srcLen, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int) srcLen); } @@ -129,18 +126,18 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { if (ret == 0) { if (compressBufLen > 0) { int len = sprintf(sLen, "%x\r\n", compressBufLen); - httpTrace("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", - buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, compressBufLen, isTheLast, buf->buf); + httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", + buf->pContext, buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); - remain = httpWriteBufNoTrace(buf->pContext, (const char *) compressBuf, (int) compressBufLen); + remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, (int)compressBufLen); } else { - httpTrace("context:%p, fd:%d, ip:%s, last:%d, compress already dumped, response:\n%s", - buf->pContext, buf->pContext->fd, buf->pContext->ipstr, isTheLast, buf->buf); + httpTrace("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext, + buf->pContext->fd, isTheLast, buf->buf); return 0; // there is no data to dump. } } else { - httpError("context:%p, fd:%d, ip:%s, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s", - buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, isTheLast, ret, buf->buf); + httpError("context:%p, fd:%d, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s", + buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf); return 0; } } @@ -173,7 +170,7 @@ void httpWriteJsonBufHead(JsonBuf* buf) { void httpWriteJsonBufEnd(JsonBuf* buf) { if (buf->pContext->fd <= 0) { - httpTrace("context:%p, fd:%d, ip:%s, json buf fd is 0", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); + httpTrace("context:%p, fd:%d, json buf fd is 0", buf->pContext, buf->pContext->fd); buf->pContext->fd = -1; } @@ -192,7 +189,7 @@ void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext) { httpGzipCompressInit(buf->pContext); } - httpDebug("context:%p, fd:%d, ip:%s, json buffer initialized", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); + httpDebug("context:%p, fd:%d, json buffer initialized", buf->pContext, buf->pContext->fd); } void httpJsonItemToken(JsonBuf* buf) { diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index f53aff7831..a7c17dfdbb 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -46,7 +46,7 @@ const char *httpRespTemplate[] = { }; static void httpSendErrorRespImp(HttpContext *pContext, int httpCode, char *httpCodeStr, int errNo, char *desc) { - httpError("context:%p, fd:%d, ip:%s, code:%d, error:%s", pContext, pContext->fd, pContext->ipstr, httpCode, desc); + httpError("context:%p, fd:%d, code:%d, error:%s", pContext, pContext->fd, httpCode, desc); char head[512] = {0}; char body[512] = {0}; @@ -169,7 +169,7 @@ void httpSendErrorRespWithDesc(HttpContext *pContext, int errNo, char *desc) { httpCodeStr = "Bad Request"; break; default: - httpError("context:%p, fd:%d, ip:%s, error:%d not recognized", pContext, pContext->fd, pContext->ipstr, errNo); + httpError("context:%p, fd:%d, error:%d not recognized", pContext, pContext->fd, errNo); break; } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 6802d3624a..614cc92700 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -31,7 +31,7 @@ #define EPOLLWAKEUP (1u << 29) #endif -static bool ehttpReadData(HttpContext *pContext); +static bool httpReadData(HttpContext *pContext); static void httpStopThread(HttpThread* pThread) { pThread->stop = true; @@ -73,43 +73,9 @@ void httpCleanUpConnect() { httpDebug("http server:%s is cleaned up", pServer->label); } -int httpReadDataImp(HttpContext *pContext) { - HttpParser *pParser = &pContext->parser; - - while (pParser->bufsize <= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) { - int nread = (int)taosReadSocket(pContext->fd, pParser->buffer + pParser->bufsize, HTTP_STEP_SIZE); - if (nread >= 0 && nread < HTTP_STEP_SIZE) { - pParser->bufsize += nread; - break; - } else if (nread < 0) { - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { - httpDebug("context:%p, fd:%d, ip:%s, read from socket error:%d, wait another event", - pContext, pContext->fd, pContext->ipstr, errno); - break; - } else { - httpError("context:%p, fd:%d, ip:%s, read from socket error:%d, close connect", - pContext, pContext->fd, pContext->ipstr, errno); - return HTTP_READ_DATA_FAILED; - } - } else { - pParser->bufsize += nread; - } - - if (pParser->bufsize >= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) { - httpError("context:%p, fd:%d, ip:%s, thread:%s, request big than:%d", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, HTTP_BUFFER_SIZE); - return HTTP_REQUSET_TOO_BIG; - } - } - - pParser->buffer[pParser->bufsize] = 0; - - return HTTP_READ_DATA_SUCCESS; -} - static bool httpDecompressData(HttpContext *pContext) { if (pContext->contentEncoding != HTTP_COMPRESS_GZIP) { - httpTraceL("context:%p, fd:%d, ip:%s, content:%s", pContext, pContext->fd, pContext->ipstr, pContext->parser.data.pos); + httpTraceL("context:%p, fd:%d, content:%s", pContext, pContext->fd, pContext->parser.data.pos); return true; } @@ -125,64 +91,18 @@ static bool httpDecompressData(HttpContext *pContext) { if (ret == 0) { memcpy(pContext->parser.data.pos, decompressBuf, decompressBufLen); pContext->parser.data.pos[decompressBufLen] = 0; - httpTraceL("context:%p, fd:%d, ip:%s, rawSize:%d, decompressSize:%d, content:%s", pContext, pContext->fd, - pContext->ipstr, pContext->parser.data.len, decompressBufLen, decompressBuf); + httpTraceL("context:%p, fd:%d, rawSize:%d, decompressSize:%d, content:%s", pContext, pContext->fd, + pContext->parser.data.len, decompressBufLen, decompressBuf); pContext->parser.data.len = decompressBufLen; } else { - httpError("context:%p, fd:%d, ip:%s, failed to decompress data, rawSize:%d, error:%d", - pContext, pContext->fd, pContext->ipstr, pContext->parser.data.len, ret); + httpError("context:%p, fd:%d, failed to decompress data, rawSize:%d, error:%d", pContext, pContext->fd, + pContext->parser.data.len, ret); } free(decompressBuf); return ret == 0; } -static bool httpReadData(HttpContext *pContext) { - if (!tsHttpServer.fallback) return ehttpReadData(pContext); - - if (!pContext->parsed) { - httpInitContext(pContext); - } - - int32_t code = httpReadDataImp(pContext); - if (code != HTTP_READ_DATA_SUCCESS) { - if (code == HTTP_READ_DATA_FAILED) { - httpReleaseContext(pContext); - } else { - httpSendErrorResp(pContext, code); - httpNotifyContextClose(pContext); - } - return false; - } - - if (!httpParseRequest(pContext)) { - httpNotifyContextClose(pContext); - return false; - } - - int ret = httpCheckReadCompleted(pContext); - if (ret == HTTP_CHECK_BODY_CONTINUE) { - //httpDebug("context:%p, fd:%d, ip:%s, not finished yet, wait another event", pContext, pContext->fd, pContext->ipstr); - httpReleaseContext(pContext); - return false; - } else if (ret == HTTP_CHECK_BODY_SUCCESS){ - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, read size:%d, dataLen:%d", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->parser.bufsize, pContext->parser.data.len); - if (httpDecompressData(pContext)) { - return true; - } else { - httpNotifyContextClose(pContext); - httpReleaseContext(pContext); - return false; - } - } else { - httpError("context:%p, fd:%d, ip:%s, failed to read http body, close connect", pContext, pContext->fd, pContext->ipstr); - httpNotifyContextClose(pContext); - httpReleaseContext(pContext); - return false; - } -} - static void httpProcessHttpData(void *param) { HttpServer *pServer = &tsHttpServer; HttpThread *pThread = (HttpThread *)param; @@ -194,8 +114,6 @@ static void httpProcessHttpData(void *param) { sigaddset(&set, SIGPIPE); pthread_sigmask(SIG_SETMASK, &set, NULL); - elog_set_thread_name("httpProcessHttpData"); - while (1) { struct epoll_event events[HTTP_MAX_EVENTS]; //-1 means uncertainty, 0-nowait, 1-wait 1 ms, set it from -1 to 1 @@ -215,66 +133,51 @@ static void httpProcessHttpData(void *param) { continue; } - ehttpIncContextRef(pContext); - if (events[i].events & EPOLLPRI) { - httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLPRI events occured, accessed:%d, close connect", - pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); + httpDebug("context:%p, fd:%d, state:%s, EPOLLPRI events occured, accessed:%d, close connect", pContext, + pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); httpCloseContextByServer(pContext); - if (!tsHttpServer.fallback) httpReleaseContext(pContext); - ehttpDecContextRef(&pContext); continue; } if (events[i].events & EPOLLRDHUP) { - httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLRDHUP events occured, accessed:%d, close connect", - pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); + httpDebug("context:%p, fd:%d, state:%s, EPOLLRDHUP events occured, accessed:%d, close connect", pContext, + pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); httpCloseContextByServer(pContext); - httpReleaseContext(pContext); - ehttpDecContextRef(&pContext); continue; } if (events[i].events & EPOLLERR) { - httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLERR events occured, accessed:%d, close connect", - pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); + httpDebug("context:%p, fd:%d, state:%s, EPOLLERR events occured, accessed:%d, close connect", pContext, + pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); httpCloseContextByServer(pContext); - if (!tsHttpServer.fallback) httpReleaseContext(pContext); - ehttpDecContextRef(&pContext); continue; } if (events[i].events & EPOLLHUP) { - httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLHUP events occured, accessed:%d, close connect", - pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); + httpDebug("context:%p, fd:%d, state:%s, EPOLLHUP events occured, accessed:%d, close connect", pContext, + pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); httpCloseContextByServer(pContext); - if (!tsHttpServer.fallback) httpReleaseContext(pContext); - ehttpDecContextRef(&pContext); continue; } if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) { - httpDebug("context:%p, fd:%d, ip:%s, state:%s, not in ready state, ignore read events", - pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state)); + httpDebug("context:%p, fd:%d, state:%s, not in ready state, ignore read events", pContext, pContext->fd, + httpContextStateStr(pContext->state)); httpReleaseContext(pContext); - ehttpDecContextRef(&pContext); continue; } if (pServer->status != HTTP_SERVER_RUNNING) { - httpDebug("context:%p, fd:%d, ip:%s, state:%s, server is not running, accessed:%d, close connect", pContext, - pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); + httpDebug("context:%p, fd:%d, state:%s, server is not running, accessed:%d, close connect", pContext, + pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); httpSendErrorResp(pContext, HTTP_SERVER_OFFLINE); httpNotifyContextClose(pContext); - if (!tsHttpServer.fallback) httpReleaseContext(pContext); - ehttpDecContextRef(&pContext); } else { if (httpReadData(pContext)) { (*(pThread->processData))(pContext); atomic_fetch_add_32(&pServer->requestNum, 1); } - if (!tsHttpServer.fallback) httpReleaseContext(pContext); - ehttpDecContextRef(&pContext); } } } @@ -355,8 +258,7 @@ static void *httpAcceptHttpConnection(void *arg) { httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd, pContext->ipstr, pThread->label, strerror(errno)); taosClose(pContext->fd); - if (tsHttpServer.fallback) httpReleaseContext(pContext); - ehttpDecContextRef(&pContext); + httpReleaseContext(pContext); continue; } @@ -430,12 +332,10 @@ bool httpInitConnect() { return true; } - - - -static bool ehttpReadData(HttpContext *pContext) { +static bool httpReadData(HttpContext *pContext) { HttpParser *pParser = &pContext->parser; EQ_ASSERT(!pContext->parsed); + if (!pParser->parser) { if (!pParser->inited) { httpInitContext(pContext); @@ -448,61 +348,44 @@ static bool ehttpReadData(HttpContext *pContext) { pContext->accessTimes++; pContext->lastAccessTime = taosGetTimestampSec(); - char buf[HTTP_STEP_SIZE+1] = {0}; - int nread = (int)taosReadSocket(pContext->fd, buf, sizeof(buf)); + char buf[HTTP_STEP_SIZE + 1] = {0}; + int nread = (int)taosReadSocket(pContext->fd, buf, sizeof(buf)); if (nread > 0) { buf[nread] = '\0'; - if (strstr(buf, "GET ")==buf && !strchr(buf, '\r') && !strchr(buf, '\n')) { - D("==half of request line received:\n%s\n==", buf); - } - if (ehttp_parser_parse(pParser->parser, buf, nread)) { - D("==parsing failed=="); - httpCloseContextByServer(pContext); + httpError("context:%p, fd:%d, init parse failed, close connect", pContext, pContext->fd); + httpNotifyContextClose(pContext); return false; } if (pContext->parser.failed) { - D("==parsing failed: [0x%x]==", pContext->parser.failed); + httpError("context:%p, fd:%d, parse failed, close connect", pContext, pContext->fd); httpNotifyContextClose(pContext); return false; } + if (pContext->parsed) { - // int ret = httpCheckReadCompleted(pContext); - // already done in ehttp_parser - int ret = HTTP_CHECK_BODY_SUCCESS; - if (ret == HTTP_CHECK_BODY_CONTINUE) { - //httpDebug("context:%p, fd:%d, ip:%s, not finished yet, wait another event", pContext, pContext->fd, pContext->ipstr); - return false; - } else if (ret == HTTP_CHECK_BODY_SUCCESS){ - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, read size:%d, dataLen:%d", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->parser.bufsize, pContext->parser.data.len); - if (httpDecompressData(pContext)) { - return true; - } else { - httpNotifyContextClose(pContext); - return false; - } + httpDebug("context:%p, fd:%d, read size:%d, dataLen:%d", pContext, pContext->fd, pContext->parser.bufsize, + pContext->parser.data.len); + if (httpDecompressData(pContext)) { + return true; } else { - httpError("context:%p, fd:%d, ip:%s, failed to read http body, close connect", pContext, pContext->fd, pContext->ipstr); httpNotifyContextClose(pContext); return false; } } + return pContext->parsed; } else if (nread < 0) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { - httpDebug("context:%p, fd:%d, ip:%s, read from socket error:%d, wait another event", - pContext, pContext->fd, pContext->ipstr, errno); - return false; // later again + httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno); + return false; // later again } else { - httpError("context:%p, fd:%d, ip:%s, read from socket error:%d, close connect", - pContext, pContext->fd, pContext->ipstr, errno); + httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno); return false; } } else { - // eof + httpError("context:%p, fd:%d, nread:%d, wait another event", pContext, pContext->fd, nread); return false; } } - diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index f19679e072..4549192407 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -39,15 +39,15 @@ void httpCreateSession(HttpContext *pContext, void *taos) { // taosCacheRelease(server->sessionCache, (void **)&temp, false); if (pContext->session == NULL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, error:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, + httpError("context:%p, fd:%d, user:%s, error:%s", pContext, pContext->fd, pContext->user, httpMsg[HTTP_SESSION_FULL]); taos_close(taos); pthread_mutex_unlock(&server->serverMutex); return; } - httpDebug("context:%p, fd:%d, ip:%s, user:%s, create a new session:%p:%p sessionRef:%d", pContext, pContext->fd, - pContext->ipstr, pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount); + httpDebug("context:%p, fd:%d, user:%s, create a new session:%p:%p sessionRef:%d", pContext, pContext->fd, + pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount); pthread_mutex_unlock(&server->serverMutex); } @@ -61,11 +61,10 @@ static void httpFetchSessionImp(HttpContext *pContext) { pContext->session = taosCacheAcquireByKey(server->sessionCache, sessionId, len); if (pContext->session != NULL) { atomic_add_fetch_32(&pContext->session->refCount, 1); - httpDebug("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, sessionRef:%d", pContext, pContext->fd, - pContext->ipstr, pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount); + httpDebug("context:%p, fd:%d, user:%s, find an exist session:%p:%p, sessionRef:%d", pContext, pContext->fd, + pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount); } else { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, session not found", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpDebug("context:%p, fd:%d, user:%s, session not found", pContext, pContext->fd, pContext->user); } pthread_mutex_unlock(&server->serverMutex); diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 07cdea1380..cbaa0b36d8 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -56,18 +56,18 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n if (isContinue) { // retrieve next batch of rows - httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, numOfRows, sql); + httpDebug("context:%p, fd:%d, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s", pContext, + pContext->fd, pContext->user, multiCmds->pos, numOfRows, sql); taos_fetch_rows_a(result, httpProcessMultiSqlRetrieveCallBack, param); } else { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, numOfRows, sql); + httpDebug("context:%p, fd:%d, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s", pContext, pContext->fd, + pContext->user, multiCmds->pos, numOfRows, sql); if (numOfRows < 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(numOfRows), sql); - } - + httpError("context:%p, fd:%d, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", pContext, pContext->fd, + pContext->user, multiCmds->pos, tstrerror(numOfRows), sql); + } + taos_free_result(result); if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) { @@ -94,20 +94,20 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { char * sql = httpGetCmdsString(pContext, singleCmd->sql); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - httpWarn("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, code:%s:inprogress, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(code), sql); + httpWarn("context:%p, fd:%d, user:%s, process pos:%d, code:%s:inprogress, sql:%s", pContext, pContext->fd, + pContext->user, multiCmds->pos, tstrerror(code), sql); return; } if (code < 0) { if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code)) { singleCmd->code = code; - httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos jump to:%d, last code:%s, last sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos + 1, tstrerror(code), sql); + httpDebug("context:%p, fd:%d, user:%s, process pos jump to:%d, last code:%s, last sql:%s", pContext, pContext->fd, + pContext->user, multiCmds->pos + 1, tstrerror(code), sql); } else { singleCmd->code = code; - httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, error code:%s, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(code), sql); + httpError("context:%p, fd:%d, user:%s, process pos:%d, error code:%s, sql:%s", pContext, pContext->fd, + pContext->user, multiCmds->pos, tstrerror(code), sql); if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN) { if (encode->startJsonFp) (encode->startJsonFp)(pContext, singleCmd, result); @@ -125,8 +125,8 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { if (isUpdate) { // not select or show commands int affectRows = taos_affected_rows(result); - httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, affect rows:%d, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, affectRows, sql); + httpDebug("context:%p, fd:%d, user:%s, process pos:%d, affect rows:%d, sql:%s", pContext, pContext->fd, + pContext->user, multiCmds->pos, affectRows, sql); singleCmd->code = 0; @@ -151,8 +151,8 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { taos_free_result(result); httpProcessMultiSql(pContext); } else { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, start retrieve, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, sql); + httpDebug("context:%p, fd:%d, user:%s, process pos:%d, start retrieve, sql:%s", pContext, pContext->fd, + pContext->user, multiCmds->pos, sql); if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->startJsonFp) { (encode->startJsonFp)(pContext, singleCmd, result); @@ -170,8 +170,8 @@ void httpProcessMultiSql(HttpContext *pContext) { HttpEncodeMethod *encode = pContext->encodeMethod; if (multiCmds->pos >= multiCmds->size) { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, size:%d, stop mulit-querys", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, multiCmds->size); + httpDebug("context:%p, fd:%d, user:%s, process pos:%d, size:%d, stop mulit-querys", pContext, pContext->fd, + pContext->user, multiCmds->pos, multiCmds->size); if (encode->cleanJsonFp) { (encode->cleanJsonFp)(pContext); } @@ -182,8 +182,8 @@ void httpProcessMultiSql(HttpContext *pContext) { HttpSqlCmd *cmd = multiCmds->cmds + multiCmds->pos; char *sql = httpGetCmdsString(pContext, cmd->sql); - httpTraceL("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd, - pContext->ipstr, pContext->user, multiCmds->pos, sql); + httpTraceL("context:%p, fd:%d, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd, pContext->user, + multiCmds->pos, sql); taosNotePrintHttp(sql); taos_query_a(pContext->session->taos, sql, httpProcessMultiSqlCallBack, (void *)pContext); } @@ -197,8 +197,8 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) { return; } - httpDebug("context:%p, fd:%d, ip:%s, user:%s, start multi-querys pos:%d, size:%d", pContext, pContext->fd, - pContext->ipstr, pContext->user, multiCmds->pos, multiCmds->size); + httpDebug("context:%p, fd:%d, user:%s, start multi-querys pos:%d, size:%d", pContext, pContext->fd, pContext->user, + multiCmds->pos, multiCmds->size); HttpEncodeMethod *encode = pContext->encodeMethod; if (encode->initJsonFp) { (encode->initJsonFp)(pContext); @@ -226,24 +226,23 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int #if 0 // todo refactor if (tscResultsetFetchCompleted(result)) { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, resultset fetch completed", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpDebug("context:%p, fd:%d, user:%s, resultset fetch completed", pContext, pContext->fd, pContext->user); isContinue = false; } #endif if (isContinue) { // retrieve next batch of rows - httpDebug("context:%p, fd:%d, ip:%s, user:%s, continue retrieve, numOfRows:%d", pContext, pContext->fd, - pContext->ipstr, pContext->user, numOfRows); + httpDebug("context:%p, fd:%d, user:%s, continue retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user, + numOfRows); taos_fetch_rows_a(result, httpProcessSingleSqlRetrieveCallBack, param); } else { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->ipstr, - pContext->user, numOfRows); + httpDebug("context:%p, fd:%d, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user, + numOfRows); if (numOfRows < 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->ipstr, - pContext->user, tstrerror(numOfRows)); + httpError("context:%p, fd:%d, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->user, + tstrerror(numOfRows)); } taos_free_result(result); @@ -269,20 +268,20 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo HttpEncodeMethod *encode = pContext->encodeMethod; if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s:inprogress, sqlObj:%p", - pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), (SSqlObj *)result); + httpError("context:%p, fd:%d, user:%s, query error, taos:%p, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd, + pContext->user, pContext->session->taos, tstrerror(code), (SSqlObj *)result); return; } if (code < 0) { SSqlObj *pObj = (SSqlObj *)result; if (code == TSDB_CODE_TSC_INVALID_SQL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s, sqlObj:%p, error:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj, pObj->cmd.payload); + httpError("context:%p, fd:%d, user:%s, query error, taos:%p, code:%s, sqlObj:%p, error:%s", pContext, + pContext->fd, pContext->user, pContext->session->taos, tstrerror(code), pObj, pObj->cmd.payload); httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload); } else { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s, sqlObj:%p", - pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj); + httpError("context:%p, fd:%d, user:%s, query error, taos:%p, code:%s, sqlObj:%p", pContext, pContext->fd, + pContext->user, pContext->session->taos, tstrerror(code), pObj); httpSendTaosdErrorResp(pContext, code); } taos_free_result(result); @@ -294,8 +293,8 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo // not select or show commands int affectRows = taos_affected_rows(result); - httpDebug("context:%p, fd:%d, ip:%s, user:%s, affect rows:%d, stop query, sqlObj:%p", - pContext, pContext->fd, pContext->ipstr, pContext->user, affectRows, result); + httpDebug("context:%p, fd:%d, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd, + pContext->user, affectRows, result); if (encode->startJsonFp) { (encode->startJsonFp)(pContext, &pContext->singleCmd, result); @@ -312,8 +311,7 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo taos_free_result(result); httpCloseContextByApp(pContext); } else { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, start retrieve", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpDebug("context:%p, fd:%d, user:%s, start retrieve", pContext, pContext->fd, pContext->user); if (encode->startJsonFp) { (encode->startJsonFp)(pContext, &pContext->singleCmd, result); @@ -333,14 +331,12 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) { HttpSession *pSession = pContext->session; if (sql == NULL || sql[0] == 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, error:no sql input", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpError("context:%p, fd:%d, user:%s, error:no sql input", pContext, pContext->fd, pContext->user); httpSendErrorResp(pContext, HTTP_NO_SQL_INPUT); return; } - httpTraceL("context:%p, fd:%d, ip:%s, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->ipstr, - pContext->user, sql); + httpTraceL("context:%p, fd:%d, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->user, sql); taosNotePrintHttp(sql); taos_query_a(pSession->taos, sql, httpProcessSingleSqlCallBack, (void *)pContext); } @@ -350,8 +346,8 @@ void httpProcessLoginCmd(HttpContext *pContext) { if (!httpGenTaosdAuthToken(pContext, token, 128)) { httpSendErrorResp(pContext, HTTP_GEN_TAOSD_TOKEN_ERR); } else { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, login via http, return token:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, token); + httpDebug("context:%p, fd:%d, user:%s, login via http, return token:%s", pContext, pContext->fd, pContext->user, + token); httpSendSuccResp(pContext, token); } } @@ -397,17 +393,16 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int code) { if (pContext == NULL) return; if (code < 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, login error, code:%s", pContext, pContext->fd, pContext->ipstr, - pContext->user, tstrerror(code)); + httpError("context:%p, fd:%d, user:%s, login error, code:%s", pContext, pContext->fd, pContext->user, + tstrerror(code)); httpSendTaosdErrorResp(pContext, code); return; } - httpDebug("context:%p, fd:%d, ip:%s, user:%s, connect tdengine success, taos:%p", pContext, pContext->fd, - pContext->ipstr, pContext->user, pContext->taos); + httpDebug("context:%p, fd:%d, user:%s, connect tdengine success, taos:%p", pContext, pContext->fd, pContext->user, + pContext->taos); if (pContext->taos == NULL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, login error, taos is empty", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpError("context:%p, fd:%d, user:%s, login error, taos is empty", pContext, pContext->fd, pContext->user); httpSendErrorResp(pContext, HTTP_NO_ENOUGH_SESSIONS); return; } @@ -428,8 +423,8 @@ void httpProcessRequest(HttpContext *pContext) { if (pContext->session == NULL || pContext->reqType == HTTP_REQTYPE_LOGIN) { taos_connect_a(NULL, pContext->user, pContext->pass, "", 0, httpProcessRequestCb, (void *)pContext, &(pContext->taos)); - httpDebug("context:%p, fd:%d, ip:%s, user:%s, try connect tdengine, taos:%p", pContext, pContext->fd, - pContext->ipstr, pContext->user, pContext->taos); + httpDebug("context:%p, fd:%d, user:%s, try connect tdengine, taos:%p", pContext, pContext->fd, pContext->user, + pContext->taos); } else { httpExecCmd(pContext); } diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index dc1b9a93be..e51c8dd4f7 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -40,12 +40,6 @@ HttpServer tsHttpServer; void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); int httpInitSystem() { - tsHttpServer.fallback = 0; - const char *v = getenv("FALLBACK"); - if (v) { - tsHttpServer.fallback = 1; - } - strcpy(tsHttpServer.label, "rest"); tsHttpServer.serverIp = 0; tsHttpServer.serverPort = tsHttpPort; diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index d1a0eb90f0..f6e05dc1ae 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -141,16 +141,15 @@ int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize) { bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize) { if (cmdSize > HTTP_MAX_CMD_SIZE) { - httpError("context:%p, fd:%d, ip:%s, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, - pContext->ipstr, pContext->user, cmdSize, HTTP_MAX_CMD_SIZE); + httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user, + cmdSize, HTTP_MAX_CMD_SIZE); return false; } if (pContext->multiCmds == NULL) { pContext->multiCmds = (HttpSqlCmds *)malloc(sizeof(HttpSqlCmds)); if (pContext->multiCmds == NULL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, malloc multiCmds error", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpError("context:%p, fd:%d, user:%s, malloc multiCmds error", pContext, pContext->fd, pContext->user); return false; } memset(pContext->multiCmds, 0, sizeof(HttpSqlCmds)); @@ -161,7 +160,7 @@ bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize) { free(multiCmds->cmds); multiCmds->cmds = (HttpSqlCmd *)malloc((size_t)cmdSize * sizeof(HttpSqlCmd)); if (multiCmds->cmds == NULL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->ipstr, + httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize); return false; } @@ -172,8 +171,8 @@ bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize) { free(multiCmds->buffer); multiCmds->buffer = (char *)malloc((size_t)bufferSize); if (multiCmds->buffer == NULL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->ipstr, - pContext->user, bufferSize); + httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user, + bufferSize); return false; } multiCmds->bufferSize = bufferSize; @@ -191,15 +190,14 @@ bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize) { HttpSqlCmds *multiCmds = pContext->multiCmds; if (cmdSize > HTTP_MAX_CMD_SIZE) { - httpError("context:%p, fd:%d, ip:%s, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, - pContext->ipstr, pContext->user, cmdSize, HTTP_MAX_CMD_SIZE); + httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user, + cmdSize, HTTP_MAX_CMD_SIZE); return false; } multiCmds->cmds = (HttpSqlCmd *)realloc(multiCmds->cmds, (size_t)cmdSize * sizeof(HttpSqlCmd)); if (multiCmds->cmds == NULL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->ipstr, - pContext->user, cmdSize); + httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize); return false; } memset(multiCmds->cmds + multiCmds->maxSize, 0, (size_t)(cmdSize - multiCmds->maxSize) * sizeof(HttpSqlCmd)); @@ -212,15 +210,14 @@ bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int bufferSize) { HttpSqlCmds *multiCmds = pContext->multiCmds; if (bufferSize > HTTP_MAX_BUFFER_SIZE) { - httpError("context:%p, fd:%d, ip:%s, user:%s, mulitcmd buffer size:%d large then %d", - pContext, pContext->fd, pContext->ipstr, pContext->user, bufferSize, HTTP_MAX_BUFFER_SIZE); + httpError("context:%p, fd:%d, user:%s, mulitcmd buffer size:%d large then %d", pContext, pContext->fd, + pContext->user, bufferSize, HTTP_MAX_BUFFER_SIZE); return false; } multiCmds->buffer = (char *)realloc(multiCmds->buffer, (size_t)bufferSize); if (multiCmds->buffer == NULL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->ipstr, - pContext->user, bufferSize); + httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user, bufferSize); return false; } memset(multiCmds->buffer + multiCmds->bufferSize, 0, (size_t)(bufferSize - multiCmds->bufferSize)); diff --git a/src/plugins/http/src/restHandle.c b/src/plugins/http/src/restHandle.c index f0841e2f99..8f998420de 100644 --- a/src/plugins/http/src/restHandle.c +++ b/src/plugins/http/src/restHandle.c @@ -80,15 +80,13 @@ bool restGetPassFromUrl(HttpContext* pContext) { } bool restProcessLoginRequest(HttpContext* pContext) { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, process restful login msg", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpDebug("context:%p, fd:%d, user:%s, process restful login msg", pContext, pContext->fd, pContext->user); pContext->reqType = HTTP_REQTYPE_LOGIN; return true; } bool restProcessSqlRequest(HttpContext* pContext, int timestampFmt) { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, process restful sql msg", pContext, pContext->fd, pContext->ipstr, - pContext->user); + httpDebug("context:%p, fd:%d, user:%s, process restful sql msg", pContext, pContext->fd, pContext->user); char* sql = pContext->parser.data.pos; if (sql == NULL) { diff --git a/src/plugins/http/src/restJson.c b/src/plugins/http/src/restJson.c index 7a73f6559f..4a7836ad7a 100644 --- a/src/plugins/http/src/restJson.c +++ b/src/plugins/http/src/restJson.c @@ -155,19 +155,17 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } if (cmd->numOfRows >= tsRestRowLimit) { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext, - pContext->fd, pContext->ipstr, pContext->user, cmd->numOfRows, tsRestRowLimit); + httpDebug("context:%p, fd:%d, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext, + pContext->fd, pContext->user, cmd->numOfRows, tsRestRowLimit); return false; - } - else { + } else { if (pContext->fd <= 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, connection is closed, abort retrieve", pContext, pContext->fd, - pContext->ipstr, pContext->user); + httpError("context:%p, fd:%d, user:%s, connection is closed, abort retrieve", pContext, pContext->fd, + pContext->user); return false; - } - else { - httpDebug("context:%p, fd:%d, ip:%s, user:%s, total rows:%d retrieved", pContext, pContext->fd, pContext->ipstr, - pContext->user, cmd->numOfRows); + } else { + httpDebug("context:%p, fd:%d, user:%s, total rows:%d retrieved", pContext, pContext->fd, pContext->user, + cmd->numOfRows); return true; } } diff --git a/src/plugins/http/src/tgHandle.c b/src/plugins/http/src/tgHandle.c index 48c66c4c07..cae46f222a 100644 --- a/src/plugins/http/src/tgHandle.c +++ b/src/plugins/http/src/tgHandle.c @@ -800,7 +800,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { } */ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { - httpDebug("context:%p, fd:%d, ip:%s, process telegraf query msg", pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, process telegraf query msg", pContext, pContext->fd); HttpParser *pParser = &pContext->parser; char * filter = pParser->data.pos; @@ -818,7 +818,7 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { cJSON *metrics = cJSON_GetObjectItem(root, "metrics"); if (metrics != NULL) { int size = cJSON_GetArraySize(metrics); - httpDebug("context:%p, fd:%d, ip:%s, multiple metrics:%d at one time", pContext, pContext->fd, pContext->ipstr, + httpDebug("context:%p, fd:%d, multiple metrics:%d at one time", pContext, pContext->fd, size); if (size <= 0) { httpSendErrorResp(pContext, HTTP_TG_METRICS_NULL); @@ -859,7 +859,7 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { } } } else { - httpDebug("context:%p, fd:%d, ip:%s, single metric", pContext, pContext->fd, pContext->ipstr); + httpDebug("context:%p, fd:%d, single metric", pContext, pContext->fd); if (!httpMallocMultiCmds(pContext, 3, HTTP_BUFFER_SIZE)) { httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY); diff --git a/src/plugins/http/src/tgJson.c b/src/plugins/http/src/tgJson.c index ed4ee0d7de..170a1b343e 100644 --- a/src/plugins/http/src/tgJson.c +++ b/src/plugins/http/src/tgJson.c @@ -98,8 +98,8 @@ void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { HttpSqlCmds *multiCmds = pContext->multiCmds; - httpDebug("context:%p, fd:%d, ip:%s, check telegraf command, code:%s, state:%d, type:%d, rettype:%d, tags:%d", - pContext, pContext->fd, pContext->ipstr, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum); + httpDebug("context:%p, fd:%d, check telegraf command, code:%s, state:%d, type:%d, rettype:%d, tags:%d", pContext, + pContext->fd, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum); if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) { if (cmd->cmdState == HTTP_CMD_STATE_NOT_RUN_YET) { @@ -107,16 +107,14 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; if (multiCmds->cmds[0].cmdState == HTTP_CMD_STATE_NOT_RUN_YET) { multiCmds->pos = (int16_t)-1; - httpDebug("context:%p, fd:%d, ip:%s, import failed, try create database", pContext, pContext->fd, - pContext->ipstr); + httpDebug("context:%p, fd:%d, import failed, try create database", pContext, pContext->fd); return false; } } else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; if (multiCmds->cmds[multiCmds->pos - 1].cmdState == HTTP_CMD_STATE_NOT_RUN_YET) { multiCmds->pos = (int16_t)(multiCmds->pos - 2); - httpDebug("context:%p, fd:%d, ip:%s, import failed, try create stable", pContext, pContext->fd, - pContext->ipstr); + httpDebug("context:%p, fd:%d, import failed, try create stable", pContext, pContext->fd); return false; } } else { @@ -125,11 +123,10 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { } } else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_DB) { cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; - httpDebug("context:%p, fd:%d, ip:%s, code:%s, create database failed", pContext, pContext->fd, pContext->ipstr, - tstrerror(code)); + httpDebug("context:%p, fd:%d, code:%s, create database failed", pContext, pContext->fd, tstrerror(code)); } else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_STBALE) { cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; - httpDebug("context:%p, fd:%d, ip:%s, code:%s, create stable failed", pContext, pContext->fd, pContext->ipstr, tstrerror(code)); + httpDebug("context:%p, fd:%d, code:%s, create stable failed", pContext, pContext->fd, tstrerror(code)); } else { } @@ -138,9 +135,9 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { HttpSqlCmds *multiCmds = pContext->multiCmds; - httpDebug("context:%p, fd:%d, ip:%s, get telegraf next command, pos:%d, code:%s, state:%d, type:%d, rettype:%d, tags:%d", - pContext, pContext->fd, pContext->ipstr, multiCmds->pos, tstrerror(code), cmd->cmdState, cmd->cmdType, - cmd->cmdReturnType, cmd->tagNum); + httpDebug("context:%p, fd:%d, get telegraf next command, pos:%d, code:%s, state:%d, type:%d, rettype:%d, tags:%d", + pContext, pContext->fd, multiCmds->pos, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, + cmd->tagNum); if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) { multiCmds->pos = (int16_t)(multiCmds->pos + 2); From a1527b6206b6bb4abf01abf66d899db86d78fa76 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 12 Sep 2020 08:13:12 +0000 Subject: [PATCH 029/105] TD-1311 --- src/plugins/http/src/tgHandle.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/plugins/http/src/tgHandle.c b/src/plugins/http/src/tgHandle.c index cae46f222a..84d7d13fde 100644 --- a/src/plugins/http/src/tgHandle.c +++ b/src/plugins/http/src/tgHandle.c @@ -818,8 +818,7 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { cJSON *metrics = cJSON_GetObjectItem(root, "metrics"); if (metrics != NULL) { int size = cJSON_GetArraySize(metrics); - httpDebug("context:%p, fd:%d, multiple metrics:%d at one time", pContext, pContext->fd, - size); + httpDebug("context:%p, fd:%d, multiple metrics:%d at one time", pContext, pContext->fd, size); if (size <= 0) { httpSendErrorResp(pContext, HTTP_TG_METRICS_NULL); cJSON_Delete(root); From 11aed4209e3ee9fcb681f84284544c8c68b196c7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 12 Sep 2020 14:50:29 +0000 Subject: [PATCH 030/105] TD-1311 --- src/plugins/http/inc/ehttp_parser.h | 97 +++- src/plugins/http/inc/ehttp_util_string.h | 25 +- src/plugins/http/inc/httpInt.h | 7 +- src/plugins/http/src/ehttp_parser.c | 555 +++++++++++------------ src/plugins/http/src/ehttp_util_string.c | 13 +- src/plugins/http/src/httpContext.c | 10 +- src/plugins/http/src/httpServer.c | 7 +- 7 files changed, 357 insertions(+), 357 deletions(-) diff --git a/src/plugins/http/inc/ehttp_parser.h b/src/plugins/http/inc/ehttp_parser.h index be87650128..7ddfcb32db 100644 --- a/src/plugins/http/inc/ehttp_parser.h +++ b/src/plugins/http/inc/ehttp_parser.h @@ -1,36 +1,93 @@ -#ifndef _ehttp_parser_fc7f9ac9_52da_4ee3_b556_deb2e1c3866e -#define _ehttp_parser_fc7f9ac9_52da_4ee3_b556_deb2e1c3866e +#ifndef HTTP_PARSER_H +#define HTTP_PARSER_H -#include +#include "ehttp_util_string.h" +#include "ehttp_gzip.h" -typedef struct ehttp_parser_s ehttp_parser_t; -typedef struct ehttp_parser_callbacks_s ehttp_parser_callbacks_t; -typedef struct ehttp_parser_conf_s ehttp_parser_conf_t; -typedef struct ehttp_status_code_s ehttp_status_code_t; +struct HttpContext; -struct ehttp_parser_callbacks_s { +typedef enum HTTP_PARSER_STATE { + HTTP_PARSER_BEGIN, + HTTP_PARSER_REQUEST_OR_RESPONSE, + HTTP_PARSER_METHOD, + HTTP_PARSER_TARGET, + HTTP_PARSER_HTTP_VERSION, + HTTP_PARSER_SP, + HTTP_PARSER_STATUS_CODE, + HTTP_PARSER_REASON_PHRASE, + HTTP_PARSER_CRLF, + HTTP_PARSER_HEADER, + HTTP_PARSER_HEADER_KEY, + HTTP_PARSER_HEADER_VAL, + HTTP_PARSER_CHUNK_SIZE, + HTTP_PARSER_CHUNK, + HTTP_PARSER_END, + HTTP_PARSER_ERROR, +} HTTP_PARSER_STATE; + +typedef struct HttpParserStatusObj { + int32_t status_code; + const char *status_desc; +} HttpParserStatusObj; + +typedef struct HttpParserCallbackObj { void (*on_request_line)(void *arg, const char *method, const char *target, const char *version, const char *target_raw); void (*on_status_line)(void *arg, const char *version, int status_code, const char *reason_phrase); void (*on_header_field)(void *arg, const char *key, const char *val); void (*on_body)(void *arg, const char *chunk, size_t len); void (*on_end)(void *arg); void (*on_error)(void *arg, int status_code); -}; +} HttpParserCallbackObj; -struct ehttp_parser_conf_s { - size_t flush_block_size; // <=0: immediately -}; +typedef struct HttpParserConfObj { + size_t flush_block_size; // <=0: immediately +} HttpParserConfObj; -ehttp_parser_t* ehttp_parser_create(ehttp_parser_callbacks_t callbacks, ehttp_parser_conf_t conf, void *arg); -void ehttp_parser_destroy(ehttp_parser_t *parser); -int ehttp_parser_parse(ehttp_parser_t *parser, const char *buf, size_t len); -int ehttp_parser_parse_string(ehttp_parser_t *parser, const char *str); -int ehttp_parser_parse_char(ehttp_parser_t *parser, const char c); -int ehttp_parser_parse_end(ehttp_parser_t *parser); +typedef struct HttpParseKvObj { + char *key; + char *val; +} HttpParseKvObj; + +typedef struct HttpParserObj { + HttpParserCallbackObj callbacks; + HttpParserConfObj conf; + void * arg; + char * method; + char * target; + char * target_raw; + char * version; + int http_10 : 2; + int http_11 : 2; + int accept_encoding_gzip : 2; + int accept_encoding_chunked : 2; + int transfer_gzip : 2; + int transfer_chunked : 2; + int content_length_specified : 2; + int content_chunked : 2; + int status_code; + char * reason_phrase; + char * key; + char * val; + HttpParseKvObj * kvs; + size_t kvs_count; + char * auth_basic; + char * auth_taosd; + size_t content_length; + size_t chunk_size; + size_t received_chunk_size; + size_t received_size; + ehttp_gzip_t * gzip; + HttpUtilString str; + HTTP_PARSER_STATE *stacks; + size_t stacks_count; +} HttpParserObj; + +HttpParserObj* httpParserCreate(HttpParserCallbackObj callbacks, HttpParserConfObj conf, void *arg); +void httpParserDestroy(HttpParserObj *parser); +int32_t httpParserBuf(struct HttpContext *pContext, HttpParserObj *parser, const char *buf, int32_t len); char* ehttp_parser_urldecode(const char *enc); const char* ehttp_status_code_get_desc(const int status_code); -#endif // _ehttp_parser_fc7f9ac9_52da_4ee3_b556_deb2e1c3866e - +#endif diff --git a/src/plugins/http/inc/ehttp_util_string.h b/src/plugins/http/inc/ehttp_util_string.h index 46c5a42827..eb44805f96 100644 --- a/src/plugins/http/inc/ehttp_util_string.h +++ b/src/plugins/http/inc/ehttp_util_string.h @@ -1,18 +1,13 @@ -#ifndef _ehttp_util_string_h_99dacde5_2e7d_4662_97d6_04611fde683b_ -#define _ehttp_util_string_h_99dacde5_2e7d_4662_97d6_04611fde683b_ +#ifndef HTTP_UTIL_STRING +#define HTTP_UTIL_STRING -#include +typedef struct HttpUtilString { + char * str; + size_t len; +} HttpUtilString; -typedef struct ehttp_util_string_s ehttp_util_string_t; - -struct ehttp_util_string_s { - char *str; - size_t len; -}; - -void ehttp_util_string_cleanup(ehttp_util_string_t *str); -int ehttp_util_string_append(ehttp_util_string_t *str, const char *s, size_t len); -void ehttp_util_string_clear(ehttp_util_string_t *str); - -#endif // _ehttp_util_string_h_99dacde5_2e7d_4662_97d6_04611fde683b_ +void httpParserCleanupString(HttpUtilString *str); +int32_t httpParserAppendString(HttpUtilString *str, const char *s, int32_t len); +void httpParserClearString(HttpUtilString *str); +#endif diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 044b5cc4cc..89972adc87 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -179,10 +179,9 @@ typedef struct { HttpBuf data; // body content HttpBuf token; // auth token HttpDecodeMethod *pMethod; - - ehttp_parser_t *parser; - int inited:2; - int failed:4; + HttpParserObj * parser; + int8_t inited; + int8_t failed; } HttpParser; typedef struct HttpContext { diff --git a/src/plugins/http/src/ehttp_parser.c b/src/plugins/http/src/ehttp_parser.c index 30d37f8a0d..0d0382078f 100644 --- a/src/plugins/http/src/ehttp_parser.c +++ b/src/plugins/http/src/ehttp_parser.c @@ -1,19 +1,20 @@ +#include "os.h" +#include "httpLog.h" +#include "httpContext.h" +#include "ehttp_util_string.h" #include "ehttp_parser.h" #include "ehttp_gzip.h" #include "ehttp_util_string.h" #include "elog.h" + #include #include #include -struct ehttp_status_code_s { - int status_code; - const char *status_desc; -}; -static ehttp_status_code_t status_codes[] = { +static HttpParserStatusObj status_codes[] = { {100, "Continue"}, {101, "Switching Protocol"}, {102, "Processing (WebDAV)"}, @@ -81,7 +82,7 @@ static ehttp_status_code_t status_codes[] = { }; const char* ehttp_status_code_get_desc(const int status_code) { - ehttp_status_code_t *p = status_codes; + HttpParserStatusObj *p = status_codes; while (p->status_code!=0) { if (p->status_code==status_code) return p->status_desc; ++p; @@ -89,74 +90,6 @@ const char* ehttp_status_code_get_desc(const int status_code) { return "Unknow status code"; } -typedef enum HTTP_PARSER_STATE { - HTTP_PARSER_BEGIN, - HTTP_PARSER_REQUEST_OR_RESPONSE, - HTTP_PARSER_METHOD, - HTTP_PARSER_TARGET, - HTTP_PARSER_HTTP_VERSION, - HTTP_PARSER_SP, - HTTP_PARSER_STATUS_CODE, - HTTP_PARSER_REASON_PHRASE, - HTTP_PARSER_CRLF, - HTTP_PARSER_HEADER, - HTTP_PARSER_HEADER_KEY, - HTTP_PARSER_HEADER_VAL, - HTTP_PARSER_CHUNK_SIZE, - HTTP_PARSER_CHUNK, - HTTP_PARSER_END, - HTTP_PARSER_ERROR, -} HTTP_PARSER_STATE; - -typedef struct ehttp_parser_kv_s ehttp_parser_kv_t; - -struct ehttp_parser_kv_s { - char *key; - char *val; -}; - -struct ehttp_parser_s { - ehttp_parser_callbacks_t callbacks; - void *arg; - ehttp_parser_conf_t conf; - - char *method; - char *target; - char *target_raw; - char *version; - - int http_10:2; - int http_11:2; - int accept_encoding_gzip:2; - int accept_encoding_chunked:2; - int transfer_gzip:2; - int transfer_chunked:2; - int content_length_specified:2; - int content_chunked:2; - - - int status_code; - char *reason_phrase; - - char *key; - char *val; - ehttp_parser_kv_t *kvs; - size_t kvs_count; - - char *auth_basic; - - size_t content_length; - - size_t chunk_size; - size_t received_chunk_size; - size_t received_size; - - ehttp_gzip_t *gzip; - ehttp_util_string_t str; - HTTP_PARSER_STATE *stacks; - size_t stacks_count; -}; - static void dummy_on_request_line(void *arg, const char *method, const char *target, const char *version, const char *target_raw) { } @@ -175,15 +108,14 @@ static void dummy_on_end(void *arg) { static void dummy_on_error(void *arg, int status_code) { } +static HTTP_PARSER_STATE httpParserTop(HttpParserObj *parser) { + ASSERT(parser->stacks_count >= 1); + ASSERT(parser->stacks); -static HTTP_PARSER_STATE ehttp_parser_top(ehttp_parser_t *parser) { - EQ_ASSERT(parser->stacks_count >= 1); - EQ_ASSERT(parser->stacks); - - return parser->stacks[parser->stacks_count-1]; + return parser->stacks[parser->stacks_count - 1]; } -static int ehttp_parser_push(ehttp_parser_t *parser, HTTP_PARSER_STATE state) { +static int httpParserPush(HttpParserObj *parser, HTTP_PARSER_STATE state) { size_t n = parser->stacks_count + 1; // HTTP_PARSER_STATE *stacks = (HTTP_PARSER_STATE*)reallocarray(parser->stacks, n, sizeof(*stacks)); HTTP_PARSER_STATE *stacks = (HTTP_PARSER_STATE*)realloc(parser->stacks, n * sizeof(*stacks)); @@ -196,15 +128,15 @@ static int ehttp_parser_push(ehttp_parser_t *parser, HTTP_PARSER_STATE state) { return 0; } -static int ehttp_parser_pop(ehttp_parser_t *parser) { +static int httpParserPop(HttpParserObj *parser) { if (parser->stacks_count <= 0) return -1; --parser->stacks_count; return 0; } -ehttp_parser_t *ehttp_parser_create(ehttp_parser_callbacks_t callbacks, ehttp_parser_conf_t conf, void *arg) { - ehttp_parser_t *parser = (ehttp_parser_t*)calloc(1, sizeof(*parser)); +HttpParserObj *httpParserCreate(HttpParserCallbackObj callbacks, HttpParserConfObj conf, void *arg) { + HttpParserObj *parser = (HttpParserObj*)calloc(1, sizeof(*parser)); if (!parser) return NULL; parser->callbacks = callbacks; @@ -230,16 +162,16 @@ ehttp_parser_t *ehttp_parser_create(ehttp_parser_callbacks_t callbacks, ehttp_pa parser->callbacks.on_error = dummy_on_error; } - ehttp_parser_push(parser, HTTP_PARSER_BEGIN); + httpParserPush(parser, HTTP_PARSER_BEGIN); return parser; } -static void ehttp_parser_kvs_destroy(ehttp_parser_t *parser) { +static void ehttp_parser_kvs_destroy(HttpParserObj *parser) { if (!parser->kvs) return; for (size_t i=0; ikvs_count; ++i) { - ehttp_parser_kv_t *p = &parser->kvs[i]; + HttpParseKvObj *p = &parser->kvs[i]; free(p->key); p->key = NULL; free(p->val); p->val = NULL; } @@ -251,7 +183,7 @@ static void ehttp_parser_kvs_destroy(ehttp_parser_t *parser) { parser->auth_basic = NULL; } -void ehttp_parser_destroy(ehttp_parser_t *parser) { +void httpParserDestroy(HttpParserObj *parser) { if (!parser) return; free(parser->method); parser->method = NULL; @@ -268,7 +200,7 @@ void ehttp_parser_destroy(ehttp_parser_t *parser) { ehttp_parser_kvs_destroy(parser); - ehttp_util_string_cleanup(&parser->str); + httpParserCleanupString(&parser->str); if (parser->gzip) { ehttp_gzip_destroy(parser->gzip); parser->gzip = NULL; @@ -281,52 +213,52 @@ void ehttp_parser_destroy(ehttp_parser_t *parser) { char *ehttp_parser_urldecode(const char *enc) { int ok = 1; - ehttp_util_string_t str = {0}; + HttpUtilString str = {0}; while (*enc) { char *p = strchr(enc, '%'); if (!p) break; int hex, cnt; int n = sscanf(p+1, "%2x%n", &hex, &cnt); if (n!=1 && cnt !=2) { ok = 0; break; } - if (ehttp_util_string_append(&str, enc, p-enc)) { ok = 0; break; } + if (httpParserAppendString(&str, enc, p-enc)) { ok = 0; break; } char c = (char)hex; - if (ehttp_util_string_append(&str, &c, 1)) { ok = 0; break; } + if (httpParserAppendString(&str, &c, 1)) { ok = 0; break; } enc = p+3; } char *dec = NULL; if (ok && *enc) { - if (ehttp_util_string_append(&str, enc, strlen(enc))) { ok = 0; } + if (httpParserAppendString(&str, enc, strlen(enc))) { ok = 0; } } if (ok) { dec = str.str; str.str = NULL; } - ehttp_util_string_cleanup(&str); + httpParserCleanupString(&str); return dec; } static void on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, size_t len) { - ehttp_parser_t *parser = (ehttp_parser_t*)arg; + HttpParserObj *parser = (HttpParserObj*)arg; parser->callbacks.on_body(parser->arg, buf, len); } -static int ehttp_parser_check_field(ehttp_parser_t *parser, const char *key, const char *val) { - int ok = 0; +static int32_t httpParserCheckField(HttpContext *pContext, HttpParserObj *parser, const char *key, const char *val) { + int32_t ok = 0; do { - if (0==strcasecmp(key, "Content-Length")) { - size_t len = 0; - int bytes = 0; - int n = sscanf(val, "%ld%n", &len, &bytes); - if (n==1 && bytes==strlen(val)) { + if (0 == strcasecmp(key, "Content-Length")) { + int32_t len = 0; + int32_t bytes = 0; + int32_t n = sscanf(val, "%d%n", &len, &bytes); + if (n == 1 && bytes == strlen(val)) { parser->content_length = len; - parser->chunk_size = len; + parser->chunk_size = len; parser->content_length_specified = 1; break; } ok = -1; break; } - if (0==strcasecmp(key, "Accept-Encoding")) { + if (0 == strcasecmp(key, "Accept-Encoding")) { if (strstr(val, "gzip")) { parser->accept_encoding_gzip = 1; } @@ -335,13 +267,13 @@ static int ehttp_parser_check_field(ehttp_parser_t *parser, const char *key, con } break; } - if (0==strcasecmp(key, "Content-Encoding")) { - if (0==strcmp(val, "gzip")) { + if (0 == strcasecmp(key, "Content-Encoding")) { + if (0 == strcmp(val, "gzip")) { parser->content_chunked = 1; } break; } - if (0==strcasecmp(key, "Transfer-Encoding")) { + if (0 == strcasecmp(key, "Transfer-Encoding")) { if (strstr(val, "gzip")) { parser->transfer_gzip = 1; ehttp_gzip_conf_t conf = {0}; @@ -352,7 +284,7 @@ static int ehttp_parser_check_field(ehttp_parser_t *parser, const char *key, con parser->gzip = ehttp_gzip_create_decompressor(conf, callbacks, parser); if (!parser->gzip) { - E("failed to create gzip decompressor"); + httpDebug("failed to create gzip decompressor"); ok = -1; break; } @@ -365,24 +297,37 @@ static int ehttp_parser_check_field(ehttp_parser_t *parser, const char *key, con if (0==strcasecmp(key, "Authorization")) { char *t = NULL; char *s = NULL; - int bytes = 0; - int n = sscanf(val, "%ms %ms%n", &t, &s, &bytes); - if (n==2 && t && s && bytes==strlen(val) && strcmp(t, "Basic")) { - free(parser->auth_basic); - parser->auth_basic = s; s = NULL; + int32_t bytes = 0; + int32_t n = sscanf(val, "%ms %ms%n", &t, &s, &bytes); + if (n == 2 && t && s && bytes == strlen(val)) { + if (strcmp(t, "Basic") == 0) { + free(parser->auth_basic); + parser->auth_basic = s; + s = NULL; + } else if (n == 2 && t && s && strcmp(t, "Taosd") == 0) { + free(parser->auth_taosd); + parser->auth_taosd = s; + s = NULL; + } else { + httpError("context:%p, fd:%d, invalid auth, t:%s s:%s", pContext, pContext->fd, t, s); + ok = -1; + } } else { + httpError("context:%p, fd:%d, parse auth failed, t:%s s:%s", pContext, pContext->fd, t, s); ok = -1; } - free(t); free(s); + + free(t); + free(s); break; } } while (0); return ok; } -static int ehttp_parser_kvs_append_kv(ehttp_parser_t *parser, const char *key, const char *val) { - // ehttp_parser_kv_t *kvs = (ehttp_parser_kv_t*)reallocarray(parser->kvs, parser->kvs_count + 1, sizeof(*kvs)); - ehttp_parser_kv_t *kvs = (ehttp_parser_kv_t*)realloc(parser->kvs, (parser->kvs_count + 1) * sizeof(*kvs)); +static int httpParserAppendKv(HttpParserObj *parser, const char *key, const char *val) { + // HttpParseKvObj *kvs = (HttpParseKvObj*)reallocarray(parser->kvs, parser->kvs_count + 1, sizeof(*kvs)); + HttpParseKvObj *kvs = (HttpParseKvObj*)realloc(parser->kvs, (parser->kvs_count + 1) * sizeof(*kvs)); if (!kvs) return -1; parser->kvs = kvs; @@ -403,69 +348,70 @@ static int ehttp_parser_kvs_append_kv(ehttp_parser_t *parser, const char *key, c return -1; } -static int on_begin(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnBegin(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; do { - if (c=='G' || c=='P' || c=='H' || c=='D' || c=='C' || c=='O' || c=='T') { - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (c == 'G' || c == 'P' || c == 'H' || c == 'D' || c == 'C' || c == 'O' || c == 'T') { + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; } - ehttp_parser_pop(parser); - ehttp_parser_push(parser, HTTP_PARSER_REQUEST_OR_RESPONSE); + httpParserPop(parser); + httpParserPush(parser, HTTP_PARSER_REQUEST_OR_RESPONSE); break; } - E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 400); } while (0); return ok; } -static int on_request_or_response(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnRquestOrResponse(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; do { - if (parser->str.len==1) { - if (c=='T' && parser->str.str[0]=='H') { - ehttp_parser_pop(parser); - ehttp_parser_push(parser, HTTP_PARSER_END); - ehttp_parser_push(parser, HTTP_PARSER_HEADER); - ehttp_parser_push(parser, HTTP_PARSER_CRLF); - ehttp_parser_push(parser, HTTP_PARSER_REASON_PHRASE); - ehttp_parser_push(parser, HTTP_PARSER_SP); - ehttp_parser_push(parser, HTTP_PARSER_STATUS_CODE); - ehttp_parser_push(parser, HTTP_PARSER_SP); - ehttp_parser_push(parser, HTTP_PARSER_HTTP_VERSION); + if (parser->str.len == 1) { + if (c == 'T' && parser->str.str[0] == 'H') { + httpParserPop(parser); + httpParserPush(parser, HTTP_PARSER_END); + httpParserPush(parser, HTTP_PARSER_HEADER); + httpParserPush(parser, HTTP_PARSER_CRLF); + httpParserPush(parser, HTTP_PARSER_REASON_PHRASE); + httpParserPush(parser, HTTP_PARSER_SP); + httpParserPush(parser, HTTP_PARSER_STATUS_CODE); + httpParserPush(parser, HTTP_PARSER_SP); + httpParserPush(parser, HTTP_PARSER_HTTP_VERSION); *again = 1; break; } - ehttp_parser_pop(parser); - ehttp_parser_push(parser, HTTP_PARSER_END); - ehttp_parser_push(parser, HTTP_PARSER_HEADER); - ehttp_parser_push(parser, HTTP_PARSER_CRLF); - ehttp_parser_push(parser, HTTP_PARSER_HTTP_VERSION); - ehttp_parser_push(parser, HTTP_PARSER_SP); - ehttp_parser_push(parser, HTTP_PARSER_TARGET); - ehttp_parser_push(parser, HTTP_PARSER_SP); - ehttp_parser_push(parser, HTTP_PARSER_METHOD); + httpParserPop(parser); + httpParserPush(parser, HTTP_PARSER_END); + httpParserPush(parser, HTTP_PARSER_HEADER); + httpParserPush(parser, HTTP_PARSER_CRLF); + httpParserPush(parser, HTTP_PARSER_HTTP_VERSION); + httpParserPush(parser, HTTP_PARSER_SP); + httpParserPush(parser, HTTP_PARSER_TARGET); + httpParserPush(parser, HTTP_PARSER_SP); + httpParserPush(parser, HTTP_PARSER_METHOD); *again = 1; break; } - E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 400); } while (0); return ok; } -static int on_method(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnMethod(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpDebug("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -474,24 +420,24 @@ static int on_method(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char } parser->method = strdup(parser->str.str); if (!parser->method) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; } - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); + httpParserClearString(&parser->str); + httpParserPop(parser); *again = 1; } while (0); return ok; } -static int on_target(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnTarget(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; do { - if (!isspace(c) && c!='\r' && c!='\n') { - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (!isspace(c) && c != '\r' && c != '\n') { + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -499,34 +445,34 @@ static int on_target(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char break; } parser->target_raw = strdup(parser->str.str); - parser->target = ehttp_parser_urldecode(parser->str.str); + parser->target = ehttp_parser_urldecode(parser->str.str); if (!parser->target_raw || !parser->target) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; } - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); + httpParserClearString(&parser->str); + httpParserPop(parser); *again = 1; } while (0); return ok; } -static int on_version(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnVersion(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; do { const char *prefix = "HTTP/1."; - int len = strlen(prefix); + int32_t len = strlen(prefix); if (parser->str.len < len) { if (prefix[parser->str.len]!=c) { - E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 400); break; } - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -535,13 +481,13 @@ static int on_version(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const cha } if (c!='0' && c!='1') { - E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 400); break; } - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -551,7 +497,7 @@ static int on_version(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const cha parser->version = strdup(parser->str.str); if (!parser->version) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -561,32 +507,32 @@ static int on_version(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const cha parser->callbacks.on_request_line(parser->arg, parser->method, parser->target, parser->version, parser->target_raw); } - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); + httpParserClearString(&parser->str); + httpParserPop(parser); } while (0); return ok; } -static int on_sp(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnSp(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int *again) { + int32_t ok = 0; do { - if (c==' ') { - ehttp_parser_pop(parser); + if (c == ' ') { + httpParserPop(parser); break; } - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + httpDebug("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); } while (0); return ok; } -static int on_status_code(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { +static int32_t httpParserOnStatusCode(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int *again) { int ok = 0; do { if (isdigit(c)) { - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -594,36 +540,36 @@ static int on_status_code(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const if (parser->str.len < 3) break; sscanf(parser->str.str, "%d", &parser->status_code); - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); + httpParserClearString(&parser->str); + httpParserPop(parser); break; } - E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 400); } while (0); return ok; } -static int on_reason_phrase(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { +static int32_t httpParserOnReasonPhrase(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int *again) { int ok = 0; do { if (c=='\r') { parser->reason_phrase = strdup(parser->str.str); if (!parser->reason_phrase) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; } parser->callbacks.on_status_line(parser->arg, parser->version, parser->status_code, parser->reason_phrase); - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); + httpParserClearString(&parser->str); + httpParserPop(parser); *again = 1; break; } - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -632,10 +578,10 @@ static int on_reason_phrase(ehttp_parser_t *parser, HTTP_PARSER_STATE state, con return ok; } -static int post_process(ehttp_parser_t *parser) { +static int32_t httpParserPostProcess(HttpContext *pContext, HttpParserObj *parser) { if (parser->gzip) { if (ehttp_gzip_finish(parser->gzip)) { - E("gzip failed"); + httpError("context:%p, fd:%d, gzip failed", pContext, pContext->fd); parser->callbacks.on_error(parser->arg, 507); return -1; } @@ -644,28 +590,28 @@ static int post_process(ehttp_parser_t *parser) { return 0; } -static int on_crlf(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnCrlf(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; do { const char *s = "\r\n"; - int len = strlen(s); + int32_t len = strlen(s); if (s[parser->str.len]!=c) { - E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 400); break; } - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; } if (parser->str.len == len) { - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); - if (ehttp_parser_top(parser) == HTTP_PARSER_END) { - ok = post_process(parser); + httpParserClearString(&parser->str); + httpParserPop(parser); + if (httpParserTop(parser) == HTTP_PARSER_END) { + ok = httpParserPostProcess(pContext, parser); } } break; @@ -673,49 +619,49 @@ static int on_crlf(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c return ok; } -static int on_header(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnHeader(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; do { if (c=='\r') { - ehttp_parser_pop(parser); + httpParserPop(parser); if (parser->transfer_chunked) { - ehttp_parser_push(parser, HTTP_PARSER_CHUNK_SIZE); - ehttp_parser_push(parser, HTTP_PARSER_CRLF); + httpParserPush(parser, HTTP_PARSER_CHUNK_SIZE); + httpParserPush(parser, HTTP_PARSER_CRLF); } else { if (parser->content_length > 0) { - ehttp_parser_push(parser, HTTP_PARSER_CHUNK); + httpParserPush(parser, HTTP_PARSER_CHUNK); } - ehttp_parser_push(parser, HTTP_PARSER_CRLF); + httpParserPush(parser, HTTP_PARSER_CRLF); } *again = 1; break; } if (c!=' ' && c!='\t' && c!=':' ) { - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; } - ehttp_parser_push(parser, HTTP_PARSER_CRLF); - ehttp_parser_push(parser, HTTP_PARSER_HEADER_VAL); - ehttp_parser_push(parser, HTTP_PARSER_SP); - ehttp_parser_push(parser, HTTP_PARSER_HEADER_KEY); + httpParserPush(parser, HTTP_PARSER_CRLF); + httpParserPush(parser, HTTP_PARSER_HEADER_VAL); + httpParserPush(parser, HTTP_PARSER_SP); + httpParserPush(parser, HTTP_PARSER_HEADER_KEY); break; } - E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 400); } while (0); return ok; } -static int on_header_key(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { +static int httpParserOnHeaderKey(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int *again) { int ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -725,61 +671,62 @@ static int on_header_key(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const if (c==':') { parser->key = strdup(parser->str.str); if (!parser->key) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; } - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); + httpParserClearString(&parser->str); + httpParserPop(parser); break; } - E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 400); } while (0); return ok; } -static int on_header_val(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnHeaderVal(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; do { if (c != '\r' && c != '\n' && (!isspace(c) || parser->str.len>0)) { - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; } break; } - const char *val = parser->str.str; - ok = ehttp_parser_check_field(parser, parser->key, val); - if (ehttp_parser_kvs_append_kv(parser, parser->key, val)) { + const char *val = parser->str.str; + ok = httpParserCheckField(pContext, parser, parser->key, val); + if (httpParserAppendKv(parser, parser->key, val)) { ok = -1; parser->callbacks.on_error(parser->arg, 507); } else { parser->callbacks.on_header_field(parser->arg, parser->key, val); } - free(parser->key); parser->key = NULL; + free(parser->key); + parser->key = NULL; val = NULL; - if (ok==-1) break; - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); + if (ok == -1) break; + httpParserClearString(&parser->str); + httpParserPop(parser); *again = 1; } while (0); return ok; } -static int on_chunk_size(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; - int bytes; - size_t len; +static int32_t httpParserOnChunkSize(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; + int32_t bytes; + int32_t len; int n; do { if (isxdigit(c)) { - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -787,44 +734,44 @@ static int on_chunk_size(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const break; } if (c=='\r') { - n = sscanf(parser->str.str, "%lx%n", &len, &bytes); + n = sscanf(parser->str.str, "%x%n", &len, &bytes); if (n==1 && bytes==strlen(parser->str.str) && len>=0) { if (len==0) { if (parser->content_length_specified == 0 || parser->received_size == parser->content_length) { - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); - ehttp_parser_push(parser, HTTP_PARSER_CRLF); - ehttp_parser_push(parser, HTTP_PARSER_CRLF); + httpParserClearString(&parser->str); + httpParserPop(parser); + httpParserPush(parser, HTTP_PARSER_CRLF); + httpParserPush(parser, HTTP_PARSER_CRLF); *again = 1; break; } } else { if (parser->content_length_specified == 0 || parser->received_size + len <= parser->content_length) { parser->chunk_size = len; - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); - ehttp_parser_push(parser, HTTP_PARSER_CHUNK_SIZE); - ehttp_parser_push(parser, HTTP_PARSER_CRLF); - ehttp_parser_push(parser, HTTP_PARSER_CHUNK); - ehttp_parser_push(parser, HTTP_PARSER_CRLF); + httpParserClearString(&parser->str); + httpParserPop(parser); + httpParserPush(parser, HTTP_PARSER_CHUNK_SIZE); + httpParserPush(parser, HTTP_PARSER_CRLF); + httpParserPush(parser, HTTP_PARSER_CHUNK); + httpParserPush(parser, HTTP_PARSER_CRLF); *again = 1; break; } } } } - E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 400); } while (0); return ok; } -static int on_chunk(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { +static int httpParserOnChunk(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int *again) { int ok = 0; do { - if (ehttp_util_string_append(&parser->str, &c, 1)) { - E("parser state: %d, char: [%c]%02x, oom", state, c, c); + if (httpParserAppendString(&parser->str, &c, 1)) { + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -835,7 +782,7 @@ static int on_chunk(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char if (parser->gzip) { if (ehttp_gzip_write(parser->gzip, parser->str.str, parser->str.len)) { - E("gzip failed"); + httpError("context:%p, fd:%d, gzip failed", pContext, pContext->fd); ok = -1; parser->callbacks.on_error(parser->arg, 507); break; @@ -844,124 +791,128 @@ static int on_chunk(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char parser->callbacks.on_body(parser->arg, parser->str.str, parser->str.len); } parser->received_chunk_size = 0; - ehttp_util_string_clear(&parser->str); - ehttp_parser_pop(parser); - if (ehttp_parser_top(parser) == HTTP_PARSER_END) { - ok = post_process(parser); + httpParserClearString(&parser->str); + httpParserPop(parser); + if (httpParserTop(parser) == HTTP_PARSER_END) { + ok = httpParserPostProcess(pContext, parser); } } while (0); return ok; } -static int on_end(ehttp_parser_t *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnEnd(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + int32_t ok = 0; do { - E("parser state: %d, unexpected char: [%c]%02x", state, c, c); + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; parser->callbacks.on_error(parser->arg, 507); } while (0); return ok; } -static int parse_char(ehttp_parser_t *parser, const char c, int *again) { - int ok = 0; - HTTP_PARSER_STATE state = ehttp_parser_top(parser); +static int32_t httpParseChar(HttpContext *pContext, HttpParserObj *parser, const char c, int32_t *again) { + int32_t ok = 0; + HTTP_PARSER_STATE state = httpParserTop(parser); do { if (state == HTTP_PARSER_BEGIN) { - ok = on_begin(parser, state, c, again); + ok = httpParserOnBegin(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_REQUEST_OR_RESPONSE) { - ok = on_request_or_response(parser, state, c, again); + ok = httpParserOnRquestOrResponse(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_METHOD) { - ok = on_method(parser, state, c, again); + ok = httpParserOnMethod(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_TARGET) { - ok = on_target(parser, state, c, again); + ok = httpParserOnTarget(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_HTTP_VERSION) { - ok = on_version(parser, state, c, again); + ok = httpParserOnVersion(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_SP) { - ok = on_sp(parser, state, c, again); + ok = httpParserOnSp(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_STATUS_CODE) { - ok = on_status_code(parser, state, c, again); + ok = httpParserOnStatusCode(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_REASON_PHRASE) { - ok = on_reason_phrase(parser, state, c, again); + ok = httpParserOnReasonPhrase(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_CRLF) { - ok = on_crlf(parser, state, c, again); + ok = httpParserOnCrlf(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_HEADER) { - ok = on_header(parser, state, c, again); + ok = httpParserOnHeader(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_HEADER_KEY) { - ok = on_header_key(parser, state, c, again); + ok = httpParserOnHeaderKey(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_HEADER_VAL) { - ok = on_header_val(parser, state, c, again); + ok = httpParserOnHeaderVal(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_CHUNK_SIZE) { - ok = on_chunk_size(parser, state, c, again); + ok = httpParserOnChunkSize(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_CHUNK) { - ok = on_chunk(parser, state, c, again); + ok = httpParserOnChunk(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_END) { - ok = on_end(parser, state, c, again); + ok = httpParserOnEnd(pContext, parser, state, c, again); break; } if (state == HTTP_PARSER_ERROR) { ok = -2; break; } - E("unknown parser state: %d", state); + + httpError("context:%p, fd:%d, unknown parse state:%d", pContext, pContext->fd, state); ok = -1; parser->callbacks.on_error(parser->arg, 500); } while (0); - if (ok==-1) { - ehttp_parser_push(parser, HTTP_PARSER_ERROR); + + if (ok == -1) { + httpError("context:%p, fd:%d, failed to parse, state:%d ok:%d", pContext, pContext->fd, state, ok); + httpParserPush(parser, HTTP_PARSER_ERROR); } - if (ok==-2) ok = -1; + + if (ok == -2) { + httpError("context:%p, fd:%d, failed to parse, state:%d ok:%d", pContext, pContext->fd, state, ok); + ok = -1; + } + return ok; } -int ehttp_parser_parse_string(ehttp_parser_t *parser, const char *str) { - return ehttp_parser_parse(parser, str, str?strlen(str):0); -} - -int ehttp_parser_parse_char(ehttp_parser_t *parser, const char c) { - return ehttp_parser_parse(parser, &c, 1); -} - -int ehttp_parser_parse(ehttp_parser_t *parser, const char *buf, size_t len) { +int32_t httpParserBuf(HttpContext *pContext, HttpParserObj *parser, const char *buf, int32_t len) { const char *p = buf; - int ret = 0; - size_t i = 0; + int32_t ret = 0; + int32_t i = 0; + while (i < len) { - int again = 0; - ret = parse_char(parser, *p, &again); - if (ret) break; + int32_t again = 0; + ret = httpParseChar(pContext, parser, *p, &again); + if (ret != 0) { + httpError("context:%p, fd:%d, parse failed, ret:%d i:%d len:%d buf:%s", pContext, pContext->fd, ret, i, len, buf); + break; + } if (again) continue; ++p; ++i; } + return ret; } - diff --git a/src/plugins/http/src/ehttp_util_string.c b/src/plugins/http/src/ehttp_util_string.c index 94ebaaafa6..8900aa89bb 100644 --- a/src/plugins/http/src/ehttp_util_string.c +++ b/src/plugins/http/src/ehttp_util_string.c @@ -1,17 +1,14 @@ +#include "os.h" #include "ehttp_util_string.h" -#include -#include - -void ehttp_util_string_cleanup(ehttp_util_string_t *str) { +void httpParserCleanupString(HttpUtilString *str) { free(str->str); str->str = NULL; str->len = 0; } -int ehttp_util_string_append(ehttp_util_string_t *str, const char *s, size_t len) { - // int n = str->str?strlen(str->str):0; - int n = str->len; +int32_t httpParserAppendString(HttpUtilString *str, const char *s, int32_t len) { + int32_t n = str->len; char *p = (char*)realloc(str->str, n + len + 1); if (!p) return -1; strncpy(p+n, s, len); @@ -21,7 +18,7 @@ int ehttp_util_string_append(ehttp_util_string_t *str, const char *s, size_t len return 0; } -void ehttp_util_string_clear(ehttp_util_string_t *str) { +void httpParserClearString(HttpUtilString *str) { if (str->str) { str->str[0] = '\0'; str->len = 0; diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 59c81f5960..987de519c7 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -26,11 +26,9 @@ #include "httpResp.h" #include "httpSql.h" #include "httpSession.h" - #include "httpContext.h" #include "elog.h" -// dirty tweak extern bool httpGetHttpMethod(HttpContext* pContext); extern bool httpParseURL(HttpContext* pContext); extern bool httpParseHttpVersion(HttpContext* pContext); @@ -74,7 +72,7 @@ static void httpDestroyContext(void *data) { httpFreeMultiCmds(pContext); if (pContext->parser.parser) { - ehttp_parser_destroy(pContext->parser.parser); + httpParserDestroy(pContext->parser.parser); pContext->parser.parser = NULL; } @@ -195,7 +193,7 @@ bool httpInitContext(HttpContext *pContext) { memset(pParser, 0, sizeof(HttpParser)); pParser->pCur = pParser->pLast = pParser->buffer; - ehttp_parser_callbacks_t callbacks = { + HttpParserCallbackObj callbacks = { httpParseOnRequestLine, httpParseOnStatusLine, httpParseOnHeaderField, @@ -203,10 +201,10 @@ bool httpInitContext(HttpContext *pContext) { httpParseOnEnd, httpParseOnError }; - ehttp_parser_conf_t conf = { + HttpParserConfObj conf = { .flush_block_size = 0 }; - pParser->parser = ehttp_parser_create(callbacks, conf, pContext); + pParser->parser = httpParserCreate(callbacks, conf, pContext); pParser->inited = 1; httpDebug("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed); diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 614cc92700..b42e2cb0aa 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -352,8 +352,11 @@ static bool httpReadData(HttpContext *pContext) { int nread = (int)taosReadSocket(pContext->fd, buf, sizeof(buf)); if (nread > 0) { buf[nread] = '\0'; - if (ehttp_parser_parse(pParser->parser, buf, nread)) { - httpError("context:%p, fd:%d, init parse failed, close connect", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf); + int ok = httpParserBuf(pContext, pParser->parser, buf, nread); + + if (ok) { + httpError("context:%p, fd:%d, init parse failed, reason:%d close connect", pContext, pContext->fd, ok); httpNotifyContextClose(pContext); return false; } From 01c69a637754d9724cc29719b77b79195ec66916 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 12 Sep 2020 15:02:21 +0000 Subject: [PATCH 031/105] TD-1225 rename files --- src/plugins/http/inc/ehttp_util_string.h | 13 --- .../http/inc/{gcHandle.h => httpGcHandle.h} | 0 .../http/inc/{gcJson.h => httpGcJson.h} | 0 .../http/inc/{ehttp_gzip.h => httpGzip.h} | 0 src/plugins/http/inc/httpInt.h | 3 +- .../http/inc/{ehttp_parser.h => httpParser.h} | 15 ++- .../inc/{restHandle.h => httpRestHandle.h} | 0 .../http/inc/{restJson.h => httpRestJson.h} | 0 .../http/inc/{tgHandle.h => httpTgHandle.h} | 0 .../http/inc/{tgJson.h => httpTgJson.h} | 0 src/plugins/http/src/ehttp_util_string.c | 27 ------ src/plugins/http/src/httpContext.c | 5 +- .../http/src/{gcHandle.c => httpGcHandle.c} | 6 +- .../http/src/{gcJson.c => httpGcJson.c} | 4 +- .../http/src/{ehttp_gzip.c => httpGzip.c} | 5 +- .../http/src/{ehttp_parser.c => httpParser.c} | 40 +++++--- .../src/{restHandle.c => httpRestHandle.c} | 4 +- .../http/src/{restJson.c => httpRestJson.c} | 4 +- src/plugins/http/src/httpServer.c | 4 +- src/plugins/http/src/httpSystem.c | 6 +- .../http/src/{tgHandle.c => httpTgHandle.c} | 4 +- .../http/src/{tgJson.c => httpTgJson.c} | 4 +- src/util/inc/elog.h | 71 -------------- src/util/src/elog.c | 95 ------------------- 24 files changed, 60 insertions(+), 250 deletions(-) delete mode 100644 src/plugins/http/inc/ehttp_util_string.h rename src/plugins/http/inc/{gcHandle.h => httpGcHandle.h} (100%) rename src/plugins/http/inc/{gcJson.h => httpGcJson.h} (100%) rename src/plugins/http/inc/{ehttp_gzip.h => httpGzip.h} (100%) rename src/plugins/http/inc/{ehttp_parser.h => httpParser.h} (89%) rename src/plugins/http/inc/{restHandle.h => httpRestHandle.h} (100%) rename src/plugins/http/inc/{restJson.h => httpRestJson.h} (100%) rename src/plugins/http/inc/{tgHandle.h => httpTgHandle.h} (100%) rename src/plugins/http/inc/{tgJson.h => httpTgJson.h} (100%) delete mode 100644 src/plugins/http/src/ehttp_util_string.c rename src/plugins/http/src/{gcHandle.c => httpGcHandle.c} (99%) rename src/plugins/http/src/{gcJson.c => httpGcJson.c} (99%) rename src/plugins/http/src/{ehttp_gzip.c => httpGzip.c} (99%) rename src/plugins/http/src/{ehttp_parser.c => httpParser.c} (98%) rename src/plugins/http/src/{restHandle.c => httpRestHandle.c} (98%) rename src/plugins/http/src/{restJson.c => httpRestJson.c} (99%) rename src/plugins/http/src/{tgHandle.c => httpTgHandle.c} (99%) rename src/plugins/http/src/{tgJson.c => httpTgJson.c} (99%) delete mode 100644 src/util/inc/elog.h delete mode 100644 src/util/src/elog.c diff --git a/src/plugins/http/inc/ehttp_util_string.h b/src/plugins/http/inc/ehttp_util_string.h deleted file mode 100644 index eb44805f96..0000000000 --- a/src/plugins/http/inc/ehttp_util_string.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef HTTP_UTIL_STRING -#define HTTP_UTIL_STRING - -typedef struct HttpUtilString { - char * str; - size_t len; -} HttpUtilString; - -void httpParserCleanupString(HttpUtilString *str); -int32_t httpParserAppendString(HttpUtilString *str, const char *s, int32_t len); -void httpParserClearString(HttpUtilString *str); - -#endif diff --git a/src/plugins/http/inc/gcHandle.h b/src/plugins/http/inc/httpGcHandle.h similarity index 100% rename from src/plugins/http/inc/gcHandle.h rename to src/plugins/http/inc/httpGcHandle.h diff --git a/src/plugins/http/inc/gcJson.h b/src/plugins/http/inc/httpGcJson.h similarity index 100% rename from src/plugins/http/inc/gcJson.h rename to src/plugins/http/inc/httpGcJson.h diff --git a/src/plugins/http/inc/ehttp_gzip.h b/src/plugins/http/inc/httpGzip.h similarity index 100% rename from src/plugins/http/inc/ehttp_gzip.h rename to src/plugins/http/inc/httpGzip.h diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 89972adc87..4348e5e090 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -27,8 +27,7 @@ #include "httpCode.h" #include "httpLog.h" #include "httpJson.h" - -#include "ehttp_parser.h" +#include "httpParser.h" #define HTTP_MAX_CMD_SIZE 1024 #define HTTP_MAX_BUFFER_SIZE 1024*1024 diff --git a/src/plugins/http/inc/ehttp_parser.h b/src/plugins/http/inc/httpParser.h similarity index 89% rename from src/plugins/http/inc/ehttp_parser.h rename to src/plugins/http/inc/httpParser.h index 7ddfcb32db..4facea0286 100644 --- a/src/plugins/http/inc/ehttp_parser.h +++ b/src/plugins/http/inc/httpParser.h @@ -1,8 +1,7 @@ #ifndef HTTP_PARSER_H #define HTTP_PARSER_H -#include "ehttp_util_string.h" -#include "ehttp_gzip.h" +#include "httpGzip.h" struct HttpContext; @@ -25,6 +24,11 @@ typedef enum HTTP_PARSER_STATE { HTTP_PARSER_ERROR, } HTTP_PARSER_STATE; +typedef struct HttpParserString { + char * str; + size_t len; +} HttpParserString; + typedef struct HttpParserStatusObj { int32_t status_code; const char *status_desc; @@ -77,11 +81,16 @@ typedef struct HttpParserObj { size_t received_chunk_size; size_t received_size; ehttp_gzip_t * gzip; - HttpUtilString str; + HttpParserString str; HTTP_PARSER_STATE *stacks; size_t stacks_count; } HttpParserObj; +void httpParserCleanupString(HttpParserString *str); +int32_t httpParserAppendString(HttpParserString *str, const char *s, int32_t len); +void httpParserClearString(HttpParserString *str); + + HttpParserObj* httpParserCreate(HttpParserCallbackObj callbacks, HttpParserConfObj conf, void *arg); void httpParserDestroy(HttpParserObj *parser); int32_t httpParserBuf(struct HttpContext *pContext, HttpParserObj *parser, const char *buf, int32_t len); diff --git a/src/plugins/http/inc/restHandle.h b/src/plugins/http/inc/httpRestHandle.h similarity index 100% rename from src/plugins/http/inc/restHandle.h rename to src/plugins/http/inc/httpRestHandle.h diff --git a/src/plugins/http/inc/restJson.h b/src/plugins/http/inc/httpRestJson.h similarity index 100% rename from src/plugins/http/inc/restJson.h rename to src/plugins/http/inc/httpRestJson.h diff --git a/src/plugins/http/inc/tgHandle.h b/src/plugins/http/inc/httpTgHandle.h similarity index 100% rename from src/plugins/http/inc/tgHandle.h rename to src/plugins/http/inc/httpTgHandle.h diff --git a/src/plugins/http/inc/tgJson.h b/src/plugins/http/inc/httpTgJson.h similarity index 100% rename from src/plugins/http/inc/tgJson.h rename to src/plugins/http/inc/httpTgJson.h diff --git a/src/plugins/http/src/ehttp_util_string.c b/src/plugins/http/src/ehttp_util_string.c deleted file mode 100644 index 8900aa89bb..0000000000 --- a/src/plugins/http/src/ehttp_util_string.c +++ /dev/null @@ -1,27 +0,0 @@ -#include "os.h" -#include "ehttp_util_string.h" - -void httpParserCleanupString(HttpUtilString *str) { - free(str->str); - str->str = NULL; - str->len = 0; -} - -int32_t httpParserAppendString(HttpUtilString *str, const char *s, int32_t len) { - int32_t n = str->len; - char *p = (char*)realloc(str->str, n + len + 1); - if (!p) return -1; - strncpy(p+n, s, len); - p[n+len] = '\0'; - str->str = p; - str->len = n+len; - return 0; -} - -void httpParserClearString(HttpUtilString *str) { - if (str->str) { - str->str[0] = '\0'; - str->len = 0; - } -} - diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 987de519c7..d310f27ee6 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -27,7 +27,6 @@ #include "httpSql.h" #include "httpSession.h" #include "httpContext.h" -#include "elog.h" extern bool httpGetHttpMethod(HttpContext* pContext); extern bool httpParseURL(HttpContext* pContext); @@ -144,8 +143,8 @@ HttpContext *httpCreateContext(int32_t fd) { HttpContext *httpGetContext(void *ptr) { uint64_t handleVal = (uint64_t)ptr; HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, sizeof(HttpContext *)); - EQ_ASSERT(ppContext); - EQ_ASSERT(*ppContext); + ASSERT(ppContext); + ASSERT(*ppContext); if (ppContext) { HttpContext *pContext = *ppContext; diff --git a/src/plugins/http/src/gcHandle.c b/src/plugins/http/src/httpGcHandle.c similarity index 99% rename from src/plugins/http/src/gcHandle.c rename to src/plugins/http/src/httpGcHandle.c index 4aed6eb5cc..09f17cae66 100644 --- a/src/plugins/http/src/gcHandle.c +++ b/src/plugins/http/src/httpGcHandle.c @@ -15,11 +15,11 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "taosdef.h" #include "cJSON.h" #include "httpLog.h" -#include "gcHandle.h" -#include "gcJson.h" -#include "taosdef.h" +#include "httpGcHandle.h" +#include "httpGcJson.h" static HttpDecodeMethod gcDecodeMethod = {"grafana", gcProcessRequest}; static HttpEncodeMethod gcHeartBeatMethod = { diff --git a/src/plugins/http/src/gcJson.c b/src/plugins/http/src/httpGcJson.c similarity index 99% rename from src/plugins/http/src/gcJson.c rename to src/plugins/http/src/httpGcJson.c index 94d53db6ef..85cae24201 100644 --- a/src/plugins/http/src/gcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -15,8 +15,8 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "gcHandle.h" -#include "gcJson.h" +#include "httpGcHandle.h" +#include "httpGcJson.h" #include "httpJson.h" #include "httpResp.h" diff --git a/src/plugins/http/src/ehttp_gzip.c b/src/plugins/http/src/httpGzip.c similarity index 99% rename from src/plugins/http/src/ehttp_gzip.c rename to src/plugins/http/src/httpGzip.c index ded344dfea..5712aff7ec 100644 --- a/src/plugins/http/src/ehttp_gzip.c +++ b/src/plugins/http/src/httpGzip.c @@ -1,9 +1,6 @@ -#include "ehttp_gzip.h" - #include "os.h" #include "zlib.h" - -#include +#include "httpGzip.h" typedef enum { EHTTP_GZIP_INITING, diff --git a/src/plugins/http/src/ehttp_parser.c b/src/plugins/http/src/httpParser.c similarity index 98% rename from src/plugins/http/src/ehttp_parser.c rename to src/plugins/http/src/httpParser.c index 0d0382078f..c1fd481efd 100644 --- a/src/plugins/http/src/ehttp_parser.c +++ b/src/plugins/http/src/httpParser.c @@ -1,18 +1,8 @@ #include "os.h" #include "httpLog.h" #include "httpContext.h" -#include "ehttp_util_string.h" -#include "ehttp_parser.h" - -#include "ehttp_gzip.h" -#include "ehttp_util_string.h" -#include "elog.h" - - -#include -#include -#include - +#include "httpParser.h" +#include "httpGzip.h" static HttpParserStatusObj status_codes[] = { {100, "Continue"}, @@ -213,7 +203,7 @@ void httpParserDestroy(HttpParserObj *parser) { char *ehttp_parser_urldecode(const char *enc) { int ok = 1; - HttpUtilString str = {0}; + HttpParserString str = {0}; while (*enc) { char *p = strchr(enc, '%'); if (!p) break; @@ -916,3 +906,27 @@ int32_t httpParserBuf(HttpContext *pContext, HttpParserObj *parser, const char * return ret; } + +void httpParserCleanupString(HttpParserString *str) { + free(str->str); + str->str = NULL; + str->len = 0; +} + +int32_t httpParserAppendString(HttpParserString *str, const char *s, int32_t len) { + int32_t n = str->len; + char *p = (char*)realloc(str->str, n + len + 1); + if (!p) return -1; + strncpy(p+n, s, len); + p[n+len] = '\0'; + str->str = p; + str->len = n+len; + return 0; +} + +void httpParserClearString(HttpParserString *str) { + if (str->str) { + str->str[0] = '\0'; + str->len = 0; + } +} diff --git a/src/plugins/http/src/restHandle.c b/src/plugins/http/src/httpRestHandle.c similarity index 98% rename from src/plugins/http/src/restHandle.c rename to src/plugins/http/src/httpRestHandle.c index 8f998420de..b285b19c3f 100644 --- a/src/plugins/http/src/restHandle.c +++ b/src/plugins/http/src/httpRestHandle.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "os.h" #include "httpLog.h" -#include "restHandle.h" -#include "restJson.h" +#include "httpRestHandle.h" +#include "httpRestJson.h" static HttpDecodeMethod restDecodeMethod = {"rest", restProcessRequest}; static HttpDecodeMethod restDecodeMethod2 = {"restful", restProcessRequest}; diff --git a/src/plugins/http/src/restJson.c b/src/plugins/http/src/httpRestJson.c similarity index 99% rename from src/plugins/http/src/restJson.c rename to src/plugins/http/src/httpRestJson.c index 4a7836ad7a..c16727faa0 100644 --- a/src/plugins/http/src/restJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -18,8 +18,8 @@ #include "tglobal.h" #include "httpLog.h" #include "httpJson.h" -#include "restHandle.h" -#include "restJson.h" +#include "httpRestHandle.h" +#include "httpRestJson.h" void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect_rows) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index b42e2cb0aa..e0d2a90994 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -25,8 +25,6 @@ #include "httpResp.h" #include "httpUtil.h" -#include "elog.h" - #ifndef EPOLLWAKEUP #define EPOLLWAKEUP (1u << 29) #endif @@ -334,7 +332,7 @@ bool httpInitConnect() { static bool httpReadData(HttpContext *pContext) { HttpParser *pParser = &pContext->parser; - EQ_ASSERT(!pContext->parsed); + ASSERT(!pContext->parsed); if (!pParser->parser) { if (!pParser->inited) { diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index e51c8dd4f7..ee43b3c826 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -27,9 +27,9 @@ #include "httpResp.h" #include "httpHandle.h" #include "httpQueue.h" -#include "gcHandle.h" -#include "restHandle.h" -#include "tgHandle.h" +#include "httpGcHandle.h" +#include "httpRestHandle.h" +#include "httpTgHandle.h" #ifndef _ADMIN void adminInitHandle(HttpServer* pServer) {} diff --git a/src/plugins/http/src/tgHandle.c b/src/plugins/http/src/httpTgHandle.c similarity index 99% rename from src/plugins/http/src/tgHandle.c rename to src/plugins/http/src/httpTgHandle.c index 84d7d13fde..a7444676ae 100644 --- a/src/plugins/http/src/tgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -19,8 +19,8 @@ #include "taosdef.h" #include "taosmsg.h" #include "httpInt.h" -#include "tgHandle.h" -#include "tgJson.h" +#include "httpTgHandle.h" +#include "httpTgJson.h" #include "cJSON.h" /* diff --git a/src/plugins/http/src/tgJson.c b/src/plugins/http/src/httpTgJson.c similarity index 99% rename from src/plugins/http/src/tgJson.c rename to src/plugins/http/src/httpTgJson.c index 170a1b343e..5c6985cd69 100644 --- a/src/plugins/http/src/tgJson.c +++ b/src/plugins/http/src/httpTgJson.c @@ -19,8 +19,8 @@ #include "httpLog.h" #include "httpJson.h" #include "httpResp.h" -#include "tgHandle.h" -#include "tgJson.h" +#include "httpTgHandle.h" +#include "httpTgJson.h" void tgInitQueryJson(HttpContext *pContext) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); diff --git a/src/util/inc/elog.h b/src/util/inc/elog.h deleted file mode 100644 index 8e11c3e761..0000000000 --- a/src/util/inc/elog.h +++ /dev/null @@ -1,71 +0,0 @@ -#ifndef _elog_h_8897be44_dda8_45b6_9d37_8d8691cb05fb_ -#define _elog_h_8897be44_dda8_45b6_9d37_8d8691cb05fb_ - -#include - -typedef enum { - ELOG_DEBUG, - ELOG_INFO, - ELOG_WARN, - ELOG_ERROR, - ELOG_CRITICAL, - ELOG_VERBOSE, - ELOG_ABORT, -} ELOG_LEVEL; - -void elog_set_level(ELOG_LEVEL base); // only log those not less than base -void elog_set_thread_name(const char *name); - -void elog(ELOG_LEVEL level, int fd, const char *file, int line, const char *func, const char *fmt, ...) -#ifdef __GNUC__ - __attribute__((format(printf, 6, 7))) -#endif -; - -#define DLOG(fd, fmt, ...) elog(ELOG_DEBUG, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define ILOG(fd, fmt, ...) elog(ELOG_INFO, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define WLOG(fd, fmt, ...) elog(ELOG_WARN, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define ELOG(fd, fmt, ...) elog(ELOG_ERROR, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define CLOG(fd, fmt, ...) elog(ELOG_CRITICAL, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define VLOG(fd, fmt, ...) elog(ELOG_VERBOSE, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define ALOG(fd, fmt, ...) elog(ELOG_ABORT, fd, __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) - -#define D(fmt, ...) elog(ELOG_DEBUG, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define I(fmt, ...) elog(ELOG_INFO, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define W(fmt, ...) elog(ELOG_WARN, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define E(fmt, ...) elog(ELOG_ERROR, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define C(fmt, ...) elog(ELOG_CRITICAL, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define V(fmt, ...) elog(ELOG_VERBOSE, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) -#define A(fmt, ...) elog(ELOG_ABORT, fileno(stdout), __FILE__, __LINE__, __FUNCTION__, fmt, ##__VA_ARGS__) - - - -// NOTE: https://en.wikipedia.org/wiki/Fail-fast -// for the sake of simplicity, both implementation and usage, -// we'll follow `fail-fast` or `let-it-crash` philosophy. - -// assertion in both debug/release build -#define EQ_ABORT(fmt, ...) A("Assertion failure: "fmt, ##__VA_ARGS__) - -#define EQ_ASSERT(statement) do { \ - if (statement) break; \ - A("Assertion failure: %s", #statement); \ -} while (0) - -#define EQ_ASSERT_EXT(statement, fmt, ...) do { \ - if (statement) break; \ - A("Assertion failure: %s: "fmt, #statement, ##__VA_ARGS__); \ -} while (0) - -#define EQ_ASSERT_API0(statement) do { \ - if (statement) break; \ - A("Assertion failure: %s failed: [%d]%s", #statement, errno, strerror(errno)); \ -} while (0) - -#define EQ_ASSERT_API(api) do { \ - A("Assertion failure: %s failed: [%d]%s", #api, errno, strerror(errno)); \ -} while (0) - - -#endif // _elog_h_8897be44_dda8_45b6_9d37_8d8691cb05fb_ - diff --git a/src/util/src/elog.c b/src/util/src/elog.c deleted file mode 100644 index 95b580962c..0000000000 --- a/src/util/src/elog.c +++ /dev/null @@ -1,95 +0,0 @@ -#include "elog.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#define gettid() syscall(__NR_gettid) - -static ELOG_LEVEL elog_level_base = ELOG_DEBUG; - -static __thread long elog_thread_id; -static __thread char elog_thread_name[24] = {0}; - -void elog_set_level(ELOG_LEVEL base) { - elog_level_base = base; -} - -void elog_set_thread_name(const char *name) { - elog_thread_id = gettid(); - snprintf(elog_thread_name, sizeof(elog_thread_name), "%s", name); -} - -void elog(ELOG_LEVEL level, int fd, const char *file, int line, const char *func, const char *fmt, ...) -{ - if (level < elog_level_base) return; - if (fd == -1) return; - - if (elog_thread_name[0]=='\0') { - elog_set_thread_name("unknown"); - } - - char *p; - int n; - size_t bytes; - - char buf[4096]; - snprintf(buf, sizeof(buf), "%s", file); - - char fn[1024]; - snprintf(fn, sizeof(fn), "%s", basename(buf)); - - char C; - switch (level) { - case ELOG_DEBUG: C = 'D'; break; - case ELOG_INFO: C = 'I'; break; - case ELOG_WARN: C = 'W'; break; - case ELOG_ERROR: C = 'E'; break; - case ELOG_CRITICAL: C = 'C'; break; - case ELOG_VERBOSE: C = 'V'; break; - case ELOG_ABORT: C = 'A'; break; - default: return; - } - - struct tm t; - struct timeval tv; - - if (gettimeofday(&tv, NULL)) return; - if (!localtime_r(&tv.tv_sec, &t)) return; - - p = buf; - bytes = sizeof(buf); - - n = snprintf(p, bytes, "%c[%02d/%02d %02d:%02d:%02d.%06ld][%06ld]: ==", - C, - t.tm_mon + 1, t.tm_mday, - t.tm_hour, t.tm_min, t.tm_sec, - tv.tv_usec, - elog_thread_id); - p += n; bytes -= n; - - va_list arg; - va_start(arg, fmt); - if (bytes>0) { - n = vsnprintf(p, bytes, fmt, arg); - p += n; bytes -= n; - } - va_end(arg); - - if (bytes>0) { - n = snprintf(p, bytes, "== t:%s#%s[%d]#%s()", - elog_thread_name, fn, line, func); - } - - dprintf(fd, "%s\n", buf); - - if (level == ELOG_ABORT) { - abort(); - } -} - From 6e3affb9231a1fe8e250aa51d91a08600a3a0a1f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 13 Sep 2020 00:52:14 +0800 Subject: [PATCH 032/105] [td-1319] --- src/client/src/tscServer.c | 22 +++++++----- src/client/src/tscSql.c | 73 +++++++++++++++++++++++--------------- src/client/src/tscUtil.c | 50 +++++++++++++++++++++++--- 3 files changed, 103 insertions(+), 42 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index eda6db8a2b..26fe19d8be 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -248,6 +248,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; + assert(*pSql->self == pSql); + if (pObj->signature != pObj) { tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature); @@ -263,6 +265,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature); + void** p1 = p; + taosCacheRelease(tscObjCache, (void**) &p1, false); + taosCacheRelease(tscObjCache, (void**) &p, true); rpcFreeCont(rpcMsg->pCont); return; @@ -368,21 +373,20 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); } - bool shouldFree = false; + bool shouldFree = tscShouldBeFreed(pSql);; if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; - - shouldFree = tscShouldBeFreed(pSql); (*pSql->fp)(pSql->param, pSql, rpcMsg->code); - - if (shouldFree) { - void** p1 = p; - taosCacheRelease(tscObjCache, (void **)&p1, true); - tscDebug("%p sqlObj is automatically freed", pSql); - } } + void** p1 = p; taosCacheRelease(tscObjCache, (void**) &p, false); + + if (shouldFree) { + taosCacheRelease(tscObjCache, (void **)&p1, true); + tscDebug("%p sqlObj is automatically freed", pSql); + } + rpcFreeCont(rpcMsg->pCont); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 33996307ad..58859e944c 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -26,6 +26,7 @@ #include "tsclient.h" #include "ttokendef.h" #include "tutil.h" +#include "tscProfile.h" static bool validImpl(const char* str, size_t maxsize) { if (str == NULL) { @@ -536,39 +537,53 @@ int taos_select_db(TAOS *taos, const char *db) { } // send free message to vnode to free qhandle and corresponding resources in vnode -//static bool tscKillQueryInVnode(SSqlObj* pSql) { -// SSqlCmd* pCmd = &pSql->cmd; -// SSqlRes* pRes = &pSql->res; -// -// SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); -// STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); -// -// if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { -// return false; -// } -// -// if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && (pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL) && -// (pCmd->command == TSDB_SQL_SELECT || -// pCmd->command == TSDB_SQL_SHOW || -// pCmd->command == TSDB_SQL_RETRIEVE || -// pCmd->command == TSDB_SQL_FETCH)) { -// -// pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; -// tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]); -// tscProcessSql(pSql); -// return true; -// } -// -// return false; -//} +static bool tscKillQueryInDnode(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + + if (pRes == NULL || pRes->qhandle == 0) { + return true; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) { + return true; + } + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + tscRemoveFromSqlList(pSql); + + int32_t cmd = pCmd->command; + if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && + (cmd == TSDB_SQL_SELECT || + cmd == TSDB_SQL_SHOW || + cmd == TSDB_SQL_RETRIEVE || + cmd == TSDB_SQL_FETCH)) { + pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s, ", pSql, sqlCmd[pCmd->command]); + + tscProcessSql(pSql); + return false; + } + + return true; +} void taos_free_result(TAOS_RES *res) { - if (res == NULL) { + SSqlObj* pSql = (SSqlObj*) res; + if (pSql == NULL || pSql->signature != pSql) { + tscError("%p already released sqlObj", res); return; } - SSqlObj* pSql = (SSqlObj*) res; - taosCacheRelease(tscObjCache, (void**) &pSql->self, true); + assert(pSql->self != 0 && *pSql->self == pSql); + + bool freeNow = tscKillQueryInDnode(pSql); + if (freeNow) { + tscDebug("%p free sqlObj in cache", pSql); + taosCacheRelease(tscObjCache, (void**) &pSql->self, true); + } } //static void doFreeResult(TAOS_RES *res) { @@ -596,7 +611,7 @@ void taos_free_result(TAOS_RES *res) { // } // // pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; -// if (!tscKillQueryInVnode(pSql)) { +// if (!tscKillQueryInDnode(pSql)) { // tscFreeSqlObj(pSql); // tscDebug("%p sqlObj is freed by app", pSql); // } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 47abe60ddd..7d4369816a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -363,6 +363,7 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { taosTFree(pSql->pSubs); pSql->numOfSubs = 0; + pSql->self = 0; tscResetSqlCmdObj(pCmd, false); } @@ -390,12 +391,36 @@ void tscFreeSqlObjInCache(void *pSql) { tscFreeSqlObj(*p); } +static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { + return false; + } + + int32_t cmd = pCmd->command; + if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && + (cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_SHOW || cmd == TSDB_SQL_RETRIEVE || cmd == TSDB_SQL_FETCH)) { + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]); + tscProcessSql(pSql); + return true; + } + + return false; +} + void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; } - tscDebug("%p start to free sql object", pSql); + tscDebug("%p start to free sqlObj", pSql); + tscFreeSubobj(pSql); tscPartiallyFreeSqlObj(pSql); @@ -412,6 +437,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { tsem_destroy(&pSql->rspSem); free(pSql); + tscDebug("%p free sqlObj completed", pSql); } void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { @@ -423,7 +449,10 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { taosTFree(pDataBlock->params); // free the refcount for metermeta - taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pTableMeta), false); + if (pDataBlock->pTableMeta != NULL) { + taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pTableMeta), false); + } + taosTFree(pDataBlock); } @@ -478,7 +507,10 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name)); - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false); + + if (pTableMetaInfo->pTableMeta != NULL) { + taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false); + } pTableMetaInfo->pTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pDataBlock->pTableMeta); } else { @@ -758,6 +790,13 @@ void tscCloseTscObj(STscObj* pObj) { if (p == NULL) { break; } + + tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p); + taosMsleep(100); + + // todo fix me!! two threads call taos_free_result will cause problem. + tscDebug("%p free :%p", pObj, p); + taos_free_result(p); } if (pObj->pDnodeConn != NULL) { @@ -1703,7 +1742,10 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) return; } - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); + if (pTableMetaInfo->pTableMeta != NULL) { + taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); + } + taosTFree(pTableMetaInfo->vgroupList); tscColumnListDestroy(pTableMetaInfo->tagColList); From ada7c188fa92bed67f4db28cef4e8b7d648b41ca Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sun, 13 Sep 2020 23:15:51 +0800 Subject: [PATCH 033/105] [TD-1329] add test case for incorrect count query --- tests/pytest/queryCount.py | 91 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tests/pytest/queryCount.py diff --git a/tests/pytest/queryCount.py b/tests/pytest/queryCount.py new file mode 100644 index 0000000000..3281c09269 --- /dev/null +++ b/tests/pytest/queryCount.py @@ -0,0 +1,91 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import threading +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class QueryCountMultiThread: + def initConnection(self): + self.records = 10000000 + self.numOfTherads = 50 + self.ts = 1537146000000 + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = "/home/xp/git/TDengine/sim/dnode1/cfg" + self.conn = taos.connect( + self.host, + self.user, + self.password, + self.config) + + def insertData(self, threadID): + cursor = self.conn.cursor() + print("Thread %d: starting" % threadID) + base = 200000 * threadID + for i in range(200): + query = "insert into tb values" + for j in range(1000): + query += "(%d, %d, 'test')" % (self.ts + base + i * 1000 + j, base + i * 1000 + j) + cursor.execute(query) + cursor.close() + print("Thread %d: finishing" % threadID) + + def run(self): + tdDnodes.init("") + tdDnodes.setTestCluster(False) + tdDnodes.setValgrind(False) + + tdDnodes.stopAll() + tdDnodes.deploy(1) + tdDnodes.start(1) + + cursor = self.conn.cursor() + cursor.execute("drop database if exists db") + cursor.execute("create database db") + cursor.execute("use db") + cursor.execute("create table tb (ts timestamp, id int, name nchar(30))") + cursor.close() + + threads = [] + for i in range(50): + thread = threading.Thread(target=self.insertData, args=(i,)) + threads.append(thread) + thread.start() + + for i in range(50): + threads[i].join() + + cursor = self.conn.cursor() + cursor.execute("use db") + sql = "select count(*) from tb" + cursor.execute(sql) + data = cursor.fetchall() + + if(data[0][0] == 10000000): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%d" % (sql, 0, 0, data[0][0], 10000000)) + else: + tdLog.exit("sql:%s, row:%d col:%d data:%d == expect:%d" % (sql, 0, 0, data[0][0], 10000000)) + + cursor.close() + self.conn.close() + +q = QueryCountMultiThread() +q.initConnection() +q.run() \ No newline at end of file From eff4b5807a9a98b4d2c5767448510fa7098f3771 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Sep 2020 04:50:45 +0000 Subject: [PATCH 034/105] TD-1311 --- src/inc/taoserror.h | 37 +- src/plugins/http/inc/httpAuth.h | 6 +- src/plugins/http/inc/httpGcJson.h | 2 +- src/plugins/http/inc/httpGzip.h | 29 +- src/plugins/http/inc/httpInt.h | 118 +-- src/plugins/http/inc/httpJson.h | 62 +- src/plugins/http/inc/httpParser.h | 155 ++-- src/plugins/http/inc/httpResp.h | 6 +- src/plugins/http/inc/httpRestJson.h | 8 +- src/plugins/http/inc/httpServer.h | 5 +- src/plugins/http/inc/httpSql.h | 14 +- src/plugins/http/inc/httpTgJson.h | 6 +- src/plugins/http/inc/httpUtil.h | 18 +- src/plugins/http/src/httpAuth.c | 16 +- src/plugins/http/src/httpContext.c | 183 +---- src/plugins/http/src/httpGcHandle.c | 25 +- src/plugins/http/src/httpGcJson.c | 30 +- src/plugins/http/src/httpGzip.c | 34 +- src/plugins/http/src/httpHandle.c | 229 +----- src/plugins/http/src/httpJson.c | 111 +-- src/plugins/http/src/httpParser.c | 1027 +++++++++++++++---------- src/plugins/http/src/httpQueue.c | 6 +- src/plugins/http/src/httpResp.c | 40 +- src/plugins/http/src/httpRestHandle.c | 16 +- src/plugins/http/src/httpRestJson.c | 22 +- src/plugins/http/src/httpServer.c | 87 +-- src/plugins/http/src/httpSystem.c | 6 +- src/plugins/http/src/httpTgHandle.c | 103 ++- src/plugins/http/src/httpTgJson.c | 14 +- src/plugins/http/src/httpUtil.c | 68 +- 30 files changed, 1145 insertions(+), 1338 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index d8e5c8f1d7..3f7995f25e 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -97,8 +97,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, 0, 0x0211, "Applicatio TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, 0, 0x0216, "Syntax errr in SQL") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, 0, 0x0216, "Syntax errr in SQL") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed") @@ -247,6 +247,39 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync modul // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") +// http +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_UNSUPPORT_URL, 0, 0x1100, "http url is not support") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_ENOUGH_MEMORY, 0, 0x1101, "no enough memory") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_VERSION, 0, 0x1102, "invalid http version") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH, 0, 0x1103, "invalid content length") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_CREATE_GZIP_FAILED, 0, 0x1104, "failed to create gzip") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_FINISH_GZIP_FAILED, 0, 0x1105, "failed to finish gzip") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_TYPE, 0, 0x1106, "invalid type of Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_FORMAT, 0, 0x1107, "invalid format of Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_BASIC_AUTH, 0, 0x1108, "invalid basic Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_TAOSD_AUTH, 0, 0x1109, "invalid taosd Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_METHOD_FAILED, 0, 0x110A, "failed to parse method") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_TARGET_FAILED, 0, 0x110B, "failed to parse target") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_VERSION_FAILED, 0, 0x110C, "failed to parse http version") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_SP_FAILED, 0, 0x110D, "failed to parse sp") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_STATUS_FAILED, 0, 0x110E, "failed to parse status") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_PHRASE_FAILED, 0, 0x110F, "failed to parse phrase") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CRLF_FAILED, 0, 0x1110, "failed to parse crlf") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_FAILED, 0, 0x1111, "failed to parse header") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED, 0, 0x1112, "failed to parse header key") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED, 0, 0x1113, "failed to parse header val") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED, 0, 0x1114, "failed to parse chunk size") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_FAILED, 0, 0x1115, "failed to parse chunk") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_END_FAILED, 0, 0x1116, "failed to parse end section") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_INVALID_STATE, 0, 0x1117, "invalid parse state") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_ERROR_STATE, 0, 0x1118, "failed to parse error section") + + + + + + + #ifdef TAOS_ERROR_C }; #endif diff --git a/src/plugins/http/inc/httpAuth.h b/src/plugins/http/inc/httpAuth.h index b8fabbe1ec..4becae6332 100644 --- a/src/plugins/http/inc/httpAuth.h +++ b/src/plugins/http/inc/httpAuth.h @@ -16,8 +16,8 @@ #ifndef TDENGINE_HTTP_TOKEN_H #define TDENGINE_HTTP_TOKEN_H -bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len); -bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len); -bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen); +bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len); +bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len); +bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen); #endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpGcJson.h b/src/plugins/http/inc/httpGcJson.h index 609bb9b95e..0ba860687d 100644 --- a/src/plugins/http/inc/httpGcJson.h +++ b/src/plugins/http/inc/httpGcJson.h @@ -24,7 +24,7 @@ void gcCleanQueryJson(HttpContext *pContext); void gcStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result); void gcStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd); -bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows); +bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); void gcSendHeartBeatResp(HttpContext *pContext, HttpSqlCmd *cmd); diff --git a/src/plugins/http/inc/httpGzip.h b/src/plugins/http/inc/httpGzip.h index b2d6ace9b0..aeac79c975 100644 --- a/src/plugins/http/inc/httpGzip.h +++ b/src/plugins/http/inc/httpGzip.h @@ -1,7 +1,20 @@ -#ifndef _ehttp_gzip_h_9196791b_ac2a_4d73_9979_f4b41abbc4c0_ -#define _ehttp_gzip_h_9196791b_ac2a_4d73_9979_f4b41abbc4c0_ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ -#include +#ifndef HTTP_GZIP_H +#define HTTP_GZIP_H #define EHTTP_GZIP_CHUNK_SIZE_DEFAULT (1024*16) @@ -11,20 +24,20 @@ typedef struct ehttp_gzip_callbacks_s ehttp_gzip_callbacks_t; typedef struct ehttp_gzip_conf_s ehttp_gzip_conf_t; struct ehttp_gzip_callbacks_s { - void (*on_data)(ehttp_gzip_t *gzip, void *arg, const char *buf, size_t len); + void (*on_data)(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len); }; struct ehttp_gzip_conf_s { - int get_header:2; // 0: not fetching header info - size_t chunk_size; // 0: fallback to default: EHTTP_GZIP_CHUNK_SIZE_DEFAULT + int32_t get_header:2; // 0: not fetching header info + int32_t chunk_size; // 0: fallback to default: EHTTP_GZIP_CHUNK_SIZE_DEFAULT }; ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); void ehttp_gzip_destroy(ehttp_gzip_t *gzip); -int ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, size_t len); -int ehttp_gzip_finish(ehttp_gzip_t *gzip); +int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len); +int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip); #endif // _ehttp_gzip_h_9196791b_ac2a_4d73_9979_f4b41abbc4c0_ diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 4348e5e090..86e1eeb6d6 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -30,51 +30,26 @@ #include "httpParser.h" #define HTTP_MAX_CMD_SIZE 1024 -#define HTTP_MAX_BUFFER_SIZE 1024*1024 - +#define HTTP_MAX_BUFFER_SIZE 1024*1024*8 #define HTTP_LABEL_SIZE 8 #define HTTP_MAX_EVENTS 10 -#define HTTP_BUFFER_SIZE 1024*65 //65k -#define HTTP_DECOMPRESS_BUF_SIZE 1024*64 +#define HTTP_BUFFER_INIT 8192 +#define HTTP_BUFFER_SIZE 8192000 #define HTTP_STEP_SIZE 1024 //http message get process step by step -#define HTTP_MAX_URL 5 //http url stack size #define HTTP_METHOD_SCANNER_SIZE 7 //http method fp size #define HTTP_GC_TARGET_SIZE 512 - -#define HTTP_VERSION_10 0 -#define HTTP_VERSION_11 1 -//#define HTTP_VERSION_12 2 - -#define HTTP_UNCUNKED 0 -#define HTTP_CHUNKED 1 - -#define HTTP_KEEPALIVE_NO_INPUT 0 -#define HTTP_KEEPALIVE_ENABLE 1 -#define HTTP_KEEPALIVE_DISABLE 2 - -#define HTTP_REQTYPE_OTHERS 0 -#define HTTP_REQTYPE_LOGIN 1 -#define HTTP_REQTYPE_HEARTBEAT 2 -#define HTTP_REQTYPE_SINGLE_SQL 3 -#define HTTP_REQTYPE_MULTI_SQL 4 - -#define HTTP_CHECK_BODY_ERROR -1 -#define HTTP_CHECK_BODY_CONTINUE 0 -#define HTTP_CHECK_BODY_SUCCESS 1 - -#define HTTP_READ_DATA_SUCCESS 0 -#define HTTP_READ_DATA_FAILED 1 - #define HTTP_WRITE_RETRY_TIMES 500 #define HTTP_WRITE_WAIT_TIME_MS 5 -#define HTTP_EXPIRED_TIME 60000 -#define HTTP_DELAY_CLOSE_TIME_MS 500 - -#define HTTP_COMPRESS_IDENTITY 0 -#define HTTP_COMPRESS_GZIP 2 - #define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + TSDB_PASSWORD_LEN) +typedef enum HttpReqType { + HTTP_REQTYPE_OTHERS = 0, + HTTP_REQTYPE_LOGIN = 1, + HTTP_REQTYPE_HEARTBEAT = 2, + HTTP_REQTYPE_SINGLE_SQL = 3, + HTTP_REQTYPE_MULTI_SQL = 4 +} HttpReqType; + typedef enum { HTTP_SERVER_INIT, HTTP_SERVER_RUNNING, @@ -83,21 +58,12 @@ typedef enum { } HttpServerStatus; typedef enum { - HTTP_CONTEXT_STATE_READY, - HTTP_CONTEXT_STATE_HANDLING, - HTTP_CONTEXT_STATE_DROPPING, - HTTP_CONTEXT_STATE_CLOSED + HTTP_CONTEXT_STATE_READY, + HTTP_CONTEXT_STATE_HANDLING, + HTTP_CONTEXT_STATE_DROPPING, + HTTP_CONTEXT_STATE_CLOSED } HttpContextState; -struct HttpContext; -struct HttpThread; - -typedef struct { - char id[HTTP_SESSION_ID_LEN]; - int refCount; - void *taos; -} HttpSession; - typedef enum { HTTP_CMD_TYPE_UN_SPECIFIED, HTTP_CMD_TYPE_CREATE_DB, @@ -109,6 +75,15 @@ typedef enum { HTTP_CMD_STATE_NOT_RUN_YET, HTTP_CMD_STATE_RUN_FINISHED } HttpSql typedef enum { HTTP_CMD_RETURN_TYPE_WITH_RETURN, HTTP_CMD_RETURN_TYPE_NO_RETURN } HttpSqlCmdReturnType; +struct HttpContext; +struct HttpThread; + +typedef struct { + char id[HTTP_SESSION_ID_LEN]; + int32_t refCount; + void * taos; +} HttpSession; + typedef struct { // used by single cmd char *nativSql; @@ -158,42 +133,17 @@ typedef struct { void (*setNextCmdFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code); } HttpEncodeMethod; -typedef struct { - char *pos; - int32_t len; -} HttpBuf; - typedef enum { EHTTP_CONTEXT_PROCESS_FAILED = 0x01, EHTTP_CONTEXT_PARSER_FAILED = 0x02 } EHTTP_CONTEXT_FAILED_CAUSE; -typedef struct { - char buffer[HTTP_BUFFER_SIZE]; - int bufsize; - char *pLast; - char *pCur; - HttpBuf method; - HttpBuf path[HTTP_MAX_URL]; // url: dbname/meter/query - HttpBuf data; // body content - HttpBuf token; // auth token - HttpDecodeMethod *pMethod; - HttpParserObj * parser; - int8_t inited; - int8_t failed; -} HttpParser; - typedef struct HttpContext { int32_t refCount; - int fd; + int32_t fd; uint32_t accessTimes; uint32_t lastAccessTime; int32_t state; - uint8_t httpVersion; - uint8_t httpChunked; - uint8_t httpKeepAlive; // http1.0 and not keep-alive, close connection immediately - uint8_t acceptEncoding; - uint8_t contentEncoding; uint8_t reqType; uint8_t parsed; char ipstr[22]; @@ -203,12 +153,12 @@ typedef struct HttpContext { void * ppContext; HttpSession *session; z_stream gzipStream; - HttpParser parser; + HttpParser *parser; HttpSqlCmd singleCmd; HttpSqlCmds *multiCmds; JsonBuf * jsonBuf; - void * timer; - HttpEncodeMethod * encodeMethod; + HttpEncodeMethod *encodeMethod; + HttpDecodeMethod *decodeMethod; struct HttpThread *pThread; } HttpContext; @@ -217,9 +167,9 @@ typedef struct HttpThread { HttpContext * pHead; pthread_mutex_t threadMutex; bool stop; - int pollFd; - int numOfContexts; - int threadId; + int32_t pollFd; + int32_t numOfContexts; + int32_t threadId; char label[HTTP_LABEL_SIZE]; bool (*processData)(HttpContext *pContext); } HttpThread; @@ -228,9 +178,9 @@ typedef struct HttpServer { char label[HTTP_LABEL_SIZE]; uint32_t serverIp; uint16_t serverPort; - int fd; - int numOfThreads; - int methodScannerLen; + int32_t fd; + int32_t numOfThreads; + int32_t methodScannerLen; int32_t requestNum; int32_t status; pthread_t thread; diff --git a/src/plugins/http/inc/httpJson.h b/src/plugins/http/inc/httpJson.h index 905460c67b..ac0a632137 100644 --- a/src/plugins/http/inc/httpJson.h +++ b/src/plugins/http/inc/httpJson.h @@ -37,65 +37,65 @@ extern char JsonTrueTkn[]; extern char JsonFalseTkn[]; typedef struct { - int size; - int total; - char* lst; - char buf[JSON_BUFFER_SIZE]; - struct HttpContext* pContext; + int32_t size; + int32_t total; + char* lst; + char buf[JSON_BUFFER_SIZE]; + struct HttpContext* pContext; } JsonBuf; // http response -int httpWriteBuf(struct HttpContext* pContext, const char* buf, int sz); -int httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int sz); -int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz); +int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz); +int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz); +int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t sz); // builder callback typedef void (*httpJsonBuilder)(JsonBuf* buf, void* jsnHandle); // buffer -void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext); -void httpWriteJsonBufHead(JsonBuf* buf); -int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast); -void httpWriteJsonBufEnd(JsonBuf* buf); +void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext); +void httpWriteJsonBufHead(JsonBuf* buf); +int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast); +void httpWriteJsonBufEnd(JsonBuf* buf); // value -void httpJsonString(JsonBuf* buf, char* sVal, int len); -void httpJsonOriginString(JsonBuf* buf, char* sVal, int len); -void httpJsonStringForTransMean(JsonBuf* buf, char* SVal, int maxLen); +void httpJsonString(JsonBuf* buf, char* sVal, int32_t len); +void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len); +void httpJsonStringForTransMean(JsonBuf* buf, char* SVal, int32_t maxLen); void httpJsonInt64(JsonBuf* buf, int64_t num); void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us); void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us); -void httpJsonInt(JsonBuf* buf, int num); +void httpJsonInt(JsonBuf* buf, int32_t num); void httpJsonFloat(JsonBuf* buf, float num); void httpJsonDouble(JsonBuf* buf, double num); void httpJsonNull(JsonBuf* buf); -void httpJsonBool(JsonBuf* buf, int val); +void httpJsonBool(JsonBuf* buf, int32_t val); // pair -void httpJsonPair(JsonBuf* buf, char* name, int nameLen, char* sVal, int valLen); -void httpJsonPairOriginString(JsonBuf* buf, char* name, int nameLen, char* sVal, int valLen); -void httpJsonPairHead(JsonBuf* buf, char* name, int len); -void httpJsonPairIntVal(JsonBuf* buf, char* name, int nNameLen, int num); -void httpJsonPairInt64Val(JsonBuf* buf, char* name, int nNameLen, int64_t num); -void httpJsonPairBoolVal(JsonBuf* buf, char* name, int nNameLen, int num); -void httpJsonPairFloatVal(JsonBuf* buf, char* name, int nNameLen, float num); -void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int nNameLen, double num); -void httpJsonPairNullVal(JsonBuf* buf, char* name, int nNameLen); +void httpJsonPair(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen); +void httpJsonPairOriginString(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen); +void httpJsonPairHead(JsonBuf* buf, char* name, int32_t len); +void httpJsonPairIntVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num); +void httpJsonPairInt64Val(JsonBuf* buf, char* name, int32_t nNameLen, int64_t num); +void httpJsonPairBoolVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num); +void httpJsonPairFloatVal(JsonBuf* buf, char* name, int32_t nNameLen, float num); +void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int32_t nNameLen, double num); +void httpJsonPairNullVal(JsonBuf* buf, char* name, int32_t nNameLen); // object -void httpJsonPairArray(JsonBuf* buf, char* name, int nLen, httpJsonBuilder builder, void* dsHandle); -void httpJsonPairObject(JsonBuf* buf, char* name, int nLen, httpJsonBuilder builder, void* dsHandle); +void httpJsonPairArray(JsonBuf* buf, char* name, int32_t nLen, httpJsonBuilder builder, void* dsHandle); +void httpJsonPairObject(JsonBuf* buf, char* name, int32_t nLen, httpJsonBuilder builder, void* dsHandle); void httpJsonObject(JsonBuf* buf, httpJsonBuilder fnBuilder, void* dsHandle); void httpJsonArray(JsonBuf* buf, httpJsonBuilder fnBuidler, void* jsonHandle); // print -void httpJsonTestBuf(JsonBuf* buf, int safety); +void httpJsonTestBuf(JsonBuf* buf, int32_t safety); void httpJsonToken(JsonBuf* buf, char c); void httpJsonItemToken(JsonBuf* buf); -void httpJsonPrint(JsonBuf* buf, const char* json, int len); +void httpJsonPrint(JsonBuf* buf, const char* json, int32_t len); // quick -void httpJsonPairStatus(JsonBuf* buf, int code); +void httpJsonPairStatus(JsonBuf* buf, int32_t code); // http json printer JsonBuf* httpMallocJsonBuf(struct HttpContext* pContext); diff --git a/src/plugins/http/inc/httpParser.h b/src/plugins/http/inc/httpParser.h index 4facea0286..5694c8f0cd 100644 --- a/src/plugins/http/inc/httpParser.h +++ b/src/plugins/http/inc/httpParser.h @@ -1,9 +1,23 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #ifndef HTTP_PARSER_H #define HTTP_PARSER_H - #include "httpGzip.h" -struct HttpContext; +#define HTTP_MAX_URL 5 // http url stack size typedef enum HTTP_PARSER_STATE { HTTP_PARSER_BEGIN, @@ -24,79 +38,82 @@ typedef enum HTTP_PARSER_STATE { HTTP_PARSER_ERROR, } HTTP_PARSER_STATE; -typedef struct HttpParserString { - char * str; - size_t len; -} HttpParserString; +typedef enum HTTP_AUTH_TYPE { + HTTP_INVALID_AUTH, + HTTP_BASIC_AUTH, + HTTP_TAOSD_AUTH +} HTTP_AUTH_TYPE; -typedef struct HttpParserStatusObj { - int32_t status_code; - const char *status_desc; -} HttpParserStatusObj; +typedef enum HTTP_VERSION { + HTTP_VERSION_10 = 0, + HTTP_VERSION_11 = 1, + HTTP_VERSION_12 = 2, + HTTP_INVALID_VERSION +} HTTP_VERSION; -typedef struct HttpParserCallbackObj { - void (*on_request_line)(void *arg, const char *method, const char *target, const char *version, const char *target_raw); - void (*on_status_line)(void *arg, const char *version, int status_code, const char *reason_phrase); - void (*on_header_field)(void *arg, const char *key, const char *val); - void (*on_body)(void *arg, const char *chunk, size_t len); - void (*on_end)(void *arg); - void (*on_error)(void *arg, int status_code); -} HttpParserCallbackObj; +typedef enum HTTP_KEEPALIVE { + HTTP_KEEPALIVE_NO_INPUT = 0, + HTTP_KEEPALIVE_ENABLE = 1, + HTTP_KEEPALIVE_DISABLE = 2 +} HTTP_KEEPALIVE; -typedef struct HttpParserConfObj { - size_t flush_block_size; // <=0: immediately -} HttpParserConfObj; +typedef struct HttpString { + char * str; + int32_t pos; + int32_t size; +} HttpString; -typedef struct HttpParseKvObj { - char *key; - char *val; -} HttpParseKvObj; +typedef struct HttpStatus { + int32_t code; + char * desc; +} HttpStatus; -typedef struct HttpParserObj { - HttpParserCallbackObj callbacks; - HttpParserConfObj conf; - void * arg; - char * method; - char * target; - char * target_raw; - char * version; - int http_10 : 2; - int http_11 : 2; - int accept_encoding_gzip : 2; - int accept_encoding_chunked : 2; - int transfer_gzip : 2; - int transfer_chunked : 2; - int content_length_specified : 2; - int content_chunked : 2; - int status_code; - char * reason_phrase; - char * key; - char * val; - HttpParseKvObj * kvs; - size_t kvs_count; - char * auth_basic; - char * auth_taosd; - size_t content_length; - size_t chunk_size; - size_t received_chunk_size; - size_t received_size; - ehttp_gzip_t * gzip; - HttpParserString str; - HTTP_PARSER_STATE *stacks; - size_t stacks_count; -} HttpParserObj; +typedef struct HttpStack{ + int8_t *stacks; + int32_t pos; + int32_t size; +} HttpStack; -void httpParserCleanupString(HttpParserString *str); -int32_t httpParserAppendString(HttpParserString *str, const char *s, int32_t len); -void httpParserClearString(HttpParserString *str); +struct HttpContext; +typedef struct HttpParser { + struct HttpContext *pContext; + ehttp_gzip_t *gzip; + HttpStack stacks; + HttpString str; + HttpString body; + HttpString path[HTTP_MAX_URL]; + char * method; + char * target; + char * target_raw; + char * version; + char * reasonPhrase; + char * key; + char * val; + char * authContent; + int8_t httpVersion; + int8_t acceptEncodingGzip; + int8_t acceptEncodingChunked; + int8_t contentLengthSpecified; + int8_t contentChunked; + int8_t transferGzip; + int8_t transferChunked; + int8_t keepAlive; + int8_t authType; + int32_t contentLength; + int32_t chunkSize; + int32_t receivedChunkSize; + int32_t receivedSize; + int32_t statusCode; + int8_t inited; + int8_t parsed; + int16_t httpCode; + int32_t parseCode; +} HttpParser; - -HttpParserObj* httpParserCreate(HttpParserCallbackObj callbacks, HttpParserConfObj conf, void *arg); -void httpParserDestroy(HttpParserObj *parser); -int32_t httpParserBuf(struct HttpContext *pContext, HttpParserObj *parser, const char *buf, int32_t len); - -char* ehttp_parser_urldecode(const char *enc); - -const char* ehttp_status_code_get_desc(const int status_code); +void httpInitParser(HttpParser *parser); +HttpParser *httpCreateParser(struct HttpContext *pContext); +void httpDestroyParser(HttpParser *parser); +int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len); +char * httpGetStatusDesc(int32_t statusCode); #endif diff --git a/src/plugins/http/inc/httpResp.h b/src/plugins/http/inc/httpResp.h index 5eaaa2a037..9086d0c9d3 100644 --- a/src/plugins/http/inc/httpResp.h +++ b/src/plugins/http/inc/httpResp.h @@ -32,9 +32,9 @@ enum _httpRespTempl { extern const char *httpRespTemplate[]; -void httpSendErrorResp(HttpContext *pContext, int errNo); -void httpSendErrorRespWithDesc(HttpContext *pContext, int errNo, char *desc); -void httpSendTaosdErrorResp(HttpContext *pContext, int errCode); +void httpSendErrorResp(HttpContext *pContext, int32_t errNo); +void httpSendErrorRespWithDesc(HttpContext *pContext, int32_t errNo, char *desc); +void httpSendTaosdErrorResp(HttpContext *pContext, int32_t errCode); void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg); void httpSendSuccResp(HttpContext *pContext, char *desc); void httpSendOptionResp(HttpContext *pContext, char *desc); diff --git a/src/plugins/http/inc/httpRestJson.h b/src/plugins/http/inc/httpRestJson.h index 7cff21d190..112e845f36 100644 --- a/src/plugins/http/inc/httpRestJson.h +++ b/src/plugins/http/inc/httpRestJson.h @@ -43,12 +43,12 @@ #define REST_TIMESTAMP_FMT_TIMESTAMP 1 #define REST_TIMESTAMP_FMT_UTC_STRING 2 -void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect_rows); +void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows); void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result); -bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows); -bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows); -bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows); +bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); +bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); +bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd); #endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpServer.h b/src/plugins/http/inc/httpServer.h index 508baa6112..58ed3545f3 100644 --- a/src/plugins/http/inc/httpServer.h +++ b/src/plugins/http/inc/httpServer.h @@ -21,8 +21,7 @@ bool httpInitConnect(); void httpCleanUpConnect(); -void *httpInitServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); -void httpCleanUpServer(HttpServer *pServer); -int httpReadDataImp(HttpContext *pContext); +void *httpInitServer(char *ip, uint16_t port, char *label, int32_t numOfThreads, void *fp, void *shandle); +void httpCleanUpServer(HttpServer *pServer); #endif diff --git a/src/plugins/http/inc/httpSql.h b/src/plugins/http/inc/httpSql.h index 09f5b142fb..660a65e44d 100644 --- a/src/plugins/http/inc/httpSql.h +++ b/src/plugins/http/inc/httpSql.h @@ -19,20 +19,20 @@ int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...); int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const format, ...); -int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize); +int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int32_t mallocSize); int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext); -bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize); -bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize); -bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int bufferSize); +bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferSize); +bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize); +bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize); void httpFreeMultiCmds(HttpContext *pContext); HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext); HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext); -int httpCurSqlCmdPos(HttpContext *pContext); +int32_t httpCurSqlCmdPos(HttpContext *pContext); void httpTrimTableName(char *name); -int httpShrinkTableName(HttpContext *pContext, int pos, char *name); -char *httpGetCmdsString(HttpContext *pContext, int pos); +int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name); +char *httpGetCmdsString(HttpContext *pContext, int32_t pos); #endif diff --git a/src/plugins/http/inc/httpTgJson.h b/src/plugins/http/inc/httpTgJson.h index bf3aa093ae..6b7d0681b6 100644 --- a/src/plugins/http/inc/httpTgJson.h +++ b/src/plugins/http/inc/httpTgJson.h @@ -24,8 +24,8 @@ void tgInitQueryJson(HttpContext *pContext); void tgCleanQueryJson(HttpContext *pContext); void tgStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result); void tgStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd); -void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect_rows); -bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code); -void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int code); +void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows); +bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code); +void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code); #endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpUtil.h b/src/plugins/http/inc/httpUtil.h index c82f702ebc..61cd50a77a 100644 --- a/src/plugins/http/inc/httpUtil.h +++ b/src/plugins/http/inc/httpUtil.h @@ -17,21 +17,19 @@ #define TDENGINE_HTTP_UTIL_H bool httpCheckUsedbSql(char *sql); -void httpTimeToString(time_t t, char *buf, int buflen); +void httpTimeToString(time_t t, char *buf, int32_t buflen); -bool httpUrlMatch(HttpContext *pContext, int pos, char *cmp); -bool httpParseRequest(HttpContext *pContext); -int httpCheckReadCompleted(HttpContext *pContext); -void httpReadDirtyData(HttpContext *pContext); +bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp); +bool httpParseRequest(HttpContext *pContext); +int32_t httpCheckReadCompleted(HttpContext *pContext); +void httpReadDirtyData(HttpContext *pContext); -int httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData); -int httpGzipCompressInit(HttpContext *pContext); -int httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLen, +int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData); +int32_t httpGzipCompressInit(HttpContext *pContext); +int32_t httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLen, char *outDestData, int32_t *outDestDataLen, bool isTheLast); // http request parser void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod); - - #endif diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c index dd4d14c709..40bb691b5d 100644 --- a/src/plugins/http/src/httpAuth.c +++ b/src/plugins/http/src/httpAuth.c @@ -23,9 +23,9 @@ #define KEY_DES_4 4971256377704625728L -bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) { +bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; - int outlen = 0; + int32_t outlen = 0; char *base64 = (char *)base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token); @@ -40,7 +40,7 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) { return false; } - int user_len = (int)(user - base64); + int32_t user_len = (int32_t)(user - base64); if (user_len < 1 || user_len >= TSDB_USER_LEN) { httpError("context:%p, fd:%d, basic token:%s parse user error", pContext, pContext->fd, token); free(base64); @@ -50,7 +50,7 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) { pContext->user[user_len] = 0; char *password = user + 1; - int pass_len = (int)((base64 + outlen) - password); + int32_t pass_len = (int32_t)((base64 + outlen) - password); if (pass_len < 1 || pass_len >= TSDB_PASSWORD_LEN) { httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token); free(base64); @@ -64,9 +64,9 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) { return true; } -bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) { +bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; - int outlen = 0; + int32_t outlen = 0; unsigned char *base64 = base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token); @@ -96,7 +96,7 @@ bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) { } } -bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen) { +bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) { char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; size_t size = sizeof(pContext->user); tstrncpy(buffer, pContext->user, size); @@ -111,7 +111,7 @@ bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen) { free(encrypt); free(base64); - httpDebug("context:%p, fd:%d, gen taosd token:%s", pContext, pContext->fd, token); + httpDebug("context:%p, fd:%d, generate taosd token:%s", pContext, pContext->fd, token); return true; } diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index d310f27ee6..2c65b9d16a 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -27,19 +27,7 @@ #include "httpSql.h" #include "httpSession.h" #include "httpContext.h" - -extern bool httpGetHttpMethod(HttpContext* pContext); -extern bool httpParseURL(HttpContext* pContext); -extern bool httpParseHttpVersion(HttpContext* pContext); -extern bool httpGetDecodeMethod(HttpContext* pContext); -extern bool httpParseHead(HttpContext* pContext); - -static void httpParseOnRequestLine(void *arg, const char *method, const char *target, const char *version, const char *target_raw); -static void httpParseOnStatusLine(void *arg, const char *version, int status_code, const char *reason_phrase); -static void httpParseOnHeaderField(void *arg, const char *key, const char *val); -static void httpParseOnBody(void *arg, const char *chunk, size_t len); -static void httpParseOnEnd(void *arg); -static void httpParseOnError(void *arg, int status_code); +#include "httpParser.h" static void httpDestroyContext(void *data); @@ -70,9 +58,9 @@ static void httpDestroyContext(void *data) { httpFreeJsonBuf(pContext); httpFreeMultiCmds(pContext); - if (pContext->parser.parser) { - httpParserDestroy(pContext->parser.parser); - pContext->parser.parser = NULL; + if (pContext->parser) { + httpDestroyParser(pContext->parser); + pContext->parser = NULL; } taosTFree(pContext); @@ -125,9 +113,9 @@ HttpContext *httpCreateContext(int32_t fd) { if (pContext == NULL) return NULL; pContext->fd = fd; - pContext->httpVersion = HTTP_VERSION_10; pContext->lastAccessTime = taosGetTimestampSec(); pContext->state = HTTP_CONTEXT_STATE_READY; + pContext->parser = httpCreateParser(pContext); uint64_t handleVal = (uint64_t)pContext; HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, sizeof(int64_t), &pContext, sizeof(int64_t), 3000); @@ -164,6 +152,7 @@ void httpReleaseContext(HttpContext *pContext) { return; } + pContext->parser->inited = 0; HttpContext **ppContext = pContext->ppContext; httpDebug("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount); @@ -178,45 +167,24 @@ void httpReleaseContext(HttpContext *pContext) { bool httpInitContext(HttpContext *pContext) { pContext->accessTimes++; pContext->lastAccessTime = taosGetTimestampSec(); - pContext->httpVersion = HTTP_VERSION_10; - pContext->httpKeepAlive = HTTP_KEEPALIVE_NO_INPUT; - pContext->httpChunked = HTTP_UNCUNKED; - pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY; - pContext->contentEncoding = HTTP_COMPRESS_IDENTITY; + pContext->reqType = HTTP_REQTYPE_OTHERS; pContext->encodeMethod = NULL; - pContext->timer = NULL; memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); - HttpParser *pParser = &pContext->parser; - memset(pParser, 0, sizeof(HttpParser)); - pParser->pCur = pParser->pLast = pParser->buffer; - - HttpParserCallbackObj callbacks = { - httpParseOnRequestLine, - httpParseOnStatusLine, - httpParseOnHeaderField, - httpParseOnBody, - httpParseOnEnd, - httpParseOnError - }; - HttpParserConfObj conf = { - .flush_block_size = 0 - }; - pParser->parser = httpParserCreate(callbacks, conf, pContext); - pParser->inited = 1; httpDebug("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed); return true; } void httpCloseContextByApp(HttpContext *pContext) { + HttpParser *parser = pContext->parser; pContext->parsed = false; bool keepAlive = true; - if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) { + if (parser->httpVersion == HTTP_VERSION_10 && parser->keepAlive != HTTP_KEEPALIVE_ENABLE) { keepAlive = false; - } else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) { + } else if (parser->httpVersion != HTTP_VERSION_10 && parser->keepAlive == HTTP_KEEPALIVE_DISABLE) { keepAlive = false; } else { } @@ -262,134 +230,3 @@ void httpCloseContextByServer(HttpContext *pContext) { pContext->parsed = false; httpRemoveContextFromEpoll(pContext); } - -static void httpParseOnRequestLine(void *arg, const char *method, const char *target, const char *version, const char *target_raw) { - HttpContext *pContext = (HttpContext*)arg; - HttpParser *pParser = &pContext->parser; - - int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); - int n = snprintf(pParser->pLast, avail, "%s %s %s\r\n", method, target_raw, version); - char *last = pParser->pLast; - - do { - if (n >= avail) { - httpDebug("context:%p, fd:%d, request line(%s,%s,%s,%s), exceeding buffer size", pContext, pContext->fd, method, - target, version, target_raw); - break; - } - pParser->bufsize += n; - - if (!httpGetHttpMethod(pContext)) { - httpDebug("context:%p, fd:%d, request line(%s,%s,%s,%s), parse http method failed", pContext, pContext->fd, - method, target, version, target_raw); - break; - } - if (!httpParseURL(pContext)) { - httpDebug("context:%p, fd:%d, request line(%s,%s,%s,%s), parse http url failed", pContext, pContext->fd, method, - target, version, target_raw); - break; - } - if (!httpParseHttpVersion(pContext)) { - httpDebug("context:%p, fd:%d, request line(%s,%s,%s,%s), parse http version failed", pContext, pContext->fd, - method, target, version, target_raw); - break; - } - if (!httpGetDecodeMethod(pContext)) { - httpDebug("context:%p, fd:%d, request line(%s,%s,%s,%s), get decode method failed", pContext, pContext->fd, - method, target, version, target_raw); - break; - } - - last += n; - pParser->pLast = last; - return; - } while (0); - - pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; -} - -static void httpParseOnStatusLine(void *arg, const char *version, int status_code, const char *reason_phrase) { - HttpContext *pContext = (HttpContext*)arg; - HttpParser *pParser = &pContext->parser; - - httpDebug("context:%p, fd:%d, failed to parse status line ", pContext, pContext->fd); - pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; -} - -static void httpParseOnHeaderField(void *arg, const char *key, const char *val) { - HttpContext *pContext = (HttpContext*)arg; - HttpParser *pParser = &pContext->parser; - - if (pParser->failed) return; - - httpDebug("context:%p, fd:%d, key:%s val:%s", pContext, pContext->fd, key, val); - int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); - int n = snprintf(pParser->pLast, avail, "%s: %s\r\n", key, val); - char *last = pParser->pLast; - - do { - if (n >= avail) { - httpDebug("context:%p, fd:%d, header field(%s,%s), exceeding buffer size", pContext, pContext->fd, key, val); - break; - } - pParser->bufsize += n; - pParser->pCur = pParser->pLast + n; - - if (!httpParseHead(pContext)) { - httpDebug("context:%p, fd:%d, header field(%s,%s), parse failed", pContext, pContext->fd, key, val); - break; - } - - last += n; - pParser->pLast = last; - return; - } while (0); - - pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; -} - -static void httpParseOnBody(void *arg, const char *chunk, size_t len) { - HttpContext *pContext = (HttpContext*)arg; - HttpParser *pParser = &pContext->parser; - - if (pParser->failed) return; - - if (pParser->data.pos == 0) { - pParser->data.pos = pParser->pLast; - pParser->data.len = 0; - } - - int avail = sizeof(pParser->buffer) - (pParser->pLast - pParser->buffer); - if (len + 1 >= avail) { - httpError("context:%p, fd:%d, failed parse body, exceeding buffer size", pContext, pContext->fd); - pParser->failed |= EHTTP_CONTEXT_PROCESS_FAILED; - return; - } - - memcpy(pParser->pLast, chunk, len); - pParser->pLast += len; - pParser->data.len += len; -} - -static void httpParseOnEnd(void *arg) { - HttpContext *pContext = (HttpContext*)arg; - HttpParser *pParser = &pContext->parser; - - if (pParser->failed) return; - - if (pParser->data.pos == 0) pParser->data.pos = pParser->pLast; - - if (!pContext->parsed) { - pContext->parsed = true; - } - - httpDebug("context:%p, fd:%d, parse success", pContext, pContext->fd); -} - -static void httpParseOnError(void *arg, int status_code) { - HttpContext *pContext = (HttpContext *)arg; - HttpParser * pParser = &pContext->parser; - - httpError("context:%p, fd:%d, failed to parse, status_code:%d", pContext, pContext->fd, status_code); - pParser->failed |= EHTTP_CONTEXT_PARSER_FAILED; -} diff --git a/src/plugins/http/src/httpGcHandle.c b/src/plugins/http/src/httpGcHandle.c index 09f17cae66..e010c77ffd 100644 --- a/src/plugins/http/src/httpGcHandle.c +++ b/src/plugins/http/src/httpGcHandle.c @@ -47,22 +47,22 @@ static HttpEncodeMethod gcQueryMethod = { void gcInitHandle(HttpServer* pServer) { httpAddMethod(pServer, &gcDecodeMethod); } bool gcGetUserFromUrl(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - if (pParser->path[GC_USER_URL_POS].len >= TSDB_USER_LEN || pParser->path[GC_USER_URL_POS].len <= 0) { + HttpParser* pParser = pContext->parser; + if (pParser->path[GC_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[GC_USER_URL_POS].pos <= 0) { return false; } - tstrncpy(pContext->user, pParser->path[GC_USER_URL_POS].pos, TSDB_USER_LEN); + tstrncpy(pContext->user, pParser->path[GC_USER_URL_POS].str, TSDB_USER_LEN); return true; } bool gcGetPassFromUrl(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - if (pParser->path[GC_PASS_URL_POS].len >= TSDB_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].len <= 0) { + HttpParser* pParser = pContext->parser; + if (pParser->path[GC_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].pos <= 0) { return false; } - tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].pos, TSDB_PASSWORD_LEN); + tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].str, TSDB_PASSWORD_LEN); return true; } @@ -144,8 +144,7 @@ bool gcProcessLoginRequest(HttpContext* pContext) { bool gcProcessQueryRequest(HttpContext* pContext) { httpDebug("context:%p, fd:%d, process grafana query msg", pContext, pContext->fd); - HttpParser* pParser = &pContext->parser; - char* filter = pParser->data.pos; + char* filter = pContext->parser->body.str; if (filter == NULL) { httpSendErrorResp(pContext, HTTP_NO_MSG_INPUT); return false; @@ -157,7 +156,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { return false; } - int size = cJSON_GetArraySize(root); + int32_t size = cJSON_GetArraySize(root); if (size <= 0) { httpSendErrorResp(pContext, HTTP_GC_QUERY_NULL); cJSON_Delete(root); @@ -176,7 +175,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { return false; } - for (int i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { cJSON* query = cJSON_GetArrayItem(root, i); if (query == NULL) continue; @@ -186,14 +185,14 @@ bool gcProcessQueryRequest(HttpContext* pContext) { continue; } - int refIdBuffer = httpAddToSqlCmdBuffer(pContext, refId->valuestring); + int32_t refIdBuffer = httpAddToSqlCmdBuffer(pContext, refId->valuestring); if (refIdBuffer == -1) { httpWarn("context:%p, fd:%d, user:%s, refId buffer is full", pContext, pContext->fd, pContext->user); break; } cJSON* alias = cJSON_GetObjectItem(query, "alias"); - int aliasBuffer = -1; + int32_t aliasBuffer = -1; if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) { aliasBuffer = httpAddToSqlCmdBuffer(pContext, alias->valuestring); if (aliasBuffer == -1) { @@ -211,7 +210,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { continue; } - int sqlBuffer = httpAddToSqlCmdBuffer(pContext, sql->valuestring); + int32_t sqlBuffer = httpAddToSqlCmdBuffer(pContext, sql->valuestring); if (sqlBuffer == -1) { httpWarn("context:%p, fd:%d, user:%s, sql buffer is full", pContext, pContext->fd, pContext->user); break; diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index 85cae24201..a291641dc3 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -54,8 +54,8 @@ void gcWriteTargetStartJson(JsonBuf *jsonBuf, char *refId, char *target) { httpJsonToken(jsonBuf, JsonObjStt); // target section - httpJsonPair(jsonBuf, "refId", 5, refId, (int)strlen(refId)); - httpJsonPair(jsonBuf, "target", 6, target, (int)strlen(target)); + httpJsonPair(jsonBuf, "refId", 5, refId, (int32_t)strlen(refId)); + httpJsonPair(jsonBuf, "target", 6, target, (int32_t)strlen(target)); // data begin httpJsonPairHead(jsonBuf, "datapoints", 10); @@ -82,25 +82,25 @@ void gcStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) { } } -bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows) { +bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; - int num_fields = taos_num_fields(result); + int32_t num_fields = taos_num_fields(result); TAOS_FIELD *fields = taos_fetch_fields(result); if (num_fields == 0) { return false; } - int precision = taos_result_precision(result); + int32_t precision = taos_result_precision(result); // such as select count(*) from sys.cpu // such as select count(*) from sys.cpu group by ipaddr // such as select count(*) from sys.cpu interval(1d) // such as select count(*) from sys.cpu interval(1d) group by ipaddr // such as select count(*) count(*) from sys.cpu group by ipaddr interval(1d) - int dataFields = -1; - int groupFields = -1; + int32_t dataFields = -1; + int32_t groupFields = -1; bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP; if (hasTimestamp) { dataFields = 1; @@ -119,7 +119,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } cmd->numOfRows += numOfRows; - for (int k = 0; k < numOfRows; ++k) { + for (int32_t k = 0; k < numOfRows; ++k) { TAOS_ROW row = taos_fetch_row(result); if (row == NULL) { cmd->numOfRows--; @@ -130,9 +130,9 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, // for group by if (groupFields != -1) { char target[HTTP_GC_TARGET_SIZE] = {0}; - int len; + int32_t len; len = snprintf(target,HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer); - for (int i = dataFields + 1; i= 0; i--) { + for (int32_t i = dataFields; i >= 0; i--) { httpJsonItemToken(jsonBuf); if (row[i] == NULL) { httpJsonOriginString(jsonBuf, "null", 4); @@ -253,13 +253,13 @@ void gcSendHeartBeatResp(HttpContext *pContext, HttpSqlCmd *cmd) { httpInitJsonBuf(jsonBuf, pContext); httpJsonToken(jsonBuf, JsonObjStt); - httpJsonPair(jsonBuf, "message", (int)strlen("message"), desc, (int)strlen(desc)); + httpJsonPair(jsonBuf, "message", (int32_t)strlen("message"), desc, (int32_t)strlen(desc)); httpJsonToken(jsonBuf, JsonObjEnd); char head[1024]; - int hLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_GRAFANA], httpVersionStr[pContext->httpVersion], - httpKeepAliveStr[pContext->httpKeepAlive], (jsonBuf->lst - jsonBuf->buf)); + int32_t hLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_GRAFANA], httpVersionStr[pContext->parser->httpVersion], + httpKeepAliveStr[pContext->parser->keepAlive], (jsonBuf->lst - jsonBuf->buf)); httpWriteBuf(pContext, head, hLen); - httpWriteBuf(pContext, jsonBuf->buf, (int)(jsonBuf->lst - jsonBuf->buf)); + httpWriteBuf(pContext, jsonBuf->buf, (int32_t)(jsonBuf->lst - jsonBuf->buf)); } diff --git a/src/plugins/http/src/httpGzip.c b/src/plugins/http/src/httpGzip.c index 5712aff7ec..54f900c755 100644 --- a/src/plugins/http/src/httpGzip.c +++ b/src/plugins/http/src/httpGzip.c @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE #include "os.h" #include "zlib.h" #include "httpGzip.h" @@ -16,10 +32,10 @@ struct ehttp_gzip_s { gz_header *header; char *chunk; - int state; + int32_t state; }; -static void dummy_on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, size_t len) { +static void dummy_on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) { } static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) { @@ -72,7 +88,7 @@ ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_ // 868 below), inflate() will not automatically decode concatenated gzip streams. // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state // 870 would need to be reset to continue decoding a subsequent gzip stream. - int ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS + int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS if (ret != Z_OK) break; if (gzip->header) { ret = inflateGetHeader(gzip->gzip, gzip->header); @@ -97,7 +113,7 @@ void ehttp_gzip_destroy(ehttp_gzip_t *gzip) { free(gzip); } -int ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, size_t len) { +int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { if (gzip->state != EHTTP_GZIP_READY) return -1; if (len <= 0) return 0; @@ -105,7 +121,7 @@ int ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, size_t len) { gzip->gzip->avail_in = len; while (gzip->gzip->avail_in) { - int ret; + int32_t ret; if (gzip->header) { ret = inflate(gzip->gzip, Z_BLOCK); } else { @@ -117,7 +133,7 @@ int ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, size_t len) { if (ret!=Z_STREAM_END) continue; } - size_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); @@ -128,18 +144,18 @@ int ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, size_t len) { return 0; } -int ehttp_gzip_finish(ehttp_gzip_t *gzip) { +int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) { if (gzip->state != EHTTP_GZIP_READY) return -1; gzip->gzip->next_in = NULL; gzip->gzip->avail_in = 0; - int ret; + int32_t ret; ret = inflate(gzip->gzip, Z_FINISH); if (ret != Z_STREAM_END) return -1; - size_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 59b3268392..9fb46c5734 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -15,240 +15,17 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taos.h" -#include "tglobal.h" -#include "tsocket.h" -#include "ttimer.h" #include "httpInt.h" #include "httpResp.h" -#include "httpAuth.h" -#include "httpServer.h" #include "httpContext.h" #include "httpHandle.h" -void httpToLowerUrl(char* url) { - /*ignore case */ - while (*url) { - if (*url >= 'A' && *url <= 'Z') { - *url = *url | 0x20; - } - url++; - } -} - -bool httpUrlMatch(HttpContext* pContext, int pos, char* cmp) { - HttpParser* pParser = &pContext->parser; - - if (pos < 0 || pos >= HTTP_MAX_URL) { - return false; - } - - if (pParser->path[pos].len <= 0) { - return false; - } - - if (strcmp(pParser->path[pos].pos, cmp) != 0) { - return false; - } - - return true; -} - -// /account/db/meter HTTP/1.1\r\nHost -bool httpParseURL(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - char* pSeek; - char* pEnd = strchr(pParser->pLast, ' '); - if (pEnd == NULL) { - httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL); - return false; - } - - if (*pParser->pLast != '/') { - httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL); - return false; - } - pParser->pLast++; - - for (int i = 0; i < HTTP_MAX_URL; i++) { - pSeek = strchr(pParser->pLast, '/'); - if (pSeek == NULL) { - break; - } - pParser->path[i].pos = pParser->pLast; - if (pSeek <= pEnd) { - pParser->path[i].len = (int16_t)(pSeek - pParser->pLast); - pParser->path[i].pos[pParser->path[i].len] = 0; - httpToLowerUrl(pParser->path[i].pos); - pParser->pLast = pSeek + 1; - } else { - pParser->path[i].len = (int16_t)(pEnd - pParser->pLast); - pParser->path[i].pos[pParser->path[i].len] = 0; - httpToLowerUrl(pParser->path[i].pos); - pParser->pLast = pEnd + 1; - break; - } - } - pParser->pLast = pEnd + 1; - - if (pParser->path[0].len == 0) { - httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL); - return false; - } - - return true; -} - -bool httpParseHttpVersion(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - char* pEnd = strchr(pParser->pLast, '1'); - if (pEnd == NULL) { - httpError("context:%p, fd:%d, can't find http version at position:%s", pContext, pContext->fd, pParser->pLast); - httpSendErrorResp(pContext, HTTP_PARSE_HTTP_VERSION_ERROR); - return false; - } - - if (*(pEnd + 1) != '.') { - httpError("context:%p, fd:%d, can't find http version at position:%s", pContext, pContext->fd, pParser->pLast); - httpSendErrorResp(pContext, HTTP_PARSE_HTTP_VERSION_ERROR); - return false; - } - - if (*(pEnd + 2) == '0') - pContext->httpVersion = HTTP_VERSION_10; - else if (*(pEnd + 2) == '1') - pContext->httpVersion = HTTP_VERSION_11; - else if (*(pEnd + 2) == '2') - pContext->httpVersion = HTTP_VERSION_11; - else - pContext->httpVersion = HTTP_VERSION_10; - - httpDebug("context:%p, fd:%d, httpVersion:1.%d", pContext, pContext->fd, pContext->httpVersion); - return true; -} - -bool httpGetNextLine(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - while (pParser->buffer + pParser->bufsize - pParser->pCur++ > 0) { - if (*(pParser->pCur) == '\n' && *(pParser->pCur - 1) == '\r') { - // cut the string - *pParser->pCur = 0; - return true; - } - } - - httpSendErrorResp(pContext, HTTP_PARSE_HEAD_ERROR); - - return false; -} - -bool httpGetHttpMethod(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - char* pSeek = strchr(pParser->pLast, ' '); - - if (pSeek == NULL) { - httpError("context:%p, fd:%d, failed to parse httpMethod", pContext, pContext->fd); - httpSendErrorResp(pContext, HTTP_PARSE_HTTP_METHOD_ERROR); - return false; - } - - pParser->method.pos = pParser->pLast; - pParser->method.len = (int16_t)(pSeek - pParser->pLast); - pParser->method.pos[pParser->method.len] = 0; - pParser->pLast = pSeek + 1; - - httpTrace("context:%p, fd:%d, httpMethod:%s", pContext, pContext->fd, pParser->method.pos); - return true; -} - -bool httpGetDecodeMethod(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - - HttpServer* pServer = &tsHttpServer; - int methodLen = pServer->methodScannerLen; - for (int i = 0; i < methodLen; i++) { - HttpDecodeMethod* method = pServer->methodScanner[i]; - if (strcmp(method->module, pParser->path[0].pos) != 0) { - continue; - } - pParser->pMethod = method; - return true; - } - - httpError("context:%p, fd:%d, error:the url is not support, method:%s, path:%s", - pContext, pContext->fd, pParser->method.pos, pParser->path[0].pos); - httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL); - - return false; -} - -bool httpParseHead(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - if (strncasecmp(pParser->pLast, "Content-Length: ", 16) == 0) { - pParser->data.len = (int32_t)atoi(pParser->pLast + 16); - httpTrace("context:%p, fd:%d, Content-Length:%d", pContext, pContext->fd, - pParser->data.len); - } else if (strncasecmp(pParser->pLast, "Accept-Encoding: ", 17) == 0) { - if (tsHttpEnableCompress && strstr(pParser->pLast + 17, "gzip") != NULL) { - pContext->acceptEncoding = HTTP_COMPRESS_GZIP; - httpTrace("context:%p, fd:%d, Accept-Encoding:gzip", pContext, pContext->fd); - } else { - pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY; - httpTrace("context:%p, fd:%d, Accept-Encoding:identity", pContext, pContext->fd); - } - } else if (strncasecmp(pParser->pLast, "Content-Encoding: ", 18) == 0) { - if (strstr(pParser->pLast + 18, "gzip") != NULL) { - pContext->contentEncoding = HTTP_COMPRESS_GZIP; - httpTrace("context:%p, fd:%d, Content-Encoding:gzip", pContext, pContext->fd); - } else { - pContext->contentEncoding = HTTP_COMPRESS_IDENTITY; - httpTrace("context:%p, fd:%d, Content-Encoding:identity", pContext, pContext->fd); - } - } else if (strncasecmp(pParser->pLast, "Connection: ", 12) == 0) { - if (strncasecmp(pParser->pLast + 12, "Keep-Alive", 10) == 0) { - pContext->httpKeepAlive = HTTP_KEEPALIVE_ENABLE; - } else { - pContext->httpKeepAlive = HTTP_KEEPALIVE_DISABLE; - } - httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->httpKeepAlive); - } else if (strncasecmp(pParser->pLast, "Transfer-Encoding: ", 19) == 0) { - if (strncasecmp(pParser->pLast + 19, "chunked", 7) == 0) { - pContext->httpChunked = HTTP_CHUNKED; - } - } else if (strncasecmp(pParser->pLast, "Authorization: ", 15) == 0) { - if (strncasecmp(pParser->pLast + 15, "Basic ", 6) == 0) { - pParser->token.pos = pParser->pLast + 21; - pParser->token.len = (int16_t)(pParser->pCur - pParser->token.pos - 1); - bool parsed = httpParseBasicAuthToken(pContext, pParser->token.pos, pParser->token.len); - if (!parsed) { - httpSendErrorResp(pContext, HTTP_INVALID_BASIC_AUTH_TOKEN); - return false; - } - } else if (strncasecmp(pParser->pLast + 15, "Taosd ", 6) == 0) { - pParser->token.pos = pParser->pLast + 21; - pParser->token.len = (int16_t)(pParser->pCur - pParser->token.pos - 1); - bool parsed = httpParseTaosdAuthToken(pContext, pParser->token.pos, pParser->token.len); - if (!parsed) { - httpSendErrorResp(pContext, HTTP_INVALID_TAOSD_AUTH_TOKEN); - return false; - } - } else { - httpSendErrorResp(pContext, HTTP_INVALID_AUTH_TOKEN); - return false; - } - } else { - } - - return true; -} - bool httpDecodeRequest(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - if (pParser->pMethod->decodeFp == NULL) { + if (pContext->decodeMethod->decodeFp == NULL) { return false; } - return (*pParser->pMethod->decodeFp)(pContext); + return (*pContext->decodeMethod->decodeFp)(pContext); } /** @@ -263,7 +40,7 @@ bool httpProcessData(HttpContext* pContext) { } // handle Cross-domain request - if (strcmp(pContext->parser.method.pos, "OPTIONS") == 0) { + if (strcmp(pContext->parser->method, "OPTIONS") == 0) { httpDebug("context:%p, fd:%d, process options request", pContext, pContext->fd); httpSendOptionResp(pContext, "process options request success"); } else { diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index e200efbcef..e02782357c 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -17,6 +17,7 @@ #include "os.h" #include "taosmsg.h" #include "taoserror.h" +#include "tglobal.h" #include "http.h" #include "httpLog.h" #include "httpCode.h" @@ -38,14 +39,14 @@ char JsonNulTkn[] = "null"; char JsonTrueTkn[] = "true"; char JsonFalseTkn[] = "false"; -int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz) { - int len; - int countWait = 0; - int writeLen = 0; +int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t sz) { + int32_t len; + int32_t countWait = 0; + int32_t writeLen = 0; do { if (pContext->fd > 2){ - len = (int)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL); + len = (int32_t)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL); } else { return sz; @@ -68,8 +69,8 @@ int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz) { return writeLen; } -int httpWriteBuf(struct HttpContext* pContext, const char* buf, int sz) { - int writeSz = httpWriteBufByFd(pContext, buf, sz); +int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz) { + int32_t writeSz = httpWriteBufByFd(pContext, buf, sz); if (writeSz != sz) { httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response:\n%s", pContext, pContext->fd, sz, writeSz, buf); @@ -80,8 +81,8 @@ int httpWriteBuf(struct HttpContext* pContext, const char* buf, int sz) { return writeSz; } -int httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int sz) { - int writeSz = httpWriteBufByFd(pContext, buf, sz); +int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32_t sz) { + int32_t writeSz = httpWriteBufByFd(pContext, buf, sz); if (writeSz != sz) { httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response", pContext, pContext->fd, sz, writeSz); @@ -90,8 +91,8 @@ int httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int sz) { return writeSz; } -int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { - int remain = 0; +int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { + int32_t remain = 0; char sLen[24]; uint64_t srcLen = (uint64_t) (buf->lst - buf->buf); @@ -108,28 +109,28 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { * The remote endpoint then decodes the stream by concatenating the chunks and uncompressing the result. */ - if (buf->pContext->acceptEncoding == HTTP_COMPRESS_IDENTITY) { + if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) { if (buf->lst == buf->buf) { httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd); return 0; // there is no data to dump. } else { - int len = sprintf(sLen, "%lx\r\n", srcLen); + int32_t len = sprintf(sLen, "%lx\r\n", srcLen); httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd, srcLen, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); - remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int) srcLen); + remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int32_t)srcLen); } } else { char compressBuf[JSON_BUFFER_SIZE] = {0}; int32_t compressBufLen = JSON_BUFFER_SIZE; - int ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast); + int32_t ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast); if (ret == 0) { if (compressBufLen > 0) { - int len = sprintf(sLen, "%x\r\n", compressBufLen); + int32_t len = sprintf(sLen, "%x\r\n", compressBufLen); httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); - remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, (int)compressBufLen); + remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen); } else { httpTrace("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext, buf->pContext->fd, isTheLast, buf->buf); @@ -143,9 +144,9 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { } httpWriteBufNoTrace(buf->pContext, "\r\n", 2); - buf->total += (int) (buf->lst - buf->buf); + buf->total += (int32_t)(buf->lst - buf->buf); buf->lst = buf->buf; - memset(buf->buf, 0, (size_t) buf->size); + memset(buf->buf, 0, (size_t)buf->size); return remain; } @@ -155,14 +156,14 @@ void httpWriteJsonBufHead(JsonBuf* buf) { } char msg[1024] = {0}; - int len = -1; + int32_t len = -1; - if (buf->pContext->acceptEncoding == HTTP_COMPRESS_IDENTITY) { - len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->httpVersion], - httpKeepAliveStr[buf->pContext->httpKeepAlive]); + if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) { + len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion], + httpKeepAliveStr[buf->pContext->parser->keepAlive]); } else { - len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_COMPRESS], httpVersionStr[buf->pContext->httpVersion], - httpKeepAliveStr[buf->pContext->httpKeepAlive]); + len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion], + httpKeepAliveStr[buf->pContext->parser->keepAlive]); } httpWriteBuf(buf->pContext, (const char*)msg, len); @@ -185,7 +186,7 @@ void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext) { buf->pContext = pContext; memset(buf->lst, 0, JSON_BUFFER_SIZE); - if (pContext->acceptEncoding == HTTP_COMPRESS_GZIP) { + if (pContext->parser->acceptEncodingGzip == 1 && tsHttpEnableCompress) { httpGzipCompressInit(buf->pContext); } @@ -200,19 +201,19 @@ void httpJsonItemToken(JsonBuf* buf) { if (buf->lst > buf->buf) httpJsonToken(buf, JsonItmTkn); } -void httpJsonString(JsonBuf* buf, char* sVal, int len) { +void httpJsonString(JsonBuf* buf, char* sVal, int32_t len) { httpJsonItemToken(buf); httpJsonToken(buf, JsonStrStt); httpJsonPrint(buf, sVal, len); httpJsonToken(buf, JsonStrEnd); } -void httpJsonOriginString(JsonBuf* buf, char* sVal, int len) { +void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len) { httpJsonItemToken(buf); httpJsonPrint(buf, sVal, len); } -void httpJsonStringForTransMean(JsonBuf* buf, char* sVal, int maxLen) { +void httpJsonStringForTransMean(JsonBuf* buf, char* sVal, int32_t maxLen) { httpJsonItemToken(buf); httpJsonToken(buf, JsonStrStt); @@ -221,18 +222,18 @@ void httpJsonStringForTransMean(JsonBuf* buf, char* sVal, int maxLen) { char* lastPos = sVal; char* curPos = sVal; - for (int i = 0; i < maxLen; ++i) { + for (int32_t i = 0; i < maxLen; ++i) { if (*curPos == 0) { break; } if (*curPos == '\"') { - httpJsonPrint(buf, lastPos, (int)(curPos - lastPos)); + httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos)); curPos++; lastPos = curPos; httpJsonPrint(buf, "\\\"", 2); } else if (*curPos == '\\') { - httpJsonPrint(buf, lastPos, (int)(curPos - lastPos)); + httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos)); curPos++; lastPos = curPos; httpJsonPrint(buf, "\\\\", 2); @@ -242,7 +243,7 @@ void httpJsonStringForTransMean(JsonBuf* buf, char* sVal, int maxLen) { } if (*lastPos) { - httpJsonPrint(buf, lastPos, (int)(curPos - lastPos)); + httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos)); } } @@ -258,14 +259,14 @@ void httpJsonInt64(JsonBuf* buf, int64_t num) { void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { char ts[35] = {0}; struct tm *ptm; - int precision = 1000; + int32_t precision = 1000; if (us) { precision = 1000000; } time_t tt = t / precision; ptm = localtime(&tt); - int length = (int) strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); + int32_t length = (int32_t) strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); if (us) { length += snprintf(ts + length, 8, ".%06ld", t % precision); } else { @@ -278,25 +279,25 @@ void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us) { char ts[40] = {0}; struct tm *ptm; - int precision = 1000; + int32_t precision = 1000; if (us) { precision = 1000000; } time_t tt = t / precision; ptm = localtime(&tt); - int length = (int) strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm); + int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm); if (us) { length += snprintf(ts + length, 8, ".%06ld", t % precision); } else { length += snprintf(ts + length, 5, ".%03ld", t % precision); } - length += (int) strftime(ts + length, 40 - length, "%z", ptm); + length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm); httpJsonString(buf, ts, length); } -void httpJsonInt(JsonBuf* buf, int num) { +void httpJsonInt(JsonBuf* buf, int32_t num) { httpJsonItemToken(buf); httpJsonTestBuf(buf, MAX_NUM_STR_SZ); buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%d", num); @@ -328,65 +329,65 @@ void httpJsonDouble(JsonBuf* buf, double num) { void httpJsonNull(JsonBuf* buf) { httpJsonString(buf, "null", 4); } -void httpJsonBool(JsonBuf* buf, int val) { +void httpJsonBool(JsonBuf* buf, int32_t val) { if (val == 0) httpJsonPrint(buf, JsonFalseTkn, sizeof(JsonFalseTkn)); else httpJsonPrint(buf, JsonTrueTkn, sizeof(JsonTrueTkn)); } -void httpJsonPairHead(JsonBuf* buf, char* name, int len) { +void httpJsonPairHead(JsonBuf* buf, char* name, int32_t len) { httpJsonItemToken(buf); httpJsonString(buf, name, len); httpJsonToken(buf, JsonPairTkn); } -void httpJsonPair(JsonBuf* buf, char* name, int nameLen, char* sVal, int valLen) { +void httpJsonPair(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen) { httpJsonPairHead(buf, name, nameLen); httpJsonString(buf, sVal, valLen); } -void httpJsonPairOriginString(JsonBuf* buf, char* name, int nameLen, char* sVal, int valLen) { +void httpJsonPairOriginString(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen) { httpJsonPairHead(buf, name, nameLen); httpJsonOriginString(buf, sVal, valLen); } -void httpJsonPairIntVal(JsonBuf* buf, char* name, int nNameLen, int num) { +void httpJsonPairIntVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num) { httpJsonPairHead(buf, name, nNameLen); httpJsonInt(buf, num); } -void httpJsonPairInt64Val(JsonBuf* buf, char* name, int nNameLen, int64_t num) { +void httpJsonPairInt64Val(JsonBuf* buf, char* name, int32_t nNameLen, int64_t num) { httpJsonPairHead(buf, name, nNameLen); httpJsonInt64(buf, num); } -void httpJsonPairBoolVal(JsonBuf* buf, char* name, int nNameLen, int num) { +void httpJsonPairBoolVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num) { httpJsonPairHead(buf, name, nNameLen); httpJsonBool(buf, num); } -void httpJsonPairFloatVal(JsonBuf* buf, char* name, int nNameLen, float num) { +void httpJsonPairFloatVal(JsonBuf* buf, char* name, int32_t nNameLen, float num) { httpJsonPairHead(buf, name, nNameLen); httpJsonFloat(buf, num); } -void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int nNameLen, double num) { +void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int32_t nNameLen, double num) { httpJsonPairHead(buf, name, nNameLen); httpJsonDouble(buf, num); } -void httpJsonPairNullVal(JsonBuf* buf, char* name, int nNameLen) { +void httpJsonPairNullVal(JsonBuf* buf, char* name, int32_t nNameLen) { httpJsonPairHead(buf, name, nNameLen); httpJsonNull(buf); } -void httpJsonPairArray(JsonBuf* buf, char* name, int len, httpJsonBuilder fnBuilder, void* dsHandle) { +void httpJsonPairArray(JsonBuf* buf, char* name, int32_t len, httpJsonBuilder fnBuilder, void* dsHandle) { httpJsonPairHead(buf, name, len); httpJsonArray(buf, fnBuilder, dsHandle); } -void httpJsonPairObject(JsonBuf* buf, char* name, int len, httpJsonBuilder fnBuilder, void* dsHandle) { +void httpJsonPairObject(JsonBuf* buf, char* name, int32_t len, httpJsonBuilder fnBuilder, void* dsHandle) { httpJsonPairHead(buf, name, len); httpJsonObject(buf, fnBuilder, dsHandle); } @@ -405,7 +406,7 @@ void httpJsonArray(JsonBuf* buf, httpJsonBuilder fnBuilder, void* jsonHandle) { httpJsonToken(buf, JsonArrEnd); } -void httpJsonTestBuf(JsonBuf* buf, int safety) { +void httpJsonTestBuf(JsonBuf* buf, int32_t safety) { if ((buf->lst - buf->buf + safety) < buf->size) return; // buf->slot = *buf->lst; httpWriteJsonBufBody(buf, false); @@ -416,7 +417,7 @@ void httpJsonToken(JsonBuf* buf, char c) { *buf->lst++ = c; } -void httpJsonPrint(JsonBuf* buf, const char* json, int len) { +void httpJsonPrint(JsonBuf* buf, const char* json, int32_t len) { if (len == 0 || len >= JSON_BUFFER_SIZE) { return; } @@ -432,7 +433,7 @@ void httpJsonPrint(JsonBuf* buf, const char* json, int len) { buf->lst += len; } -void httpJsonPairStatus(JsonBuf* buf, int code) { +void httpJsonPairStatus(JsonBuf* buf, int32_t code) { if (code == 0) { httpJsonPair(buf, "status", 6, "succ", 4); } else { @@ -445,7 +446,7 @@ void httpJsonPairStatus(JsonBuf* buf, int code) { } else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { httpJsonPair(buf, "desc", 4, "failed to create table", 22); } else { - httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int)strlen(tstrerror(code))); + httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int32_t)strlen(tstrerror(code))); } } } diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index c1fd481efd..0238c39eb8 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -1,10 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE #include "os.h" +#include "taoserror.h" #include "httpLog.h" #include "httpContext.h" #include "httpParser.h" #include "httpGzip.h" +#include "httpAuth.h" -static HttpParserStatusObj status_codes[] = { +static void httpOnData(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len); + +static HttpStatus httpStatusCodes[] = { {100, "Continue"}, {101, "Switching Protocol"}, {102, "Processing (WebDAV)"}, @@ -71,127 +91,411 @@ static HttpParserStatusObj status_codes[] = { {0, NULL} }; -const char* ehttp_status_code_get_desc(const int status_code) { - HttpParserStatusObj *p = status_codes; - while (p->status_code!=0) { - if (p->status_code==status_code) return p->status_desc; +char *httpGetStatusDesc(int32_t statusCode) { + HttpStatus *p = httpStatusCodes; + while (p->code != 0) { + if (p->code == statusCode) return p->desc; ++p; } return "Unknow status code"; } -static void dummy_on_request_line(void *arg, const char *method, const char *target, const char *version, const char *target_raw) { +static void httpCleanupString(HttpString *str) { + free(str->str); + str->str = NULL; + str->pos = 0; + str->size = 0; } -static void dummy_on_status_line(void *arg, const char *version, int status_code, const char *reason_phrase) { +static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { + if (str->size == 0) { + str->pos = 0; + str->size = 32; + str->str = malloc(str->size); + } else if (str->pos + len + 1 > str->size) { + str->size *= 10; + str->str = realloc(str->str, str->size); + } else { + } + + if (str->str == NULL) return -1; + + memcpy(str->str + str->pos, s, len); + str->pos += len; + str->str[str->pos] = 0; + return 0; } -static void dummy_on_header_field(void *arg, const char *key, const char *val) { +static void httpClearString(HttpString *str) { + if (str->str) { + str->str[0] = '\0'; + str->pos = 0; + str->size = 0; + } } -static void dummy_on_body(void *arg, const char *chunk, size_t len) { +static int32_t httpOnError(HttpParser *parser, int32_t httpCode, int32_t parseCode) { + HttpContext *pContext = parser->pContext; + if (httpCode != 0) parser->httpCode = httpCode; + if (parseCode != 0) parser->parseCode = parseCode; + httpError("context:%p, fd:%d, parse failed, httpCode:%d parseCode:%d", pContext, pContext->fd, httpCode, parseCode); + return 0; } -static void dummy_on_end(void *arg) { -} +static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target, char *version) { + HttpContext *pContext = pParser->pContext; + httpDebug("context:%p, fd:%d, method:%s target:%s version:%s", pContext, pContext->fd, method, target, version); -static void dummy_on_error(void *arg, int status_code) { -} + // parse url + char *pStart = target + 1; + for (int32_t i = 0; i < HTTP_MAX_URL; i++) { + char *pSeek = strchr(pStart, '/'); + if (pSeek == NULL) { + pParser->path[i].str = strdup(pStart); + pParser->path[i].size = strlen(pStart); + pParser->path[i].pos = pParser->path[i].size; + break; + } else { + int32_t len = (int32_t)(pSeek - pStart); + pParser->path[i].str = malloc(len + 1); + memcpy(pParser->path[i].str, pStart, len); + pParser->path[i].str[len] = 0; + pParser->path[i].size = len; + pParser->path[i].pos = len; + } + pStart = pSeek + 1; + } -static HTTP_PARSER_STATE httpParserTop(HttpParserObj *parser) { - ASSERT(parser->stacks_count >= 1); - ASSERT(parser->stacks); + // parse decode method + for (int32_t i = 0; i < tsHttpServer.methodScannerLen; i++) { + HttpDecodeMethod *method = tsHttpServer.methodScanner[i]; + if (strcmp(method->module, pParser->path[0].str) == 0) { + pContext->decodeMethod = method; + break; + } + } - return parser->stacks[parser->stacks_count - 1]; -} + if (pContext->decodeMethod != NULL) { + httpDebug("context:%p, fd:%d, decode method is %s", pContext, pContext->fd, pContext->decodeMethod->module); + } else { + httpError("context:%p, fd:%d, the url is not support, target:%s", pContext, pContext->fd, target); + httpOnError(pParser, 0, TSDB_CODE_HTTP_UNSUPPORT_URL); + return -1; + } -static int httpParserPush(HttpParserObj *parser, HTTP_PARSER_STATE state) { - size_t n = parser->stacks_count + 1; - // HTTP_PARSER_STATE *stacks = (HTTP_PARSER_STATE*)reallocarray(parser->stacks, n, sizeof(*stacks)); - HTTP_PARSER_STATE *stacks = (HTTP_PARSER_STATE*)realloc(parser->stacks, n * sizeof(*stacks)); - if (!stacks) return -1; - - parser->stacks_count = n; - parser->stacks = stacks; - parser->stacks[n-1] = state; + // parse version + if (pParser->httpVersion < HTTP_VERSION_10 || pParser->httpVersion > HTTP_VERSION_12) { + httpError("context:%p, fd:%d, unsupport httpVersion %d", pContext, pContext->fd, pParser->httpVersion); + httpOnError(pParser, 0, TSDB_CODE_HTTP_INVALID_VERSION); + } else { + httpDebug("context:%p, fd:%d, httpVersion:1.%d", pContext, pContext->fd, pParser->httpVersion); + } return 0; } -static int httpParserPop(HttpParserObj *parser) { - if (parser->stacks_count <= 0) return -1; - --parser->stacks_count; +static int32_t httpOnStatusLine(HttpParser *pParser, int32_t code, const char *reason) { + HttpContext *pContext = pParser->pContext; + httpError("context:%p, fd:%d, status line, code:%d reason:%s", pContext, pContext->fd, code, reason); + return 0; +} + +static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const char *val) { + HttpContext *pContext = parser->pContext; + httpDebug("context:%p, fd:%d, key:%s val:%s", pContext, pContext->fd, key, val); + + if (0 == strcasecmp(key, "Content-Length")) { + int32_t len = 0; + int32_t bytes = 0; + int32_t n = sscanf(val, "%d%n", &len, &bytes); + if (n == 1 && bytes == strlen(val)) { + parser->contentLength = len; + parser->chunkSize = len; + parser->contentLengthSpecified = 1; + httpDebug("context:%p, fd:%d, contentLength:%d chunkSize:%d contentLengthSpecified:%d", pContext, pContext->fd, + parser->contentLength, parser->chunkSize, parser->contentLengthSpecified); + return 0; + } else { + httpError("context:%p, fd:%d, failed to parser %s:%s", pContext, pContext->fd, key, val); + httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH); + return -1; + } + } + + else if (0 == strcasecmp(key, "Accept-Encoding")) { + if (strstr(val, "gzip")) { + parser->acceptEncodingGzip = 1; + httpDebug("context:%p, fd:%d, acceptEncodingGzip:%d", pContext, pContext->fd, parser->acceptEncodingGzip); + } + if (strstr(val, "chunked")) { + parser->acceptEncodingChunked = 1; + httpDebug("context:%p, fd:%d, acceptEncodingChunked:%d", pContext, pContext->fd, parser->acceptEncodingChunked); + } + return 0; + } + + else if (strncasecmp(key, "Connection: ", 12) == 0) { + if (strncasecmp(val, "Keep-Alive", 10) == 0) { + parser->keepAlive = HTTP_KEEPALIVE_ENABLE; + } else { + parser->keepAlive = HTTP_KEEPALIVE_DISABLE; + } + httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive); + } + + else if (0 == strcasecmp(key, "Content-Encoding")) { + if (0 == strcmp(val, "gzip")) { + parser->contentChunked = 1; + httpDebug("context:%p, fd:%d, contentChunked:%d", pContext, pContext->fd, parser->contentChunked); + } + return 0; + } + + else if (0 == strcasecmp(key, "Transfer-Encoding")) { + if (strstr(val, "gzip")) { + parser->transferGzip = 1; + ehttp_gzip_conf_t conf = {0}; + ehttp_gzip_callbacks_t callbacks = {0}; + + callbacks.on_data = httpOnData; + + parser->gzip = ehttp_gzip_create_decompressor(conf, callbacks, parser); + + if (!parser->gzip) { + httpError("context:%p, fd:%d, failed to create gzip decompressor", pContext, pContext->fd); + httpOnError(parser, 0, TSDB_CODE_HTTP_CREATE_GZIP_FAILED); + return -1; + } + } + if (strstr(val, "chunked")) { + parser->transferChunked = 1; + httpDebug("context:%p, fd:%d, transferChunked:%d", pContext, pContext->fd, parser->transferChunked); + } + return 0; + } + + else if (0 == strcasecmp(key, "Authorization")) { + char * t = NULL; + char * s = NULL; + int32_t bytes = 0; + int32_t n = sscanf(val, "%ms %ms%n", &t, &s, &bytes); + if (n == 2 && t && s && bytes == strlen(val)) { + if (strcmp(t, "Basic") == 0) { + free(parser->authContent); + parser->authContent = s; + parser->authType = HTTP_BASIC_AUTH; + s = NULL; + free(t); + free(s); + httpDebug("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent); + int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, strlen(parser->authContent)); + if (ok != 0) { + httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_BASIC_AUTH); + return -1; + } + return 0; + } else if (strcmp(t, "Taosd") == 0) { + free(parser->authContent); + parser->authContent = s; + parser->authType = HTTP_TAOSD_AUTH; + s = NULL; + free(t); + free(s); + httpDebug("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent); + int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, strlen(parser->authContent)); + if (ok != 0) { + httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_TAOSD_AUTH); + return -1; + } + return 0; + } else { + free(t); + free(s); + parser->authType = HTTP_INVALID_AUTH; + httpError("context:%p, fd:%d, invalid auth, t:%s s:%s", pContext, pContext->fd, t, s); + httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_TYPE); + return -1; + } + } else { + free(t); + free(s); + parser->authType = HTTP_INVALID_AUTH; + httpError("context:%p, fd:%d, parse auth failed, t:%s s:%s", pContext, pContext->fd, t, s); + httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_FORMAT); + return -1; + } + } return 0; } -HttpParserObj *httpParserCreate(HttpParserCallbackObj callbacks, HttpParserConfObj conf, void *arg) { - HttpParserObj *parser = (HttpParserObj*)calloc(1, sizeof(*parser)); +static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { + HttpContext *pContext = parser->pContext; + HttpString * buf = &parser->body; + if (parser->parseCode != TSDB_CODE_SUCCESS) return -1; + + int32_t avail = buf->size - buf->pos; + if (len + 1 >= avail) { + if (buf->size >= HTTP_BUFFER_SIZE) { + httpError("context:%p, fd:%d, failed parse body, exceeding buffer size %d", pContext, pContext->fd, buf->size); + httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); + return -1; + } else { + int32_t newSize = buf->size * 10; + newSize = MIN(newSize, HTTP_BUFFER_SIZE); + buf->str = realloc(buf->str, newSize); + if (buf->str == NULL) { + httpError("context:%p, fd:%d, failed parse body, realloc %d failed", pContext, pContext->fd, newSize); + httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); + return -1; + } + buf->size = newSize; + } + } + + memcpy(buf->str + buf->pos, chunk, len); + buf->pos += len; + buf->str[buf->pos] = 0; + + return 0; +} + +static int32_t httpOnEnd(HttpParser *parser) { + HttpContext *pContext = parser->pContext; + parser->parsed = true; + + if (parser->parseCode != TSDB_CODE_SUCCESS) { + return -1; + } + + httpDebug("context:%p, fd:%d, parse success", pContext, pContext->fd); + return 0; +} + +static HTTP_PARSER_STATE httpTopStack(HttpParser *parser) { + HttpStack *stack = &parser->stacks; + ASSERT(stack->pos >= 1); + + return stack->stacks[stack->pos - 1]; +} + +static int32_t httpPushStack(HttpParser *parser, HTTP_PARSER_STATE state) { + HttpStack *stack = &parser->stacks; + if (stack->size == 0) { + stack->pos = 0; + stack->size = 32; + stack->stacks = malloc(stack->size * sizeof(int8_t)); + } else if (stack->pos + 1 > stack->size) { + stack->size *= 10; + stack->stacks = realloc(stack->stacks, stack->size * sizeof(int8_t)); + } else { + } + + if (stack->stacks == NULL) return -1; + + stack->stacks[stack->pos] = state; + stack->pos++; + + return 0; +} + +static int32_t httpPopStack(HttpParser *parser) { + HttpStack *stack = &parser->stacks; + ASSERT(stack->pos >= 1); + stack->pos--; + return 0; +} + +static void httpClearStack(HttpStack *stack) { + stack->pos = 0; +} + +static int32_t httpCleanupStack(HttpStack *stack) { + free(stack->stacks); + memset(stack, 0, sizeof(HttpStack)); + + return 0; +} + +void httpInitParser(HttpParser *parser) { + HttpContext *pContext = parser->pContext; + httpTrace("context:%p, fd:%d, free parser", pContext, pContext->fd); + + parser->parsed = false; + parser->inited = 1; + parser->httpVersion = 0; + parser->acceptEncodingGzip = 0; + parser->acceptEncodingChunked = 0; + parser->contentLengthSpecified = 0; + parser->contentChunked = 0; + parser->transferGzip = 0; + parser->transferChunked = 0; + parser->keepAlive = 0; + parser->authType = 0; + parser->contentLength = 0; + parser->chunkSize = 0; + parser->receivedChunkSize = 0; + parser->receivedSize = 0; + parser->statusCode = 0; + parser->httpCode = 0; + parser->parseCode = 0; + + free(parser->method); parser->method = NULL; + free(parser->target); parser->target = NULL; + free(parser->target_raw); parser->target_raw = NULL; + free(parser->version); parser->version = NULL; + free(parser->reasonPhrase); parser->reasonPhrase = NULL; + free(parser->key); parser->key = NULL; + free(parser->val); parser->val = NULL; + free(parser->authContent); parser->authContent = NULL; + + httpClearStack(&parser->stacks); + httpClearString(&parser->str); + httpClearString(&parser->body); + for (int32_t i = 0; i < HTTP_MAX_URL; ++i) { + httpClearString(&parser->path[i]); + } + + if (parser->gzip != NULL) { + ehttp_gzip_destroy(parser->gzip); + parser->gzip = NULL; + } + + httpPushStack(parser, HTTP_PARSER_BEGIN); +} + +HttpParser *httpCreateParser(HttpContext *pContext) { + HttpParser *parser = calloc(1, sizeof(HttpParser)); if (!parser) return NULL; + httpTrace("context:%p, fd:%d, create parser", pContext, pContext->fd); - parser->callbacks = callbacks; - parser->arg = arg; - parser->conf = conf; - - if (parser->callbacks.on_request_line == NULL) { - parser->callbacks.on_request_line = dummy_on_request_line; - } - if (parser->callbacks.on_status_line == NULL) { - parser->callbacks.on_status_line = dummy_on_status_line; - } - if (parser->callbacks.on_header_field == NULL) { - parser->callbacks.on_header_field = dummy_on_header_field; - } - if (parser->callbacks.on_body == NULL) { - parser->callbacks.on_body = dummy_on_body; - } - if (parser->callbacks.on_end == NULL) { - parser->callbacks.on_end = dummy_on_end; - } - if (parser->callbacks.on_error == NULL) { - parser->callbacks.on_error = dummy_on_error; - } - - httpParserPush(parser, HTTP_PARSER_BEGIN); - + parser->pContext = pContext; return parser; } -static void ehttp_parser_kvs_destroy(HttpParserObj *parser) { - if (!parser->kvs) return; +void httpDestroyParser(HttpParser *parser) { + HttpContext *pContext = parser->pContext; + httpTrace("context:%p, fd:%d, free parser", pContext, pContext->fd); - for (size_t i=0; ikvs_count; ++i) { - HttpParseKvObj *p = &parser->kvs[i]; - free(p->key); p->key = NULL; - free(p->val); p->val = NULL; - } - free(parser->kvs); - parser->kvs = NULL; - parser->kvs_count = 0; - - free(parser->auth_basic); - parser->auth_basic = NULL; -} - -void httpParserDestroy(HttpParserObj *parser) { if (!parser) return; free(parser->method); parser->method = NULL; free(parser->target); parser->target = NULL; free(parser->target_raw); parser->target_raw = NULL; free(parser->version); parser->version = NULL; - free(parser->reason_phrase); parser->reason_phrase = NULL; + free(parser->reasonPhrase); parser->reasonPhrase = NULL; free(parser->key); parser->key = NULL; free(parser->val); parser->val = NULL; - free(parser->auth_basic); parser->auth_basic = NULL; - free(parser->stacks); parser->stacks = NULL; + free(parser->authContent); parser->authContent = NULL; + + httpCleanupStack(&parser->stacks); + httpCleanupString(&parser->str); + httpCleanupString(&parser->body); + for (int32_t i = 0; i < HTTP_MAX_URL; ++i) { + httpCleanupString(&parser->path[i]); + } - parser->stacks_count = 0; - - ehttp_parser_kvs_destroy(parser); - - httpParserCleanupString(&parser->str); - if (parser->gzip) { + if (parser->gzip != NULL) { ehttp_gzip_destroy(parser->gzip); parser->gzip = NULL; } @@ -201,209 +505,106 @@ void httpParserDestroy(HttpParserObj *parser) { #define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) -char *ehttp_parser_urldecode(const char *enc) { - int ok = 1; - HttpParserString str = {0}; +char *httpDecodeUrl(const char *enc) { + int32_t ok = 1; + HttpString str = {0}; while (*enc) { char *p = strchr(enc, '%'); if (!p) break; - int hex, cnt; - int n = sscanf(p+1, "%2x%n", &hex, &cnt); + int32_t hex, cnt; + int32_t n = sscanf(p+1, "%2x%n", &hex, &cnt); if (n!=1 && cnt !=2) { ok = 0; break; } - if (httpParserAppendString(&str, enc, p-enc)) { ok = 0; break; } + if (httpAppendString(&str, enc, p-enc)) { ok = 0; break; } char c = (char)hex; - if (httpParserAppendString(&str, &c, 1)) { ok = 0; break; } + if (httpAppendString(&str, &c, 1)) { ok = 0; break; } enc = p+3; } char *dec = NULL; if (ok && *enc) { - if (httpParserAppendString(&str, enc, strlen(enc))) { ok = 0; } + if (httpAppendString(&str, enc, strlen(enc))) { ok = 0; } } if (ok) { dec = str.str; str.str = NULL; } - httpParserCleanupString(&str); + httpCleanupString(&str); return dec; } -static void on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, size_t len) { - HttpParserObj *parser = (HttpParserObj*)arg; - parser->callbacks.on_body(parser->arg, buf, len); +static void httpOnData(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) { + HttpParser *parser = (HttpParser*)arg; + httpOnBody(parser, buf, len); } -static int32_t httpParserCheckField(HttpContext *pContext, HttpParserObj *parser, const char *key, const char *val) { - int32_t ok = 0; - do { - if (0 == strcasecmp(key, "Content-Length")) { - int32_t len = 0; - int32_t bytes = 0; - int32_t n = sscanf(val, "%d%n", &len, &bytes); - if (n == 1 && bytes == strlen(val)) { - parser->content_length = len; - parser->chunk_size = len; - parser->content_length_specified = 1; - break; - } - ok = -1; - break; - } - if (0 == strcasecmp(key, "Accept-Encoding")) { - if (strstr(val, "gzip")) { - parser->accept_encoding_gzip = 1; - } - if (strstr(val, "chunked")) { - parser->accept_encoding_chunked = 1; - } - break; - } - if (0 == strcasecmp(key, "Content-Encoding")) { - if (0 == strcmp(val, "gzip")) { - parser->content_chunked = 1; - } - break; - } - if (0 == strcasecmp(key, "Transfer-Encoding")) { - if (strstr(val, "gzip")) { - parser->transfer_gzip = 1; - ehttp_gzip_conf_t conf = {0}; - ehttp_gzip_callbacks_t callbacks = {0}; - - callbacks.on_data = on_data; - - parser->gzip = ehttp_gzip_create_decompressor(conf, callbacks, parser); - - if (!parser->gzip) { - httpDebug("failed to create gzip decompressor"); - ok = -1; - break; - } - } - if (strstr(val, "chunked")) { - parser->transfer_chunked = 1; - } - break; - } - if (0==strcasecmp(key, "Authorization")) { - char *t = NULL; - char *s = NULL; - int32_t bytes = 0; - int32_t n = sscanf(val, "%ms %ms%n", &t, &s, &bytes); - if (n == 2 && t && s && bytes == strlen(val)) { - if (strcmp(t, "Basic") == 0) { - free(parser->auth_basic); - parser->auth_basic = s; - s = NULL; - } else if (n == 2 && t && s && strcmp(t, "Taosd") == 0) { - free(parser->auth_taosd); - parser->auth_taosd = s; - s = NULL; - } else { - httpError("context:%p, fd:%d, invalid auth, t:%s s:%s", pContext, pContext->fd, t, s); - ok = -1; - } - } else { - httpError("context:%p, fd:%d, parse auth failed, t:%s s:%s", pContext, pContext->fd, t, s); - ok = -1; - } - - free(t); - free(s); - break; - } - } while (0); - return ok; -} - -static int httpParserAppendKv(HttpParserObj *parser, const char *key, const char *val) { - // HttpParseKvObj *kvs = (HttpParseKvObj*)reallocarray(parser->kvs, parser->kvs_count + 1, sizeof(*kvs)); - HttpParseKvObj *kvs = (HttpParseKvObj*)realloc(parser->kvs, (parser->kvs_count + 1) * sizeof(*kvs)); - if (!kvs) return -1; - - parser->kvs = kvs; - - kvs[parser->kvs_count].key = strdup(key); - kvs[parser->kvs_count].val = strdup(val); - - if (kvs[parser->kvs_count].key && kvs[parser->kvs_count].val) { - ++parser->kvs_count; - return 0; - } - - free(kvs[parser->kvs_count].key); - kvs[parser->kvs_count].key = NULL; - free(kvs[parser->kvs_count].val); - kvs[parser->kvs_count].val = NULL; - - return -1; -} - -static int32_t httpParserOnBegin(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { +static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; do { if (c == 'G' || c == 'P' || c == 'H' || c == 'D' || c == 'C' || c == 'O' || c == 'T') { - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); break; } - httpParserPop(parser); - httpParserPush(parser, HTTP_PARSER_REQUEST_OR_RESPONSE); + httpPopStack(parser); + httpPushStack(parser, HTTP_PARSER_REQUEST_OR_RESPONSE); break; } httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 400); + httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); } while (0); return ok; } -static int32_t httpParserOnRquestOrResponse(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { +static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; do { - if (parser->str.len == 1) { + if (parser->str.pos == 1) { if (c == 'T' && parser->str.str[0] == 'H') { - httpParserPop(parser); - httpParserPush(parser, HTTP_PARSER_END); - httpParserPush(parser, HTTP_PARSER_HEADER); - httpParserPush(parser, HTTP_PARSER_CRLF); - httpParserPush(parser, HTTP_PARSER_REASON_PHRASE); - httpParserPush(parser, HTTP_PARSER_SP); - httpParserPush(parser, HTTP_PARSER_STATUS_CODE); - httpParserPush(parser, HTTP_PARSER_SP); - httpParserPush(parser, HTTP_PARSER_HTTP_VERSION); + httpPopStack(parser); + httpPushStack(parser, HTTP_PARSER_END); + httpPushStack(parser, HTTP_PARSER_HEADER); + httpPushStack(parser, HTTP_PARSER_CRLF); + httpPushStack(parser, HTTP_PARSER_REASON_PHRASE); + httpPushStack(parser, HTTP_PARSER_SP); + httpPushStack(parser, HTTP_PARSER_STATUS_CODE); + httpPushStack(parser, HTTP_PARSER_SP); + httpPushStack(parser, HTTP_PARSER_HTTP_VERSION); *again = 1; break; } - httpParserPop(parser); - httpParserPush(parser, HTTP_PARSER_END); - httpParserPush(parser, HTTP_PARSER_HEADER); - httpParserPush(parser, HTTP_PARSER_CRLF); - httpParserPush(parser, HTTP_PARSER_HTTP_VERSION); - httpParserPush(parser, HTTP_PARSER_SP); - httpParserPush(parser, HTTP_PARSER_TARGET); - httpParserPush(parser, HTTP_PARSER_SP); - httpParserPush(parser, HTTP_PARSER_METHOD); + httpPopStack(parser); + httpPushStack(parser, HTTP_PARSER_END); + httpPushStack(parser, HTTP_PARSER_HEADER); + httpPushStack(parser, HTTP_PARSER_CRLF); + httpPushStack(parser, HTTP_PARSER_HTTP_VERSION); + httpPushStack(parser, HTTP_PARSER_SP); + httpPushStack(parser, HTTP_PARSER_TARGET); + httpPushStack(parser, HTTP_PARSER_SP); + httpPushStack(parser, HTTP_PARSER_METHOD); *again = 1; break; } httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 400); + httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); } while (0); return ok; } -static int32_t httpParserOnMethod(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { +static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpDebug("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); break; } break; @@ -412,59 +613,63 @@ static int32_t httpParserOnMethod(HttpContext *pContext, HttpParserObj *parser, if (!parser->method) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); break; + } else { + httpTrace("context:%p, fd:%d, httpMethod:%s", pContext, pContext->fd, parser->method); } - httpParserClearString(&parser->str); - httpParserPop(parser); + httpClearString(&parser->str); + httpPopStack(parser); *again = 1; } while (0); return ok; } -static int32_t httpParserOnTarget(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { +static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; do { if (!isspace(c) && c != '\r' && c != '\n') { - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_TARGET_FAILED); break; } break; } parser->target_raw = strdup(parser->str.str); - parser->target = ehttp_parser_urldecode(parser->str.str); + parser->target = httpDecodeUrl(parser->str.str); if (!parser->target_raw || !parser->target) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_TARGET_FAILED); break; } - httpParserClearString(&parser->str); - httpParserPop(parser); + httpClearString(&parser->str); + httpPopStack(parser); *again = 1; } while (0); return ok; } -static int32_t httpParserOnVersion(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { +static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; do { const char *prefix = "HTTP/1."; int32_t len = strlen(prefix); - if (parser->str.len < len) { - if (prefix[parser->str.len]!=c) { + if (parser->str.pos < len) { + if (prefix[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 400); + httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); break; } - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); break; } break; @@ -473,135 +678,143 @@ static int32_t httpParserOnVersion(HttpContext *pContext, HttpParserObj *parser, if (c!='0' && c!='1') { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 400); + httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); break; } - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); break; } - if (c=='0') parser->http_10 = 1; - if (c=='1') parser->http_11 = 1; + + if (c == '0') parser->httpVersion = HTTP_VERSION_10; + else if (c == '1') parser->httpVersion = HTTP_VERSION_11; + else if (c == '2') parser->httpVersion = HTTP_VERSION_12; + else parser->httpVersion = HTTP_INVALID_VERSION; parser->version = strdup(parser->str.str); if (!parser->version) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); break; } if (parser->method) { - parser->callbacks.on_request_line(parser->arg, parser->method, parser->target, parser->version, parser->target_raw); + ok = httpOnRequestLine(parser, parser->method, parser->target, parser->version); } - httpParserClearString(&parser->str); - httpParserPop(parser); + httpClearString(&parser->str); + httpPopStack(parser); } while (0); return ok; } -static int32_t httpParserOnSp(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int *again) { +static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; do { if (c == ' ') { - httpParserPop(parser); + httpPopStack(parser); break; } httpDebug("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_SP_FAILED); } while (0); return ok; } -static int32_t httpParserOnStatusCode(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; + int32_t ok = 0; do { if (isdigit(c)) { - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_STATUS_FAILED); break; } - if (parser->str.len < 3) break; + if (parser->str.pos < 3) break; - sscanf(parser->str.str, "%d", &parser->status_code); - httpParserClearString(&parser->str); - httpParserPop(parser); + sscanf(parser->str.str, "%d", &parser->statusCode); + httpClearString(&parser->str); + httpPopStack(parser); break; } httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 400); + httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_STATUS_FAILED); } while (0); return ok; } -static int32_t httpParserOnReasonPhrase(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnReasonPhrase(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; + int32_t ok = 0; do { - if (c=='\r') { - parser->reason_phrase = strdup(parser->str.str); - if (!parser->reason_phrase) { + if (c == '\r') { + parser->reasonPhrase = strdup(parser->str.str); + if (!parser->reasonPhrase) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_PHRASE_FAILED); break; } - parser->callbacks.on_status_line(parser->arg, parser->version, parser->status_code, parser->reason_phrase); - httpParserClearString(&parser->str); - httpParserPop(parser); + ok = httpOnStatusLine(parser, parser->statusCode, parser->reasonPhrase); + httpClearString(&parser->str); + httpPopStack(parser); *again = 1; break; } - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_PHRASE_FAILED); break; } } while (0); return ok; } -static int32_t httpParserPostProcess(HttpContext *pContext, HttpParserObj *parser) { +static int32_t httpParserPostProcess(HttpParser *parser) { + HttpContext *pContext = parser->pContext; if (parser->gzip) { if (ehttp_gzip_finish(parser->gzip)) { httpError("context:%p, fd:%d, gzip failed", pContext, pContext->fd); - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_FINISH_GZIP_FAILED); return -1; } } - parser->callbacks.on_end(parser->arg); + httpOnEnd(parser); return 0; } -static int32_t httpParserOnCrlf(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { +static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; do { const char *s = "\r\n"; int32_t len = strlen(s); - if (s[parser->str.len]!=c) { + if (s[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 400); + httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_CRLF_FAILED); break; } - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_CRLF_FAILED); break; } - if (parser->str.len == len) { - httpParserClearString(&parser->str); - httpParserPop(parser); - if (httpParserTop(parser) == HTTP_PARSER_END) { - ok = httpParserPostProcess(pContext, parser); + if (parser->str.pos == len) { + httpClearString(&parser->str); + httpPopStack(parser); + if (httpTopStack(parser) == HTTP_PARSER_END) { + ok = httpParserPostProcess(parser); } } break; @@ -609,141 +822,140 @@ static int32_t httpParserOnCrlf(HttpContext *pContext, HttpParserObj *parser, HT return ok; } -static int32_t httpParserOnHeader(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { +static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; do { - if (c=='\r') { - httpParserPop(parser); - if (parser->transfer_chunked) { - httpParserPush(parser, HTTP_PARSER_CHUNK_SIZE); - httpParserPush(parser, HTTP_PARSER_CRLF); + if (c == '\r') { + httpPopStack(parser); + if (parser->transferChunked) { + httpPushStack(parser, HTTP_PARSER_CHUNK_SIZE); + httpPushStack(parser, HTTP_PARSER_CRLF); } else { - if (parser->content_length > 0) { - httpParserPush(parser, HTTP_PARSER_CHUNK); + if (parser->contentLength > 0) { + httpPushStack(parser, HTTP_PARSER_CHUNK); } - httpParserPush(parser, HTTP_PARSER_CRLF); + httpPushStack(parser, HTTP_PARSER_CRLF); } *again = 1; break; } - if (c!=' ' && c!='\t' && c!=':' ) { - if (httpParserAppendString(&parser->str, &c, 1)) { + if (c != ' ' && c != '\t' && c != ':') { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_HEADER_FAILED); break; } - httpParserPush(parser, HTTP_PARSER_CRLF); - httpParserPush(parser, HTTP_PARSER_HEADER_VAL); - httpParserPush(parser, HTTP_PARSER_SP); - httpParserPush(parser, HTTP_PARSER_HEADER_KEY); + httpPushStack(parser, HTTP_PARSER_CRLF); + httpPushStack(parser, HTTP_PARSER_HEADER_VAL); + httpPushStack(parser, HTTP_PARSER_SP); + httpPushStack(parser, HTTP_PARSER_HEADER_KEY); break; } httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 400); + httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_HEADER_FAILED); } while (0); return ok; } -static int httpParserOnHeaderKey(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; + int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED); break; } break; } - if (c==':') { + if (c == ':') { parser->key = strdup(parser->str.str); if (!parser->key) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED); break; } - httpParserClearString(&parser->str); - httpParserPop(parser); + httpClearString(&parser->str); + httpPopStack(parser); break; } httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 400); + httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED); } while (0); return ok; } -static int32_t httpParserOnHeaderVal(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { +static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; do { - if (c != '\r' && c != '\n' && (!isspace(c) || parser->str.len>0)) { - if (httpParserAppendString(&parser->str, &c, 1)) { + if (c != '\r' && c != '\n' && (!isspace(c) || parser->str.pos > 0)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + parser->parseCode = TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED; + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED); break; } break; } const char *val = parser->str.str; - ok = httpParserCheckField(pContext, parser, parser->key, val); - if (httpParserAppendKv(parser, parser->key, val)) { - ok = -1; - parser->callbacks.on_error(parser->arg, 507); - } else { - parser->callbacks.on_header_field(parser->arg, parser->key, val); - } + ok = httpOnParseHeaderField(parser, parser->key, val); free(parser->key); parser->key = NULL; val = NULL; if (ok == -1) break; - httpParserClearString(&parser->str); - httpParserPop(parser); + httpClearString(&parser->str); + httpPopStack(parser); *again = 1; } while (0); return ok; } -static int32_t httpParserOnChunkSize(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { +static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; int32_t bytes; int32_t len; - int n; + int32_t n; do { if (isxdigit(c)) { - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED); break; } break; } - if (c=='\r') { + if (c == '\r') { n = sscanf(parser->str.str, "%x%n", &len, &bytes); - if (n==1 && bytes==strlen(parser->str.str) && len>=0) { - if (len==0) { - if (parser->content_length_specified == 0 || parser->received_size == parser->content_length) { - httpParserClearString(&parser->str); - httpParserPop(parser); - httpParserPush(parser, HTTP_PARSER_CRLF); - httpParserPush(parser, HTTP_PARSER_CRLF); + if (n == 1 && bytes == strlen(parser->str.str) && len >= 0) { + if (len == 0) { + if (parser->contentLengthSpecified == 0 || parser->receivedSize == parser->contentLength) { + httpClearString(&parser->str); + httpPopStack(parser); + httpPushStack(parser, HTTP_PARSER_CRLF); + httpPushStack(parser, HTTP_PARSER_CRLF); *again = 1; break; } } else { - if (parser->content_length_specified == 0 || parser->received_size + len <= parser->content_length) { - parser->chunk_size = len; - httpParserClearString(&parser->str); - httpParserPop(parser); - httpParserPush(parser, HTTP_PARSER_CHUNK_SIZE); - httpParserPush(parser, HTTP_PARSER_CRLF); - httpParserPush(parser, HTTP_PARSER_CHUNK); - httpParserPush(parser, HTTP_PARSER_CRLF); + if (parser->contentLengthSpecified == 0 || parser->receivedSize + len <= parser->contentLength) { + parser->chunkSize = len; + httpClearString(&parser->str); + httpPopStack(parser); + httpPushStack(parser, HTTP_PARSER_CHUNK_SIZE); + httpPushStack(parser, HTTP_PARSER_CRLF); + httpPushStack(parser, HTTP_PARSER_CHUNK); + httpPushStack(parser, HTTP_PARSER_CRLF); *again = 1; break; } @@ -752,116 +964,119 @@ static int32_t httpParserOnChunkSize(HttpContext *pContext, HttpParserObj *parse } httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 400); + httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED); } while (0); return ok; } -static int httpParserOnChunk(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int *again) { - int ok = 0; +static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; + int32_t ok = 0; do { - if (httpParserAppendString(&parser->str, &c, 1)) { + if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_CHUNK_FAILED); break; } - ++parser->received_size; - ++parser->received_chunk_size; - if (parser->received_chunk_size < parser->chunk_size) break; + ++parser->receivedSize; + ++parser->receivedChunkSize; + if (parser->receivedChunkSize < parser->chunkSize) break; if (parser->gzip) { - if (ehttp_gzip_write(parser->gzip, parser->str.str, parser->str.len)) { + if (ehttp_gzip_write(parser->gzip, parser->str.str, parser->str.pos)) { httpError("context:%p, fd:%d, gzip failed", pContext, pContext->fd); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_CHUNK_FAILED); break; } } else { - parser->callbacks.on_body(parser->arg, parser->str.str, parser->str.len); + httpOnBody(parser, parser->str.str, parser->str.pos); } - parser->received_chunk_size = 0; - httpParserClearString(&parser->str); - httpParserPop(parser); - if (httpParserTop(parser) == HTTP_PARSER_END) { - ok = httpParserPostProcess(pContext, parser); + parser->receivedChunkSize = 0; + httpClearString(&parser->str); + httpPopStack(parser); + if (httpTopStack(parser) == HTTP_PARSER_END) { + ok = httpParserPostProcess(parser); } } while (0); return ok; } -static int32_t httpParserOnEnd(HttpContext *pContext, HttpParserObj *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { +static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; do { - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; - parser->callbacks.on_error(parser->arg, 507); + httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); + httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_END_FAILED); } while (0); return ok; } -static int32_t httpParseChar(HttpContext *pContext, HttpParserObj *parser, const char c, int32_t *again) { +static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { + HttpContext *pContext = parser->pContext; int32_t ok = 0; - HTTP_PARSER_STATE state = httpParserTop(parser); + HTTP_PARSER_STATE state = httpTopStack(parser); do { if (state == HTTP_PARSER_BEGIN) { - ok = httpParserOnBegin(pContext, parser, state, c, again); + ok = httpParserOnBegin(parser, state, c, again); break; } if (state == HTTP_PARSER_REQUEST_OR_RESPONSE) { - ok = httpParserOnRquestOrResponse(pContext, parser, state, c, again); + ok = httpParserOnRquestOrResponse(parser, state, c, again); break; } if (state == HTTP_PARSER_METHOD) { - ok = httpParserOnMethod(pContext, parser, state, c, again); + ok = httpParserOnMethod(parser, state, c, again); break; } if (state == HTTP_PARSER_TARGET) { - ok = httpParserOnTarget(pContext, parser, state, c, again); + ok = httpParserOnTarget(parser, state, c, again); break; } if (state == HTTP_PARSER_HTTP_VERSION) { - ok = httpParserOnVersion(pContext, parser, state, c, again); + ok = httpParserOnVersion(parser, state, c, again); break; } if (state == HTTP_PARSER_SP) { - ok = httpParserOnSp(pContext, parser, state, c, again); + ok = httpParserOnSp(parser, state, c, again); break; } if (state == HTTP_PARSER_STATUS_CODE) { - ok = httpParserOnStatusCode(pContext, parser, state, c, again); + ok = httpParserOnStatusCode(parser, state, c, again); break; } if (state == HTTP_PARSER_REASON_PHRASE) { - ok = httpParserOnReasonPhrase(pContext, parser, state, c, again); + ok = httpParserOnReasonPhrase(parser, state, c, again); break; } if (state == HTTP_PARSER_CRLF) { - ok = httpParserOnCrlf(pContext, parser, state, c, again); + ok = httpParserOnCrlf(parser, state, c, again); break; } if (state == HTTP_PARSER_HEADER) { - ok = httpParserOnHeader(pContext, parser, state, c, again); + ok = httpParserOnHeader(parser, state, c, again); break; } if (state == HTTP_PARSER_HEADER_KEY) { - ok = httpParserOnHeaderKey(pContext, parser, state, c, again); + ok = httpParserOnHeaderKey(parser, state, c, again); break; } if (state == HTTP_PARSER_HEADER_VAL) { - ok = httpParserOnHeaderVal(pContext, parser, state, c, again); + ok = httpParserOnHeaderVal(parser, state, c, again); break; } if (state == HTTP_PARSER_CHUNK_SIZE) { - ok = httpParserOnChunkSize(pContext, parser, state, c, again); + ok = httpParserOnChunkSize(parser, state, c, again); break; } if (state == HTTP_PARSER_CHUNK) { - ok = httpParserOnChunk(pContext, parser, state, c, again); + ok = httpParserOnChunk(parser, state, c, again); break; } if (state == HTTP_PARSER_END) { - ok = httpParserOnEnd(pContext, parser, state, c, again); + ok = httpParserOnEnd(parser, state, c, again); break; } if (state == HTTP_PARSER_ERROR) { @@ -869,32 +1084,34 @@ static int32_t httpParseChar(HttpContext *pContext, HttpParserObj *parser, const break; } - httpError("context:%p, fd:%d, unknown parse state:%d", pContext, pContext->fd, state); ok = -1; - parser->callbacks.on_error(parser->arg, 500); + httpError("context:%p, fd:%d, unknown parse state:%d", pContext, pContext->fd, state); + httpOnError(parser, 500, TSDB_CODE_HTTP_PARSE_INVALID_STATE); } while (0); if (ok == -1) { - httpError("context:%p, fd:%d, failed to parse, state:%d ok:%d", pContext, pContext->fd, state, ok); - httpParserPush(parser, HTTP_PARSER_ERROR); + httpError("context:%p, fd:%d, failed to parse, state:%d", pContext, pContext->fd, state); + httpPushStack(parser, HTTP_PARSER_ERROR); } if (ok == -2) { - httpError("context:%p, fd:%d, failed to parse, state:%d ok:%d", pContext, pContext->fd, state, ok); ok = -1; + httpError("context:%p, fd:%d, failed to parse, invalid state", pContext, pContext->fd); + httpOnError(parser, 500, TSDB_CODE_HTTP_PARSE_ERROR_STATE); } return ok; } -int32_t httpParserBuf(HttpContext *pContext, HttpParserObj *parser, const char *buf, int32_t len) { +int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len) { + HttpContext *pContext = parser->pContext; const char *p = buf; int32_t ret = 0; int32_t i = 0; while (i < len) { int32_t again = 0; - ret = httpParseChar(pContext, parser, *p, &again); + ret = httpParseChar(parser, *p, &again); if (ret != 0) { httpError("context:%p, fd:%d, parse failed, ret:%d i:%d len:%d buf:%s", pContext, pContext->fd, ret, i, len, buf); break; @@ -906,27 +1123,3 @@ int32_t httpParserBuf(HttpContext *pContext, HttpParserObj *parser, const char * return ret; } - -void httpParserCleanupString(HttpParserString *str) { - free(str->str); - str->str = NULL; - str->len = 0; -} - -int32_t httpParserAppendString(HttpParserString *str, const char *s, int32_t len) { - int32_t n = str->len; - char *p = (char*)realloc(str->str, n + len + 1); - if (!p) return -1; - strncpy(p+n, s, len); - p[n+len] = '\0'; - str->str = p; - str->len = n+len; - return 0; -} - -void httpParserClearString(HttpParserString *str) { - if (str->str) { - str->str[0] = '\0'; - str->len = 0; - } -} diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 9625102f74..76632eb508 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -39,15 +39,15 @@ typedef struct { typedef struct { void *param; void *result; - int numOfRows; - void (*fp)(void *param, void *result, int numOfRows); + int32_t numOfRows; + void (*fp)(void *param, void *result, int32_t numOfRows); } SHttpResult; static SHttpWorkerPool tsHttpPool; static taos_qset tsHttpQset; static taos_queue tsHttpQueue; -void httpDispatchToResultQueue(void *param, TAOS_RES *result, int numOfRows, void (*fp)(void *param, void *result, int numOfRows)) { +void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t numOfRows, void (*fp)(void *param, void *result, int32_t numOfRows)) { if (tsHttpQueue != NULL) { SHttpResult *pMsg = (SHttpResult *)taosAllocateQitem(sizeof(SHttpResult)); pMsg->param = param; diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index a7c17dfdbb..def8ecf5e1 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -45,23 +45,23 @@ const char *httpRespTemplate[] = { "%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sAccess-Control-Allow-Methods:POST, GET, OPTIONS, DELETE, PUT\r\nAccess-Control-Allow-Headers:Accept, Content-Type\r\nContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n" }; -static void httpSendErrorRespImp(HttpContext *pContext, int httpCode, char *httpCodeStr, int errNo, char *desc) { +static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char *httpCodeStr, int32_t errNo, char *desc) { httpError("context:%p, fd:%d, code:%d, error:%s", pContext, pContext->fd, httpCode, desc); char head[512] = {0}; char body[512] = {0}; - int bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_ERROR], errNo, desc); - int headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[pContext->httpVersion], httpCode, - httpCodeStr, httpKeepAliveStr[pContext->httpKeepAlive], bodyLen); + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_ERROR], errNo, desc); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[pContext->parser->httpVersion], + httpCode, httpCodeStr, httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); httpCloseContextByApp(pContext); } -void httpSendErrorRespWithDesc(HttpContext *pContext, int errNo, char *desc) { - int httpCode = 500; +void httpSendErrorRespWithDesc(HttpContext *pContext, int32_t errNo, char *desc) { + int32_t httpCode = 500; char *httpCodeStr = "Internal Server Error"; switch (errNo) { case HTTP_SUCCESS: @@ -180,20 +180,20 @@ void httpSendErrorRespWithDesc(HttpContext *pContext, int errNo, char *desc) { } } -void httpSendErrorResp(HttpContext *pContext, int errNo) { httpSendErrorRespWithDesc(pContext, errNo, NULL); } +void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { httpSendErrorRespWithDesc(pContext, errNo, NULL); } -void httpSendTaosdErrorResp(HttpContext *pContext, int errCode) { - int httpCode = 400; +void httpSendTaosdErrorResp(HttpContext *pContext, int32_t errCode) { + int32_t httpCode = 400; httpSendErrorRespImp(pContext, httpCode, "Bad Request", errCode & 0XFFFF, (char*)tstrerror(errCode)); } -void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { - int httpCode = 400; - char temp[512] = {0}; - int len = sprintf(temp, "invalid SQL: %s", errMsg); +void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char *errMsg) { + int32_t httpCode = 400; + char temp[512] = {0}; + int32_t len = sprintf(temp, "invalid SQL: %s", errMsg); - for (int i = 0; i < len; ++i) { + for (int32_t i = 0; i < len; ++i) { if (temp[i] == '\"') { temp[i] = '\''; } else if (temp[i] == '\n') { @@ -208,9 +208,9 @@ void httpSendSuccResp(HttpContext *pContext, char *desc) { char head[1024] = {0}; char body[1024] = {0}; - int bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], HTTP_SUCCESS, desc); - int headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[pContext->httpVersion], - httpKeepAliveStr[pContext->httpKeepAlive], bodyLen); + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], HTTP_SUCCESS, desc); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[pContext->parser->httpVersion], + httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); @@ -221,9 +221,9 @@ void httpSendOptionResp(HttpContext *pContext, char *desc) { char head[1024] = {0}; char body[1024] = {0}; - int bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], HTTP_SUCCESS, desc); - int headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[pContext->httpVersion], - httpKeepAliveStr[pContext->httpKeepAlive], bodyLen); + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], HTTP_SUCCESS, desc); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[pContext->parser->httpVersion], + httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); diff --git a/src/plugins/http/src/httpRestHandle.c b/src/plugins/http/src/httpRestHandle.c index b285b19c3f..fbdce566f0 100644 --- a/src/plugins/http/src/httpRestHandle.c +++ b/src/plugins/http/src/httpRestHandle.c @@ -60,22 +60,22 @@ void restInitHandle(HttpServer* pServer) { } bool restGetUserFromUrl(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - if (pParser->path[REST_USER_URL_POS].len >= TSDB_USER_LEN || pParser->path[REST_USER_URL_POS].len <= 0) { + HttpParser* pParser = pContext->parser; + if (pParser->path[REST_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[REST_USER_URL_POS].pos <= 0) { return false; } - tstrncpy(pContext->user, pParser->path[REST_USER_URL_POS].pos, TSDB_USER_LEN); + tstrncpy(pContext->user, pParser->path[REST_USER_URL_POS].str, TSDB_USER_LEN); return true; } bool restGetPassFromUrl(HttpContext* pContext) { - HttpParser* pParser = &pContext->parser; - if (pParser->path[REST_PASS_URL_POS].len >= TSDB_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].len <= 0) { + HttpParser* pParser = pContext->parser; + if (pParser->path[REST_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].pos <= 0) { return false; } - tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].pos, TSDB_PASSWORD_LEN); + tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].str, TSDB_PASSWORD_LEN); return true; } @@ -85,10 +85,10 @@ bool restProcessLoginRequest(HttpContext* pContext) { return true; } -bool restProcessSqlRequest(HttpContext* pContext, int timestampFmt) { +bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) { httpDebug("context:%p, fd:%d, user:%s, process restful sql msg", pContext, pContext->fd, pContext->user); - char* sql = pContext->parser.data.pos; + char* sql = pContext->parser->body.str; if (sql == NULL) { httpSendErrorResp(pContext, HTTP_NO_SQL_INPUT); return false; diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index c16727faa0..26f0441519 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -21,7 +21,7 @@ #include "httpRestHandle.h" #include "httpRestJson.h" -void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect_rows) { +void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return; @@ -43,7 +43,7 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) if (jsonBuf == NULL) return; TAOS_FIELD *fields = taos_fetch_fields(result); - int num_fields = taos_num_fields(result); + int32_t num_fields = taos_num_fields(result); httpInitJsonBuf(jsonBuf, pContext); httpWriteJsonBufHead(jsonBuf); @@ -66,9 +66,9 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) httpJsonItemToken(jsonBuf); httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN); } else { - for (int i = 0; i < num_fields; ++i) { + for (int32_t i = 0; i < num_fields; ++i) { httpJsonItemToken(jsonBuf); - httpJsonString(jsonBuf, fields[i].name, (int)strlen(fields[i].name)); + httpJsonString(jsonBuf, fields[i].name, (int32_t)strlen(fields[i].name)); } } @@ -83,16 +83,16 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) httpJsonToken(jsonBuf, JsonArrStt); } -bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows, int timestampFormat) { +bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, int32_t timestampFormat) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; cmd->numOfRows += numOfRows; - int num_fields = taos_num_fields(result); + int32_t num_fields = taos_num_fields(result); TAOS_FIELD *fields = taos_fetch_fields(result); - for (int k = 0; k < numOfRows; ++k) { + for (int32_t k = 0; k < numOfRows; ++k) { TAOS_ROW row = taos_fetch_row(result); if (row == NULL) { cmd->numOfRows--; @@ -104,7 +104,7 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, httpJsonItemToken(jsonBuf); httpJsonToken(jsonBuf, JsonArrStt); - for (int i = 0; i < num_fields; i++) { + for (int32_t i = 0; i < num_fields; i++) { httpJsonItemToken(jsonBuf); if (row[i] == NULL) { @@ -171,15 +171,15 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } } -bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows) { +bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP); } -bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows) { +bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING); } -bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows) { +bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING); } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index e0d2a90994..212ce46473 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -61,7 +61,7 @@ void httpCleanUpConnect() { if (pServer->pThreads == NULL) return; pthread_join(pServer->thread, NULL); - for (int i = 0; i < pServer->numOfThreads; ++i) { + for (int32_t i = 0; i < pServer->numOfThreads; ++i) { HttpThread* pThread = pServer->pThreads + i; if (pThread != NULL) { httpStopThread(pThread); @@ -71,41 +71,11 @@ void httpCleanUpConnect() { httpDebug("http server:%s is cleaned up", pServer->label); } -static bool httpDecompressData(HttpContext *pContext) { - if (pContext->contentEncoding != HTTP_COMPRESS_GZIP) { - httpTraceL("context:%p, fd:%d, content:%s", pContext, pContext->fd, pContext->parser.data.pos); - return true; - } - - char *decompressBuf = calloc(HTTP_DECOMPRESS_BUF_SIZE, 1); - int32_t decompressBufLen = HTTP_DECOMPRESS_BUF_SIZE; - size_t bufsize = sizeof(pContext->parser.buffer) - (pContext->parser.data.pos - pContext->parser.buffer) - 1; - if (decompressBufLen > (int)bufsize) { - decompressBufLen = (int)bufsize; - } - - int ret = httpGzipDeCompress(pContext->parser.data.pos, pContext->parser.data.len, decompressBuf, &decompressBufLen); - - if (ret == 0) { - memcpy(pContext->parser.data.pos, decompressBuf, decompressBufLen); - pContext->parser.data.pos[decompressBufLen] = 0; - httpTraceL("context:%p, fd:%d, rawSize:%d, decompressSize:%d, content:%s", pContext, pContext->fd, - pContext->parser.data.len, decompressBufLen, decompressBuf); - pContext->parser.data.len = decompressBufLen; - } else { - httpError("context:%p, fd:%d, failed to decompress data, rawSize:%d, error:%d", pContext, pContext->fd, - pContext->parser.data.len, ret); - } - - free(decompressBuf); - return ret == 0; -} - static void httpProcessHttpData(void *param) { HttpServer *pServer = &tsHttpServer; HttpThread *pThread = (HttpThread *)param; HttpContext *pContext; - int fdNum; + int32_t fdNum; sigset_t set; sigemptyset(&set); @@ -122,7 +92,7 @@ static void httpProcessHttpData(void *param) { } if (fdNum <= 0) continue; - for (int i = 0; i < fdNum; ++i) { + for (int32_t i = 0; i < fdNum; ++i) { pContext = httpGetContext(events[i].data.ptr); if (pContext == NULL) { httpError("context:%p, is already released, close connect", events[i].data.ptr); @@ -182,13 +152,13 @@ static void httpProcessHttpData(void *param) { } static void *httpAcceptHttpConnection(void *arg) { - int connFd = -1; + int32_t connFd = -1; struct sockaddr_in clientAddr; - int threadId = 0; + int32_t threadId = 0; HttpServer * pServer = &tsHttpServer; HttpThread * pThread = NULL; HttpContext * pContext = NULL; - int totalFds = 0; + int32_t totalFds = 0; sigset_t set; sigemptyset(&set); @@ -208,7 +178,7 @@ static void *httpAcceptHttpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(clientAddr); - connFd = (int)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); + connFd = (int32_t)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); if (connFd == -1) { if (errno == EINVAL) { httpDebug("http server:%s socket was shutdown, exiting...", pServer->label); @@ -219,7 +189,7 @@ static void *httpAcceptHttpConnection(void *arg) { } totalFds = 1; - for (int i = 0; i < pServer->numOfThreads; ++i) { + for (int32_t i = 0; i < pServer->numOfThreads; ++i) { totalFds += pServer->pThreads[i].numOfContexts; } @@ -283,7 +253,7 @@ bool httpInitConnect() { } HttpThread *pThread = pServer->pThreads; - for (int i = 0; i < pServer->numOfThreads; ++i) { + for (int32_t i = 0; i < pServer->numOfThreads; ++i) { sprintf(pThread->label, "%s%d", pServer->label, i); pThread->processData = pServer->processData; pThread->threadId = i; @@ -331,52 +301,39 @@ bool httpInitConnect() { } static bool httpReadData(HttpContext *pContext) { - HttpParser *pParser = &pContext->parser; - ASSERT(!pContext->parsed); - - if (!pParser->parser) { - if (!pParser->inited) { - httpInitContext(pContext); - } - if (!pParser->parser) { - return false; - } + HttpParser *pParser = pContext->parser; + ASSERT(!pParser->parsed); + if (!pParser->inited) { + httpInitParser(pParser); } pContext->accessTimes++; pContext->lastAccessTime = taosGetTimestampSec(); - char buf[HTTP_STEP_SIZE + 1] = {0}; - int nread = (int)taosReadSocket(pContext->fd, buf, sizeof(buf)); + char buf[HTTP_STEP_SIZE + 1] = {0}; + int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, sizeof(buf)); if (nread > 0) { buf[nread] = '\0'; httpTrace("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf); - int ok = httpParserBuf(pContext, pParser->parser, buf, nread); + int32_t ok = httpParseBuf(pParser, buf, nread); if (ok) { - httpError("context:%p, fd:%d, init parse failed, reason:%d close connect", pContext, pContext->fd, ok); + httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok, pParser->parseCode); httpNotifyContextClose(pContext); return false; } - if (pContext->parser.failed) { - httpError("context:%p, fd:%d, parse failed, close connect", pContext, pContext->fd); + if (pParser->parseCode) { + httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode); httpNotifyContextClose(pContext); return false; } - if (pContext->parsed) { - httpDebug("context:%p, fd:%d, read size:%d, dataLen:%d", pContext, pContext->fd, pContext->parser.bufsize, - pContext->parser.data.len); - if (httpDecompressData(pContext)) { - return true; - } else { - httpNotifyContextClose(pContext); - return false; - } + if (pParser->parsed) { + httpDebug("context:%p, fd:%d, len:%d, body:%s", pContext, pContext->fd, pParser->body.pos, pParser->body.str); } - return pContext->parsed; + return true; } else if (nread < 0) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno); diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index ee43b3c826..8993b233dd 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -37,9 +37,9 @@ void opInitHandle(HttpServer* pServer) {} #endif HttpServer tsHttpServer; -void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); +void taosInitNote(int32_t numOfNoteLines, int32_t maxNotes, char* lable); -int httpInitSystem() { +int32_t httpInitSystem() { strcpy(tsHttpServer.label, "rest"); tsHttpServer.serverIp = 0; tsHttpServer.serverPort = tsHttpPort; @@ -60,7 +60,7 @@ int httpInitSystem() { return 0; } -int httpStartSystem() { +int32_t httpStartSystem() { httpInfo("start http server ..."); if (tsHttpServer.status != HTTP_SERVER_INIT) { diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index a7444676ae..83b31f652b 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -83,16 +83,16 @@ static const char DEFAULT_TELEGRAF_CFG[] = "]}"; typedef struct { - char *name; - char *tbName; - char **fields; - int fieldNum; + char * name; + char * tbName; + char ** fields; + int32_t fieldNum; } STgSchema; typedef struct { STgSchema *schemas; - int size; - int pos; + int32_t size; + int32_t pos; } STgSchemas; static STgSchemas tgSchemas = {0}; @@ -107,7 +107,7 @@ void tgFreeSchema(STgSchema *schema) { schema->tbName = NULL; } if (schema->fields != NULL) { - for (int f = 0; f < schema->fieldNum; ++f) { + for (int32_t f = 0; f < schema->fieldNum; ++f) { if (schema->fields[f] != NULL) { free(schema->fields[f]); schema->fields[f] = NULL; @@ -121,7 +121,7 @@ void tgFreeSchema(STgSchema *schema) { void tgFreeSchemas() { if (tgSchemas.schemas != NULL) { - for (int s = 0; s < tgSchemas.size; ++s) { + for (int32_t s = 0; s < tgSchemas.size; ++s) { tgFreeSchema(&tgSchemas.schemas[s]); } free(tgSchemas.schemas); @@ -130,7 +130,7 @@ void tgFreeSchemas() { } } -void tgInitSchemas(int size) { +void tgInitSchemas(int32_t size) { tgFreeSchemas(); tgSchemas.schemas = calloc(sizeof(STgSchema), size); tgSchemas.size = 0; @@ -154,7 +154,7 @@ void tgParseSchemaMetric(cJSON *metric) { parsedOk = false; goto ParseEnd; } - int nameLen = (int)strlen(name->valuestring); + int32_t nameLen = (int32_t)strlen(name->valuestring); if (nameLen == 0) { parsedOk = false; goto ParseEnd; @@ -177,7 +177,7 @@ void tgParseSchemaMetric(cJSON *metric) { parsedOk = false; goto ParseEnd; } - int tbnameLen = (int)strlen(tbname->valuestring); + int32_t tbnameLen = (int32_t)strlen(tbname->valuestring); if (tbnameLen == 0) { parsedOk = false; goto ParseEnd; @@ -191,7 +191,7 @@ void tgParseSchemaMetric(cJSON *metric) { if (fields == NULL) { goto ParseEnd; } - int fieldSize = cJSON_GetArraySize(fields); + int32_t fieldSize = cJSON_GetArraySize(fields); if (fieldSize <= 0 || fieldSize > TSDB_MAX_COLUMNS) { goto ParseEnd; } @@ -199,7 +199,7 @@ void tgParseSchemaMetric(cJSON *metric) { if (fieldSize > 0) { schema.fields = calloc(sizeof(STgSchema), (size_t)fieldSize); schema.fieldNum = fieldSize; - for (int i = 0; i < fieldSize; i++) { + for (int32_t i = 0; i < fieldSize; i++) { cJSON *field = cJSON_GetArrayItem(fields, i); if (field == NULL) { parsedOk = false; @@ -209,7 +209,7 @@ void tgParseSchemaMetric(cJSON *metric) { parsedOk = false; goto ParseEnd; } - int nameLen = (int)strlen(field->valuestring); + int32_t nameLen = (int32_t)strlen(field->valuestring); if (nameLen == 0 || nameLen >= TSDB_TABLE_NAME_LEN) { parsedOk = false; goto ParseEnd; @@ -227,13 +227,13 @@ ParseEnd: } } -int tgParseSchema(const char *content, char*fileName) { +int32_t tgParseSchema(const char *content, char*fileName) { cJSON *root = cJSON_Parse(content); if (root == NULL) { httpError("failed to parse telegraf schema file:%s, invalid json format, content:%s", fileName, content); return -1; } - int size = 0; + int32_t size = 0; cJSON *metrics = cJSON_GetObjectItem(root, "metrics"); if (metrics != NULL) { size = cJSON_GetArraySize(metrics); @@ -244,7 +244,7 @@ int tgParseSchema(const char *content, char*fileName) { } tgInitSchemas(size); - for (int i = 0; i < size; i++) { + for (int32_t i = 0; i < size; i++) { cJSON *metric = cJSON_GetArrayItem(metrics, i); if (metric != NULL) { tgParseSchemaMetric(metric); @@ -260,7 +260,7 @@ int tgParseSchema(const char *content, char*fileName) { return size; } -int tgReadSchema(char *fileName) { +int32_t tgReadSchema(char *fileName) { FILE *fp = fopen(fileName, "r"); if (fp == NULL) { return -1; @@ -286,7 +286,7 @@ int tgReadSchema(char *fileName) { } content[contentSize] = 0; - int schemaNum = tgParseSchema(content, fileName); + int32_t schemaNum = tgParseSchema(content, fileName); free(content); fclose(fp); @@ -313,53 +313,53 @@ void tgCleanupHandle() { } bool tgGetUserFromUrl(HttpContext *pContext) { - HttpParser *pParser = &pContext->parser; - if (pParser->path[TG_USER_URL_POS].len >= TSDB_USER_LEN || pParser->path[TG_USER_URL_POS].len <= 0) { + HttpParser *pParser = pContext->parser; + if (pParser->path[TG_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[TG_USER_URL_POS].pos <= 0) { return false; } - tstrncpy(pContext->user, pParser->path[TG_USER_URL_POS].pos, sizeof(pContext->user)); + tstrncpy(pContext->user, pParser->path[TG_USER_URL_POS].str, sizeof(pContext->user)); return true; } bool tgGetPassFromUrl(HttpContext *pContext) { - HttpParser *pParser = &pContext->parser; - if (pParser->path[TG_PASS_URL_POS].len >= TSDB_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].len <= 0) { + HttpParser *pParser = pContext->parser; + if (pParser->path[TG_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].pos <= 0) { return false; } - tstrncpy(pContext->pass, pParser->path[TG_PASS_URL_POS].pos, sizeof(pContext->pass)); + tstrncpy(pContext->pass, pParser->path[TG_PASS_URL_POS].str, sizeof(pContext->pass)); return true; } char *tgGetDbFromUrl(HttpContext *pContext) { - HttpParser *pParser = &pContext->parser; - if (pParser->path[TG_DB_URL_POS].len <= 0) { + HttpParser *pParser = pContext->parser; + if (pParser->path[TG_DB_URL_POS].pos <= 0) { httpSendErrorResp(pContext, HTTP_TG_DB_NOT_INPUT); return NULL; } - if (pParser->path[TG_DB_URL_POS].len >= TSDB_DB_NAME_LEN) { + if (pParser->path[TG_DB_URL_POS].pos >= TSDB_DB_NAME_LEN) { httpSendErrorResp(pContext, HTTP_TG_DB_TOO_LONG); return NULL; } - return pParser->path[TG_DB_URL_POS].pos; + return pParser->path[TG_DB_URL_POS].str; } -char *tgGetStableName(char *stname, cJSON *fields, int fieldsSize) { - for (int s = 0; s < tgSchemas.size; ++s) { +char *tgGetStableName(char *stname, cJSON *fields, int32_t fieldsSize) { + for (int32_t s = 0; s < tgSchemas.size; ++s) { STgSchema *schema = &tgSchemas.schemas[s]; if (strcasecmp(schema->name, stname) != 0) { continue; } bool schemaMatched = true; - for (int f = 0; f < schema->fieldNum; ++f) { + for (int32_t f = 0; f < schema->fieldNum; ++f) { char *fieldName = schema->fields[f]; bool fieldMatched = false; - for (int i = 0; i < fieldsSize; i++) { + for (int32_t i = 0; i < fieldsSize; i++) { cJSON *field = cJSON_GetArrayItem(fields, i); if (strcasecmp(field->string, fieldName) == 0) { fieldMatched = true; @@ -412,7 +412,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_NULL); return false; } - int nameLen = (int)strlen(name->valuestring); + int32_t nameLen = (int32_t)strlen(name->valuestring); if (nameLen == 0) { httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_NULL); return false; @@ -444,7 +444,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { return false; } - int tagsSize = cJSON_GetArraySize(tags); + int32_t tagsSize = cJSON_GetArraySize(tags); if (tagsSize <= 0) { httpSendErrorResp(pContext, HTTP_TG_TAGS_SIZE_0); return false; @@ -457,7 +457,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { cJSON *host = NULL; - for (int i = 0; i < tagsSize; i++) { + for (int32_t i = 0; i < tagsSize; i++) { cJSON *tag = cJSON_GetArrayItem(tags, i); if (tag == NULL) { httpSendErrorResp(pContext, HTTP_TG_TAG_NULL); @@ -518,7 +518,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { return false; } - int fieldsSize = cJSON_GetArraySize(fields); + int32_t fieldsSize = cJSON_GetArraySize(fields); if (fieldsSize <= 0) { httpSendErrorResp(pContext, HTTP_TG_FIELDS_SIZE_0); return false; @@ -529,7 +529,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { return false; } - for (int i = 0; i < fieldsSize; i++) { + for (int32_t i = 0; i < fieldsSize; i++) { cJSON *field = cJSON_GetArrayItem(fields, i); if (field == NULL) { httpSendErrorResp(pContext, HTTP_TG_FIELD_NULL); @@ -579,11 +579,11 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { // order by tag name cJSON *orderedTags[TG_MAX_SORT_TAG_SIZE] = {0}; - int orderTagsLen = 0; - for (int i = 0; i < tagsSize; ++i) { + int32_t orderTagsLen = 0; + for (int32_t i = 0; i < tagsSize; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); orderedTags[orderTagsLen++] = tag; - for (int j = orderTagsLen - 1; j >= 1; --j) { + for (int32_t j = orderTagsLen - 1; j >= 1; --j) { cJSON *tag1 = orderedTags[j]; cJSON *tag2 = orderedTags[j - 1]; if (strcasecmp(tag1->string, "host") == 0 || strcmp(tag1->string, tag2->string) < 0) { @@ -609,7 +609,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { httpShrinkTableName(pContext, table_cmd->stable, httpGetCmdsString(pContext, table_cmd->stable)); // stable tag for detail - for (int i = 0; i < orderTagsLen; ++i) { + for (int32_t i = 0; i < orderTagsLen; ++i) { cJSON *tag = orderedTags[i]; stable_cmd->tagNames[i] = table_cmd->tagNames[i] = httpAddToSqlCmdBuffer(pContext, tag->string); @@ -631,7 +631,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { } else { table_cmd->table = stable_cmd->table = httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring); } - for (int i = 0; i < orderTagsLen; ++i) { + for (int32_t i = 0; i < orderTagsLen; ++i) { cJSON *tag = orderedTags[i]; if (tag == host) continue; if (tag->type == cJSON_String) @@ -653,7 +653,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { // assembling create stable sql stable_cmd->sql = httpAddToSqlCmdBufferNoTerminal(pContext, "create table if not exists %s.%s(ts timestamp", db, httpGetCmdsString(pContext, table_cmd->stable)); - for (int i = 0; i < fieldsSize; ++i) { + for (int32_t i = 0; i < fieldsSize; ++i) { cJSON *field = cJSON_GetArrayItem(fields, i); char * field_type = "double"; if (field->type == cJSON_String) @@ -668,7 +668,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { } httpAddToSqlCmdBufferNoTerminal(pContext, ") tags("); - for (int i = 0; i < orderTagsLen; ++i) { + for (int32_t i = 0; i < orderTagsLen; ++i) { cJSON *tag = orderedTags[i]; char * tag_type = "bigint"; if (tag->type == cJSON_String) @@ -689,7 +689,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { table_cmd->sql = httpAddToSqlCmdBufferNoTerminal(pContext, "import into %s.%s using %s.%s tags(", db, httpGetCmdsString(pContext, table_cmd->table), db, httpGetCmdsString(pContext, table_cmd->stable)); - for (int i = 0; i < orderTagsLen; ++i) { + for (int32_t i = 0; i < orderTagsLen; ++i) { cJSON *tag = orderedTags[i]; if (i != orderTagsLen - 1) { if (tag->type == cJSON_Number) @@ -719,7 +719,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { } httpAddToSqlCmdBufferNoTerminal(pContext, " values(%" PRId64 ",", timestamp->valueint); - for (int i = 0; i < fieldsSize; ++i) { + for (int32_t i = 0; i < fieldsSize; ++i) { cJSON *field = cJSON_GetArrayItem(fields, i); if (i != fieldsSize - 1) { if (field->type == cJSON_Number) @@ -802,8 +802,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { bool tgProcessQueryRequest(HttpContext *pContext, char *db) { httpDebug("context:%p, fd:%d, process telegraf query msg", pContext, pContext->fd); - HttpParser *pParser = &pContext->parser; - char * filter = pParser->data.pos; + char *filter = pContext->parser->body.str; if (filter == NULL) { httpSendErrorResp(pContext, HTTP_NO_MSG_INPUT); return false; @@ -817,7 +816,7 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { cJSON *metrics = cJSON_GetObjectItem(root, "metrics"); if (metrics != NULL) { - int size = cJSON_GetArraySize(metrics); + int32_t size = cJSON_GetArraySize(metrics); httpDebug("context:%p, fd:%d, multiple metrics:%d at one time", pContext, pContext->fd, size); if (size <= 0) { httpSendErrorResp(pContext, HTTP_TG_METRICS_NULL); @@ -825,7 +824,7 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { return false; } - int cmdSize = size * 2 + 1; + int32_t cmdSize = size * 2 + 1; if (cmdSize > HTTP_MAX_CMD_SIZE) { httpSendErrorResp(pContext, HTTP_TG_METRICS_SIZE); cJSON_Delete(root); @@ -848,7 +847,7 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { cmd->cmdReturnType = HTTP_CMD_RETURN_TYPE_NO_RETURN; cmd->sql = httpAddToSqlCmdBuffer(pContext, "create database if not exists %s", db); - for (int i = 0; i < size; i++) { + for (int32_t i = 0; i < size; i++) { cJSON *metric = cJSON_GetArrayItem(metrics, i); if (metric != NULL) { if (!tgProcessSingleMetric(pContext, metric, db)) { diff --git a/src/plugins/http/src/httpTgJson.c b/src/plugins/http/src/httpTgJson.c index 5c6985cd69..603092f09d 100644 --- a/src/plugins/http/src/httpTgJson.c +++ b/src/plugins/http/src/httpTgJson.c @@ -61,19 +61,19 @@ void tgStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) // data httpJsonItemToken(jsonBuf); httpJsonPair(jsonBuf, "metric", 6, httpGetCmdsString(pContext, cmd->stable), - (int)strlen(httpGetCmdsString(pContext, cmd->metric))); + (int32_t)strlen(httpGetCmdsString(pContext, cmd->metric))); httpJsonItemToken(jsonBuf); httpJsonPair(jsonBuf, "stable", 6, httpGetCmdsString(pContext, cmd->stable), - (int)strlen(httpGetCmdsString(pContext, cmd->stable))); + (int32_t)strlen(httpGetCmdsString(pContext, cmd->stable))); httpJsonItemToken(jsonBuf); httpJsonPair(jsonBuf, "table", 5, httpGetCmdsString(pContext, cmd->table), - (int)strlen(httpGetCmdsString(pContext, cmd->table))); + (int32_t)strlen(httpGetCmdsString(pContext, cmd->table))); httpJsonItemToken(jsonBuf); httpJsonPair(jsonBuf, "timestamp", 9, httpGetCmdsString(pContext, cmd->timestamp), - (int)strlen(httpGetCmdsString(pContext, cmd->timestamp))); // hack way + (int32_t)strlen(httpGetCmdsString(pContext, cmd->timestamp))); // hack way } void tgStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) { @@ -88,7 +88,7 @@ void tgStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) { httpJsonToken(jsonBuf, JsonObjEnd); } -void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect_rows) { +void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return; @@ -96,7 +96,7 @@ void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect httpJsonPairIntVal(jsonBuf, "affected_rows", 13, affect_rows); } -bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { +bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code) { HttpSqlCmds *multiCmds = pContext->multiCmds; httpDebug("context:%p, fd:%d, check telegraf command, code:%s, state:%d, type:%d, rettype:%d, tags:%d", pContext, pContext->fd, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum); @@ -133,7 +133,7 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { return true; } -void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { +void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code) { HttpSqlCmds *multiCmds = pContext->multiCmds; httpDebug("context:%p, fd:%d, get telegraf next command, pos:%d, code:%s, state:%d, type:%d, rettype:%d, tags:%d", pContext, pContext->fd, multiCmds->pos, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index f6e05dc1ae..2c8879f880 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -29,7 +29,7 @@ bool httpCheckUsedbSql(char *sql) { return false; } -void httpTimeToString(time_t t, char *buf, int buflen) { +void httpTimeToString(time_t t, char *buf, int32_t buflen) { memset(buf, 0, (size_t)buflen); char ts[32] = {0}; @@ -44,13 +44,13 @@ int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, . HttpSqlCmds *cmd = pContext->multiCmds; if (cmd->buffer == NULL) return -1; - int remainLength = cmd->bufferSize - cmd->bufferPos; + int32_t remainLength = cmd->bufferSize - cmd->bufferPos; if (remainLength < 4096) { if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1; } - char *buffer = cmd->buffer + cmd->bufferPos; - int len = 0; + char * buffer = cmd->buffer + cmd->bufferPos; + int32_t len = 0; va_list argpointer; va_start(argpointer, format); @@ -76,13 +76,13 @@ int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const HttpSqlCmds *cmd = pContext->multiCmds; if (cmd->buffer == NULL) return -1; - int remainLength = cmd->bufferSize - cmd->bufferPos; + int32_t remainLength = cmd->bufferSize - cmd->bufferPos; if (remainLength < 4096) { if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1; } - char *buffer = cmd->buffer + cmd->bufferPos; - int len = 0; + char * buffer = cmd->buffer + cmd->bufferPos; + int32_t len = 0; va_list argpointer; va_start(argpointer, format); @@ -107,7 +107,7 @@ int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext) { HttpSqlCmds *cmd = pContext->multiCmds; if (cmd->buffer == NULL) return -1; - int remainLength = cmd->bufferSize - cmd->bufferPos; + int32_t remainLength = cmd->bufferSize - cmd->bufferPos; if (remainLength < 4096) { if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1; } @@ -124,7 +124,7 @@ int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext) { return (int32_t)(buffer - cmd->buffer); } -int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize) { +int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int32_t mallocSize) { HttpSqlCmds *cmd = pContext->multiCmds; if (cmd->buffer == NULL) return -1; @@ -139,7 +139,7 @@ int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize) { return (int32_t)(buffer - cmd->buffer); } -bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize) { +bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferSize) { if (cmdSize > HTTP_MAX_CMD_SIZE) { httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user, cmdSize, HTTP_MAX_CMD_SIZE); @@ -186,7 +186,7 @@ bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize) { return true; } -bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize) { +bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize) { HttpSqlCmds *multiCmds = pContext->multiCmds; if (cmdSize > HTTP_MAX_CMD_SIZE) { @@ -206,7 +206,7 @@ bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize) { return true; } -bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int bufferSize) { +bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize) { HttpSqlCmds *multiCmds = pContext->multiCmds; if (bufferSize > HTTP_MAX_BUFFER_SIZE) { @@ -258,7 +258,7 @@ bool httpCompareMethod(HttpDecodeMethod *pSrc, HttpDecodeMethod *pCmp) { } void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod) { - int pos = 0; + int32_t pos = 0; for (pos = 0; pos < pServer->methodScannerLen; ++pos) { if (httpCompareMethod(pServer->methodScanner[pos], pMethod)) { break; @@ -293,13 +293,13 @@ HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext) { return multiCmds->cmds + multiCmds->size - 1; } -int httpNextSqlCmdPos(HttpContext *pContext) { +int32_t httpNextSqlCmdPos(HttpContext *pContext) { HttpSqlCmds *multiCmds = pContext->multiCmds; return multiCmds->size; } void httpTrimTableName(char *name) { - for (int i = 0; name[i] != 0; i++) { + for (int32_t i = 0; name[i] != 0; i++) { if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'') name[i] = '_'; if (i == TSDB_TABLE_NAME_LEN) { @@ -309,9 +309,9 @@ void httpTrimTableName(char *name) { } } -int httpShrinkTableName(HttpContext *pContext, int pos, char *name) { - int len = 0; - for (int i = 0; name[i] != 0; i++) { +int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name) { + int32_t len = 0; + for (int32_t i = 0; name[i] != 0; i++) { if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'' || name[i] == '\"') name[i] = '_'; @@ -327,7 +327,7 @@ int httpShrinkTableName(HttpContext *pContext, int pos, char *name) { MD5Update(&context, (uint8_t *)name, (uint32_t)len); MD5Final(&context); - int table_name = httpAddToSqlCmdBuffer( + int32_t table_name = httpAddToSqlCmdBuffer( pContext, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], @@ -340,7 +340,7 @@ int httpShrinkTableName(HttpContext *pContext, int pos, char *name) { return table_name; } -char *httpGetCmdsString(HttpContext *pContext, int pos) { +char *httpGetCmdsString(HttpContext *pContext, int32_t pos) { HttpSqlCmds *multiCmds = pContext->multiCmds; if (pos < 0 || pos >= multiCmds->bufferSize) { return ""; @@ -349,8 +349,8 @@ char *httpGetCmdsString(HttpContext *pContext, int pos) { return multiCmds->buffer + pos; } -int httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData) { - int err = 0; +int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData) { + int32_t err = 0; z_stream gzipStream = {0}; static char dummyHead[2] = { @@ -393,7 +393,7 @@ int httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t return 0; } -int httpGzipCompressInit(HttpContext *pContext) { +int32_t httpGzipCompressInit(HttpContext *pContext) { pContext->gzipStream.zalloc = (alloc_func) 0; pContext->gzipStream.zfree = (free_func) 0; pContext->gzipStream.opaque = (voidpf) 0; @@ -404,8 +404,8 @@ int httpGzipCompressInit(HttpContext *pContext) { return 0; } -int httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) { - int err = 0; +int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) { + int32_t err = 0; pContext->gzipStream.next_in = (Bytef *) srcData; pContext->gzipStream.avail_in = (uLong) nSrcData; pContext->gzipStream.next_out = (Bytef *) destData; @@ -439,3 +439,21 @@ int httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, cha *nDestData = (int32_t) (pContext->gzipStream.total_out); return 0; } + +bool httpUrlMatch(HttpContext* pContext, int32_t pos, char* cmp) { + HttpParser* pParser = pContext->parser; + + if (pos < 0 || pos >= HTTP_MAX_URL) { + return false; + } + + if (pParser->path[pos].pos <= 0) { + return false; + } + + if (strcmp(pParser->path[pos].str, cmp) != 0) { + return false; + } + + return true; +} From a39fbb1fd6df0bf072a3035cabfe7eeb305139a1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 13:23:12 +0800 Subject: [PATCH 035/105] [td-1319] --- src/client/inc/tsclient.h | 3 +- src/client/src/tscAsync.c | 8 ++- src/client/src/tscLocal.c | 2 +- src/client/src/tscSQLParser.c | 3 +- src/client/src/tscServer.c | 60 +++++++++-------- src/client/src/tscSql.c | 68 +++++++++---------- src/client/src/tscStream.c | 4 +- src/client/src/tscSystem.c | 14 ++-- src/client/src/tscUtil.c | 122 +++++++++++++++++++--------------- src/util/src/tcache.c | 50 +++++++++----- 10 files changed, 189 insertions(+), 145 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 11b7815586..74a0e8c11c 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -334,6 +334,7 @@ typedef struct STscObj { struct SSqlStream *streamList; void* pDnodeConn; pthread_mutex_t mutex; + T_REF_DECLARE(); } STscObj; typedef struct SSqlObj { @@ -503,7 +504,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField } } -extern SCacheObj* tscCacheHandle; +extern SCacheObj* tscMetaCache; extern SCacheObj* tscObjCache; extern void * tscTmr; extern void * tscQhandle; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 5e9aa1b1f8..09610575f6 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -51,6 +51,11 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->fp = fp; pSql->fetchFp = fp; + uint64_t handle = (uint64_t) pSql; + pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); + + T_REF_INC(pSql->pTscObj); + pSql->sqlstr = calloc(1, sqlLen + 1); if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); @@ -64,9 +69,6 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const tscDebugL("%p SQL: %s", pSql, pSql->sqlstr); pCmd->curSql = pSql->sqlstr; - uint64_t handle = (uint64_t) pSql; - pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); - int32_t code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 4b6174e13d..50f82ae662 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -429,7 +429,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { pRes->qhandle = 0x1; pRes->numOfRows = 0; } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { - taosCacheEmpty(tscCacheHandle); + taosCacheEmpty(tscMetaCache); pRes->code = TSDB_CODE_SUCCESS; } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { pRes->code = tscProcessServerVer(pSql); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index de00827c3a..e8f1a2d758 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1181,13 +1181,14 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } END_TRY len = tbufTell(&bw); - char* c = tbufGetData(&bw, true); + char* c = tbufGetData(&bw, false); // set the serialized binary string as the parameter of arithmetic expression addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len, index.tableIndex); insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); + tbufCloseWriter(&bw); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); } else { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 26fe19d8be..1b6a77bdc7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -373,17 +373,17 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); } - bool shouldFree = tscShouldBeFreed(pSql);; + bool shouldFree = tscShouldBeFreed(pSql); if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; (*pSql->fp)(pSql->param, pSql, rpcMsg->code); } void** p1 = p; - taosCacheRelease(tscObjCache, (void**) &p, false); + taosCacheRelease(tscObjCache, (void**) &p1, false); - if (shouldFree) { - taosCacheRelease(tscObjCache, (void **)&p1, true); + if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it + taosCacheRelease(tscObjCache, (void **)&p, true); tscDebug("%p sqlObj is automatically freed", pSql); } @@ -1718,7 +1718,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); - pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, + pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); // todo handle out of memory case @@ -1830,7 +1830,7 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { // int32_t size = (int32_t)(rsp - ((char *)pMeta)); // Consistent with STableMeta in cache // // pMeta->index = 0; - // (void)taosCachePut(tscCacheHandle, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer); + // (void)taosCachePut(tscMetaCache, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer); // } } @@ -1917,12 +1917,14 @@ int tscProcessShowRsp(SSqlObj *pSql) { key[0] = pCmd->msgType + 'a'; strcpy(key + 1, "showlist"); - taosCacheRelease(tscCacheHandle, (void *)&(pTableMetaInfo->pTableMeta), false); - + if (pTableMetaInfo->pTableMeta != NULL) { + taosCacheRelease(tscMetaCache, (void *)&(pTableMetaInfo->pTableMeta), false); + } + size_t size = 0; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size, + pTableMetaInfo->pTableMeta = taosCachePut(tscMetaCache, key, strlen(key), (char *)pTableMeta, size, tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); @@ -1981,6 +1983,8 @@ static void createHBObj(STscObj* pObj) { pSql->pTscObj = pObj; pSql->signature = pSql; pObj->pHb = pSql; + T_REF_INC(pObj); + tscAddSubqueryInfo(&pObj->pHb->cmd); tscDebug("%p HB is allocated, pObj:%p", pObj->pHb, pObj); @@ -2025,14 +2029,14 @@ int tscProcessUseDbRsp(SSqlObj *pSql) { int tscProcessDropDbRsp(SSqlObj *pSql) { pSql->pTscObj->db[0] = 0; - taosCacheEmpty(tscCacheHandle); + taosCacheEmpty(tscMetaCache); return 0; } int tscProcessDropTableRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); + STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); if (pTableMeta == NULL) { /* not in cache, abort */ return 0; } @@ -2045,10 +2049,10 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { * instead. */ tscDebug("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true); + taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); if (pTableMetaInfo->pTableMeta) { - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); + taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); } return 0; @@ -2057,21 +2061,21 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); + STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); if (pTableMeta == NULL) { /* not in cache, abort */ return 0; } tscDebug("%p force release metermeta in cache after alter-table: %s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true); + taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); if (pTableMetaInfo->pTableMeta) { bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); + taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); if (isSuperTable) { // if it is a super table, reset whole query cache tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); - taosCacheEmpty(tscCacheHandle); + taosCacheEmpty(tscMetaCache); } } @@ -2156,6 +2160,12 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_META; + T_REF_INC(pNew->pTscObj); + + // TODO add test case on x86 platform + uint64_t adr = (uint64_t) pNew; + pNew->self = taosCachePut(tscObjCache, &adr, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2*60*1000); + tscAddSubqueryInfo(&pNew->cmd); SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); @@ -2179,10 +2189,6 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->fp = tscTableMetaCallBack; pNew->param = pSql; - // TODO add test case on x86 platform - uint64_t adr = (uint64_t) pNew; - pNew->self = taosCachePut(tscObjCache, &adr, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2*60*1000); - int32_t code = tscProcessSql(pNew); if (code == TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated @@ -2196,10 +2202,10 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { // If this STableMetaInfo owns a table meta, release it first if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false); + taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), false); } - pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); + pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); if (pTableMetaInfo->pTableMeta != NULL) { STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns, @@ -2234,7 +2240,7 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta); } - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); + taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); return getTableMetaFromMgmt(pSql, pTableMetaInfo); } @@ -2264,7 +2270,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - + + // TODO TEST IT SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); if (pNewQueryInfo == NULL) { tscFreeSqlObj(pNew); @@ -2274,7 +2281,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta *pTableMeta = taosCacheAcquireByData(tscCacheHandle, pMInfo->pTableMeta); + STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta); tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList); } @@ -2284,6 +2291,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { } pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; + T_REF_INC(pNew->pTscObj); uint64_t p = (uint64_t) pNew; pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 58859e944c..025f0a2b14 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -102,6 +102,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con pObj->signature = pObj; pObj->pDnodeConn = pDnodeConn; + T_REF_INIT_VAL(pObj, 1); tstrncpy(pObj->user, user, sizeof(pObj->user)); secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); @@ -155,6 +156,8 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con *taos = pObj; } + T_REF_INC(pSql->pTscObj); + uint64_t key = (uint64_t) pSql; pSql->self = taosCachePut(tscObjCache, &key, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); @@ -261,6 +264,31 @@ void taos_close(TAOS *taos) { tscFreeSqlObj(pObj->pHb); } + // free all sqlObjs created by using this connect before free the STscObj + while(1) { + pthread_mutex_lock(&pObj->mutex); + void* p = pObj->sqlList; + pthread_mutex_unlock(&pObj->mutex); + + if (p == NULL) { + break; + } + + tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p); + taosMsleep(100); + + // todo fix me!! two threads call taos_free_result will cause problem. + tscDebug("%p free :%p", pObj, p); + taos_free_result(p); + } + + int32_t ref = T_REF_DEC(pObj); + assert(ref >= 0); + + if (ref > 0) { + return; + } + tscCloseTscObj(pObj); } @@ -537,7 +565,7 @@ int taos_select_db(TAOS *taos, const char *db) { } // send free message to vnode to free qhandle and corresponding resources in vnode -static bool tscKillQueryInDnode(SSqlObj* pSql) { +static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; @@ -561,7 +589,7 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) { cmd == TSDB_SQL_FETCH)) { pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s, ", pSql, sqlCmd[pCmd->command]); + tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s", pSql, sqlCmd[pCmd->command]); tscProcessSql(pSql); return false; @@ -577,46 +605,16 @@ void taos_free_result(TAOS_RES *res) { return; } - assert(pSql->self != 0 && *pSql->self == pSql); +// assert(pSql->self != 0 && *pSql->self == pSql); bool freeNow = tscKillQueryInDnode(pSql); if (freeNow) { tscDebug("%p free sqlObj in cache", pSql); - taosCacheRelease(tscObjCache, (void**) &pSql->self, true); + SSqlObj** p = pSql->self; + taosCacheRelease(tscObjCache, (void**) &p, true); } } -//static void doFreeResult(TAOS_RES *res) { -// SSqlObj *pSql = (SSqlObj *)res; -// -// if (pSql == NULL || pSql->signature != pSql) { -// tscDebug("%p sqlObj has been freed", pSql); -// return; -// } -// -// // The semaphore can not be changed while freeing async sub query objects. -// SSqlRes *pRes = &pSql->res; -// if (pRes == NULL || pRes->qhandle == 0) { -// tscFreeSqlObj(pSql); -// tscDebug("%p SqlObj is freed by app, qhandle is null", pSql); -// return; -// } -// -// // set freeFlag to 1 in retrieve message if there are un-retrieved results data in node -// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); -// if (pQueryInfo == NULL) { -// tscFreeSqlObj(pSql); -// tscDebug("%p SqlObj is freed by app", pSql); -// return; -// } -// -// pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; -// if (!tscKillQueryInDnode(pSql)) { -// tscFreeSqlObj(pSql); -// tscDebug("%p sqlObj is freed by app", pSql); -// } -//} - int taos_errno(TAOS_RES *tres) { SSqlObj *pSql = (SSqlObj *) tres; if (pSql == NULL || pSql->signature != pSql) { diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 79e0011093..71ba76dc6c 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -167,7 +167,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), true); + taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true); taosTFree(pTableMetaInfo->vgroupList); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); @@ -275,7 +275,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf // release the metric/meter meta information reference, so data in cache can be updated - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false); + taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); tscFreeSqlResult(pSql); taosTFree(pSql->pSubs); pSql->numOfSubs = 0; diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 2c7fcf05c9..82ce1d3679 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -30,7 +30,7 @@ #include "tlocale.h" // global, not configurable -SCacheObj* tscCacheHandle; +SCacheObj* tscMetaCache; SCacheObj* tscObjCache; void * tscTmr; void * tscQhandle; @@ -145,9 +145,9 @@ void taos_init_imp(void) { refreshTime = refreshTime > 10 ? 10 : refreshTime; refreshTime = refreshTime < 10 ? 10 : refreshTime; - if (tscCacheHandle == NULL) { - tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta"); - tscObjCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, refreshTime, false, tscFreeSqlObjInCache, "sqlObjHandle"); + if (tscMetaCache == NULL) { + tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta"); + tscObjCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, refreshTime/2, false, tscFreeSqlObjInCache, "sqlObj"); } tscDebug("client is initialized successfully"); @@ -156,9 +156,9 @@ void taos_init_imp(void) { void taos_init() { pthread_once(&tscinit, taos_init_imp); } void taos_cleanup() { - if (tscCacheHandle != NULL) { - taosCacheCleanup(tscCacheHandle); - tscCacheHandle = NULL; + if (tscMetaCache != NULL) { + taosCacheCleanup(tscMetaCache); + tscMetaCache = NULL; taosCacheCleanup(tscObjCache); tscObjCache = NULL; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7d4369816a..7f2c1716a6 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -344,8 +344,6 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { } SSqlCmd* pCmd = &pSql->cmd; - STscObj* pObj = pSql->pTscObj; - int32_t cmd = pCmd->command; if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { @@ -353,11 +351,11 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { } // pSql->sqlstr will be used by tscBuildQueryStreamDesc - if (pObj->signature == pObj) { +// if (pObj->signature == pObj) { //pthread_mutex_lock(&pObj->mutex); taosTFree(pSql->sqlstr); //pthread_mutex_unlock(&pObj->mutex); - } +// } tscFreeSqlResult(pSql); @@ -384,42 +382,61 @@ static void tscFreeSubobj(SSqlObj* pSql) { pSql->numOfSubs = 0; } +//static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) { +// SSqlCmd* pCmd = &pSql->cmd; +// SSqlRes* pRes = &pSql->res; +// +// if (pRes == NULL || pRes->qhandle == 0) { +// return true; +// } +// +// SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); +// if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) { +// return true; +// } +// +// STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); +// tscRemoveFromSqlList(pSql); +// +// int32_t cmd = pCmd->command; +// if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && +// (cmd == TSDB_SQL_SELECT || +// cmd == TSDB_SQL_SHOW || +// cmd == TSDB_SQL_RETRIEVE || +// cmd == TSDB_SQL_FETCH)) { +// pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; +// pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; +// tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s, ", pSql, sqlCmd[pCmd->command]); +// +// tscProcessSql(pSql); +// return false; +// } +// +// return true; +//} + +/** + * The free operation will cause the pSql to be removed from hash table and free it in + * the function of processmsgfromserver is impossible in this case, since it will fail + * to retrieve pSqlObj in hashtable. + * + * @param pSql + */ void tscFreeSqlObjInCache(void *pSql) { assert(pSql != NULL); - SSqlObj** p = (SSqlObj**) pSql; + SSqlObj** p = (SSqlObj**)pSql; + assert((*p)->self != 0 && (*p)->self == (p)); tscFreeSqlObj(*p); } -static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - return false; - } - - int32_t cmd = pCmd->command; - if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && - (cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_SHOW || cmd == TSDB_SQL_RETRIEVE || cmd == TSDB_SQL_FETCH)) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]); - tscProcessSql(pSql); - return true; - } - - return false; -} - void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; } tscDebug("%p start to free sqlObj", pSql); + STscObj* pTscObj = pSql->pTscObj; tscFreeSubobj(pSql); tscPartiallyFreeSqlObj(pSql); @@ -438,6 +455,13 @@ void tscFreeSqlObj(SSqlObj* pSql) { free(pSql); tscDebug("%p free sqlObj completed", pSql); + + int32_t ref = T_REF_DEC(pTscObj); + assert(ref >= 0); + + if (ref == 0) { + tscCloseTscObj(pTscObj); + } } void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { @@ -450,7 +474,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pTableMeta), false); + taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false); } taosTFree(pDataBlock); @@ -509,10 +533,10 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name)); if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false); + taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); } - pTableMetaInfo->pTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pDataBlock->pTableMeta); + pTableMetaInfo->pTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pDataBlock->pTableMeta); } else { assert(strncmp(pTableMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0); } @@ -583,7 +607,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff * due to operation such as drop database. So here we add the reference count directly instead of invoke * taosGetDataFromCache, which may return NULL value. */ - dataBuf->pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMeta); + dataBuf->pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMeta); assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; @@ -777,28 +801,10 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { // TODO: all subqueries should be freed correctly before close this connection. void tscCloseTscObj(STscObj* pObj) { assert(pObj != NULL); - + pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); - // wait for all sqlObjs created according to this connect closed - while(1) { - pthread_mutex_lock(&pObj->mutex); - void* p = pObj->sqlList; - pthread_mutex_unlock(&pObj->mutex); - - if (p == NULL) { - break; - } - - tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p); - taosMsleep(100); - - // todo fix me!! two threads call taos_free_result will cause problem. - tscDebug("%p free :%p", pObj, p); - taos_free_result(p); - } - if (pObj->pDnodeConn != NULL) { rpcClose(pObj->pDnodeConn); pObj->pDnodeConn = NULL; @@ -1743,7 +1749,7 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) } if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); + taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); } taosTFree(pTableMetaInfo->vgroupList); @@ -1769,6 +1775,8 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm } pNew->pTscObj = pSql->pTscObj; + T_REF_INC(pNew->pTscObj); + pNew->signature = pNew; SSqlCmd* pCmd = &pNew->cmd; @@ -1800,6 +1808,8 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL); + T_REF_INC(pNew->pTscObj); + uint64_t p = (uint64_t) pNew; pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000); return pNew; @@ -1890,6 +1900,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; + T_REF_INC(pNew->pTscObj); pNew->sqlstr = strdup(pSql->sqlstr); if (pNew->sqlstr == NULL) { @@ -1994,14 +2005,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMetaInfo* pFinalInfo = NULL; if (pPrevSql == NULL) { - STableMeta* pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup + STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup assert(pTableMeta != NULL); pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); - STableMeta* pPrevTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); + STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList); @@ -2041,6 +2052,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void tscDebug("%p new sub insertion: %p, vnodeIdx:%d", pSql, pNew, pTableMetaInfo->vgroupIndex); } + T_REF_INC(pNew->pTscObj); + uint64_t p = (uint64_t) pNew; pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 10); return pNew; @@ -2154,6 +2167,7 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* s return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } + int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) { const char* msgFormat1 = "invalid SQL: %s"; const char* msgFormat2 = "invalid SQL: \'%s\' (%s)"; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 7fda057483..4d737ebe66 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -71,7 +71,7 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const * @param pCacheObj Cache object * @param pNode Cache slot object */ -static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode); +static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode); /** * remove nodes in trash with refCount == 0 in cache @@ -80,7 +80,7 @@ static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode); * @param force force model, if true, remove data in trash without check refcount. * may cause corruption. So, forece model only applys before cache is closed */ -static void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force); +static void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force); /** * release node @@ -222,7 +222,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v taosTFree(p); } else { - taosAddToTrash(pCacheObj, p); + taosAddToTrashcan(pCacheObj, p); uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data); } } @@ -322,11 +322,11 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { } void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { - if (taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) { + if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) { return; } - if (pCacheObj == NULL || (*data) == NULL) { + if ((*data) == NULL) { uError("cache:%s, NULL data to release", pCacheObj->name); return; } @@ -399,19 +399,19 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { "others already", pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data); assert(p->pTNodeHeader == NULL); - taosAddToTrash(pCacheObj, p); + taosAddToTrashcan(pCacheObj, p); } else { uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); if (ref > 0) { assert(pNode->pTNodeHeader == NULL); - taosAddToTrash(pCacheObj, pNode); + taosAddToTrashcan(pCacheObj, pNode); } else { // ref == 0 atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size); int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable); - uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, num:%d size:%" PRId64 "bytes", + uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, totalNum:%d size:%" PRId64 "bytes", pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->totalSize); if (pCacheObj->freeFp) { @@ -432,6 +432,26 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { char* key = pNode->key; char* p = pNode->data; +// int32_t ref = T_REF_VAL_GET(pNode); +// +// if (ref == 1 && inTrashcan) { +// // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may be +// // destroyed by refresh worker if decrease ref count before removing it from linked-list. +// assert(pNode->pTNodeHeader->pData == pNode); +// +// __cache_wr_lock(pCacheObj); +// doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); +// __cache_unlock(pCacheObj); +// +// ref = T_REF_DEC(pNode); +// assert(ref == 0); +// +// doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); +// } else { +// ref = T_REF_DEC(pNode); +// assert(ref >= 0); +// } + int32_t ref = T_REF_DEC(pNode); uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trashcan:%d", pCacheObj->name, key, p, ref, inTrashcan); } @@ -452,7 +472,7 @@ static bool travHashTableEmptyFn(void* param, void* data) { if (T_REF_VAL_GET(pNode) == 0) { taosCacheReleaseNode(pCacheObj, pNode); } else { // do add to trashcan - taosAddToTrash(pCacheObj, pNode); + taosAddToTrashcan(pCacheObj, pNode); } // this node should be remove from hash table @@ -463,7 +483,7 @@ void taosCacheEmpty(SCacheObj *pCacheObj) { SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); - taosTrashCanEmpty(pCacheObj, false); + taosTrashcanEmpty(pCacheObj, false); } void taosCacheCleanup(SCacheObj *pCacheObj) { @@ -503,7 +523,7 @@ SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char * return pNewNode; } -void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { +void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { if (pNode->inTrashcan) { /* node is already in trash */ assert(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); return; @@ -525,11 +545,11 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pCacheObj->numOfElemsInTrash++; __cache_unlock(pCacheObj); - uDebug("cache:%s key:%p, %p move to trash, numOfElem in trash:%d", pCacheObj->name, pNode->key, pNode->data, + uDebug("cache:%s key:%p, %p move to trashcan, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, pNode->data, pCacheObj->numOfElemsInTrash); } -void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { +void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { __cache_wr_lock(pCacheObj); if (pCacheObj->numOfElemsInTrash == 0) { @@ -573,7 +593,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { // todo memory leak if there are object with refcount greater than 0 in hash table? taosHashCleanup(pCacheObj->pHashTable); - taosTrashCanEmpty(pCacheObj, true); + taosTrashcanEmpty(pCacheObj, true); __cache_lock_destroy(pCacheObj); @@ -648,7 +668,7 @@ void* taosCacheTimedRefresh(void *handle) { doCacheRefresh(pCacheObj, now, NULL); } - taosTrashCanEmpty(pCacheObj, false); + taosTrashcanEmpty(pCacheObj, false); } return NULL; From 76bf908f3b112f334649406e1f87f617c6a72daa Mon Sep 17 00:00:00 2001 From: Steven Li Date: Mon, 14 Sep 2020 05:36:53 +0000 Subject: [PATCH 036/105] Fixed TD-1430 for the crash_gen tool, ensuring no client side error for the test --- tests/pytest/crash_gen.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index c0a8fd1f00..d9fdbea5fc 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -1763,7 +1763,7 @@ class TaskCreateDb(StateTransitionTask): return state.canCreateDb() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - self.execWtSql(wt, "create database db") + self.execWtSql(wt, "create database db replica {}".format(Dice.throw(3)+1)) class TaskDropDb(StateTransitionTask): @@ -1832,7 +1832,7 @@ class TdSuperTable: return dbc.query("SELECT * FROM db.{}".format(self._stName)) > 0 def ensureTable(self, dbc: DbConn, regTableName: str): - sql = "select tbname from {} where tbname in ('{}')".format(self._stName, regTableName) + sql = "select tbname from db.{} where tbname in ('{}')".format(self._stName, regTableName) if dbc.query(sql) >= 1 : # reg table exists already return sql = "CREATE TABLE {} USING {} tags ({})".format( @@ -1916,9 +1916,9 @@ class TaskReadData(StateTransitionTask): 'max(speed)', 'first(speed)', 'last(speed)', - # 'top(speed)', # TODO: not supported? - # 'bottom(speed)', # TODO: not supported? - # 'percentile(speed, 10)', # TODO: TD-1316 + 'top(speed, 50)', # TODO: not supported? + 'bottom(speed, 50)', # TODO: not supported? + 'apercentile(speed, 10)', # TODO: TD-1316 'last_row(speed)', # Transformation Functions # 'diff(speed)', # TODO: no supported?! @@ -1928,7 +1928,9 @@ class TaskReadData(StateTransitionTask): None ]) try: + # Run the query against the regular table first dbc.execute("select {} from db.{}".format(aggExpr, rTbName)) + # Then run it against the super table if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?! dbc.execute("select {} from db.{}".format(aggExpr, sTable.getName())) except taos.error.ProgrammingError as err: @@ -2022,7 +2024,7 @@ class TaskRestartService(StateTransitionTask): return state.canDropFixedSuperTable() # Basicallly when we have the super table return False # don't run this otherwise - CHANCE_TO_RESTART_SERVICE = 100 + CHANCE_TO_RESTART_SERVICE = 200 def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): if not gConfig.auto_start_service: # only execute when we are in -a mode print("_a", end="", flush=True) From dbc45370173325151371b1479ed32373af899ba4 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Mon, 14 Sep 2020 05:40:12 +0000 Subject: [PATCH 037/105] Backtrack cluster testing from crash_gen tool, for now --- tests/pytest/crash_gen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index d9fdbea5fc..1ea19dfac3 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -1763,8 +1763,8 @@ class TaskCreateDb(StateTransitionTask): return state.canCreateDb() def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - self.execWtSql(wt, "create database db replica {}".format(Dice.throw(3)+1)) - + # self.execWtSql(wt, "create database db replica {}".format(Dice.throw(3)+1)) + self.execWtSql(wt, "create database db") class TaskDropDb(StateTransitionTask): @classmethod From 2bf6b3de2d462c188df4589ea3ce3446d4834329 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 13:54:52 +0800 Subject: [PATCH 038/105] [td-225] update test --- src/util/tests/cacheTest.cpp | 42 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp index 51221e0b35..0a4791f6a9 100644 --- a/src/util/tests/cacheTest.cpp +++ b/src/util/tests/cacheTest.cpp @@ -12,65 +12,65 @@ int32_t tsMaxMeterConnections = 200; // test cache TEST(testCase, client_cache_test) { const int32_t REFRESH_TIME_IN_SEC = 2; - SCacheObj* tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test"); + SCacheObj* tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test"); const char* key1 = "test1"; char data1[] = "test11"; - char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, strlen(key1), data1, strlen(data1)+1, 1); + char* cachedObj = (char*) taosCachePut(tscMetaCache, key1, strlen(key1), data1, strlen(data1)+1, 1); sleep(REFRESH_TIME_IN_SEC+1); printf("obj is still valid: %s\n", cachedObj); char data2[] = "test22"; - taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + taosCacheRelease(tscMetaCache, (void**) &cachedObj, false); /* the object is cleared by cache clean operation */ - cachedObj = (char*) taosCachePut(tscCacheHandle, key1, strlen(key1), data2, strlen(data2)+1, 20); + cachedObj = (char*) taosCachePut(tscMetaCache, key1, strlen(key1), data2, strlen(data2)+1, 20); printf("after updated: %s\n", cachedObj); printf("start to remove data from cache\n"); - taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + taosCacheRelease(tscMetaCache, (void**) &cachedObj, false); printf("end of removing data from cache\n"); const char* key3 = "test2"; const char* data3 = "kkkkkkk"; - char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, strlen(key3), data3, strlen(data3) + 1, 1); + char* cachedObj2 = (char*) taosCachePut(tscMetaCache, key3, strlen(key3), data3, strlen(data3) + 1, 1); printf("%s\n", cachedObj2); - taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + taosCacheRelease(tscMetaCache, (void**) &cachedObj2, false); sleep(3); - char* d = (char*) taosCacheAcquireByKey(tscCacheHandle, key3, strlen(key3)); + char* d = (char*) taosCacheAcquireByKey(tscMetaCache, key3, strlen(key3)); // assert(d == NULL); char key5[] = "test5"; char data5[] = "data5kkkkk"; - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data5, strlen(data5) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data5, strlen(data5) + 1, 20); const char* data6= "new Data after updated"; - taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + taosCacheRelease(tscMetaCache, (void**) &cachedObj2, false); - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data6, strlen(data6) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data6, strlen(data6) + 1, 20); printf("%s\n", cachedObj2); - taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); + taosCacheRelease(tscMetaCache, (void**) &cachedObj2, true); const char* data7 = "add call update procedure"; - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data7, strlen(data7) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data7, strlen(data7) + 1, 20); printf("%s\n=======================================\n\n", cachedObj2); - char* cc = (char*) taosCacheAcquireByKey(tscCacheHandle, key5, strlen(key5)); + char* cc = (char*) taosCacheAcquireByKey(tscMetaCache, key5, strlen(key5)); - taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); - taosCacheRelease(tscCacheHandle, (void**) &cc, false); + taosCacheRelease(tscMetaCache, (void**) &cachedObj2, true); + taosCacheRelease(tscMetaCache, (void**) &cc, false); const char* data8 = "ttft"; const char* key6 = "key6"; - char* ft = (char*) taosCachePut(tscCacheHandle, key6, strlen(key6), data8, strlen(data8), 20); - taosCacheRelease(tscCacheHandle, (void**) &ft, false); + char* ft = (char*) taosCachePut(tscMetaCache, key6, strlen(key6), data8, strlen(data8), 20); + taosCacheRelease(tscMetaCache, (void**) &ft, false); /** * 140ns @@ -78,14 +78,14 @@ TEST(testCase, client_cache_test) { uint64_t startTime = taosGetTimestampUs(); printf("Cache Performance Test\nstart time:%" PRIu64 "\n", startTime); for(int32_t i=0; i<1000; ++i) { - char* dd = (char*) taosCacheAcquireByKey(tscCacheHandle, key6, strlen(key6)); + char* dd = (char*) taosCacheAcquireByKey(tscMetaCache, key6, strlen(key6)); if (dd != NULL) { // printf("get the data\n"); } else { printf("data has been released\n"); } - taosCacheRelease(tscCacheHandle, (void**) &dd, false); + taosCacheRelease(tscMetaCache, (void**) &dd, false); } uint64_t endTime = taosGetTimestampUs(); @@ -93,7 +93,7 @@ TEST(testCase, client_cache_test) { printf("End of Test, %" PRIu64 "\nTotal Elapsed Time:%" PRIu64 " us.avg:%f us\n", endTime, el, el/1000.0); - taosCacheCleanup(tscCacheHandle); + taosCacheCleanup(tscMetaCache); } TEST(testCase, cache_resize_test) { From 0f081ce06d6a943278b96ed9a651cdc46f26678a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 14 Sep 2020 14:19:31 +0800 Subject: [PATCH 039/105] fix TD-1431 --- src/tsdb/src/tsdbMeta.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index a84bb69777..564d7f5db5 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -27,7 +27,8 @@ static int tsdbCompareSchemaVersion(const void *key1, const void *key2); static int tsdbRestoreTable(void *pHandle, void *cont, int contLen); static void tsdbOrgMeta(void *pHandle); static char * getTagIndexKey(const void *pData); -static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper); +static STable *tsdbNewTable(); +static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper); static void tsdbFreeTable(STable *pTable); static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock); static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock); @@ -92,7 +93,7 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { super = tsdbGetTableByUid(pMeta, pCfg->superUid); if (super == NULL) { // super table not exists, try to create it newSuper = 1; - super = tsdbNewTable(pCfg, true); + super = tsdbCreateTableFromCfg(pCfg, true); if (super == NULL) goto _err; } else { if (TABLE_TYPE(super) != TSDB_SUPER_TABLE || TABLE_UID(super) != pCfg->superUid) { @@ -102,7 +103,7 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { } } - table = tsdbNewTable(pCfg, false); + table = tsdbCreateTableFromCfg(pCfg, false); if (table == NULL) goto _err; // Register to meta @@ -654,15 +655,24 @@ static char *getTagIndexKey(const void *pData) { return res; } -static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { +static STable *tsdbNewTable() { + STable *pTable = (STable *)calloc(1, sizeof(*pTable)); + if (pTable == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + pTable->lastKey = TSKEY_INITIAL_VAL; + + return pTable; +} + +static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { STable *pTable = NULL; size_t tsize = 0; - pTable = (STable *)calloc(1, sizeof(STable)); - if (pTable == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } + pTable = tsdbNewTable(); + if (pTable == NULL) goto _err; if (isSuper) { pTable->type = TSDB_SUPER_TABLE; @@ -731,8 +741,6 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { } } } - - pTable->lastKey = TSKEY_INITIAL_VAL; } T_REF_INC(pTable); @@ -1139,11 +1147,9 @@ static int tsdbEncodeTable(void **buf, STable *pTable) { } static void *tsdbDecodeTable(void *buf, STable **pRTable) { - STable *pTable = (STable *)calloc(1, sizeof(STable)); - if (pTable == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } + STable *pTable = tsdbNewTable(); + if (pTable == NULL) return NULL; + uint8_t type = 0; buf = taosDecodeFixedU8(buf, &type); From 7d56810a6ed816e3d93079c67fac76932bd3e45d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 14:37:58 +0800 Subject: [PATCH 040/105] [td-225] update test --- src/query/src/qParserImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 1e8ceffae6..c1b0bd5706 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -179,7 +179,7 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - if (pRight != NULL && pLeft != NULL) { + if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { char* endPos = pRight->token.z + pRight->token.n; pExpr->token.z = pLeft->token.z; pExpr->token.n = (uint32_t)(endPos - pExpr->token.z); From 7dd32d26cf843f29ec75ebc111161525e5f9eb4b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Sep 2020 06:49:00 +0000 Subject: [PATCH 041/105] TD-1311 --- src/inc/taoserror.h | 121 +++++++++++---- src/plugins/http/inc/httpCode.h | 112 -------------- src/plugins/http/inc/httpInt.h | 3 +- src/plugins/http/inc/httpResp.h | 2 - src/plugins/http/src/httpCode.c | 108 ------------- src/plugins/http/src/httpGcHandle.c | 17 ++- src/plugins/http/src/httpGcJson.c | 26 ++-- src/plugins/http/src/httpJson.c | 1 - src/plugins/http/src/httpParser.c | 2 +- src/plugins/http/src/httpResp.c | 210 ++++++++++---------------- src/plugins/http/src/httpRestHandle.c | 9 +- src/plugins/http/src/httpServer.c | 2 +- src/plugins/http/src/httpSession.c | 3 +- src/plugins/http/src/httpSql.c | 15 +- src/plugins/http/src/httpTgHandle.c | 80 +++++----- 15 files changed, 255 insertions(+), 456 deletions(-) delete mode 100644 src/plugins/http/inc/httpCode.h delete mode 100644 src/plugins/http/src/httpCode.c diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 3f7995f25e..d8bd293dba 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -248,37 +248,104 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync modul TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") // http -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_UNSUPPORT_URL, 0, 0x1100, "http url is not support") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_ENOUGH_MEMORY, 0, 0x1101, "no enough memory") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_VERSION, 0, 0x1102, "invalid http version") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH, 0, 0x1103, "invalid content length") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_CREATE_GZIP_FAILED, 0, 0x1104, "failed to create gzip") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_FINISH_GZIP_FAILED, 0, 0x1105, "failed to finish gzip") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_TYPE, 0, 0x1106, "invalid type of Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_FORMAT, 0, 0x1107, "invalid format of Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_BASIC_AUTH, 0, 0x1108, "invalid basic Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_TAOSD_AUTH, 0, 0x1109, "invalid taosd Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_METHOD_FAILED, 0, 0x110A, "failed to parse method") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_TARGET_FAILED, 0, 0x110B, "failed to parse target") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_VERSION_FAILED, 0, 0x110C, "failed to parse http version") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_SP_FAILED, 0, 0x110D, "failed to parse sp") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_STATUS_FAILED, 0, 0x110E, "failed to parse status") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_PHRASE_FAILED, 0, 0x110F, "failed to parse phrase") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CRLF_FAILED, 0, 0x1110, "failed to parse crlf") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_FAILED, 0, 0x1111, "failed to parse header") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED, 0, 0x1112, "failed to parse header key") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED, 0, 0x1113, "failed to parse header val") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED, 0, 0x1114, "failed to parse chunk size") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_FAILED, 0, 0x1115, "failed to parse chunk") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_END_FAILED, 0, 0x1116, "failed to parse end section") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_INVALID_STATE, 0, 0x1117, "invalid parse state") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_ERROR_STATE, 0, 0x1118, "failed to parse error section") - - +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, 0, 0x1100, "http server is not onlin") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_UNSUPPORT_URL, 0, 0x1101, "url is not support") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVLALID_URL, 0, 0x1102, "invalid url format") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_ENOUGH_MEMORY, 0, 0x1103, "no enough memory") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_REQUSET_TOO_BIG, 0, 0x1104, "request size is too big") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_AUTH_INFO, 0, 0x1105, "no auth info input") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_MSG_INPUT, 0, 0x1106, "request is empty") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_SQL_INPUT, 0, 0x1107, "no sql input") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_EXEC_USEDB, 0, 0x1108, "no need to execute use db cmd") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SESSION_FULL, 0, 0x1109, "session list was full") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR, 0, 0x110A, "generate taosd token error") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_MULTI_REQUEST, 0, 0x110B, "size of multi request is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_CREATE_GZIP_FAILED, 0, 0x110C, "failed to create gzip") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_FINISH_GZIP_FAILED, 0, 0x110D, "failed to finish gzip") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_LOGIN_FAILED, 0, 0x110E, "failed to login") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_VERSION, 0, 0x1120, "invalid http version") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH, 0, 0x1121, "invalid content length") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_TYPE, 0, 0x1122, "invalid type of Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_FORMAT, 0, 0x1123, "invalid format of Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_BASIC_AUTH, 0, 0x1124, "invalid basic Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_TAOSD_AUTH, 0, 0x1125, "invalid taosd Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_METHOD_FAILED, 0, 0x1126, "failed to parse method") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_TARGET_FAILED, 0, 0x1127, "failed to parse target") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_VERSION_FAILED, 0, 0x1128, "failed to parse http version") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_SP_FAILED, 0, 0x1129, "failed to parse sp") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_STATUS_FAILED, 0, 0x112A, "failed to parse status") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_PHRASE_FAILED, 0, 0x112B, "failed to parse phrase") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CRLF_FAILED, 0, 0x112C, "failed to parse crlf") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_FAILED, 0, 0x112D, "failed to parse header") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED, 0, 0x112E, "failed to parse header key") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED, 0, 0x112F, "failed to parse header val") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED, 0, 0x1130, "failed to parse chunk size") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_FAILED, 0, 0x1131, "failed to parse chunk") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_END_FAILED, 0, 0x1132, "failed to parse end section") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_INVALID_STATE, 0, 0x1134, "invalid parse state") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_ERROR_STATE, 0, 0x1135, "failed to parse error section") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_NULL, 0, 0x1150, "query size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_SIZE, 0, 0x1151, "query size can not more than 100") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR, 0, 0x1152, "parse grafana json error") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_NOT_INPUT, 0, 0x1160, "database name can not be null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_TOO_LONG, 0, 0x1161, "database name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_INVALID_JSON, 0, 0x1162, "invalid telegraf json fromat") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_NULL, 0, 0x1163, "metrics size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_SIZE, 0, 0x1164, "metrics size can not more than 1K") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NULL, 0, 0x1165, "metric name not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_TYPE, 0, 0x1166, "metric name type should be string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_NULL, 0, 0x1167, "metric name length is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_LONG, 0, 0x1168, "metric name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_NULL, 0, 0x1169, "timestamp not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE, 0, 0x116A, "timestamp type should be integer") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL, 0, 0x116B, "timestamp value smaller than 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_NULL, 0, 0x116C, "tags not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_0, 0, 0x116D, "tags size too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG, 0, 0x116E, "tags size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NULL, 0, 0x116F, "tag is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_NULL, 0, 0x1170, "tag name is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_SIZE, 0, 0x1171, "tag name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE, 0, 0x1172, "tag value type should be number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_NULL, 0, 0x1173, "tag value is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_NULL, 0, 0x1174, "table is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_SIZE, 0, 0x1175, "table name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_NULL, 0, 0x1176, "fields not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_0, 0, 0x1177, "fields size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG, 0, 0x1178, "fields size too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NULL, 0, 0x1179, "field is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_NULL, 0, 0x117A, "field name is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE, 0, 0x117B, "field name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE, 0, 0x117C, "field value type should be number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL, 0, 0x117D, "field value is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_HOST_NOT_STRING, 0, 0x117E, "host type should be string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST, 0, 0x117F, "stable not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_NOT_INPUT, 0, 0x1190, "database name can not be null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_TOO_LONG, 0, 0x1191, "database name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_INVALID_JSON, 0, 0x1192, "invalid opentsdb json fromat") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_NULL, 0, 0x1193, "metrics size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_SIZE, 0, 0x1194, "metrics size can not more than 10K") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NULL, 0, 0x1195, "metric name not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_TYPE, 0, 0x1196, "metric name type should be string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_NULL, 0, 0x1197, "metric name length is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_LONG, 0, 0x1198, "metric name length can not more than 22") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_NULL, 0, 0x1199, "timestamp not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE, 0, 0x119A, "timestamp type should be integer") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL, 0, 0x119B, "timestamp value smaller than 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_NULL, 0, 0x119C, "tags not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_0, 0, 0x119D, "tags size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG, 0, 0x119E, "tags size too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NULL, 0, 0x119F, "tag is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_NULL, 0, 0x11A0, "tag name is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_SIZE, 0, 0x11A1, "tag name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE, 0, 0x11A2, "tag value type should be boolean, number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_NULL, 0, 0x11A3, "tag value is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, 0, 0x11A4, "tag value can not more than 64") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, 0, 0x11A5, "value not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A5, "value type should be boolean, number or string") #ifdef TAOS_ERROR_C }; diff --git a/src/plugins/http/inc/httpCode.h b/src/plugins/http/inc/httpCode.h deleted file mode 100644 index 08111260e9..0000000000 --- a/src/plugins/http/inc/httpCode.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_CODE_H -#define TDENGINE_HTTP_CODE_H - -//for fixed msg info -#define HTTP_SUCCESS 0 -#define HTTP_SERVER_OFFLINE 1 -#define HTTP_UNSUPPORT_URL 2 -#define HTTP_PARSE_HTTP_METHOD_ERROR 3 -#define HTTP_PARSE_HTTP_VERSION_ERROR 4 -#define HTTP_PARSE_HEAD_ERROR 5 -#define HTTP_REQUSET_TOO_BIG 6 -#define HTTP_PARSE_BODY_ERROR 7 -#define HTTP_PARSE_CHUNKED_BODY_ERROR 8 -#define HTTP_PARSE_URL_ERROR 9 -#define HTTP_INVALID_AUTH_TOKEN 10 -#define HTTP_PARSE_USR_ERROR 11 -#define HTTP_NO_SQL_INPUT 12 -#define HTTP_SESSION_FULL 13 -#define HTTP_NO_ENOUGH_MEMORY 14 -#define HTTP_GEN_TAOSD_TOKEN_ERR 15 -#define HTTP_INVALID_DB_TABLE 16 -#define HTTP_NO_EXEC_USEDB 17 -#define HTTP_PARSE_GC_REQ_ERROR 18 -#define HTTP_INVALID_MULTI_REQUEST 19 -#define HTTP_NO_MSG_INPUT 20 -#define HTTP_NO_ENOUGH_SESSIONS 21 - -//telegraf -#define HTTP_TG_DB_NOT_INPUT 22 -#define HTTP_TG_DB_TOO_LONG 23 -#define HTTP_TG_INVALID_JSON 24 -#define HTTP_TG_METRICS_NULL 25 -#define HTTP_TG_METRICS_SIZE 26 -#define HTTP_TG_METRIC_NULL 27 -#define HTTP_TG_METRIC_TYPE 28 -#define HTTP_TG_METRIC_NAME_NULL 29 -#define HTTP_TG_METRIC_NAME_LONG 30 -#define HTTP_TG_TIMESTAMP_NULL 31 -#define HTTP_TG_TIMESTAMP_TYPE 32 -#define HTTP_TG_TIMESTAMP_VAL_NULL 33 -#define HTTP_TG_TAGS_NULL 34 -#define HTTP_TG_TAGS_SIZE_0 35 -#define HTTP_TG_TAGS_SIZE_LONG 36 -#define HTTP_TG_TAG_NULL 37 -#define HTTP_TG_TAG_NAME_NULL 38 -#define HTTP_TG_TAG_NAME_SIZE 39 -#define HTTP_TG_TAG_VALUE_TYPE 40 -#define HTTP_TG_TAG_VALUE_NULL 41 -#define HTTP_TG_TABLE_NULL 42 -#define HTTP_TG_TABLE_SIZE 43 -#define HTTP_TG_FIELDS_NULL 44 -#define HTTP_TG_FIELDS_SIZE_0 45 -#define HTTP_TG_FIELDS_SIZE_LONG 46 -#define HTTP_TG_FIELD_NULL 47 -#define HTTP_TG_FIELD_NAME_NULL 48 -#define HTTP_TG_FIELD_NAME_SIZE 49 -#define HTTP_TG_FIELD_VALUE_TYPE 50 -#define HTTP_TG_FIELD_VALUE_NULL 51 -#define HTTP_INVALID_BASIC_AUTH_TOKEN 52 -#define HTTP_INVALID_TAOSD_AUTH_TOKEN 53 -#define HTTP_TG_HOST_NOT_STRING 54 - -//grafana -#define HTTP_GC_QUERY_NULL 55 -#define HTTP_GC_QUERY_SIZE 56 - -//opentsdb -#define HTTP_OP_DB_NOT_INPUT 57 -#define HTTP_OP_DB_TOO_LONG 58 -#define HTTP_OP_INVALID_JSON 59 -#define HTTP_OP_METRICS_NULL 60 -#define HTTP_OP_METRICS_SIZE 61 -#define HTTP_OP_METRIC_NULL 62 -#define HTTP_OP_METRIC_TYPE 63 -#define HTTP_OP_METRIC_NAME_NULL 64 -#define HTTP_OP_METRIC_NAME_LONG 65 -#define HTTP_OP_TIMESTAMP_NULL 66 -#define HTTP_OP_TIMESTAMP_TYPE 67 -#define HTTP_OP_TIMESTAMP_VAL_NULL 68 -#define HTTP_OP_TAGS_NULL 69 -#define HTTP_OP_TAGS_SIZE_0 70 -#define HTTP_OP_TAGS_SIZE_LONG 71 -#define HTTP_OP_TAG_NULL 72 -#define HTTP_OP_TAG_NAME_NULL 73 -#define HTTP_OP_TAG_NAME_SIZE 74 -#define HTTP_OP_TAG_VALUE_TYPE 75 -#define HTTP_OP_TAG_VALUE_NULL 76 -#define HTTP_OP_TAG_VALUE_TOO_LONG 77 -#define HTTP_OP_VALUE_NULL 78 -#define HTTP_OP_VALUE_TYPE 79 - -//tgf -#define HTTP_TG_STABLE_NOT_EXIST 80 - -extern char *httpMsg[]; - -#endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 86e1eeb6d6..38a52356f8 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -24,7 +24,6 @@ #include "tutil.h" #include "zlib.h" #include "http.h" -#include "httpCode.h" #include "httpLog.h" #include "httpJson.h" #include "httpParser.h" @@ -37,7 +36,7 @@ #define HTTP_BUFFER_SIZE 8192000 #define HTTP_STEP_SIZE 1024 //http message get process step by step #define HTTP_METHOD_SCANNER_SIZE 7 //http method fp size -#define HTTP_GC_TARGET_SIZE 512 +#define TSDB_CODE_HTTP_GC_TARGET_SIZE 512 #define HTTP_WRITE_RETRY_TIMES 500 #define HTTP_WRITE_WAIT_TIME_MS 5 #define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + TSDB_PASSWORD_LEN) diff --git a/src/plugins/http/inc/httpResp.h b/src/plugins/http/inc/httpResp.h index 9086d0c9d3..a528bcc39e 100644 --- a/src/plugins/http/inc/httpResp.h +++ b/src/plugins/http/inc/httpResp.h @@ -33,8 +33,6 @@ enum _httpRespTempl { extern const char *httpRespTemplate[]; void httpSendErrorResp(HttpContext *pContext, int32_t errNo); -void httpSendErrorRespWithDesc(HttpContext *pContext, int32_t errNo, char *desc); -void httpSendTaosdErrorResp(HttpContext *pContext, int32_t errCode); void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg); void httpSendSuccResp(HttpContext *pContext, char *desc); void httpSendOptionResp(HttpContext *pContext, char *desc); diff --git a/src/plugins/http/src/httpCode.c b/src/plugins/http/src/httpCode.c deleted file mode 100644 index 9ec07fd851..0000000000 --- a/src/plugins/http/src/httpCode.c +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE - - -char* httpMsg[] = { - "success", // 0 - "http server is not online", // 1 - "http url is not support", // 2 - "http method parse error", // 3 - "http version should be 1.0, 1.1 or 1.2", // 4 - "http head parse error", // 5 - "request size is too big", - "http body size invalid", - "http chunked body parse error", // 8 - "http url parse error", // 9 - "invalid type of Authorization", - "no auth info input", - "no sql input", - "session list was full", - "no enough memory to alloc sqls", - "generate taosd token error", - "db and table can not be null", - "no need to execute use db cmd", - "parse grafana json error", - "size of multi request is 0", // 19 - "request is empty", // 20 - "no enough connections for http", // 21 - - // telegraf - "database name can not be null", // 22 - "database name too long", - "invalid telegraf json fromat", - "metrics size is 0", - "metrics size can not more than 1K", // 26 - "metric name not find", - "metric name type should be string", - "metric name length is 0", - "metric name length too long", - "timestamp not find", // 31 - "timestamp type should be integer", - "timestamp value smaller than 0", - "tags not find", - "tags size is 0", - "tags size too long", // 36 - "tag is null", - "tag name is null", - "tag name length too long", // 39 - "tag value type should be number or string", - "tag value is null", - "table is null", // 42 - "table name length too long", - "fields not find", // 44 - "fields size is 0", - "fields size too long", - "field is null", // 47 - "field name is null", - "field name length too long", // 49 - "field value type should be number or string", - "field value is null", // 51 - "parse basic auth token error", - "parse http auth token error", - "host type should be string", - - // grafana - "query size is 0", // 55 - "query size can not more than 100", - - // opentsdb - "database name can not be null", // 57 - "database name too long", - "invalid opentsdb json fromat", // 59 - "metrics size is 0", - "metrics size can not more than 10K", // 61 - "metric name not find", - "metric name type should be string", - "metric name length is 0", - "metric name length can not more than 22", - "timestamp not find", - "timestamp type should be integer", - "timestamp value smaller than 0", - "tags not find", - "tags size is 0", - "tags size too long", // 71 - "tag is null", - "tag name is null", - "tag name length too long", // 74 - "tag value type should be boolean, number or string", - "tag value is null", - "tag value can not more than 64", // 77 - "value not find", - "value type should be boolean, number or string", - "stable not exist", - -}; diff --git a/src/plugins/http/src/httpGcHandle.c b/src/plugins/http/src/httpGcHandle.c index e010c77ffd..01be301637 100644 --- a/src/plugins/http/src/httpGcHandle.c +++ b/src/plugins/http/src/httpGcHandle.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosdef.h" +#include "taoserror.h" #include "cJSON.h" #include "httpLog.h" #include "httpGcHandle.h" @@ -146,31 +147,31 @@ bool gcProcessQueryRequest(HttpContext* pContext) { char* filter = pContext->parser->body.str; if (filter == NULL) { - httpSendErrorResp(pContext, HTTP_NO_MSG_INPUT); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_MSG_INPUT); return false; } cJSON* root = cJSON_Parse(filter); if (root == NULL) { - httpSendErrorResp(pContext, HTTP_PARSE_GC_REQ_ERROR); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR); return false; } int32_t size = cJSON_GetArraySize(root); if (size <= 0) { - httpSendErrorResp(pContext, HTTP_GC_QUERY_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_QUERY_NULL); cJSON_Delete(root); return false; } if (size > 100) { - httpSendErrorResp(pContext, HTTP_GC_QUERY_SIZE); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_QUERY_SIZE); cJSON_Delete(root); return false; } if (!httpMallocMultiCmds(pContext, size, HTTP_BUFFER_SIZE)) { - httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); cJSON_Delete(root); return false; } @@ -218,7 +219,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { HttpSqlCmd* cmd = httpNewSqlCmd(pContext); if (cmd == NULL) { - httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); cJSON_Delete(root); return false; } @@ -227,7 +228,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { cmd->values = refIdBuffer; cmd->table = aliasBuffer; cmd->numOfRows = 0; // hack way as target flags - cmd->timestamp = httpAddToSqlCmdBufferWithSize(pContext, HTTP_GC_TARGET_SIZE + 1); // hack way + cmd->timestamp = httpAddToSqlCmdBufferWithSize(pContext, TSDB_CODE_HTTP_GC_TARGET_SIZE + 1); // hack way if (cmd->timestamp == -1) { httpWarn("context:%p, fd:%d, user:%s, cant't malloc target size, sql buffer is full", pContext, pContext->fd, @@ -260,7 +261,7 @@ bool gcProcessRequest(struct HttpContext* pContext) { } if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) { - httpSendErrorResp(pContext, HTTP_PARSE_USR_ERROR); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO); return false; } diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index a291641dc3..e864d54ac0 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -129,48 +129,48 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, // for group by if (groupFields != -1) { - char target[HTTP_GC_TARGET_SIZE] = {0}; + char target[TSDB_CODE_HTTP_GC_TARGET_SIZE] = {0}; int32_t len; - len = snprintf(target,HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer); + len = snprintf(target,TSDB_CODE_HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer); for (int32_t i = dataFields + 1; i= avail) { if (buf->size >= HTTP_BUFFER_SIZE) { httpError("context:%p, fd:%d, failed parse body, exceeding buffer size %d", pContext, pContext->fd, buf->size); - httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); + httpOnError(parser, 0, TSDB_CODE_HTTP_REQUSET_TOO_BIG); return -1; } else { int32_t newSize = buf->size * 10; diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index def8ecf5e1..77dfdb1bf4 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -19,7 +19,6 @@ #include "taosmsg.h" #include "httpLog.h" #include "httpResp.h" -#include "httpCode.h" #include "httpJson.h" #include "httpContext.h" @@ -45,7 +44,7 @@ const char *httpRespTemplate[] = { "%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sAccess-Control-Allow-Methods:POST, GET, OPTIONS, DELETE, PUT\r\nAccess-Control-Allow-Headers:Accept, Content-Type\r\nContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n" }; -static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char *httpCodeStr, int32_t errNo, char *desc) { +static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char *httpCodeStr, int32_t errNo, const char *desc) { httpError("context:%p, fd:%d, code:%d, error:%s", pContext, pContext->fd, httpCode, desc); char head[512] = {0}; @@ -60,132 +59,85 @@ static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char * httpCloseContextByApp(pContext); } -void httpSendErrorRespWithDesc(HttpContext *pContext, int32_t errNo, char *desc) { - int32_t httpCode = 500; - char *httpCodeStr = "Internal Server Error"; - switch (errNo) { - case HTTP_SUCCESS: - httpCode = 200; - httpCodeStr = "OK"; - break; - case HTTP_SERVER_OFFLINE: - case HTTP_UNSUPPORT_URL: - httpCode = 404; - httpCodeStr = "Not Found"; - break; - case HTTP_PARSE_HTTP_METHOD_ERROR: - httpCode = 405; - httpCodeStr = "Method Not Allowed"; - break; - case HTTP_PARSE_HTTP_VERSION_ERROR: - httpCode = 505; - httpCodeStr = "HTTP Version Not Supported"; - break; - case HTTP_PARSE_HEAD_ERROR: - httpCode = 406; - httpCodeStr = "Not Acceptable"; - break; - case HTTP_REQUSET_TOO_BIG: - httpCode = 413; - httpCodeStr = "Request Entity Too Large"; - break; - case HTTP_PARSE_BODY_ERROR: - case HTTP_PARSE_CHUNKED_BODY_ERROR: - httpCode = 409; - httpCodeStr = "Conflict"; - break; - case HTTP_PARSE_URL_ERROR: - httpCode = 414; - httpCodeStr = "Request-URI Invalid"; - break; - case HTTP_INVALID_AUTH_TOKEN: - case HTTP_PARSE_USR_ERROR: - httpCode = 401; - httpCodeStr = "Unauthorized"; - break; - case HTTP_NO_SQL_INPUT: - httpCode = 400; - httpCodeStr = "Bad Request"; - break; - case HTTP_SESSION_FULL: - httpCode = 421; - httpCodeStr = "Too many connections"; - break; - case HTTP_NO_ENOUGH_MEMORY: - case HTTP_GEN_TAOSD_TOKEN_ERR: - httpCode = 507; - httpCodeStr = "Insufficient Storage"; - break; - case HTTP_INVALID_DB_TABLE: - case HTTP_NO_EXEC_USEDB: - case HTTP_PARSE_GC_REQ_ERROR: - case HTTP_INVALID_MULTI_REQUEST: - case HTTP_NO_MSG_INPUT: - httpCode = 400; - httpCodeStr = "Bad Request"; - break; - case HTTP_NO_ENOUGH_SESSIONS: - httpCode = 421; - httpCodeStr = "Too many connections"; - break; - // telegraf - case HTTP_TG_DB_NOT_INPUT: - case HTTP_TG_DB_TOO_LONG: - case HTTP_TG_INVALID_JSON: - case HTTP_TG_METRICS_NULL: - case HTTP_TG_METRICS_SIZE: - case HTTP_TG_METRIC_NULL: - case HTTP_TG_METRIC_TYPE: - case HTTP_TG_METRIC_NAME_NULL: - case HTTP_TG_METRIC_NAME_LONG: - case HTTP_TG_TIMESTAMP_NULL: - case HTTP_TG_TIMESTAMP_TYPE: - case HTTP_TG_TIMESTAMP_VAL_NULL: - case HTTP_TG_TAGS_NULL: - case HTTP_TG_TAGS_SIZE_0: - case HTTP_TG_TAGS_SIZE_LONG: - case HTTP_TG_TAG_NULL: - case HTTP_TG_TAG_NAME_NULL: - case HTTP_TG_TAG_NAME_SIZE: - case HTTP_TG_TAG_VALUE_TYPE: - case HTTP_TG_TAG_VALUE_NULL: - case HTTP_TG_TABLE_NULL: - case HTTP_TG_TABLE_SIZE: - case HTTP_TG_FIELDS_NULL: - case HTTP_TG_FIELDS_SIZE_0: - case HTTP_TG_FIELDS_SIZE_LONG: - case HTTP_TG_FIELD_NULL: - case HTTP_TG_FIELD_NAME_NULL: - case HTTP_TG_FIELD_NAME_SIZE: - case HTTP_TG_FIELD_VALUE_TYPE: - case HTTP_TG_FIELD_VALUE_NULL: - case HTTP_INVALID_BASIC_AUTH_TOKEN: - case HTTP_INVALID_TAOSD_AUTH_TOKEN: - case HTTP_TG_HOST_NOT_STRING: - // grafana - case HTTP_GC_QUERY_NULL: - case HTTP_GC_QUERY_SIZE: - httpCode = 400; - httpCodeStr = "Bad Request"; - break; - default: - httpError("context:%p, fd:%d, error:%d not recognized", pContext, pContext->fd, errNo); - break; - } +void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { + int32_t httpCode = 500; + if (errNo == TSDB_CODE_SUCCESS) + httpCode = 200; + else if (errNo == TSDB_CODE_HTTP_SERVER_OFFLINE) + httpCode = 404; + else if (errNo == TSDB_CODE_HTTP_UNSUPPORT_URL) + httpCode = 404; + else if (errNo == TSDB_CODE_HTTP_INVLALID_URL) + httpCode = 404; + else if (errNo == TSDB_CODE_HTTP_NO_ENOUGH_MEMORY) + httpCode = 507; + else if (errNo == TSDB_CODE_HTTP_REQUSET_TOO_BIG) + httpCode = 413; + else if (errNo == TSDB_CODE_HTTP_NO_AUTH_INFO) + httpCode = 401; + else if (errNo == TSDB_CODE_HTTP_NO_MSG_INPUT) + httpCode = 400; + else if (errNo == TSDB_CODE_HTTP_NO_SQL_INPUT) + httpCode = 400; + else if (errNo == TSDB_CODE_HTTP_NO_EXEC_USEDB) + httpCode = 400; + else if (errNo == TSDB_CODE_HTTP_SESSION_FULL) + httpCode = 421; + else if (errNo == TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR) + httpCode = 507; + else if (errNo == TSDB_CODE_HTTP_INVALID_MULTI_REQUEST) + httpCode = 400; + else if (errNo == TSDB_CODE_HTTP_CREATE_GZIP_FAILED) + httpCode = 507; + else if (errNo == TSDB_CODE_HTTP_FINISH_GZIP_FAILED) + httpCode = 507; + else if (errNo == TSDB_CODE_HTTP_INVALID_VERSION) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_INVALID_AUTH_TYPE) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_INVALID_AUTH_FORMAT) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_INVALID_BASIC_AUTH) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_INVALID_TAOSD_AUTH) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_METHOD_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_TARGET_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_VERSION_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_SP_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_STATUS_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_PHRASE_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_CRLF_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_CHUNK_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_END_FAILED) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_INVALID_STATE) + httpCode = 406; + else if (errNo == TSDB_CODE_HTTP_PARSE_ERROR_STATE) + httpCode = 406; + else + httpCode = 400; - if (desc == NULL) { - httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 5000, httpMsg[errNo]); - } else { - httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 5000, desc); - } -} - -void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { httpSendErrorRespWithDesc(pContext, errNo, NULL); } - -void httpSendTaosdErrorResp(HttpContext *pContext, int32_t errCode) { - int32_t httpCode = 400; - - httpSendErrorRespImp(pContext, httpCode, "Bad Request", errCode & 0XFFFF, (char*)tstrerror(errCode)); + char *httpCodeStr = httpGetStatusDesc(httpCode); + httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo & 0XFFFF, tstrerror(errNo)); } void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char *errMsg) { @@ -208,7 +160,7 @@ void httpSendSuccResp(HttpContext *pContext, char *desc) { char head[1024] = {0}; char body[1024] = {0}; - int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], HTTP_SUCCESS, desc); + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc); int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[pContext->parser->httpVersion], httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); @@ -221,7 +173,7 @@ void httpSendOptionResp(HttpContext *pContext, char *desc) { char head[1024] = {0}; char body[1024] = {0}; - int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], HTTP_SUCCESS, desc); + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc); int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[pContext->parser->httpVersion], httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); diff --git a/src/plugins/http/src/httpRestHandle.c b/src/plugins/http/src/httpRestHandle.c index fbdce566f0..0a28c431ef 100644 --- a/src/plugins/http/src/httpRestHandle.c +++ b/src/plugins/http/src/httpRestHandle.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "taoserror.h" #include "httpLog.h" #include "httpRestHandle.h" #include "httpRestJson.h" @@ -90,7 +91,7 @@ bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) { char* sql = pContext->parser->body.str; if (sql == NULL) { - httpSendErrorResp(pContext, HTTP_NO_SQL_INPUT); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT); return false; } @@ -99,7 +100,7 @@ bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) { * for async test * if (httpCheckUsedbSql(sql)) { - httpSendErrorResp(pContext, HTTP_NO_EXEC_USEDB); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_EXEC_USEDB); return false; } */ @@ -126,7 +127,7 @@ bool restProcessRequest(struct HttpContext* pContext) { } if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) { - httpSendErrorResp(pContext, HTTP_PARSE_USR_ERROR); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO); return false; } @@ -141,6 +142,6 @@ bool restProcessRequest(struct HttpContext* pContext) { } else { } - httpSendErrorResp(pContext, HTTP_PARSE_URL_ERROR); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVLALID_URL); return false; } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 212ce46473..2a558e25d5 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -139,7 +139,7 @@ static void httpProcessHttpData(void *param) { if (pServer->status != HTTP_SERVER_RUNNING) { httpDebug("context:%p, fd:%d, state:%s, server is not running, accessed:%d, close connect", pContext, pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); - httpSendErrorResp(pContext, HTTP_SERVER_OFFLINE); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_SERVER_OFFLINE); httpNotifyContextClose(pContext); } else { if (httpReadData(pContext)) { diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index 4549192407..a96e4433b0 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taos.h" +#include "taoserror.h" #include "tglobal.h" #include "tcache.h" #include "httpInt.h" @@ -40,7 +41,7 @@ void httpCreateSession(HttpContext *pContext, void *taos) { if (pContext->session == NULL) { httpError("context:%p, fd:%d, user:%s, error:%s", pContext, pContext->fd, pContext->user, - httpMsg[HTTP_SESSION_FULL]); + tstrerror(TSDB_CODE_HTTP_SESSION_FULL)); taos_close(taos); pthread_mutex_unlock(&server->serverMutex); return; diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index cbaa0b36d8..c60124a991 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -17,6 +17,7 @@ #include "os.h" #include "tnote.h" #include "taos.h" +#include "taoserror.h" #include "tsclient.h" #include "httpInt.h" #include "httpContext.h" @@ -193,7 +194,7 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) { HttpSqlCmds *multiCmds = pContext->multiCmds; if (multiCmds == NULL || multiCmds->size <= 0 || multiCmds->pos >= multiCmds->size || multiCmds->pos < 0) { - httpSendErrorResp(pContext, HTTP_INVALID_MULTI_REQUEST); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_MULTI_REQUEST); return; } @@ -282,7 +283,7 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo } else { httpError("context:%p, fd:%d, user:%s, query error, taos:%p, code:%s, sqlObj:%p", pContext, pContext->fd, pContext->user, pContext->session->taos, tstrerror(code), pObj); - httpSendTaosdErrorResp(pContext, code); + httpSendErrorResp(pContext, code); } taos_free_result(result); return; @@ -332,7 +333,7 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) { if (sql == NULL || sql[0] == 0) { httpError("context:%p, fd:%d, user:%s, error:no sql input", pContext, pContext->fd, pContext->user); - httpSendErrorResp(pContext, HTTP_NO_SQL_INPUT); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT); return; } @@ -344,7 +345,7 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) { void httpProcessLoginCmd(HttpContext *pContext) { char token[128] = {0}; if (!httpGenTaosdAuthToken(pContext, token, 128)) { - httpSendErrorResp(pContext, HTTP_GEN_TAOSD_TOKEN_ERR); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR); } else { httpDebug("context:%p, fd:%d, user:%s, login via http, return token:%s", pContext, pContext->fd, pContext->user, token); @@ -395,7 +396,7 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int code) { if (code < 0) { httpError("context:%p, fd:%d, user:%s, login error, code:%s", pContext, pContext->fd, pContext->user, tstrerror(code)); - httpSendTaosdErrorResp(pContext, code); + httpSendErrorResp(pContext, code); return; } @@ -403,14 +404,14 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int code) { pContext->taos); if (pContext->taos == NULL) { httpError("context:%p, fd:%d, user:%s, login error, taos is empty", pContext, pContext->fd, pContext->user); - httpSendErrorResp(pContext, HTTP_NO_ENOUGH_SESSIONS); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_LOGIN_FAILED); return; } httpCreateSession(pContext, pContext->taos); if (pContext->session == NULL) { - httpSendErrorResp(pContext, HTTP_SESSION_FULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_SESSION_FULL); httpCloseContextByApp(pContext); } else { httpExecCmd(pContext); diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index 83b31f652b..e2b57b87bb 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -335,12 +335,12 @@ bool tgGetPassFromUrl(HttpContext *pContext) { char *tgGetDbFromUrl(HttpContext *pContext) { HttpParser *pParser = pContext->parser; if (pParser->path[TG_DB_URL_POS].pos <= 0) { - httpSendErrorResp(pContext, HTTP_TG_DB_NOT_INPUT); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_DB_NOT_INPUT); return NULL; } if (pParser->path[TG_DB_URL_POS].pos >= TSDB_DB_NAME_LEN) { - httpSendErrorResp(pContext, HTTP_TG_DB_TOO_LONG); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_DB_TOO_LONG); return NULL; } @@ -401,57 +401,57 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { // metric name cJSON *name = cJSON_GetObjectItem(metric, "name"); if (name == NULL) { - httpSendErrorResp(pContext, HTTP_TG_METRIC_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NULL); return false; } if (name->type != cJSON_String) { - httpSendErrorResp(pContext, HTTP_TG_METRIC_TYPE); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_TYPE); return false; } if (name->valuestring == NULL) { - httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_NULL); return false; } int32_t nameLen = (int32_t)strlen(name->valuestring); if (nameLen == 0) { - httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_NULL); return false; } if (nameLen >= TSDB_TABLE_NAME_LEN - 8) { - httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_LONG); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_LONG); return false; } // timestamp cJSON *timestamp = cJSON_GetObjectItem(metric, "timestamp"); if (timestamp == NULL) { - httpSendErrorResp(pContext, HTTP_TG_TIMESTAMP_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_NULL); return false; } if (timestamp->type != cJSON_Number) { - httpSendErrorResp(pContext, HTTP_TG_TIMESTAMP_TYPE); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE); return false; } if (timestamp->valueint <= 0) { - httpSendErrorResp(pContext, HTTP_TG_TIMESTAMP_VAL_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL); return false; } // tags cJSON *tags = cJSON_GetObjectItem(metric, "tags"); if (tags == NULL) { - httpSendErrorResp(pContext, HTTP_TG_TAGS_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_NULL); return false; } int32_t tagsSize = cJSON_GetArraySize(tags); if (tagsSize <= 0) { - httpSendErrorResp(pContext, HTTP_TG_TAGS_SIZE_0); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_SIZE_0); return false; } if (tagsSize > TG_MAX_SORT_TAG_SIZE) { - httpSendErrorResp(pContext, HTTP_TG_TAGS_SIZE_LONG); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG); return false; } @@ -460,11 +460,11 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { for (int32_t i = 0; i < tagsSize; i++) { cJSON *tag = cJSON_GetArrayItem(tags, i); if (tag == NULL) { - httpSendErrorResp(pContext, HTTP_TG_TAG_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NULL); return false; } if (tag->string == NULL || strlen(tag->string) == 0) { - httpSendErrorResp(pContext, HTTP_TG_TAG_NAME_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_NULL); return false; } @@ -474,19 +474,19 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { */ if (0) { if (strlen(tag->string) >= TSDB_COL_NAME_LEN) { - httpSendErrorResp(pContext, HTTP_TG_TAG_NAME_SIZE); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_SIZE); return false; } } if (tag->type != cJSON_Number && tag->type != cJSON_String) { - httpSendErrorResp(pContext, HTTP_TG_TAG_VALUE_TYPE); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE); return false; } if (tag->type == cJSON_String) { if (tag->valuestring == NULL || strlen(tag->valuestring) == 0) { - httpSendErrorResp(pContext, HTTP_TG_TAG_VALUE_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_VALUE_NULL); return false; } } @@ -497,46 +497,46 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { } if (host == NULL) { - httpSendErrorResp(pContext, HTTP_TG_TABLE_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TABLE_NULL); return false; } if (host->type != cJSON_String) { - httpSendErrorResp(pContext, HTTP_TG_HOST_NOT_STRING); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_HOST_NOT_STRING); return false; } if (strlen(host->valuestring) >= TSDB_TABLE_NAME_LEN - 1) { - httpSendErrorResp(pContext, HTTP_TG_TABLE_SIZE); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TABLE_SIZE); return false; } // fields cJSON *fields = cJSON_GetObjectItem(metric, "fields"); if (fields == NULL) { - httpSendErrorResp(pContext, HTTP_TG_FIELDS_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_NULL); return false; } int32_t fieldsSize = cJSON_GetArraySize(fields); if (fieldsSize <= 0) { - httpSendErrorResp(pContext, HTTP_TG_FIELDS_SIZE_0); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_SIZE_0); return false; } if (fieldsSize > (TSDB_MAX_COLUMNS - TSDB_MAX_TAGS - 1)) { - httpSendErrorResp(pContext, HTTP_TG_FIELDS_SIZE_LONG); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG); return false; } for (int32_t i = 0; i < fieldsSize; i++) { cJSON *field = cJSON_GetArrayItem(fields, i); if (field == NULL) { - httpSendErrorResp(pContext, HTTP_TG_FIELD_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NULL); return false; } if (field->string == NULL || strlen(field->string) == 0) { - httpSendErrorResp(pContext, HTTP_TG_FIELD_NAME_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_NULL); return false; } /* @@ -545,17 +545,17 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { */ if (0) { if (strlen(field->string) >= TSDB_COL_NAME_LEN) { - httpSendErrorResp(pContext, HTTP_TG_FIELD_NAME_SIZE); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE); return false; } } if (field->type != cJSON_Number && field->type != cJSON_String) { - httpSendErrorResp(pContext, HTTP_TG_FIELD_VALUE_TYPE); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE); return false; } if (field->type == cJSON_String) { if (field->valuestring == NULL || strlen(field->valuestring) == 0) { - httpSendErrorResp(pContext, HTTP_TG_FIELD_VALUE_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL); return false; } } @@ -564,7 +564,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { // assembling cmds HttpSqlCmd *stable_cmd = httpNewSqlCmd(pContext); if (stable_cmd == NULL) { - httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); return false; } stable_cmd->cmdType = HTTP_CMD_TYPE_CREATE_STBALE; @@ -572,7 +572,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { HttpSqlCmd *table_cmd = httpNewSqlCmd(pContext); if (table_cmd == NULL) { - httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); return false; } table_cmd->cmdType = HTTP_CMD_TYPE_INSERT; @@ -804,13 +804,13 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { char *filter = pContext->parser->body.str; if (filter == NULL) { - httpSendErrorResp(pContext, HTTP_NO_MSG_INPUT); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_MSG_INPUT); return false; } cJSON *root = cJSON_Parse(filter); if (root == NULL) { - httpSendErrorResp(pContext, HTTP_TG_INVALID_JSON); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_INVALID_JSON); return false; } @@ -819,27 +819,27 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { int32_t size = cJSON_GetArraySize(metrics); httpDebug("context:%p, fd:%d, multiple metrics:%d at one time", pContext, pContext->fd, size); if (size <= 0) { - httpSendErrorResp(pContext, HTTP_TG_METRICS_NULL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRICS_NULL); cJSON_Delete(root); return false; } int32_t cmdSize = size * 2 + 1; if (cmdSize > HTTP_MAX_CMD_SIZE) { - httpSendErrorResp(pContext, HTTP_TG_METRICS_SIZE); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRICS_SIZE); cJSON_Delete(root); return false; } if (!httpMallocMultiCmds(pContext, cmdSize, HTTP_BUFFER_SIZE)) { - httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); cJSON_Delete(root); return false; } HttpSqlCmd *cmd = httpNewSqlCmd(pContext); if (cmd == NULL) { - httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); cJSON_Delete(root); return false; } @@ -860,14 +860,14 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { httpDebug("context:%p, fd:%d, single metric", pContext, pContext->fd); if (!httpMallocMultiCmds(pContext, 3, HTTP_BUFFER_SIZE)) { - httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); cJSON_Delete(root); return false; } HttpSqlCmd *cmd = httpNewSqlCmd(pContext); if (cmd == NULL) { - httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); cJSON_Delete(root); return false; } @@ -892,7 +892,7 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) { bool tgProcessRquest(struct HttpContext *pContext) { if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) { - httpSendErrorResp(pContext, HTTP_PARSE_USR_ERROR); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO); return false; } From 2004a042ede5d4bcdd1f6de91002dfebb103ee3e Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Mon, 14 Sep 2020 14:51:08 +0800 Subject: [PATCH 042/105] Update Model-ch.md test jenkins --- documentation20/webdocs/markdowndocs/Model-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md index 31eb6b3744..bf04ce8388 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/webdocs/markdowndocs/Model-ch.md @@ -56,4 +56,4 @@ INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 21 ## 多列模型 vs 单列模型 TDengine支持多列模型，只要物理量是一个数据采集点同时采集的（时间戳一致），这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计，单列模型，每个采集的物理量都单独建表，因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位，就建三张超级表。 -TDengine建议尽可能采用多列模型，因为插入效率以及存储效率更高。但对于有些场景，一个采集点的采集量的种类经常变化，这个时候，如果采用多列模型，就需要频繁修改超级表的结构定义，让应用变的复杂，这个时候，采用单列模型就会显得简单。 +TDengine建议尽可能采用多列模型，因为插入效率以及存储效率更高。但对于有些场景，一个采集点的采集量的种类经常变化，这个时候，如果采用多列模型，就需要频繁修改超级表的结构定义，让应用变的复杂，这个时候，采用单列模型会显得简单。 From cc7402b1a6c0f1f30efdb0212045a18e0354ee5e Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 14 Sep 2020 14:53:03 +0800 Subject: [PATCH 043/105] TD-1359: add recv timeout for accepted tcp socket this is a quick fix for td-1359, and perhaps also a fix for td-1420. --- src/rpc/src/rpcTcp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 3475e0f317..56b3fa8616 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -245,6 +245,8 @@ static void *taosAcceptTcpConnection(void *arg) { } taosKeepTcpAlive(connFd); + struct timeval to={1, 0}; + taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); // pick up the thread to handle this connection pThreadObj = pServerObj->pThreadObj + threadId; From fbb76bb9b216a8414520c0ecc3db1327152923f4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 15:54:46 +0800 Subject: [PATCH 044/105] [td-1319] --- src/client/src/tscSql.c | 34 ++++++++++++++++------------------ src/client/src/tscUtil.c | 4 ++-- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 025f0a2b14..f02e587455 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -265,22 +265,22 @@ void taos_close(TAOS *taos) { } // free all sqlObjs created by using this connect before free the STscObj - while(1) { - pthread_mutex_lock(&pObj->mutex); - void* p = pObj->sqlList; - pthread_mutex_unlock(&pObj->mutex); - - if (p == NULL) { - break; - } - - tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p); - taosMsleep(100); - - // todo fix me!! two threads call taos_free_result will cause problem. - tscDebug("%p free :%p", pObj, p); - taos_free_result(p); - } +// while(1) { +// pthread_mutex_lock(&pObj->mutex); +// void* p = pObj->sqlList; +// pthread_mutex_unlock(&pObj->mutex); +// +// if (p == NULL) { +// break; +// } +// +// tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p); +// taosMsleep(100); +// +// // todo fix me!! two threads call taos_free_result will cause problem. +// tscDebug("%p free :%p", pObj, p); +// taos_free_result(p); +// } int32_t ref = T_REF_DEC(pObj); assert(ref >= 0); @@ -605,8 +605,6 @@ void taos_free_result(TAOS_RES *res) { return; } -// assert(pSql->self != 0 && *pSql->self == pSql); - bool freeNow = tscKillQueryInDnode(pSql); if (freeNow) { tscDebug("%p free sqlObj in cache", pSql); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7f2c1716a6..979e153356 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -366,7 +366,7 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { tscResetSqlCmdObj(pCmd, false); } -static void tscFreeSubobj(SSqlObj* pSql) { +static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) { if (pSql->numOfSubs == 0) { return; } @@ -438,7 +438,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscDebug("%p start to free sqlObj", pSql); STscObj* pTscObj = pSql->pTscObj; - tscFreeSubobj(pSql); +// tscFreeSubobj(pSql); tscPartiallyFreeSqlObj(pSql); pSql->signature = NULL; From dc8b2e281f5cf2b56ab56f93a868d5f5329a26e7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 16:16:25 +0800 Subject: [PATCH 045/105] [td-1391]update the error msg. --- src/client/src/tscSQLParser.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e8f1a2d758..056ea258bd 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -360,8 +360,9 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + // additional msg has been attached already if (tscSetTableFullName(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return TSDB_CODE_TSC_INVALID_SQL; } return tscGetTableMeta(pSql, pTableMetaInfo); @@ -712,8 +713,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) { const char* msg1 = "name too long"; - const char* msg2 = "invalid db name"; - const char *msg = msg1; + const char* msg2 = "current database name is invalid"; SSqlCmd* pCmd = &pSql->cmd; int32_t code = TSDB_CODE_SUCCESS; @@ -728,17 +728,24 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa if (hasSpecifyDB(pzTableName)) { // db has been specified in sql string so we ignore current db path code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); + if (code != 0) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } } else { // get current DB name first, then set it into path SStrToken t = {0}; getCurrentDBName(pSql, &t); if (t.n == 0) { - msg = msg2; + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); + if (code != 0) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } } + if (code != TSDB_CODE_SUCCESS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); - free(oldName); + taosTFree(oldName); return code; } From 3af6d957ffc379049c294d95005c8473b4c628cf Mon Sep 17 00:00:00 2001 From: root Date: Mon, 14 Sep 2020 16:17:52 +0800 Subject: [PATCH 046/105] TD-1311 --- src/inc/taoserror.h | 6 +- src/plugins/http/inc/httpAuth.h | 6 +- src/plugins/http/inc/httpParser.h | 1 - src/plugins/http/src/httpAuth.c | 26 ++++----- src/plugins/http/src/httpParser.c | 30 +++++----- src/plugins/http/src/httpResp.c | 4 ++ src/plugins/http/src/httpServer.c | 10 +++- src/plugins/http/src/httpSql.c | 2 +- tests/script/general/http/grafana.sim | 10 ++-- tests/script/general/http/restful_full.sim | 14 ++--- tests/script/general/http/telegraf.sim | 64 +++++++++++----------- tests/script/sh/deploy.sh | 32 +++++------ tests/script/unique/http/admin.sim | 16 +++--- tests/script/unique/http/opentsdb.sim | 42 +++++++------- 14 files changed, 135 insertions(+), 128 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index d8bd293dba..67e2d43c98 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -303,8 +303,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_NULL, 0, 0x1169, "timestamp TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE, 0, 0x116A, "timestamp type should be integer") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL, 0, 0x116B, "timestamp value smaller than 0") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_NULL, 0, 0x116C, "tags not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_0, 0, 0x116D, "tags size too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG, 0, 0x116E, "tags size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_0, 0, 0x116D, "tags size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG, 0, 0x116E, "tags size too long") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NULL, 0, 0x116F, "tag is null") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_NULL, 0, 0x1170, "tag name is null") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_SIZE, 0, 0x1171, "tag name length too long") @@ -345,7 +345,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE, 0, 0x11A2, "tag value TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_NULL, 0, 0x11A3, "tag value is null") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, 0, 0x11A4, "tag value can not more than 64") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, 0, 0x11A5, "value not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A5, "value type should be boolean, number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A6, "value type should be boolean, number or string") #ifdef TAOS_ERROR_C }; diff --git a/src/plugins/http/inc/httpAuth.h b/src/plugins/http/inc/httpAuth.h index 4becae6332..2ce9725d4b 100644 --- a/src/plugins/http/inc/httpAuth.h +++ b/src/plugins/http/inc/httpAuth.h @@ -16,8 +16,8 @@ #ifndef TDENGINE_HTTP_TOKEN_H #define TDENGINE_HTTP_TOKEN_H -bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len); -bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len); -bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen); +int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len); +int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len); +int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen); #endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpParser.h b/src/plugins/http/inc/httpParser.h index 5694c8f0cd..7dd6d3e0fc 100644 --- a/src/plugins/http/inc/httpParser.h +++ b/src/plugins/http/inc/httpParser.h @@ -84,7 +84,6 @@ typedef struct HttpParser { HttpString path[HTTP_MAX_URL]; char * method; char * target; - char * target_raw; char * version; char * reasonPhrase; char * key; diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c index 40bb691b5d..8beef7c042 100644 --- a/src/plugins/http/src/httpAuth.c +++ b/src/plugins/http/src/httpAuth.c @@ -23,28 +23,28 @@ #define KEY_DES_4 4971256377704625728L -bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) { +int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; int32_t outlen = 0; char *base64 = (char *)base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token); free(base64); - return false; + return -1; } char *user = strstr(base64, ":"); if (user == NULL) { httpError("context:%p, fd:%d, basic token:%s invalid format", pContext, pContext->fd, token); free(base64); - return false; + return -1; } int32_t user_len = (int32_t)(user - base64); if (user_len < 1 || user_len >= TSDB_USER_LEN) { httpError("context:%p, fd:%d, basic token:%s parse user error", pContext, pContext->fd, token); free(base64); - return false; + return -1; } strncpy(pContext->user, base64, (size_t)user_len); pContext->user[user_len] = 0; @@ -54,36 +54,36 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) { if (pass_len < 1 || pass_len >= TSDB_PASSWORD_LEN) { httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token); free(base64); - return false; + return -1; } strncpy(pContext->pass, password, (size_t)pass_len); pContext->pass[pass_len] = 0; free(base64); httpDebug("context:%p, fd:%d, basic token parsed success, user:%s", pContext, pContext->fd, pContext->user); - return true; + return 0; } -bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) { +int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; int32_t outlen = 0; unsigned char *base64 = base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token); if (base64) free(base64); - return false; + return 01; } if (outlen != (TSDB_USER_LEN + TSDB_PASSWORD_LEN)) { httpError("context:%p, fd:%d, taosd token:%s length error", pContext, pContext->fd, token); free(base64); - return false; + return -1; } char *descrypt = taosDesDecode(KEY_DES_4, (char *)base64, outlen); if (descrypt == NULL) { httpError("context:%p, fd:%d, taosd token:%s descrypt error", pContext, pContext->fd, token); free(base64); - return false; + return -1; } else { tstrncpy(pContext->user, descrypt, sizeof(pContext->user)); tstrncpy(pContext->pass, descrypt + TSDB_USER_LEN, sizeof(pContext->pass)); @@ -92,11 +92,11 @@ bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) { pContext->user); free(base64); free(descrypt); - return true; + return 0; } } -bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) { +int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) { char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; size_t size = sizeof(pContext->user); tstrncpy(buffer, pContext->user, size); @@ -113,5 +113,5 @@ bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) { httpDebug("context:%p, fd:%d, generate taosd token:%s", pContext, pContext->fd, token); - return true; + return 0; } diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index d223b8f3a4..ec71bb2daf 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -112,7 +112,8 @@ static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { str->pos = 0; str->size = 32; str->str = malloc(str->size); - } else if (str->pos + len + 1 > str->size) { + } else if (str->pos + len + 1 >= str->size) { + str->size += len; str->size *= 10; str->str = realloc(str->str, str->size); } else { @@ -130,7 +131,6 @@ static void httpClearString(HttpString *str) { if (str->str) { str->str[0] = '\0'; str->pos = 0; - str->size = 0; } } @@ -138,7 +138,9 @@ static int32_t httpOnError(HttpParser *parser, int32_t httpCode, int32_t parseCo HttpContext *pContext = parser->pContext; if (httpCode != 0) parser->httpCode = httpCode; if (parseCode != 0) parser->parseCode = parseCode; - httpError("context:%p, fd:%d, parse failed, httpCode:%d parseCode:%d", pContext, pContext->fd, httpCode, parseCode); + + httpError("context:%p, fd:%d, parse failed, httpCode:%d parseCode:%d reason:%s", pContext, pContext->fd, httpCode, + parseCode & 0XFFFF, tstrerror(parseCode)); return 0; } @@ -309,19 +311,19 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } return 0; } else { - free(t); - free(s); parser->authType = HTTP_INVALID_AUTH; httpError("context:%p, fd:%d, invalid auth, t:%s s:%s", pContext, pContext->fd, t, s); httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_TYPE); + free(t); + free(s); return -1; } } else { - free(t); - free(s); parser->authType = HTTP_INVALID_AUTH; httpError("context:%p, fd:%d, parse auth failed, t:%s s:%s", pContext, pContext->fd, t, s); httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_FORMAT); + free(t); + free(s); return -1; } } @@ -334,14 +336,15 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { HttpString * buf = &parser->body; if (parser->parseCode != TSDB_CODE_SUCCESS) return -1; - int32_t avail = buf->size - buf->pos; - if (len + 1 >= avail) { + int32_t newSize = buf->pos + len + 1; + if (newSize >= buf->size) { if (buf->size >= HTTP_BUFFER_SIZE) { httpError("context:%p, fd:%d, failed parse body, exceeding buffer size %d", pContext, pContext->fd, buf->size); httpOnError(parser, 0, TSDB_CODE_HTTP_REQUSET_TOO_BIG); return -1; } else { - int32_t newSize = buf->size * 10; + newSize = MAX(newSize, 32); + newSize *= 10; newSize = MIN(newSize, HTTP_BUFFER_SIZE); buf->str = realloc(buf->str, newSize); if (buf->str == NULL) { @@ -442,7 +445,6 @@ void httpInitParser(HttpParser *parser) { free(parser->method); parser->method = NULL; free(parser->target); parser->target = NULL; - free(parser->target_raw); parser->target_raw = NULL; free(parser->version); parser->version = NULL; free(parser->reasonPhrase); parser->reasonPhrase = NULL; free(parser->key); parser->key = NULL; @@ -481,7 +483,6 @@ void httpDestroyParser(HttpParser *parser) { free(parser->method); parser->method = NULL; free(parser->target); parser->target = NULL; - free(parser->target_raw); parser->target_raw = NULL; free(parser->version); parser->version = NULL; free(parser->reasonPhrase); parser->reasonPhrase = NULL; free(parser->key); parser->key = NULL; @@ -638,9 +639,8 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c } break; } - parser->target_raw = strdup(parser->str.str); - parser->target = httpDecodeUrl(parser->str.str); - if (!parser->target_raw || !parser->target) { + parser->target = strdup(parser->str.str); + if (!parser->target) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_TARGET_FAILED); diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 77dfdb1bf4..755dad2d85 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -136,6 +136,10 @@ void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { else httpCode = 400; + if (pContext->parser->httpCode != 0) { + httpCode = pContext->parser->httpCode; + } + char *httpCodeStr = httpGetStatusDesc(httpCode); httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo & 0XFFFF, tstrerror(errNo)); } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 2a558e25d5..0bca8cf041 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -319,21 +319,25 @@ static bool httpReadData(HttpContext *pContext) { if (ok) { httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok, pParser->parseCode); + httpSendErrorResp(pContext, pParser->parseCode); httpNotifyContextClose(pContext); return false; } if (pParser->parseCode) { httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode); + httpSendErrorResp(pContext, pParser->parseCode); httpNotifyContextClose(pContext); return false; } - if (pParser->parsed) { + if (!pParser->parsed) { + httpDebug("context:%p, fd:%d, read not over yet, len:%d", pContext, pContext->fd, pParser->body.pos); + return false; + } else { httpDebug("context:%p, fd:%d, len:%d, body:%s", pContext, pContext->fd, pParser->body.pos, pParser->body.str); + return true; } - - return true; } else if (nread < 0) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno); diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index c60124a991..207f5d7923 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -344,7 +344,7 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) { void httpProcessLoginCmd(HttpContext *pContext) { char token[128] = {0}; - if (!httpGenTaosdAuthToken(pContext, token, 128)) { + if (httpGenTaosdAuthToken(pContext, token, 128) != 0) { httpSendErrorResp(pContext, TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR); } else { httpDebug("context:%p, fd:%d, user:%s, login via http, return token:%s", pContext, pContext->fd, pContext->user, diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim index cea804cfbb..c7866e5f4c 100644 --- a/tests/script/general/http/grafana.sim +++ b/tests/script/general/http/grafana.sim @@ -54,13 +54,13 @@ print =============== step2 - login system_content curl 127.0.0.1:7111/grafana/ print 1-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:7111/grafana/xx print 2-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi @@ -72,7 +72,7 @@ endi system_content curl 127.0.0.1:7111/grafana/root/1/123/1/1/3 print 4-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi @@ -84,13 +84,13 @@ endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/grafana/root/1/login print 6-> $system_content -if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/grafana/root/1/login print 7-> $system_content -if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index b7f98e49e0..a02140a419 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -14,26 +14,26 @@ print =============== step1 - login system_content curl 127.0.0.1:7111/rest/ print 1-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:7111/rest/xx print 2-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:7111/rest/login print 3-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi #4 system_content curl 127.0.0.1:7111/rest/login/root print 4-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi @@ -58,13 +58,13 @@ endi #8 system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 print 8-> $system_content -if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 print 9-> $system_content -if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi @@ -100,7 +100,7 @@ endi #14 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '' 127.0.0.1:7111/rest/sql print 14-> $system_content -if $system_content != @{"status":"error","code":5012,"desc":"no sql input"}@ then +if $system_content != @{"status":"error","code":4359,"desc":"no sql input"}@ then return -1 endi diff --git a/tests/script/general/http/telegraf.sim b/tests/script/general/http/telegraf.sim index e54af99ad7..4018d9661a 100644 --- a/tests/script/general/http/telegraf.sim +++ b/tests/script/general/http/telegraf.sim @@ -16,224 +16,224 @@ print =============== step1 - parse system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/ print $system_content -if $system_content != @{"status":"error","code":5022,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":4448,"desc":"database name can not be null"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_admin' -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/ print $system_content -if $system_content != @{"status":"error","code":5022,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":4448,"desc":"database name can not be null"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_admin' -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/d123456789012345678901234567890123456 print $system_content -if $system_content != @{"status":"error","code":5023,"desc":"database name too long"}@ then +if $system_content != @{"status":"error","code":4449,"desc":"database name too long"}@ then return -1 endi system_content curl -u root:taosdata -d '[]' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":4453,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":4453,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{}]' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":4453,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"metrics": []}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5025,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":4451,"desc":"metrics size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"metrics": [{}]}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":4453,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"metrics": 12}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5025,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":4451,"desc":"metrics size is 0"}@ then return -1 endi #system_content curl -u root:taosdata -d '{"metrics": [{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}]}' 127.0.0.1:7111/telegraf/db/root/taosdata1 #print $system_content -#if $system_content != @{"status":"error","code":5026,"desc":"metrics size can not more than 50"}@ then +#if $system_content != @{"status":"error","code":4452,"desc":"metrics size can not more than 50"}@ then # return -1 #endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":4453,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":111,"tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5028,"desc":"metric name type should be string"}@ then +if $system_content != @{"status":"error","code":4454,"desc":"metric name type should be string"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5029,"desc":"metric name length is 0"}@ then +if $system_content != @{"status":"error","code":4455,"desc":"metric name length is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5030,"desc":"metric name length too long"}@ then +if $system_content != @{"status":"error","code":4456,"desc":"metric name length too long"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"}}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5031,"desc":"timestamp not find"}@ then +if $system_content != @{"status":"error","code":4457,"desc":"timestamp not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":""}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5032,"desc":"timestamp type should be integer"}@ then +if $system_content != @{"status":"error","code":4458,"desc":"timestamp type should be integer"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":-1}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5033,"desc":"timestamp value smaller than 0"}@ then +if $system_content != @{"status":"error","code":4459,"desc":"timestamp value smaller than 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5034,"desc":"tags not find"}@ then +if $system_content != @{"status":"error","code":4460,"desc":"tags not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5035,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":4461,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":"","timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5035,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":4461,"desc":"tags size is 0"}@ then return -1 endi #system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor","host":"windows","instance":"1","objectname":"Processor","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata #print $system_content -#if $system_content != @{"status":"error","code":5036,"desc":"tags size too long"}@ then +#if $system_content != @{"status":"error","code":4461,"desc":"tags size too long"}@ then # return -1 #endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5035,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":4461,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"":"windows"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5038,"desc":"tag name is null"}@ then +if $system_content != @{"status":"error","code":4464,"desc":"tag name is null"}@ then return -1 endi #system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host111111111111222222222222222222222":""},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 #print $system_content -#if $system_content != @{"status":"error","code":5039,"desc":"tag name length too long"}@ then +#if $system_content != @{"status":"error","code":4465,"desc":"tag name length too long"}@ then # return -1 #endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":true},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5040,"desc":"tag value type should be number or string"}@ then +if $system_content != @{"status":"error","code":4466,"desc":"tag value type should be number or string"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":""},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5041,"desc":"tag value is null"}@ then +if $system_content != @{"status":"error","code":4467,"desc":"tag value is null"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"5022":"111"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5042,"desc":"table is null"}@ then +if $system_content != @{"status":"error","code":4468,"desc":"table is null"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5043,"desc":"table name length too long"}@ then +if $system_content != @{"status":"error","code":4469,"desc":"table name length too long"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5045,"desc":"fields size is 0"}@ then +if $system_content != @{"status":"error","code":4471,"desc":"fields size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5048,"desc":"field name is null"}@ then +if $system_content != @{"status":"error","code":4474,"desc":"field name is null"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":"","Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5051,"desc":"field value is null"}@ then +if $system_content != @{"status":"error","code":4477,"desc":"field value is null"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":true,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:7111/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":5050,"desc":"field value type should be number or string"}@ then +if $system_content != @{"status":"error","code":4476,"desc":"field value type should be number or string"}@ then return -1 endi diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 0d444a5a6e..45739b7d99 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -111,23 +111,23 @@ echo "serverPort ${NODE}" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG -echo "mDebugFlag 135" >> $TAOS_CFG -echo "sdbDebugFlag 135" >> $TAOS_CFG -echo "dDebugFlag 135" >> $TAOS_CFG -echo "vDebugFlag 135" >> $TAOS_CFG -echo "tsdbDebugFlag 135" >> $TAOS_CFG -echo "cDebugFlag 135" >> $TAOS_CFG -echo "jnidebugFlag 135" >> $TAOS_CFG -echo "odbcdebugFlag 135" >> $TAOS_CFG -echo "httpDebugFlag 135" >> $TAOS_CFG -echo "monitorDebugFlag 135" >> $TAOS_CFG -echo "mqttDebugFlag 135" >> $TAOS_CFG -echo "qdebugFlag 135" >> $TAOS_CFG -echo "rpcDebugFlag 135" >> $TAOS_CFG +echo "mDebugFlag 131" >> $TAOS_CFG +echo "sdbDebugFlag 131" >> $TAOS_CFG +echo "dDebugFlag 131" >> $TAOS_CFG +echo "vDebugFlag 131" >> $TAOS_CFG +echo "tsdbDebugFlag 131" >> $TAOS_CFG +echo "cDebugFlag 131" >> $TAOS_CFG +echo "jnidebugFlag 131" >> $TAOS_CFG +echo "odbcdebugFlag 131" >> $TAOS_CFG +echo "httpDebugFlag 207" >> $TAOS_CFG +echo "monitorDebugFlag 131" >> $TAOS_CFG +echo "mqttDebugFlag 131" >> $TAOS_CFG +echo "qdebugFlag 131" >> $TAOS_CFG +echo "rpcDebugFlag 131" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 135" >> $TAOS_CFG -echo "sdebugFlag 135" >> $TAOS_CFG -echo "wdebugFlag 135" >> $TAOS_CFG +echo "udebugFlag 131" >> $TAOS_CFG +echo "sdebugFlag 131" >> $TAOS_CFG +echo "wdebugFlag 131" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG diff --git a/tests/script/unique/http/admin.sim b/tests/script/unique/http/admin.sim index dc17520d02..8833397487 100644 --- a/tests/script/unique/http/admin.sim +++ b/tests/script/unique/http/admin.sim @@ -33,25 +33,25 @@ print =============== step1 - login system_content curl 127.0.0.1:7111/admin/ print 1-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:7111/admin/xx print 2-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:7111/admin/login print 3-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:7111/admin/login/root print 4-> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi @@ -69,13 +69,13 @@ endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.' -d 'show databases' 127.0.0.1:7111/admin/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: Taosd eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' 127.0.0.1:7111/admin/login/root/1 print 8-> $system_content -if $system_content != @{"status":"error","code":5053,"desc":"parse http auth token error"}@ then +if $system_content != @{"status":"error","code":4389,"desc":"invalid taosd Authorization"}@ then return -1 endi @@ -105,7 +105,7 @@ endi system_content curl 127.0.0.1:7111/admin/logout print 11 -----> $system_content -if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then return -1 endi @@ -168,7 +168,7 @@ print =============== step7 - use dbs system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'use d1;' 127.0.0.1:7111/admin/all print 23-> $system_content -if $system_content != @{"status":"error","code":5017,"desc":"no need to execute use db cmd"}@ then +if $system_content != @{"status":"error","code":4360,"desc":"no need to execute use db cmd"}@ then return -1 endi diff --git a/tests/script/unique/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim index 4901c5b3fd..5269817165 100644 --- a/tests/script/unique/http/opentsdb.sim +++ b/tests/script/unique/http/opentsdb.sim @@ -13,62 +13,62 @@ print ============================ dnode1 start print =============== step1 - parse system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/ print $system_content -if $system_content != @{"status":"error","code":5057,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":4496,"desc":"database name can not be null"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/db123456789012345678901234567890db print $system_content -if $system_content != @{"status":"error","code":5058,"desc":"database name too long"}@ then +if $system_content != @{"status":"error","code":4497,"desc":"database name too long"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/ print $system_content -if $system_content != @{"status":"error","code":5057,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":4496,"desc":"database name can not be null"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/db/put2 print $system_content -if $system_content != @{"status":"error","code":5009,"desc":"http url parse error"}@ then +if $system_content != @{"status":"error","code":4354,"desc":"invalid url format"}@ then return -1 endi system_content curl -u root:taosdata -d '[]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5060,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":4499,"desc":"metrics size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5059,"desc":"invalid opentsdb json fromat"}@ then +if $system_content != @{"status":"error","code":4498,"desc":"invalid opentsdb json fromat"}@ then return -1 endi system_content curl -u root:taosdata -d '{}' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5060,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":4499,"desc":"metrics size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5062,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":4501,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": 1,"timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5063,"desc":"metric name type should be string"}@ then +if $system_content != @{"status":"error","code":4502,"desc":"metric name type should be string"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5064,"desc":"metric name length is 0"}@ then +if $system_content != @{"status":"error","code":4503,"desc":"metric name length is 0"}@ then return -1 endi @@ -80,25 +80,25 @@ endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5066,"desc":"timestamp not find"}@ then +if $system_content != @{"status":"error","code":4505,"desc":"timestamp not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": "2","value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5067,"desc":"timestamp type should be integer"}@ then +if $system_content != @{"status":"error","code":4506,"desc":"timestamp type should be integer"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": -1,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5068,"desc":"timestamp value smaller than 0"}@ then +if $system_content != @{"status":"error","code":4507,"desc":"timestamp value smaller than 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5078,"desc":"value not find"}@ then +if $system_content != @{"status":"error","code":4517,"desc":"value not find"}@ then return -1 endi @@ -106,19 +106,19 @@ endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5069,"desc":"tags not find"}@ then +if $system_content != @{"status":"error","code":4508,"desc":"tags not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5070,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":4509,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": 0}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5070,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":4509,"desc":"tags size is 0"}@ then return -1 endi @@ -130,25 +130,25 @@ endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"": "web01"}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5073,"desc":"tag name is null"}@ then +if $system_content != @{"status":"error","code":4512,"desc":"tag name is null"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host01123456789001123456789001123456789001123456789001123456789001123456789": "01"}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5074,"desc":"tag name length too long"}@ then +if $system_content != @{"status":"error","code":4513,"desc":"tag name length too long"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web011234567890011234567890011234567890011234567890011234567890011234567890011234567890011234567890"}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5077,"desc":"tag value can not more than 64"}@ then +if $system_content != @{"status":"error","code":4516,"desc":"tag value can not more than 64"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": ""}}]' 127.0.0.1:7111/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":5076,"desc":"tag value is null"}@ then +if $system_content != @{"status":"error","code":4515,"desc":"tag value is null"}@ then return -1 endi From 0c43959191a2b09e017343123129f13e48a09e2f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 16:55:08 +0800 Subject: [PATCH 047/105] [td-1391] fix bugs in release obj in submit processing. --- src/client/src/tscSubquery.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e9ec272ea4..8d908b2506 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1903,7 +1903,9 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) pParentObj->res.code = pSql->res.code; } + taos_free_result(tres); taosTFree(pSupporter); + if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { return; } From 4bed0a0f3910463a6031b0a4b94332faf155381c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 17:16:23 +0800 Subject: [PATCH 048/105] [td-1391] update the error msg output info. --- src/util/src/tsocket.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 61896a86df..498fc6af6b 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -16,7 +16,7 @@ #include "os.h" #include "tulog.h" #include "tsocket.h" -#include "tutil.h" +#include "taoserror.h" int taosGetFqdn(char *fqdn) { char hostname[1024]; @@ -56,7 +56,13 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) { freeaddrinfo(result); return ip; } else { - uError("failed get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); + if (ret == EAI_SYSTEM) { + uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + } else { + uError("failed get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); + } + return 0xFFFFFFFF; } } From dfa681df3c2a7fa18d70bc9386579f7f08bbc548 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 14 Sep 2020 17:24:08 +0800 Subject: [PATCH 049/105] [TD-1291] --- src/plugins/http/inc/httpParser.h | 1 + src/plugins/http/src/httpContext.c | 26 ++++++++++---------- src/plugins/http/src/httpHandle.c | 4 ++-- src/plugins/http/src/httpJson.c | 2 +- src/plugins/http/src/httpParser.c | 38 ++++++++++++++++++------------ src/plugins/http/src/httpQueue.c | 2 +- src/plugins/http/src/httpServer.c | 6 ++--- tests/script/sh/deploy.sh | 32 ++++++++++++------------- 8 files changed, 60 insertions(+), 51 deletions(-) diff --git a/src/plugins/http/inc/httpParser.h b/src/plugins/http/inc/httpParser.h index 7dd6d3e0fc..85ba843716 100644 --- a/src/plugins/http/inc/httpParser.h +++ b/src/plugins/http/inc/httpParser.h @@ -111,6 +111,7 @@ typedef struct HttpParser { void httpInitParser(HttpParser *parser); HttpParser *httpCreateParser(struct HttpContext *pContext); +void httpClearParser(HttpParser *parser); void httpDestroyParser(HttpParser *parser); int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len); char * httpGetStatusDesc(int32_t statusCode); diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 2c65b9d16a..3114fb702e 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -138,7 +138,7 @@ HttpContext *httpGetContext(void *ptr) { HttpContext *pContext = *ppContext; if (pContext) { int32_t refCount = atomic_add_fetch_32(&pContext->refCount, 1); - httpDebug("context:%p, fd:%d, is accquired, data:%p refCount:%d", pContext, pContext->fd, ppContext, refCount); + httpTrace("context:%p, fd:%d, is accquired, data:%p refCount:%d", pContext, pContext->fd, ppContext, refCount); return pContext; } } @@ -152,9 +152,9 @@ void httpReleaseContext(HttpContext *pContext) { return; } - pContext->parser->inited = 0; + httpClearParser(pContext->parser); HttpContext **ppContext = pContext->ppContext; - httpDebug("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount); + httpTrace("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount); if (tsHttpServer.contextCache != NULL) { taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); @@ -173,7 +173,7 @@ bool httpInitContext(HttpContext *pContext) { memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); - httpDebug("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed); + httpTrace("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed); return true; } @@ -191,15 +191,15 @@ void httpCloseContextByApp(HttpContext *pContext) { if (keepAlive) { if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) { - httpDebug("context:%p, fd:%d, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) { httpRemoveContextFromEpoll(pContext); - httpDebug("context:%p, fd:%d, ast state:dropping, keepAlive:true, close connect", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, ast state:dropping, keepAlive:true, close connect", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) { - httpDebug("context:%p, fd:%d, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) { httpRemoveContextFromEpoll(pContext); - httpDebug("context:%p, fd:%d, last state:ready, keepAlive:true, close connect", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, last state:ready, keepAlive:true, close connect", pContext, pContext->fd); } else { httpRemoveContextFromEpoll(pContext); httpError("context:%p, fd:%d, last state:%s:%d, keepAlive:true, close connect", pContext, pContext->fd, @@ -207,7 +207,7 @@ void httpCloseContextByApp(HttpContext *pContext) { } } else { httpRemoveContextFromEpoll(pContext); - httpDebug("context:%p, fd:%d, ilast state:%s:%d, keepAlive:false, close context", pContext, pContext->fd, + httpTrace("context:%p, fd:%d, ilast state:%s:%d, keepAlive:false, close context", pContext, pContext->fd, httpContextStateStr(pContext->state), pContext->state); } @@ -216,13 +216,13 @@ void httpCloseContextByApp(HttpContext *pContext) { void httpCloseContextByServer(HttpContext *pContext) { if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) { - httpDebug("context:%p, fd:%d, epoll finished, still used by app", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, epoll finished, still used by app", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) { - httpDebug("context:%p, fd:%d, epoll already finished, wait app finished", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, epoll already finished, wait app finished", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) { - httpDebug("context:%p, fd:%d, epoll finished, close connect", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, epoll finished, close connect", pContext, pContext->fd); } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) { - httpDebug("context:%p, fd:%d, epoll finished, will be closed soon", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, epoll finished, will be closed soon", pContext, pContext->fd); } else { httpError("context:%p, fd:%d, unknown state:%d", pContext, pContext->fd, pContext->state); } diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 9fb46c5734..b50217cfc4 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -33,7 +33,7 @@ bool httpDecodeRequest(HttpContext* pContext) { */ bool httpProcessData(HttpContext* pContext) { if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_HANDLING)) { - httpDebug("context:%p, fd:%d, state:%s not in ready state, stop process request", pContext, pContext->fd, + httpTrace("context:%p, fd:%d, state:%s not in ready state, stop process request", pContext, pContext->fd, httpContextStateStr(pContext->state)); httpCloseContextByApp(pContext); return false; @@ -41,7 +41,7 @@ bool httpProcessData(HttpContext* pContext) { // handle Cross-domain request if (strcmp(pContext->parser->method, "OPTIONS") == 0) { - httpDebug("context:%p, fd:%d, process options request", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, process options request", pContext, pContext->fd); httpSendOptionResp(pContext, "process options request success"); } else { if (!httpDecodeRequest(pContext)) { diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 669e7c9781..e9a8947df2 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -189,7 +189,7 @@ void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext) { httpGzipCompressInit(buf->pContext); } - httpDebug("context:%p, fd:%d, json buffer initialized", buf->pContext, buf->pContext->fd); + httpTrace("context:%p, fd:%d, json buffer initialized", buf->pContext, buf->pContext->fd); } void httpJsonItemToken(JsonBuf* buf) { diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index ec71bb2daf..6f64e55b73 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -178,7 +178,7 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target } if (pContext->decodeMethod != NULL) { - httpDebug("context:%p, fd:%d, decode method is %s", pContext, pContext->fd, pContext->decodeMethod->module); + httpTrace("context:%p, fd:%d, decode method is %s", pContext, pContext->fd, pContext->decodeMethod->module); } else { httpError("context:%p, fd:%d, the url is not support, target:%s", pContext, pContext->fd, target); httpOnError(pParser, 0, TSDB_CODE_HTTP_UNSUPPORT_URL); @@ -190,7 +190,7 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target httpError("context:%p, fd:%d, unsupport httpVersion %d", pContext, pContext->fd, pParser->httpVersion); httpOnError(pParser, 0, TSDB_CODE_HTTP_INVALID_VERSION); } else { - httpDebug("context:%p, fd:%d, httpVersion:1.%d", pContext, pContext->fd, pParser->httpVersion); + httpTrace("context:%p, fd:%d, httpVersion:1.%d", pContext, pContext->fd, pParser->httpVersion); } return 0; @@ -204,7 +204,7 @@ static int32_t httpOnStatusLine(HttpParser *pParser, int32_t code, const char *r static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const char *val) { HttpContext *pContext = parser->pContext; - httpDebug("context:%p, fd:%d, key:%s val:%s", pContext, pContext->fd, key, val); + httpTrace("context:%p, fd:%d, key:%s val:%s", pContext, pContext->fd, key, val); if (0 == strcasecmp(key, "Content-Length")) { int32_t len = 0; @@ -214,7 +214,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const parser->contentLength = len; parser->chunkSize = len; parser->contentLengthSpecified = 1; - httpDebug("context:%p, fd:%d, contentLength:%d chunkSize:%d contentLengthSpecified:%d", pContext, pContext->fd, + httpTrace("context:%p, fd:%d, contentLength:%d chunkSize:%d contentLengthSpecified:%d", pContext, pContext->fd, parser->contentLength, parser->chunkSize, parser->contentLengthSpecified); return 0; } else { @@ -227,11 +227,11 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const else if (0 == strcasecmp(key, "Accept-Encoding")) { if (strstr(val, "gzip")) { parser->acceptEncodingGzip = 1; - httpDebug("context:%p, fd:%d, acceptEncodingGzip:%d", pContext, pContext->fd, parser->acceptEncodingGzip); + httpTrace("context:%p, fd:%d, acceptEncodingGzip:%d", pContext, pContext->fd, parser->acceptEncodingGzip); } if (strstr(val, "chunked")) { parser->acceptEncodingChunked = 1; - httpDebug("context:%p, fd:%d, acceptEncodingChunked:%d", pContext, pContext->fd, parser->acceptEncodingChunked); + httpTrace("context:%p, fd:%d, acceptEncodingChunked:%d", pContext, pContext->fd, parser->acceptEncodingChunked); } return 0; } @@ -248,7 +248,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const else if (0 == strcasecmp(key, "Content-Encoding")) { if (0 == strcmp(val, "gzip")) { parser->contentChunked = 1; - httpDebug("context:%p, fd:%d, contentChunked:%d", pContext, pContext->fd, parser->contentChunked); + httpTrace("context:%p, fd:%d, contentChunked:%d", pContext, pContext->fd, parser->contentChunked); } return 0; } @@ -271,7 +271,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } if (strstr(val, "chunked")) { parser->transferChunked = 1; - httpDebug("context:%p, fd:%d, transferChunked:%d", pContext, pContext->fd, parser->transferChunked); + httpTrace("context:%p, fd:%d, transferChunked:%d", pContext, pContext->fd, parser->transferChunked); } return 0; } @@ -289,7 +289,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const s = NULL; free(t); free(s); - httpDebug("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent); + httpTrace("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent); int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, strlen(parser->authContent)); if (ok != 0) { httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_BASIC_AUTH); @@ -303,7 +303,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const s = NULL; free(t); free(s); - httpDebug("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent); + httpTrace("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent); int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, strlen(parser->authContent)); if (ok != 0) { httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_TAOSD_AUTH); @@ -371,7 +371,7 @@ static int32_t httpOnEnd(HttpParser *parser) { return -1; } - httpDebug("context:%p, fd:%d, parse success", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, parse success", pContext, pContext->fd); return 0; } @@ -422,7 +422,7 @@ static int32_t httpCleanupStack(HttpStack *stack) { void httpInitParser(HttpParser *parser) { HttpContext *pContext = parser->pContext; - httpTrace("context:%p, fd:%d, free parser", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, init parser", pContext, pContext->fd); parser->parsed = false; parser->inited = 1; @@ -475,9 +475,17 @@ HttpParser *httpCreateParser(HttpContext *pContext) { return parser; } +void httpClearParser(HttpParser *parser) { + HttpContext *pContext = parser->pContext; + httpTrace("context:%p, fd:%d, clear parser", pContext, pContext->fd); + + pContext->parser->inited = 0; + pContext->parser->parsed = false; +} + void httpDestroyParser(HttpParser *parser) { HttpContext *pContext = parser->pContext; - httpTrace("context:%p, fd:%d, free parser", pContext, pContext->fd); + httpTrace("context:%p, fd:%d, destroy parser", pContext, pContext->fd); if (!parser) return; @@ -603,7 +611,7 @@ static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, c do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { if (httpAppendString(&parser->str, &c, 1)) { - httpDebug("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); break; @@ -719,7 +727,7 @@ static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const httpPopStack(parser); break; } - httpDebug("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); + httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; httpOnError(parser, 507, TSDB_CODE_HTTP_PARSE_SP_FAILED); } while (0); diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 76632eb508..86a97a6abe 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -71,7 +71,7 @@ static void *httpProcessResultQueue(void *param) { break; } - httpDebug("context:%p, res:%p will be processed in result queue", pMsg->param, pMsg->result); + httpTrace("context:%p, res:%p will be processed in result queue", pMsg->param, pMsg->result); (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->numOfRows); taosFreeQitem(pMsg); } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 0bca8cf041..a4d84b7a03 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -302,11 +302,11 @@ bool httpInitConnect() { static bool httpReadData(HttpContext *pContext) { HttpParser *pParser = pContext->parser; - ASSERT(!pParser->parsed); if (!pParser->inited) { httpInitParser(pParser); } + ASSERT(!pParser->parsed); pContext->accessTimes++; pContext->lastAccessTime = taosGetTimestampSec(); @@ -332,10 +332,10 @@ static bool httpReadData(HttpContext *pContext) { } if (!pParser->parsed) { - httpDebug("context:%p, fd:%d, read not over yet, len:%d", pContext, pContext->fd, pParser->body.pos); + httpTrace("context:%p, fd:%d, read not over yet, len:%d", pContext, pContext->fd, pParser->body.pos); return false; } else { - httpDebug("context:%p, fd:%d, len:%d, body:%s", pContext, pContext->fd, pParser->body.pos, pParser->body.str); + httpTraceL("context:%p, fd:%d, len:%d, body:%s", pContext, pContext->fd, pParser->body.pos, pParser->body.str); return true; } } else if (nread < 0) { diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 45739b7d99..0d444a5a6e 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -111,23 +111,23 @@ echo "serverPort ${NODE}" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG -echo "mDebugFlag 131" >> $TAOS_CFG -echo "sdbDebugFlag 131" >> $TAOS_CFG -echo "dDebugFlag 131" >> $TAOS_CFG -echo "vDebugFlag 131" >> $TAOS_CFG -echo "tsdbDebugFlag 131" >> $TAOS_CFG -echo "cDebugFlag 131" >> $TAOS_CFG -echo "jnidebugFlag 131" >> $TAOS_CFG -echo "odbcdebugFlag 131" >> $TAOS_CFG -echo "httpDebugFlag 207" >> $TAOS_CFG -echo "monitorDebugFlag 131" >> $TAOS_CFG -echo "mqttDebugFlag 131" >> $TAOS_CFG -echo "qdebugFlag 131" >> $TAOS_CFG -echo "rpcDebugFlag 131" >> $TAOS_CFG +echo "mDebugFlag 135" >> $TAOS_CFG +echo "sdbDebugFlag 135" >> $TAOS_CFG +echo "dDebugFlag 135" >> $TAOS_CFG +echo "vDebugFlag 135" >> $TAOS_CFG +echo "tsdbDebugFlag 135" >> $TAOS_CFG +echo "cDebugFlag 135" >> $TAOS_CFG +echo "jnidebugFlag 135" >> $TAOS_CFG +echo "odbcdebugFlag 135" >> $TAOS_CFG +echo "httpDebugFlag 135" >> $TAOS_CFG +echo "monitorDebugFlag 135" >> $TAOS_CFG +echo "mqttDebugFlag 135" >> $TAOS_CFG +echo "qdebugFlag 135" >> $TAOS_CFG +echo "rpcDebugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 131" >> $TAOS_CFG -echo "sdebugFlag 131" >> $TAOS_CFG -echo "wdebugFlag 131" >> $TAOS_CFG +echo "udebugFlag 135" >> $TAOS_CFG +echo "sdebugFlag 135" >> $TAOS_CFG +echo "wdebugFlag 135" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG From eaa6ca2701d4c02d1f1df9357e6e33d471c0ab13 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 17:44:29 +0800 Subject: [PATCH 050/105] [td-1391] upate the check of vgroup info. --- src/client/src/tscServer.c | 5 +++-- src/util/src/tsocket.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 1b6a77bdc7..32a4df4963 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1677,8 +1677,9 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfEps < 0) { - tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->sid); + if (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0) { + tscError("invalid value in table numOfEps:%d, vgId:%d sid%d", pMetaMsg->vgroup.numOfEps, + pMetaMsg->vgroup.vgId, pMetaMsg->sid); return TSDB_CODE_TSC_INVALID_VALUE; } diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 498fc6af6b..479eeaa754 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -60,7 +60,7 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) { uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); } else { - uError("failed get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); + uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); } return 0xFFFFFFFF; From 0b9e809e20c2dc26100ed21a4d7c6f6874d4725d Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 14 Sep 2020 17:47:30 +0800 Subject: [PATCH 051/105] fix TAOS SQL display issue on web site --- documentation20/webdocs/markdowndocs/TAOS SQL-ch.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index ae42582687..cb7a017411 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -110,6 +110,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic **Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。 + - **显示系统所有数据库** ```mysql SHOW DATABASES; From 89c417d6fa2b0c3b40ff7cc60ba7cf428402fd99 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 17:51:41 +0800 Subject: [PATCH 052/105] [td-1391] --- src/client/src/tscServer.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 32a4df4963..f1ef394b8f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1677,9 +1677,10 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - if (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0) { - tscError("invalid value in table numOfEps:%d, vgId:%d sid%d", pMetaMsg->vgroup.numOfEps, - pMetaMsg->vgroup.vgId, pMetaMsg->sid); + if ((pMetaMsg->tableType == TSDB_SUPER_TABLE) && + (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { + tscError("invalid value in table numOfEps:%d, vgId:%d sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, + pMetaMsg->sid); return TSDB_CODE_TSC_INVALID_VALUE; } From 5a641b058c755e51db3a7895209e33678ab60f95 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 18:01:41 +0800 Subject: [PATCH 053/105] [td-1391] update test --- src/client/src/tscServer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index f1ef394b8f..106cac9595 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1678,9 +1678,9 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if ((pMetaMsg->tableType == TSDB_SUPER_TABLE) && - (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { - tscError("invalid value in table numOfEps:%d, vgId:%d sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, - pMetaMsg->sid); + (pMetaMsg->sid < 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { + tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, + pMetaMsg->sid, pMetaMsg->tableId); return TSDB_CODE_TSC_INVALID_VALUE; } From 62d6f6c9a324eae9fef02d9ce309c2b7830743ef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 18:02:16 +0800 Subject: [PATCH 054/105] [td-1391] update test --- src/client/src/tscServer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 106cac9595..5639719c15 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1677,8 +1677,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - if ((pMetaMsg->tableType == TSDB_SUPER_TABLE) && - (pMetaMsg->sid < 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { + if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && + (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, pMetaMsg->sid, pMetaMsg->tableId); return TSDB_CODE_TSC_INVALID_VALUE; From 6bd79daa4628f97151a12a5fe7f94f7d0f0484d5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 14 Sep 2020 18:30:45 +0800 Subject: [PATCH 055/105] [TECO-19] add windows compile steps to README --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index 22984d8cfe..411dddb9f7 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,8 @@ cd TDengine ## Build TDengine +### On Linux platform + ```bash mkdir debug && cd debug cmake .. && cmake --build . @@ -109,6 +111,34 @@ aarch32: cmake .. -DCPUTYPE=aarch32 && cmake --build . ``` +### On Windows platform + +If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". +Please specify "x86_amd64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. +``` +mkdir debug && cd debug +"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < x86_amd64 | x86 > +cmake .. -G "NMake Makefiles" +nmake +``` + +If you use the Visual Studio 2019, please open a command window by executing "cmd.exe". +Please specify "x64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. +``` +mkdir debug && cd debug +"c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" < x64 | x86 > +cmake .. -G "NMake Makefiles" +nmake +``` + +Or, you can open a command window by clicking Visual Studio 2019 menu "Tools -> Command Line -> Developer Command Prompt" or "Tools -> Command Line -> Developer PowerShell" then execute commands as follows: +``` +mkdir debug && cd debug +cmake .. -G "NMake Makefiles" +nmake +``` + +# Quick Run # Quick Run To quickly start a TDengine server after building, run the command below in terminal: ```cmd From f878512588d4cbc7742476964a139c7f9d681e94 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 14 Sep 2020 18:34:22 +0800 Subject: [PATCH 056/105] [TD-1291] --- src/plugins/http/inc/httpContext.h | 2 +- src/plugins/http/inc/httpInt.h | 6 ++-- src/plugins/http/src/httpContext.c | 9 ++++-- src/plugins/http/src/httpParser.c | 45 +++++++++++++++--------------- src/plugins/http/src/httpServer.c | 8 ++++-- 5 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/plugins/http/inc/httpContext.h b/src/plugins/http/inc/httpContext.h index a2d50d6b7f..260858c5cc 100644 --- a/src/plugins/http/inc/httpContext.h +++ b/src/plugins/http/inc/httpContext.h @@ -25,7 +25,7 @@ const char *httpContextStateStr(HttpContextState state); HttpContext *httpCreateContext(int32_t fd); bool httpInitContext(HttpContext *pContext); HttpContext *httpGetContext(void * pContext); -void httpReleaseContext(HttpContext *pContext); +void httpReleaseContext(HttpContext *pContext, bool clearRes); void httpCloseContextByServer(HttpContext *pContext); void httpCloseContextByApp(HttpContext *pContext); void httpNotifyContextClose(HttpContext *pContext); diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 38a52356f8..4fae4e74c9 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -32,9 +32,9 @@ #define HTTP_MAX_BUFFER_SIZE 1024*1024*8 #define HTTP_LABEL_SIZE 8 #define HTTP_MAX_EVENTS 10 -#define HTTP_BUFFER_INIT 8192 -#define HTTP_BUFFER_SIZE 8192000 -#define HTTP_STEP_SIZE 1024 //http message get process step by step +#define HTTP_BUFFER_INIT 4096 +#define HTTP_BUFFER_SIZE 8388608 +#define HTTP_STEP_SIZE 4096 //http message get process step by step #define HTTP_METHOD_SCANNER_SIZE 7 //http method fp size #define TSDB_CODE_HTTP_GC_TARGET_SIZE 512 #define HTTP_WRITE_RETRY_TIMES 500 diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 3114fb702e..c0c0c494de 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -145,14 +145,17 @@ HttpContext *httpGetContext(void *ptr) { return NULL; } -void httpReleaseContext(HttpContext *pContext) { +void httpReleaseContext(HttpContext *pContext, bool clearRes) { int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); if (refCount < 0) { httpError("context:%p, is already released, refCount:%d", pContext, refCount); return; } - httpClearParser(pContext->parser); + if (clearRes) { + httpClearParser(pContext->parser); + } + HttpContext **ppContext = pContext->ppContext; httpTrace("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount); @@ -211,7 +214,7 @@ void httpCloseContextByApp(HttpContext *pContext) { httpContextStateStr(pContext->state), pContext->state); } - httpReleaseContext(pContext); + httpReleaseContext(pContext, true); } void httpCloseContextByServer(HttpContext *pContext) { diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index 6f64e55b73..cad5dd8f91 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -110,11 +110,11 @@ static void httpCleanupString(HttpString *str) { static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { if (str->size == 0) { str->pos = 0; - str->size = 32; + str->size = 64; str->str = malloc(str->size); } else if (str->pos + len + 1 >= str->size) { str->size += len; - str->size *= 10; + str->size *= 4; str->str = realloc(str->str, str->size); } else { } @@ -153,17 +153,10 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target for (int32_t i = 0; i < HTTP_MAX_URL; i++) { char *pSeek = strchr(pStart, '/'); if (pSeek == NULL) { - pParser->path[i].str = strdup(pStart); - pParser->path[i].size = strlen(pStart); - pParser->path[i].pos = pParser->path[i].size; + httpAppendString(pParser->path + i, pStart, strlen(pStart)); break; } else { - int32_t len = (int32_t)(pSeek - pStart); - pParser->path[i].str = malloc(len + 1); - memcpy(pParser->path[i].str, pStart, len); - pParser->path[i].str[len] = 0; - pParser->path[i].size = len; - pParser->path[i].pos = len; + httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart)); } pStart = pSeek + 1; } @@ -336,23 +329,29 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { HttpString * buf = &parser->body; if (parser->parseCode != TSDB_CODE_SUCCESS) return -1; + if (buf->size <= 0) { + buf->size = MIN(len + 2, HTTP_BUFFER_SIZE); + buf->str = malloc(buf->size); + } + int32_t newSize = buf->pos + len + 1; if (newSize >= buf->size) { if (buf->size >= HTTP_BUFFER_SIZE) { httpError("context:%p, fd:%d, failed parse body, exceeding buffer size %d", pContext, pContext->fd, buf->size); httpOnError(parser, 0, TSDB_CODE_HTTP_REQUSET_TOO_BIG); return -1; - } else { - newSize = MAX(newSize, 32); - newSize *= 10; - newSize = MIN(newSize, HTTP_BUFFER_SIZE); - buf->str = realloc(buf->str, newSize); - if (buf->str == NULL) { - httpError("context:%p, fd:%d, failed parse body, realloc %d failed", pContext, pContext->fd, newSize); - httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - return -1; - } - buf->size = newSize; + } + + newSize = MAX(newSize, HTTP_BUFFER_INIT); + newSize *= 4; + newSize = MIN(newSize, HTTP_BUFFER_SIZE); + buf->str = realloc(buf->str, newSize); + buf->size = newSize; + + if (buf->str == NULL) { + httpError("context:%p, fd:%d, failed parse body, realloc %d failed", pContext, pContext->fd, buf->size); + httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); + return -1; } } @@ -389,7 +388,7 @@ static int32_t httpPushStack(HttpParser *parser, HTTP_PARSER_STATE state) { stack->size = 32; stack->stacks = malloc(stack->size * sizeof(int8_t)); } else if (stack->pos + 1 > stack->size) { - stack->size *= 10; + stack->size *= 2; stack->stacks = realloc(stack->stacks, stack->size * sizeof(int8_t)); } else { } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index a4d84b7a03..0dfdee32da 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -132,7 +132,7 @@ static void httpProcessHttpData(void *param) { if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) { httpDebug("context:%p, fd:%d, state:%s, not in ready state, ignore read events", pContext, pContext->fd, httpContextStateStr(pContext->state)); - httpReleaseContext(pContext); + httpReleaseContext(pContext, true); continue; } @@ -145,6 +145,8 @@ static void httpProcessHttpData(void *param) { if (httpReadData(pContext)) { (*(pThread->processData))(pContext); atomic_fetch_add_32(&pServer->requestNum, 1); + } else { + httpReleaseContext(pContext, false); } } } @@ -226,7 +228,7 @@ static void *httpAcceptHttpConnection(void *arg) { httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd, pContext->ipstr, pThread->label, strerror(errno)); taosClose(pContext->fd); - httpReleaseContext(pContext); + httpReleaseContext(pContext, true); continue; } @@ -314,7 +316,7 @@ static bool httpReadData(HttpContext *pContext) { int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, sizeof(buf)); if (nread > 0) { buf[nread] = '\0'; - httpTrace("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf); + httpTrace("context:%p, fd:%d, nread:%d", pContext, pContext->fd, nread); int32_t ok = httpParseBuf(pParser, buf, nread); if (ok) { From 6e46c77f6c09787bb9a47c17ac73e937ed6ba4f0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Sep 2020 11:36:17 +0000 Subject: [PATCH 057/105] TD-1326 --- src/plugins/http/inc/httpInt.h | 2 +- src/plugins/http/inc/httpSql.h | 6 +++--- src/plugins/http/src/httpGcHandle.c | 2 +- src/plugins/http/src/httpGcJson.c | 26 +++++++++++++------------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 4fae4e74c9..36a022159f 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -36,7 +36,7 @@ #define HTTP_BUFFER_SIZE 8388608 #define HTTP_STEP_SIZE 4096 //http message get process step by step #define HTTP_METHOD_SCANNER_SIZE 7 //http method fp size -#define TSDB_CODE_HTTP_GC_TARGET_SIZE 512 +#define HTTP_GC_TARGET_SIZE 512 #define HTTP_WRITE_RETRY_TIMES 500 #define HTTP_WRITE_WAIT_TIME_MS 5 #define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + TSDB_PASSWORD_LEN) diff --git a/src/plugins/http/inc/httpSql.h b/src/plugins/http/inc/httpSql.h index 660a65e44d..db3e3a3b16 100644 --- a/src/plugins/http/inc/httpSql.h +++ b/src/plugins/http/inc/httpSql.h @@ -29,10 +29,10 @@ void httpFreeMultiCmds(HttpContext *pContext); HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext); HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext); -int32_t httpCurSqlCmdPos(HttpContext *pContext); +int32_t httpCurSqlCmdPos(HttpContext *pContext); -void httpTrimTableName(char *name); +void httpTrimTableName(char *name); int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name); -char *httpGetCmdsString(HttpContext *pContext, int32_t pos); +char * httpGetCmdsString(HttpContext *pContext, int32_t pos); #endif diff --git a/src/plugins/http/src/httpGcHandle.c b/src/plugins/http/src/httpGcHandle.c index 01be301637..5d4cb0c680 100644 --- a/src/plugins/http/src/httpGcHandle.c +++ b/src/plugins/http/src/httpGcHandle.c @@ -228,7 +228,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { cmd->values = refIdBuffer; cmd->table = aliasBuffer; cmd->numOfRows = 0; // hack way as target flags - cmd->timestamp = httpAddToSqlCmdBufferWithSize(pContext, TSDB_CODE_HTTP_GC_TARGET_SIZE + 1); // hack way + cmd->timestamp = httpAddToSqlCmdBufferWithSize(pContext, HTTP_GC_TARGET_SIZE + 1); // hack way if (cmd->timestamp == -1) { httpWarn("context:%p, fd:%d, user:%s, cant't malloc target size, sql buffer is full", pContext, pContext->fd, diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index e864d54ac0..a291641dc3 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -129,48 +129,48 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, // for group by if (groupFields != -1) { - char target[TSDB_CODE_HTTP_GC_TARGET_SIZE] = {0}; + char target[HTTP_GC_TARGET_SIZE] = {0}; int32_t len; - len = snprintf(target,TSDB_CODE_HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer); + len = snprintf(target,HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer); for (int32_t i = dataFields + 1; i Date: Mon, 14 Sep 2020 21:41:42 +0800 Subject: [PATCH 058/105] [td-1391]add free subobj --- src/client/src/tscSubquery.c | 2 -- src/client/src/tscUtil.c | 35 +---------------------------------- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 8d908b2506..e9ec272ea4 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1903,9 +1903,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) pParentObj->res.code = pSql->res.code; } - taos_free_result(tres); taosTFree(pSupporter); - if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { return; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 979e153356..f06d36e0c6 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -382,39 +382,6 @@ static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) { pSql->numOfSubs = 0; } -//static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) { -// SSqlCmd* pCmd = &pSql->cmd; -// SSqlRes* pRes = &pSql->res; -// -// if (pRes == NULL || pRes->qhandle == 0) { -// return true; -// } -// -// SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); -// if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) { -// return true; -// } -// -// STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); -// tscRemoveFromSqlList(pSql); -// -// int32_t cmd = pCmd->command; -// if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && -// (cmd == TSDB_SQL_SELECT || -// cmd == TSDB_SQL_SHOW || -// cmd == TSDB_SQL_RETRIEVE || -// cmd == TSDB_SQL_FETCH)) { -// pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; -// pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; -// tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s, ", pSql, sqlCmd[pCmd->command]); -// -// tscProcessSql(pSql); -// return false; -// } -// -// return true; -//} - /** * The free operation will cause the pSql to be removed from hash table and free it in * the function of processmsgfromserver is impossible in this case, since it will fail @@ -438,7 +405,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscDebug("%p start to free sqlObj", pSql); STscObj* pTscObj = pSql->pTscObj; -// tscFreeSubobj(pSql); + tscFreeSubobj(pSql); tscPartiallyFreeSqlObj(pSql); pSql->signature = NULL; From 125e7e934009e71d0625727b8aede68e46db37fc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Sep 2020 22:23:50 +0800 Subject: [PATCH 059/105] TD-1291 --- src/plugins/http/src/httpServer.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 0dfdee32da..2d95a0ac72 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -308,7 +308,11 @@ static bool httpReadData(HttpContext *pContext) { httpInitParser(pParser); } - ASSERT(!pParser->parsed); + if (pParser->parsed) { + httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed); + return false; + } + pContext->accessTimes++; pContext->lastAccessTime = taosGetTimestampSec(); From 9d1619c3b90b0948232913af83d8699bd40f09ab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 14 Sep 2020 23:37:45 +0800 Subject: [PATCH 060/105] [td-1391] refactor --- src/client/src/tscSql.c | 2 ++ src/client/src/tscUtil.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index f02e587455..4a001a2ce4 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -286,9 +286,11 @@ void taos_close(TAOS *taos) { assert(ref >= 0); if (ref > 0) { + tscDebug("%p %d remain sqlObjs, do not close dnodeConn:%p", pObj, ref, pObj->pDnodeConn); return; } + tscDebug("%p all sqlObj are freed, free tscObj, dnodeConn:%p", pObj, pObj->pDnodeConn); tscCloseTscObj(pObj); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index f06d36e0c6..9b3f5927ab 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -772,6 +772,7 @@ void tscCloseTscObj(STscObj* pObj) { pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); + void* p = pObj->pDnodeConn; if (pObj->pDnodeConn != NULL) { rpcClose(pObj->pDnodeConn); pObj->pDnodeConn = NULL; @@ -779,7 +780,7 @@ void tscCloseTscObj(STscObj* pObj) { pthread_mutex_destroy(&pObj->mutex); - tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn); + tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p); taosTFree(pObj); } From 95457415469356deb3c7bad6bb9d4ab9032e46b8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Sep 2020 23:38:00 +0800 Subject: [PATCH 061/105] TD-1291 --- tests/test/c/CMakeLists.txt | 14 ++++++++++++-- tests/test/c/httpTest.c | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 tests/test/c/httpTest.c diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index e1fedaee3c..5339203ec2 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -5,6 +5,12 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/plugins/http/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) IF (TD_LINUX) #add_executable(insertPerTable insertPerTable.c) @@ -28,6 +34,10 @@ IF (TD_LINUX) #add_executable(createNormalTable createNormalTable.c) #target_link_libraries(createNormalTable taos_static tutil common pthread) - add_executable(queryPerformance queryPerformance.c) - target_link_libraries(queryPerformance taos_static tutil common pthread) + #add_executable(queryPerformance queryPerformance.c) + #target_link_libraries(queryPerformance taos_static tutil common pthread) + + add_executable(httpTest httpTest.c) + target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) ENDIF() + diff --git a/tests/test/c/httpTest.c b/tests/test/c/httpTest.c new file mode 100644 index 0000000000..dd6d9e4cf8 --- /dev/null +++ b/tests/test/c/httpTest.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "os.h" +#include "tglobal.h" +#include "taoserror.h" +#include "httpSystem.h" + +int main(int argc, char *argv[]) { + // Initialize the system + if (httpInitSystem() < 0) { + exit(EXIT_FAILURE); + } + + if (httpStartSystem() < 0) { + exit(EXIT_FAILURE); + } + + while (1) { + sleep(1000); + } +} From 3a58f9f4a92a0ece691f88e5f94ef82ecb6b7e35 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 14 Sep 2020 23:36:49 +0800 Subject: [PATCH 062/105] fix test failures --- tests/pytest/import_merge/importToCommit.py | 2 +- tests/pytest/tag_lite/datatype-without-alter.py | 2 +- tests/pytest/tag_lite/datatype.py | 2 +- tests/pytest/util/sql.py | 8 ++++++-- tests/script/fullGeneralSuite.sim | 1 - 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/pytest/import_merge/importToCommit.py b/tests/pytest/import_merge/importToCommit.py index 9a17ae95fa..7bec5fcd5d 100644 --- a/tests/pytest/import_merge/importToCommit.py +++ b/tests/pytest/import_merge/importToCommit.py @@ -33,7 +33,7 @@ class TDTestCase: tdDnodes.start(1) tdSql.execute('reset query cache') tdSql.execute('drop database if exists db') - tdSql.execute('create database db cache 128 maxtables 10') + tdSql.execute('create database db cache 128') tdSql.execute('use db') tdLog.info("================= step1") diff --git a/tests/pytest/tag_lite/datatype-without-alter.py b/tests/pytest/tag_lite/datatype-without-alter.py index 42bc42bdbf..da52e149a5 100644 --- a/tests/pytest/tag_lite/datatype-without-alter.py +++ b/tests/pytest/tag_lite/datatype-without-alter.py @@ -38,7 +38,7 @@ class TDTestCase: tdLog.info("drop database db if exits") tdSql.execute('drop database if exists db') tdLog.info("================= step1") - tdSql.execute('create database db maxtables 4') + tdSql.execute('create database db') tdLog.sleep(5) tdSql.execute('use db') diff --git a/tests/pytest/tag_lite/datatype.py b/tests/pytest/tag_lite/datatype.py index bc99cf74b0..f7fa9fa3a2 100644 --- a/tests/pytest/tag_lite/datatype.py +++ b/tests/pytest/tag_lite/datatype.py @@ -38,7 +38,7 @@ class TDTestCase: tdLog.info("drop database db if exits") tdSql.execute('drop database if exists db') tdLog.info("================= step1") - tdSql.execute('create database db maxtables 4') + tdSql.execute('create database db') tdLog.sleep(5) tdSql.execute('use db') diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 627d712474..7de8efdfe9 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -123,8 +123,12 @@ class TDSql: def checkData(self, row, col, data): self.checkRowCol(row, col) - if str(self.queryResult[row][col]) != str(data): - if isinstance(data, float) and abs(self.queryResult[row][col] - data) <= 0.000001: + if self.queryResult[row][col] != data: + if str(self.queryResult[row][col]) != str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(self.queryResult[row][col] - data) <= 0.000001: tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % (self.sql, row, col, self.queryResult[row][col], data)) return diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index d137e53d27..4df7900265 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -130,7 +130,6 @@ run general/parser/join.sim run general/parser/join_multivnode.sim run general/parser/select_with_tags.sim run general/parser/groupby.sim -run general/parser/bug.sim run general/parser/tags_dynamically_specifiy.sim run general/parser/set_tag_vals.sim #unsupport run general/parser/repeatAlter.sim From c8cf0720826ec08293c0f2162ddcccbe03bee314 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Sep 2020 23:52:02 +0800 Subject: [PATCH 063/105] minor changes --- tests/test/c/httpTest.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test/c/httpTest.c b/tests/test/c/httpTest.c index dd6d9e4cf8..261546770e 100644 --- a/tests/test/c/httpTest.c +++ b/tests/test/c/httpTest.c @@ -19,7 +19,20 @@ #include "taoserror.h" #include "httpSystem.h" +void signal_handler(int signum) { + httpStopSystem(); + httpCleanUpSystem(); + exit(EXIT_SUCCESS); +} + int main(int argc, char *argv[]) { + struct sigaction act; + act.sa_handler = signal_handler; + sigaction(SIGTERM, &act, NULL); + sigaction(SIGHUP, &act, NULL); + sigaction(SIGINT, &act, NULL); + sigaction(SIGABRT, &act, NULL); + // Initialize the system if (httpInitSystem() < 0) { exit(EXIT_FAILURE); From 53b2d31c389ae6e493bae3374de7d971bc519d60 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 15 Sep 2020 03:24:05 +0000 Subject: [PATCH 064/105] minor changes --- tests/test/c/CMakeLists.txt | 3 + tests/test/c/cacheTest.c | 136 ++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 tests/test/c/cacheTest.c diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 5339203ec2..ffab39d41c 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -39,5 +39,8 @@ IF (TD_LINUX) add_executable(httpTest httpTest.c) target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) + + add_executable(cacheTest cacheTest.c) + target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) ENDIF() diff --git a/tests/test/c/cacheTest.c b/tests/test/c/cacheTest.c new file mode 100644 index 0000000000..091a6662c8 --- /dev/null +++ b/tests/test/c/cacheTest.c @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "tcache.h" +#include "tulog.h" +#include "tutil.h" + +#define MAX_REFRESH_TIME_SEC 2 +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +int32_t tsKeepTimeInSec = 3; +int32_t tsNumOfRows = 10000; +int32_t tsSizeOfRow = 64 * 1024 * 100; +void * tsCacheObj = NULL; + +typedef int64_t CacheTestKey; +typedef struct CacheTestRow { + int32_t index; + void ** ppRow; + void * data; +} CacheTestRow; + +CacheTestRow *initRow(int32_t index) { + CacheTestRow *row = calloc(sizeof(CacheTestRow), 1); + row->index = index; + row->data = malloc(tsSizeOfRow * sizeof(int8_t)); + return row; +} + +void detroyRow(void *data) { + CacheTestRow *row = *(CacheTestRow **)data; + free(row->data); + free(row); +} + +void initCache() { + tsCacheObj = taosCacheInit(TSDB_DATA_TYPE_BIGINT, MAX_REFRESH_TIME_SEC, true, detroyRow, "cachetest"); +} + +void putRowInCache() { + for (int index = 0; index < tsNumOfRows; ++index) { + CacheTestRow *row = initRow(index); + uint64_t key = (uint64_t)row; + void **ppRow = taosCachePut(tsCacheObj, &key, sizeof(int64_t), &row, sizeof(int64_t), tsKeepTimeInSec * 1000); + row->ppRow = ppRow; + taosCacheRelease(tsCacheObj, (void **)&ppRow, false); + } +} + +void cleanupCache() { + taosCacheCleanup(tsCacheObj); +} + +void initGetMemory() { + osInit(); + taos_init(); +} + +float getProcMemory() { + float procMemoryUsedMB = 0; + taosGetProcMemory(&procMemoryUsedMB); + return procMemoryUsedMB; +} + +void doTest() { + initCache(); + pPrint("%s initialize procMemory %f MB %s", GREEN, getProcMemory(), NC); + + putRowInCache(); + pPrint("%s insert %d rows, procMemory %f MB %s", GREEN, tsNumOfRows, getProcMemory(), NC); + + int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3) * 1000 + tsKeepTimeInSec * 1000; + taosMsleep(sleepMs); + pPrint("%s after sleep %d ms, procMemory %f MB %s", GREEN, sleepMs, getProcMemory(), NC); + + cleanupCache(); + taosMsleep(sleepMs); + pPrint("%s after cleanup cache, procMemory %f MB %s", GREEN, getProcMemory(), NC); +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance of cache\n"); + + printf("%s%s\n", indent, "-k"); + printf("%s%s%s%d\n", indent, indent, "KeepTimeInSec, default is ", tsKeepTimeInSec); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%d\n", indent, indent, "NumOfRows, default is ", tsNumOfRows); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%d\n", indent, indent, "SizeOfData, default is ", tsSizeOfRow); + + exit(EXIT_SUCCESS); +} + +void parseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-k") == 0) { + tsKeepTimeInSec = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + tsNumOfRows = atoi(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + tsSizeOfRow = atoi(argv[++i]); + } else { + } + } + + pPrint("%s KeepTimeInSec:%d %s", GREEN, tsKeepTimeInSec, NC); + pPrint("%s NumOfRows:%d %s", GREEN, tsNumOfRows, NC); + pPrint("%s SizeOfData:%d %s", GREEN, tsSizeOfRow, NC); +} + +int main(int argc, char *argv[]) { + initGetMemory(); + parseArgument(argc, argv); + doTest(); +} From 80eddb19df849558db6dab972bb5af2810cce387 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 15 Sep 2020 11:53:46 +0800 Subject: [PATCH 065/105] minor changes --- tests/test/c/cacheTest.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test/c/cacheTest.c b/tests/test/c/cacheTest.c index 091a6662c8..341b64638e 100644 --- a/tests/test/c/cacheTest.c +++ b/tests/test/c/cacheTest.c @@ -26,9 +26,10 @@ #define NC "\033[0m" int32_t tsKeepTimeInSec = 3; -int32_t tsNumOfRows = 10000; -int32_t tsSizeOfRow = 64 * 1024 * 100; +int32_t tsNumOfRows = 1000000; +int32_t tsSizeOfRow = 64 * 1024; void * tsCacheObj = NULL; +int32_t destroyTimes = 0; typedef int64_t CacheTestKey; typedef struct CacheTestRow { @@ -48,6 +49,10 @@ void detroyRow(void *data) { CacheTestRow *row = *(CacheTestRow **)data; free(row->data); free(row); + destroyTimes++; + if (destroyTimes % 50000 == 0) { + pPrint("%s ===> destroyTimes:%d %s", GREEN, destroyTimes, NC); + } } void initCache() { @@ -86,13 +91,15 @@ void doTest() { putRowInCache(); pPrint("%s insert %d rows, procMemory %f MB %s", GREEN, tsNumOfRows, getProcMemory(), NC); - int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3) * 1000 + tsKeepTimeInSec * 1000; + int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3 + 10) * 1000 + tsKeepTimeInSec * 1000; taosMsleep(sleepMs); pPrint("%s after sleep %d ms, procMemory %f MB %s", GREEN, sleepMs, getProcMemory(), NC); - cleanupCache(); + //cleanupCache(); taosMsleep(sleepMs); pPrint("%s after cleanup cache, procMemory %f MB %s", GREEN, getProcMemory(), NC); + + pPrint("%s finally destroyTimes:%d %s", GREEN, destroyTimes, NC); } void printHelp() { From 11cc33c575e67eba20ff89eac017523d5a378ed0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 15 Sep 2020 14:02:33 +0800 Subject: [PATCH 066/105] [td-1446] --- src/client/src/tscSql.c | 4 +- src/tsdb/src/tsdbRead.c | 56 ++++++++++++++--------- tests/script/general/parser/testSuite.sim | 1 - tests/script/general/parser/topbot.sim | 51 +++++++++++++++++++-- 4 files changed, 82 insertions(+), 30 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 4a001a2ce4..69bc69cd4a 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -286,11 +286,11 @@ void taos_close(TAOS *taos) { assert(ref >= 0); if (ref > 0) { - tscDebug("%p %d remain sqlObjs, do not close dnodeConn:%p", pObj, ref, pObj->pDnodeConn); + tscDebug("%p %d remain sqlObjs, not free tscObj and dnodeConn:%p", pObj, ref, pObj->pDnodeConn); return; } - tscDebug("%p all sqlObj are freed, free tscObj, dnodeConn:%p", pObj, pObj->pDnodeConn); + tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); tscCloseTscObj(pObj); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 24a19e83d0..3a39d5fc49 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -389,9 +389,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 ", lastKey:%" PRId64 ", %p", + "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, - pCheckInfo->lastKey, pHandle->qinfo); + pCheckInfo->lastKey, pMem->numOfRows, pHandle->qinfo); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -411,9 +411,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 ", lastKey:%" PRId64 ", %p", + "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, - pCheckInfo->lastKey, pHandle->qinfo); + pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qinfo); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -735,6 +735,7 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p return TSDB_CODE_SUCCESS; } +static int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo); static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols); static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle); @@ -791,9 +792,10 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc * Here the buffer is not enough, so only part of file block can be loaded into memory buffer */ assert(pQueryHandle->outputCapacity >= binfo.rows); + int32_t endPos = getEndPosInDataBlock(pQueryHandle, &binfo); - if ((cur->pos == 0 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->pos == (binfo.rows - 1) && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) { + if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || + (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) { pQueryHandle->realNumOfRows = binfo.rows; cur->rows = binfo.rows; @@ -809,7 +811,6 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc cur->pos = -1; } } else { // partially copy to dest buffer - int32_t endPos = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.rows - 1): 0; copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &binfo, endPos); cur->mixBlock = true; } @@ -1204,6 +1205,29 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl cur->win.ekey, cur->rows, pQueryHandle->qinfo); } +int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) { + // NOTE: reverse the order to find the end position in data block + int32_t endPos = -1; + int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + + SQueryFilePos* cur = &pQueryHandle->cur; + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + + if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) { + endPos = pBlockInfo->rows - 1; + cur->mixBlock = (cur->pos != 0); + } else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey <= pBlockInfo->window.skey) { + endPos = 0; + cur->mixBlock = (cur->pos != pBlockInfo->rows - 1); + } else { + assert(pCols->numOfRows > 0); + endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); + cur->mixBlock = true; + } + + return endPos; +} + // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { @@ -1225,19 +1249,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); STable* pTable = pCheckInfo->pTableObj; - int32_t endPos = cur->pos; - - if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { - endPos = blockInfo.rows - 1; - cur->mixBlock = (cur->pos != 0); - } else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { - endPos = 0; - cur->mixBlock = (cur->pos != blockInfo.rows - 1); - } else { - assert(pCols->numOfRows > 0); - endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); - cur->mixBlock = true; - } + int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo); tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," "end:%d, %p", @@ -1339,8 +1351,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } cur->blockCompleted = - (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); + (((pos > endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((pos < endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index f42254981c..3dd80b8e38 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -105,7 +105,6 @@ run general/parser/timestamp.sim sleep 2000 run general/parser/sliding.sim - #sleep 2000 #run general/parser/repeatStream.sim #sleep 2000 diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim index 2faee55460..3e906f3a28 100644 --- a/tests/script/general/parser/topbot.sim +++ b/tests/script/general/parser/topbot.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 200 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 1000 sql connect $dbPrefix = tb_db @@ -25,7 +25,7 @@ $stb = $stbPrefix . $i sql drop database $db -x step1 step1: -sql create database $db cache 16 +sql create database $db cache 16 maxrows 4096 keep 36500 print ====== create tables sql use $db sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int) @@ -132,15 +132,15 @@ sleep 5000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 3000 +sleep 1000 sql select count(*) from t1.test where ts>10000 and ts<90000 interval(5000a) if $rows != 3 then return -1 endi -print =========>td-1308 -sql create database db; +print ==============>td-1308 +sql create database db keep 36500; sql use db; sql create table stb (ts timestamp, c1 int, c2 binary(10)) tags(t1 binary(10)); @@ -158,4 +158,45 @@ if $rows != 1 then return -1 endi +print =======================>td-1446 +sql create table t(ts timestamp, k int) +$ts = 6000 +while $ts < 7000 + sql insert into t values ( $ts , $ts ) + $ts = $ts + 1 +endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 1000 +sql use db; + +$ts = 1000 +while $ts < 5096 + sql insert into t values ( $ts , $ts ) + $ts = $ts + 1 +endw + +sql select * from t where ts < 6500 +if $rows != 4596 then + print expect 4596, actual: $rows + return -1 +endi + +sql select * from t where ts < 7000 +if $rows != 5096 then + return -1 +endi + +sql select * from t where ts <= 6000 +if $rows != 4097 then + return -1 +endi + +sql select * from t where ts <= 6001 +if $rows != 4098 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From c233807ddc9f0e8492a5c3e9b6fd63e5df2fd4ec Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 15 Sep 2020 14:30:40 +0800 Subject: [PATCH 067/105] [TECO-20] remove docker icon --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 411dddb9f7..522fc0ebc1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -[![Docker Pulls](https://img.shields.io/docker/pulls/tdengine/tdengine)](https://hub.docker.com/repository/docker/tdengine/tdengine) [![tdengine](https://snapcraft.io//tdengine/badge.svg)](https://snapcraft.io/tdengine) [![TDengine](TDenginelogo.png)](https://www.taosdata.com) From bffd30ef80ddb2162cfcca2b7da696d916c221d1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 15 Sep 2020 15:21:22 +0800 Subject: [PATCH 068/105] [td-1290] --- src/query/src/qExecutor.c | 131 ++++++++++++++++---------------------- 1 file changed, 56 insertions(+), 75 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d48d7d5ea1..6b67c7ceb9 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1951,36 +1951,36 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo // todo handle the case the the order irrelevant query type mixed up with order critical query type // descending order query for last_row query - if (isFirstLastRowQuery(pQuery) && !QUERY_IS_ASC_QUERY(pQuery)) { + if (isFirstLastRowQuery(pQuery)) { qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), pQuery->order.order, TSDB_ORDER_ASC); - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); pQuery->order.order = TSDB_ORDER_ASC; - assert (pQuery->window.skey <= pQuery->window.ekey); + if (pQuery->window.skey > pQuery->window.ekey) { + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + return; + } + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && pQuery->order.order == TSDB_ORDER_DESC) { + pQuery->order.order = TSDB_ORDER_ASC; + if (pQuery->window.skey > pQuery->window.ekey) { + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } doExchangeTimeWindow(pQInfo, &pQuery->window); return; } - if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && !QUERY_IS_ASC_QUERY(pQuery)) { - pQuery->order.order = TSDB_ORDER_ASC; - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - assert (pQuery->window.skey <= pQuery->window.ekey); - - doExchangeTimeWindow(pQInfo, &pQuery->window); - return; - } - - if (isPointInterpoQuery(pQuery) && (pQuery->intervalTime == 0) && !QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, - pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + if (isPointInterpoQuery(pQuery) && pQuery->intervalTime == 0) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { + qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, + pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } pQuery->order.order = TSDB_ORDER_ASC; - - assert (pQuery->window.skey <= pQuery->window.ekey); - doExchangeTimeWindow(pQInfo, &pQuery->window); return; } @@ -2365,16 +2365,16 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); pQuery->sdata[i] = (tFilePage *)tmp; } - + // set the pCtx output buffer position pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes; - + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } } - + qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), newSize, pRec->capacity, newSize - pRec->rows); @@ -2920,11 +2920,11 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { STableQueryInfo *item = taosArrayGetP(pGroup, i); SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); + pageList = list; + tid = TSDB_TABLEID(item->pTable)->tid; if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { pTableList[numOfTables++] = item; - tid = TSDB_TABLEID(item->pTable)->tid; - pageList = list; } } @@ -3357,7 +3357,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - + memmove(pQuery->sdata[i]->data, (char*)pQuery->sdata[i]->data + bytes * numOfSkip, (size_t)(pQuery->rec.rows * bytes)); pRuntimeEnv->pCtx[i].aOutputBuf = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes; @@ -4354,32 +4354,6 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { return true; } -static void freeTableQueryInfo(STableGroupInfo* pTableGroupInfo) { - if (pTableGroupInfo->pGroupList == NULL) { - assert(pTableGroupInfo->numOfTables == 0); - } else { - size_t numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); - for (int32_t i = 0; i < numOfGroups; ++i) { - SArray *p = taosArrayGetP(pTableGroupInfo->pGroupList, i); - - size_t num = taosArrayGetSize(p); - for(int32_t j = 0; j < num; ++j) { - STableQueryInfo* item = taosArrayGetP(p, j); - destroyTableQueryInfo(item); - } - - taosArrayDestroy(p); - } - - taosArrayDestroy(pTableGroupInfo->pGroupList); - pTableGroupInfo->pGroupList = NULL; - pTableGroupInfo->numOfTables = 0; - } - - taosHashCleanup(pTableGroupInfo->map); - pTableGroupInfo->map = NULL; -} - static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -4415,22 +4389,20 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); - if (pRuntimeEnv->pQueryHandle == NULL) { // no data in current stable, clear all - freeTableQueryInfo(&pQInfo->tableqinfoGroupInfo); - } else { // update the query time window - pQuery->window = cond.twindow; - size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); - for (int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); + // update the query time window + pQuery->window = cond.twindow; - size_t t = taosArrayGetSize(group); - for (int32_t j = 0; j < t; ++j) { - STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pQInfo, i); - pCheckInfo->win = pQuery->window; - pCheckInfo->lastKey = pCheckInfo->win.skey; - } + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + + pCheckInfo->win = pQuery->window; + pCheckInfo->lastKey = pCheckInfo->win.skey; } } } else if (isPointInterpoQuery(pQuery)) { @@ -4484,12 +4456,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo return code; } - if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo); - setQueryStatus(pQuery, QUERY_COMPLETED); - return TSDB_CODE_SUCCESS; - } - pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; @@ -4612,7 +4578,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery* pQuery = pRuntimeEnv->pQuery; SQueryCostInfo* summary = &pRuntimeEnv->summary; - + int64_t st = taosGetTimestampMs(); TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; @@ -4622,7 +4588,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { while (tsdbNextDataBlock(pQueryHandle)) { summary->totalBlocks += 1; - + if (IS_QUERY_KILLED(pQInfo)) { longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -4659,7 +4625,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { summary->totalRows += blockInfo.rows; stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); - + qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, " "lastKey:%" PRId64, pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, @@ -6383,10 +6349,25 @@ static void freeQInfo(SQInfo *pQInfo) { taosTFree(pQuery); } - freeTableQueryInfo(&pQInfo->tableqinfoGroupInfo); + // todo refactor, extract method to destroytableDataInfo + if (pQInfo->tableqinfoGroupInfo.pGroupList != NULL) { + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); + for (int32_t i = 0; i < numOfGroups; ++i) { + SArray *p = GET_TABLEGROUP(pQInfo, i); + + size_t num = taosArrayGetSize(p); + for(int32_t j = 0; j < num; ++j) { + STableQueryInfo* item = taosArrayGetP(p, j); + destroyTableQueryInfo(item); + } + + taosArrayDestroy(p); + } + } taosTFree(pQInfo->pBuf); - + taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList); + taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); From 4d6e0ef40e11118cd24f880f7cc1d296685fed27 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 15 Sep 2020 16:09:48 +0800 Subject: [PATCH 069/105] [td-1454] --- src/client/src/tscSubquery.c | 15 +++++++-------- tests/script/general/parser/topbot.sim | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e9ec272ea4..7ea5d4183b 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2198,16 +2198,15 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { // calculate the result from several other columns if (pSup->pArithExprInfo != NULL) { if (pRes->pArithSup == NULL) { - SArithmeticSupport *sas = (SArithmeticSupport *) calloc(1, sizeof(SArithmeticSupport)); - sas->offset = 0; - sas->pArithExpr = pSup->pArithExprInfo; - sas->numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - sas->exprList = pQueryInfo->exprList; - sas->data = calloc(sas->numOfCols, POINTER_BYTES); - - pRes->pArithSup = sas; + pRes->pArithSup = (SArithmeticSupport*)calloc(1, sizeof(SArithmeticSupport)); } + pRes->pArithSup->offset = 0; + pRes->pArithSup->pArithExpr = pSup->pArithExprInfo; + pRes->pArithSup->numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + pRes->pArithSup->exprList = pQueryInfo->exprList; + pRes->pArithSup->data = calloc(pRes->pArithSup->numOfCols, POINTER_BYTES); + if (pRes->buffer[i] == NULL) { TAOS_FIELD* field = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pRes->buffer[i] = malloc(field->bytes); diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim index 3e906f3a28..5c575b6163 100644 --- a/tests/script/general/parser/topbot.sim +++ b/tests/script/general/parser/topbot.sim @@ -199,4 +199,18 @@ if $rows != 4098 then return -1 endi +print ======================>td-1454 +sql select count(*)/10, count(*)+99 from t +if $rows != 1 then + return -1 +endi + +if $data00 != 509.600000000 then + return -1 +endi + +if $data01 != 5195.000000000 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 41a21ca29ebefe2bd69b9c5a96c53498730efed0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 15 Sep 2020 16:23:37 +0800 Subject: [PATCH 070/105] TD-1225 --- src/client/src/tscParseInsert.c | 2 +- src/plugins/http/src/httpServer.c | 6 ++-- tests/script/general/http/autocreate.sim | 5 +++ tests/script/general/http/chunked.sim | 37 +++++++++++++++++++++ tests/script/general/http/gzip.sim | 27 ++++++++++++++++ tests/script/general/http/testSuite.sim | 2 ++ tests/script/jenkins/basic.txt | 2 ++ tests/test/c/cacheTest.c | 11 ++++--- tests/tsim/inc/sim.h | 2 ++ tests/tsim/src/simExe.c | 41 ++++++++++++++++++++++++ tests/tsim/src/simParse.c | 14 ++++++++ 11 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 tests/script/general/http/chunked.sim create mode 100644 tests/script/general/http/gzip.sim diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 09eb8f167e..18926a84b9 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -526,7 +526,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe int32_t index = 0; SStrToken sToken; - int16_t numOfRows = 0; + int32_t numOfRows = 0; SSchema *pSchema = tscGetTableSchema(pTableMeta); STableComInfo tinfo = tscGetTableInfo(pTableMeta); diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 2d95a0ac72..d85a236cb1 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -317,10 +317,10 @@ static bool httpReadData(HttpContext *pContext) { pContext->lastAccessTime = taosGetTimestampSec(); char buf[HTTP_STEP_SIZE + 1] = {0}; - int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, sizeof(buf)); + int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE); if (nread > 0) { buf[nread] = '\0'; - httpTrace("context:%p, fd:%d, nread:%d", pContext, pContext->fd, nread); + httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf); int32_t ok = httpParseBuf(pParser, buf, nread); if (ok) { @@ -341,7 +341,7 @@ static bool httpReadData(HttpContext *pContext) { httpTrace("context:%p, fd:%d, read not over yet, len:%d", pContext, pContext->fd, pParser->body.pos); return false; } else { - httpTraceL("context:%p, fd:%d, len:%d, body:%s", pContext, pContext->fd, pParser->body.pos, pParser->body.str); + httpDebug("context:%p, fd:%d, totalLen:%d", pContext, pContext->fd, pParser->body.pos); return true; } } else if (nread < 0) { diff --git a/tests/script/general/http/autocreate.sim b/tests/script/general/http/autocreate.sim index 6a005b028a..98d64ab839 100644 --- a/tests/script/general/http/autocreate.sim +++ b/tests/script/general/http/autocreate.sim @@ -24,5 +24,10 @@ print curl 127.0.0.1:7111/rest/sql -----> $system_content # return -1 #endi +sql select * from db.win_cpu_windows_1_processor +print rows: $rows +if $rows != 1 then + return -1 +endi #system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/chunked.sim b/tests/script/general/http/chunked.sim new file mode 100644 index 0000000000..8673655d96 --- /dev/null +++ b/tests/script/general/http/chunked.sim @@ -0,0 +1,37 @@ +system sh/stop_dnodes.sh +sleep 3000 +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 0 +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print ============================ dnode1 start + +print =============== step1 - prepare data +sql create database d1 +sql use d1 + +sql create table table_rest (ts timestamp, i int) +print sql length is 270KB +restful d1 table_rest 1591072800 10000 +restful d1 table_rest 1591172800 10000 +restful d1 table_rest 1591272800 10000 +restful d1 table_rest 1591372800 10000 +restful d1 table_rest 1591472800 10000 +restful d1 table_rest 1591572800 10000 +restful d1 table_rest 1591672800 10000 +restful d1 table_rest 1591772800 10000 +restful d1 table_rest 1591872800 10000 +restful d1 table_rest 1591972800 10000 + +sql select * from table_rest; +print rows: $rows +if $rows != 100000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/gzip.sim b/tests/script/general/http/gzip.sim new file mode 100644 index 0000000000..0289e337a6 --- /dev/null +++ b/tests/script/general/http/gzip.sim @@ -0,0 +1,27 @@ +system sh/stop_dnodes.sh +sleep 3000 +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 0 +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print ============================ dnode1 start + +print =============== step1 - prepare data +sql create database d1 +sql use d1 + +sql create table table_rest (ts timestamp, i int) +print sql length is 270KB +restful d1 table_rest 1591072800 10000 gzip +sql select * from table_rest; +print rows: $rows +if $rows != 10000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim index d91e9f452d..f35362bf07 100644 --- a/tests/script/general/http/testSuite.sim +++ b/tests/script/general/http/testSuite.sim @@ -1,3 +1,5 @@ +run general/http/autocreate.sim +run general/http/chunked.sim run general/http/restful.sim run general/http/restful_insert.sim run general/http/restful_limit.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 063d11bd9d..adb22aa265 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -77,6 +77,8 @@ cd ../../../debug; make ./test.sh -f general/field/smallint.sim ./test.sh -f general/field/tinyint.sim +./test.sh -f general/http/autocreate.sim +./test.sh -f general/http/chunked.sim ./test.sh -f general/http/restful.sim ./test.sh -f general/http/restful_insert.sim ./test.sh -f general/http/restful_limit.sim diff --git a/tests/test/c/cacheTest.c b/tests/test/c/cacheTest.c index 341b64638e..54aca0038e 100644 --- a/tests/test/c/cacheTest.c +++ b/tests/test/c/cacheTest.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include #include "os.h" #include "taos.h" #include "tcache.h" @@ -91,15 +92,17 @@ void doTest() { putRowInCache(); pPrint("%s insert %d rows, procMemory %f MB %s", GREEN, tsNumOfRows, getProcMemory(), NC); - int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3 + 10) * 1000 + tsKeepTimeInSec * 1000; + int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3) * 1000 + tsKeepTimeInSec * 1000; taosMsleep(sleepMs); pPrint("%s after sleep %d ms, procMemory %f MB %s", GREEN, sleepMs, getProcMemory(), NC); - //cleanupCache(); + cleanupCache(); taosMsleep(sleepMs); pPrint("%s after cleanup cache, procMemory %f MB %s", GREEN, getProcMemory(), NC); - - pPrint("%s finally destroyTimes:%d %s", GREEN, destroyTimes, NC); + + malloc_trim(0); + taosMsleep(sleepMs); + pPrint("%s after malloc_trim, procMemory %f MB %s", GREEN, getProcMemory(), NC); } void printHelp() { diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 6f3bc7099d..58e58a442c 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -84,6 +84,7 @@ enum { SIM_CMD_SQL, SIM_CMD_SQL_ERROR, SIM_CMD_SQL_SLOW, + SIM_CMD_RESTFUL, SIM_CMD_TEST, SIM_CMD_RETURN, SIM_CMD_END @@ -172,6 +173,7 @@ bool simExecuteReturnCmd(SScript *script, char *option); bool simExecuteSqlCmd(SScript *script, char *option); bool simExecuteSqlErrorCmd(SScript *script, char *rest); bool simExecuteSqlSlowCmd(SScript *script, char *option); +bool simExecuteRestfulCmd(SScript *script, char *rest); void simVisuallizeOption(SScript *script, char *src, char *dst); #endif \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index adc2fd0b9d..463dc33c7c 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -915,6 +915,47 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) { return simExecuteSqlImpCmd(script, rest, isSlow); } +bool simExecuteRestfulCmd(SScript *script, char *rest) { + FILE *fp = NULL; + char filename[256]; + sprintf(filename, "%s/tmp.sql", tsScriptDir); + fp = fopen(filename, "w"); + if (fp == NULL) { + fprintf(stderr, "ERROR: failed to open file: %s\n", filename); + return false; + } + + char db[64] = {0}; + char tb[64] = {0}; + char gzip[32] = {0}; + int32_t ts; + int32_t times; + sscanf(rest, "%s %s %d %d %s", db, tb, &ts, ×, gzip); + + fprintf(fp, "insert into %s.%s values ", db, tb); + for (int i = 0; i < times; ++i) { + fprintf(fp, "(%d000, %d)", ts + i, ts); + } + fprintf(fp, " \n"); + fflush(fp); + fclose(fp); + + char cmd[1024] = {0}; + if (strcmp(gzip, "gzip") == 0) { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "--compressed --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } else { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "'Transfer-Encoding: chunked' --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } + + return simExecuteSystemCmd(script, cmd); +} + bool simExecuteSqlErrorCmd(SScript *script, char *rest) { char buf[3000]; SCmdLine *line = &script->lines[script->linePos]; diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 8dcf83806f..2e6121304f 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -721,6 +721,12 @@ bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } +bool simParseRestfulCmd(char *rest, SCommand *pCmd, int lineNum) { + simParseSqlCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; + return true; +} + bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) { int expLen; @@ -1020,6 +1026,14 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = simExecuteSqlSlowCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); + cmdno = SIM_CMD_RESTFUL; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "restful"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseRestfulCmd; + simCmdList[cmdno].executeCmd = simExecuteRestfulCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + /* test is only an internal command */ cmdno = SIM_CMD_TEST; simCmdList[cmdno].cmdno = cmdno; From 2372e4decc278be058a643376eb498213081365b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 15 Sep 2020 17:26:48 +0800 Subject: [PATCH 071/105] TD-1451 --- src/plugins/http/src/httpParser.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index cad5dd8f91..0c3204687a 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -238,6 +238,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive); } +#if 0 else if (0 == strcasecmp(key, "Content-Encoding")) { if (0 == strcmp(val, "gzip")) { parser->contentChunked = 1; @@ -245,8 +246,9 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } return 0; } + #endif - else if (0 == strcasecmp(key, "Transfer-Encoding")) { + else if (0 == strcasecmp(key, "Transfer-Encoding") || 0 == strcasecmp(key, "Content-Encoding")) { if (strstr(val, "gzip")) { parser->transferGzip = 1; ehttp_gzip_conf_t conf = {0}; From 53c4f51116cd1cf66752f7ea12b4b4ec65f92288 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 15 Sep 2020 18:21:40 +0800 Subject: [PATCH 072/105] [TD-1427] --- cmake/install.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/install.inc b/cmake/install.inc index 7a92a396e3..0531d40048 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -13,7 +13,7 @@ ELSEIF (TD_WINDOWS) IF (NOT TD_GODLL) #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector) - #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) From e5443631062710ce55527b3d363963359db372d3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 15 Sep 2020 18:56:32 +0800 Subject: [PATCH 073/105] [TD-1459] : fix compile error for release build type. --- cmake/define.inc | 2 +- src/client/src/tscSQLParser.c | 1 + src/client/src/tscUtil.c | 1 + src/query/src/qExecutor.c | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/define.inc b/cmake/define.inc index 28770be254..84df6f5405 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -100,7 +100,7 @@ IF (TD_LINUX) ENDIF () SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") + SET(RELEASE_FLAGS "-O0 -Wno-unused-variable -Wunused-but-set-variable") IF (${COVER} MATCHES "true") MESSAGE(STATUS "Test coverage mode, add extra flags") diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c248b08ddd..f9f72f9174 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5271,6 +5271,7 @@ static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { } assert(qualifiedCol); + qualifiedCol = qualifiedCol; } } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 80e3828c9d..636fccd3a0 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1788,6 +1788,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p } assert(matched); + matched = matched; } tscFieldInfoUpdateOffset(pNewQueryInfo); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d48d7d5ea1..ed093e4143 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5463,6 +5463,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE } } assert(0); + return -1; } bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { From 0015100d2bda2cafa1d69ad43018eb05619772d1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 15 Sep 2020 15:40:10 +0000 Subject: [PATCH 074/105] TD-1364 --- src/common/src/tglobal.c | 11 ----- src/plugins/http/src/httpRestJson.c | 28 +++++------- src/plugins/http/src/httpServer.c | 69 +++++++++++++++-------------- src/util/src/tlog.c | 2 +- 4 files changed, 49 insertions(+), 61 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 96e8fb26c6..7e46f58a93 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -957,17 +957,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - // http configs - cfg.option = "httpCacheSessions"; - cfg.ptr = &tsHttpCacheSessions; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 100000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - cfg.option = "httpEnableRecordSql"; cfg.ptr = &tsHttpEnableRecordSql; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index 26f0441519..f8912331a3 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -87,15 +87,12 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; - cmd->numOfRows += numOfRows; - int32_t num_fields = taos_num_fields(result); TAOS_FIELD *fields = taos_fetch_fields(result); for (int32_t k = 0; k < numOfRows; ++k) { TAOS_ROW row = taos_fetch_row(result); if (row == NULL) { - cmd->numOfRows--; continue; } int32_t* length = taos_fetch_lengths(result); @@ -151,24 +148,23 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } // data row array end - httpJsonToken(jsonBuf, JsonArrEnd); - } + httpJsonToken(jsonBuf, JsonArrEnd); + cmd->numOfRows ++; - if (cmd->numOfRows >= tsRestRowLimit) { - httpDebug("context:%p, fd:%d, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext, - pContext->fd, pContext->user, cmd->numOfRows, tsRestRowLimit); - return false; - } else { if (pContext->fd <= 0) { - httpError("context:%p, fd:%d, user:%s, connection is closed, abort retrieve", pContext, pContext->fd, - pContext->user); + httpError("context:%p, fd:%d, user:%s, conn closed, abort retrieve", pContext, pContext->fd, pContext->user); + return false; + } + + if (cmd->numOfRows >= tsRestRowLimit) { + httpDebug("context:%p, fd:%d, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext, + pContext->fd, pContext->user, cmd->numOfRows, tsRestRowLimit); return false; - } else { - httpDebug("context:%p, fd:%d, user:%s, total rows:%d retrieved", pContext, pContext->fd, pContext->user, - cmd->numOfRows); - return true; } } + + httpDebug("context:%p, fd:%d, user:%s, retrieved row:%d", pContext, pContext->fd, pContext->user, cmd->numOfRows); + return true; } bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index d85a236cb1..f0a7249b51 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -315,45 +315,48 @@ static bool httpReadData(HttpContext *pContext) { pContext->accessTimes++; pContext->lastAccessTime = taosGetTimestampSec(); + char buf[HTTP_STEP_SIZE + 1] = {0}; - char buf[HTTP_STEP_SIZE + 1] = {0}; - int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE); - if (nread > 0) { - buf[nread] = '\0'; - httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf); - int32_t ok = httpParseBuf(pParser, buf, nread); + while (1) { + int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE); + if (nread > 0) { + buf[nread] = '\0'; + httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf); + int32_t ok = httpParseBuf(pParser, buf, nread); - if (ok) { - httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok, pParser->parseCode); - httpSendErrorResp(pContext, pParser->parseCode); - httpNotifyContextClose(pContext); - return false; - } + if (ok) { + httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok, + pParser->parseCode); + httpSendErrorResp(pContext, pParser->parseCode); + httpNotifyContextClose(pContext); + return false; + } - if (pParser->parseCode) { - httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode); - httpSendErrorResp(pContext, pParser->parseCode); - httpNotifyContextClose(pContext); - return false; - } + if (pParser->parseCode) { + httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode); + httpSendErrorResp(pContext, pParser->parseCode); + httpNotifyContextClose(pContext); + return false; + } - if (!pParser->parsed) { - httpTrace("context:%p, fd:%d, read not over yet, len:%d", pContext, pContext->fd, pParser->body.pos); - return false; + if (!pParser->parsed) { + httpTrace("context:%p, fd:%d, read not finished", pContext, pContext->fd); + continue; + } else { + httpDebug("context:%p, fd:%d, bodyLen:%d", pContext, pContext->fd, pParser->body.pos); + return true; + } + } else if (nread < 0) { + if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { + httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno); + return false; // later again + } else { + httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno); + return false; + } } else { - httpDebug("context:%p, fd:%d, totalLen:%d", pContext, pContext->fd, pParser->body.pos); - return true; - } - } else if (nread < 0) { - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { - httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno); - return false; // later again - } else { - httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno); + httpError("context:%p, fd:%d, nread:%d, wait another event", pContext, pContext->fd, nread); return false; } - } else { - httpError("context:%p, fd:%d, nread:%d, wait another event", pContext, pContext->fd, nread); - return false; } } diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index a8587de767..e5afe1b68e 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -433,7 +433,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . va_list argpointer; char buffer[MAX_LOGLINE_DUMP_BUFFER_SIZE]; - int32_t len; + int32_t len; struct tm Tm, *ptm; struct timeval timeSecs; time_t curTime; From 4454b077ff2c4fb444657a47fb827fd3274255b0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 15 Sep 2020 23:49:17 +0800 Subject: [PATCH 075/105] [td-1456] --- src/client/src/tscSQLParser.c | 29 +++++++++++------ src/common/src/tname.c | 5 ++- tests/script/general/parser/lastrow_query.sim | 31 +++++++++++++++++++ 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 056ea258bd..517fb32bb3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2601,7 +2601,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { continue; } - if (functionId == TSDB_FUNC_PRJ && pExpr1->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (functionId == TSDB_FUNC_PRJ && (pExpr1->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX || TSDB_COL_IS_UD_COL(pExpr1->colInfo.flag))) { continue; } @@ -5246,7 +5246,8 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) { + if ((pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) && + !(pExpr->functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExpr->colInfo.flag))) { SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex]; getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->functionId, (int32_t)pExpr->param[0].i64Key, &pExpr->resType, &pExpr->resBytes, &pExpr->interBytes, tagLength, true); @@ -5354,16 +5355,23 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) const char* msg1 = "only one selectivity function allowed in presence of tags function"; const char* msg3 = "aggregation function should not be mixed up with projection"; - bool tagColExists = false; + bool tagTsColExists = false; int16_t numOfSelectivity = 0; int16_t numOfAggregation = 0; + // todo is 0?? + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + if (!isSTable) { + return TSDB_CODE_SUCCESS; + } + size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ || (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { - tagColExists = true; // selectivity + ts/tag column + tagTsColExists = true; // selectivity + ts/tag column break; } } @@ -5384,7 +5392,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) } } - if (tagColExists) { // check if the selectivity function exists + if (tagTsColExists) { // check if the selectivity function exists // When the tag projection function on tag column that is not in the group by clause, aggregation function and // selectivity function exist in select clause is not allowed. if (numOfAggregation > 0) { @@ -5407,13 +5415,16 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) * Otherwise, return with error code. */ for (int32_t i = 0; i < numOfExprs; ++i) { - - int16_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; - if (functionId == TSDB_FUNC_TAGPRJ) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + int16_t functionId = pExpr->functionId; + if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == 0) { continue; } - if (((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) && (functionId != TSDB_FUNC_LAST_ROW)) { + if ((functionId == TSDB_FUNC_LAST_ROW) || + (functionId == TSDB_FUNC_LAST_DST && (pExpr->colInfo.flag & TSDB_COL_NULL) != 0)) { + // do nothing + } else { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 49c9e6b726..bb18cb6847 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -62,10 +62,9 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const if (name != NULL) { tstrncpy(s.name, name, sizeof(s.name)); } else { - size_t len = strdequote(exprStr->z); - size_t tlen = MIN(sizeof(s.name), len + 1); - + size_t tlen = MIN(sizeof(s.name), exprStr->n + 1); tstrncpy(s.name, exprStr->z, tlen); + strdequote(s.name); } return s; diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index 5fc47ed15d..1308ac119f 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -172,3 +172,34 @@ sql select last_row(*) from m1 where tbname in ('t1') if $rows != 0 then return -1 endi + +sql insert into t1 values('2019-1-1 1:1:1', 1); +print ===================> last_row query against normal table along with ts/tbname +sql select last_row(*),ts,'k' from t1; +if $rows != 1 then + return -1 +endi + +print ===================> last_row + user-defined column + normal tables +sql select last_row(ts), 'abc', 1234.9384, ts from t1 +if $rows != 1 then + return -1 +endi + +if $data02 != 'abc' then + return -1 +endi + +if $data03 != 1234.938400000 then + return -1 +endi + +if $data04 != @2019-01-01 01:01:01.000@ then + return -1 +endi + +print ===================> last_row + stable + ts/tag column + condition + udf +sql select last_row(*), ts, 'abc', 123.981, tbname from m1 +if $rows != 1 then + return -1 +endi From 5c055cbfcbb0cbe79f047acfad2478767bf59a89 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Sep 2020 00:04:38 +0800 Subject: [PATCH 076/105] [td-1456] update test --- tests/script/general/parser/lastrow_query.sim | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index 1308ac119f..fd2ca3cb8f 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -186,15 +186,17 @@ if $rows != 1 then return -1 endi -if $data02 != 'abc' then +if $data01 != @abc@ then + print expect abc, actual $data02 return -1 endi -if $data03 != 1234.938400000 then +if $data02 != 1234.938400000 then return -1 endi -if $data04 != @2019-01-01 01:01:01.000@ then +if $data03 != @19-01-01 01:01:01.000@ then + print expect 19-01-01 01:01:01.000, actual:$data03 return -1 endi @@ -203,3 +205,16 @@ sql select last_row(*), ts, 'abc', 123.981, tbname from m1 if $rows != 1 then return -1 endi + +if $data02 != @19-01-01 01:01:01.000@ then + return -1 +endi + +if $data03 != @abc@ then + return -1 +endi + +if $data04 != 123.981000000 then + print expect 123.981000000, actual: $data04 + return -1 +endi \ No newline at end of file From 5b208fd47989814ac8556d7b7023f354d3b3f4e1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 15 Sep 2020 16:16:02 +0000 Subject: [PATCH 077/105] TD-1291 --- src/plugins/http/src/httpContext.c | 2 -- src/plugins/http/src/httpServer.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index c0c0c494de..9238869375 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -131,8 +131,6 @@ HttpContext *httpCreateContext(int32_t fd) { HttpContext *httpGetContext(void *ptr) { uint64_t handleVal = (uint64_t)ptr; HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, sizeof(HttpContext *)); - ASSERT(ppContext); - ASSERT(*ppContext); if (ppContext) { HttpContext *pContext = *ppContext; diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index f0a7249b51..cdc6d79a75 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -341,7 +341,7 @@ static bool httpReadData(HttpContext *pContext) { if (!pParser->parsed) { httpTrace("context:%p, fd:%d, read not finished", pContext, pContext->fd); - continue; + return false; } else { httpDebug("context:%p, fd:%d, bodyLen:%d", pContext, pContext->fd, pParser->body.pos); return true; From 8381148d805302870013ae41c7035acedc8c86c2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 16 Sep 2020 00:55:38 +0000 Subject: [PATCH 078/105] TD-1364 --- src/plugins/http/src/httpServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index cdc6d79a75..f0a7249b51 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -341,7 +341,7 @@ static bool httpReadData(HttpContext *pContext) { if (!pParser->parsed) { httpTrace("context:%p, fd:%d, read not finished", pContext, pContext->fd); - return false; + continue; } else { httpDebug("context:%p, fd:%d, bodyLen:%d", pContext, pContext->fd, pParser->body.pos); return true; From 02427c1ef3c974509ab6f5fc6fecc33adfb85050 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Wed, 16 Sep 2020 09:07:35 +0800 Subject: [PATCH 079/105] Update Connections with other Tools-ch.md grafanapulgin dir is changed --- .../webdocs/markdowndocs/Connections with other Tools-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md b/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md index d62a525e28..4e55f3c5e8 100644 --- a/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md +++ b/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md @@ -11,7 +11,7 @@ TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/) ### 配置Grafana -TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafana目录下。 +TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin目录下。 以CentOS 7.2操作系统为例，将tdengine目录拷贝到/var/lib/grafana/plugins目录下，重新启动grafana即可。 From 1446d990d42147e21b52b26b8c14a2abdf80b89c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Sep 2020 09:31:50 +0800 Subject: [PATCH 080/105] [td-1456] --- tests/script/general/parser/lastrow_query.sim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index fd2ca3cb8f..e9d8ce413d 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -154,6 +154,7 @@ if $rows != 46 then endi print ========>td-1317, empty table last_row query crashed +sql drop table if exists m1; sql create table m1(ts timestamp, k int) tags (a int); sql create table t1 using m1 tags(1); sql create table t2 using m1 tags(2); From ab198e6fd7737aa87da48b2d9e32b215dd10e477 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 16 Sep 2020 10:47:47 +0800 Subject: [PATCH 081/105] disable warning for self assign. --- cmake/define.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/define.inc b/cmake/define.inc index 84df6f5405..5069fb0512 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -118,7 +118,7 @@ IF (TD_DARWIN_64) ADD_DEFINITIONS(-DDARWIN) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-self-assign -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") SET(DEBUG_FLAGS "-O0 -DDEBUG") SET(RELEASE_FLAGS "-O0") ENDIF () From ce458cf037c0dae18a5d552baa8b136adb0d4fda Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 16 Sep 2020 11:00:48 +0800 Subject: [PATCH 082/105] [TD-1468] --- src/kit/shell/src/shellEngine.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 277dc45f8e..1d1ca1c42b 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -765,7 +765,9 @@ void read_history() { FILE *f = fopen(f_history, "r"); if (f == NULL) { #ifndef WINDOWS - fprintf(stderr, "Failed to open file %s\n", f_history); + if (errno != ENOENT) { + fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno)); + } #endif return; } @@ -792,7 +794,7 @@ void write_history() { FILE *f = fopen(f_history, "w"); if (f == NULL) { #ifndef WINDOWS - fprintf(stderr, "Failed to open file %s for write\n", f_history); + fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); #endif return; } From e7099d55181d3c6e06c125ff93069733ba4545a2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 16 Sep 2020 11:02:36 +0800 Subject: [PATCH 083/105] add void cast to solve unused variables. --- cmake/define.inc | 2 +- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscUtil.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 5069fb0512..84df6f5405 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -118,7 +118,7 @@ IF (TD_DARWIN_64) ADD_DEFINITIONS(-DDARWIN) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-self-assign -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") SET(DEBUG_FLAGS "-O0 -DDEBUG") SET(RELEASE_FLAGS "-O0") ENDIF () diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f9f72f9174..165ee7acc5 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5271,7 +5271,7 @@ static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { } assert(qualifiedCol); - qualifiedCol = qualifiedCol; + (void) qualifiedCol; } } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 636fccd3a0..a78bedb3d9 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1788,7 +1788,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p } assert(matched); - matched = matched; + (void)matched; } tscFieldInfoUpdateOffset(pNewQueryInfo); From 5e6006f533ae57240c2c6771cb95192dcf553487 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Sep 2020 11:28:17 +0800 Subject: [PATCH 084/105] [td-225] fix compiler errors. --- src/client/src/tscSubquery.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 7ea5d4183b..5e912d7f6e 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1947,16 +1947,22 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) { } int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; - + SSqlRes *pRes = &pSql->res; + size_t size = taosArrayGetSize(pCmd->pDataBlocks); assert(size > 0); // the number of already initialized subqueries int32_t numOfSub = 0; + SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + pState->numOfTotal = pSql->numOfSubs; + pState->numOfRemain = pSql->numOfSubs; + pSql->numOfSubs = (uint16_t)size; + pRes->code = TSDB_CODE_SUCCESS; + pSql->pSubs = calloc(size, POINTER_BYTES); if (pSql->pSubs == NULL) { goto _error; @@ -1964,12 +1970,6 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); - SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); - pState->numOfTotal = pSql->numOfSubs; - pState->numOfRemain = pSql->numOfSubs; - - pRes->code = TSDB_CODE_SUCCESS; - while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); if (pSupporter == NULL) { From b002f0be5107e27fa6c141ac1aed4f5cf2935119 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Sep 2020 12:54:00 +0800 Subject: [PATCH 085/105] [td-225] --- src/client/src/tscSubquery.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 5e912d7f6e..0706e10aa9 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1950,8 +1950,10 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - size_t size = taosArrayGetSize(pCmd->pDataBlocks); - assert(size > 0); + pSql->numOfSubs = taosArrayGetSize(pCmd->pDataBlocks); + assert(pSql->numOfSubs > 0); + + pRes->code = TSDB_CODE_SUCCESS; // the number of already initialized subqueries int32_t numOfSub = 0; @@ -1960,15 +1962,12 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { pState->numOfTotal = pSql->numOfSubs; pState->numOfRemain = pSql->numOfSubs; - pSql->numOfSubs = (uint16_t)size; - pRes->code = TSDB_CODE_SUCCESS; - - pSql->pSubs = calloc(size, POINTER_BYTES); + pSql->pSubs = calloc(pSql->numOfSubs, POINTER_BYTES); if (pSql->pSubs == NULL) { goto _error; } - tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); + tscDebug("%p submit data to %d vnode(s)", pSql, pSql->numOfSubs); while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); @@ -1999,8 +1998,8 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { tscDebug("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, numOfSub); numOfSub++; } else { - tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%" PRIzu ", code:%s", pSql, numOfSub, - size, tstrerror(pRes->code)); + tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql, numOfSub, + pSql->numOfSubs, tstrerror(pRes->code)); goto _error; } } From d545eb9bb412901138b0573399fb9fc5815311e1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Sep 2020 15:01:10 +0800 Subject: [PATCH 086/105] [td-225] fix compiler error in win platform --- src/util/src/tsocket.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 479eeaa754..4cf73e6dff 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -56,13 +56,16 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) { freeaddrinfo(result); return ip; } else { +#ifdef EAI_SYSTEM if (ret == EAI_SYSTEM) { uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); } else { uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); } - +#else + uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); +#endif return 0xFFFFFFFF; } } From 848a37aafe759b40d5ef1bd853ded2b09ac59c38 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Sep 2020 15:05:57 +0800 Subject: [PATCH 087/105] [td-225] fix compiler error in win platform --- src/query/src/qPercentile.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index ccc451108e..8d74dbbdc2 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -79,13 +79,13 @@ static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, d case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: - range->iMinVal = minval; - range->iMaxVal = maxval; + range->iMinVal = (int32_t) minval; + range->iMaxVal = (int32_t) maxval; break; case TSDB_DATA_TYPE_BIGINT: - range->i64MinVal = minval; - range->i64MaxVal = maxval; + range->i64MinVal = (int64_t) minval; + range->i64MaxVal = (int64_t) maxval; break; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: From d7f409fd9ac27c8b7a2b2f952f608ab73936be0c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Sep 2020 15:09:43 +0800 Subject: [PATCH 088/105] [td-225] fix compiler error in win platform --- src/query/src/qPercentile.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index 8d74dbbdc2..3a8be781d5 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -740,8 +740,10 @@ double getPercentile(tMemBucket *pMemBucket, double percent) { case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: return fabs(percent - 100) < DBL_EPSILON? pRange->iMaxVal:pRange->iMinVal; - case TSDB_DATA_TYPE_BIGINT: - return fabs(percent - 100) < DBL_EPSILON? pRange->i64MaxVal:pRange->i64MinVal; + case TSDB_DATA_TYPE_BIGINT: { + double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal); + return v; + } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal; From b9db2aafb66a52d1e9f70b94878a568f4c1ffe34 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 16 Sep 2020 15:35:38 +0800 Subject: [PATCH 089/105] Update Taos error code --- documentation20/webdocs/markdowndocs/Taos Error Code-ch.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md b/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md index 6ba0decfd3..a9a3dee432 100644 --- a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md +++ b/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md @@ -1,7 +1,7 @@ # TDengine 2.0 错误码以及对应的十进制码 -| Code | bit | error code | 错误描述 | 十进制错误码 | +| 状态码 | 模 | 错误码（十六进制） | 错误描述 | 错误码（十进制） | |-----------------------| :---: | :---------: | :------------------------ | ---------------- | |TSDB_CODE_RPC_ACTION_IN_PROGRESS| 0 | 0x0001| "Action in progress"| -2147483647| |TSDB_CODE_RPC_AUTH_REQUIRED| 0 | 0x0002 | "Authentication required"| -2147483646| @@ -87,7 +87,7 @@ |TSDB_CODE_MND_INVALID_ACCT_OPTION| 0 | 0x0342 | "Invalid account options"| -2147482814| |TSDB_CODE_MND_USER_ALREADY_EXIST| 0 | 0x0350 | "User already exists"| -2147482800| |TSDB_CODE_MND_INVALID_USER |0 | 0x0351 | "Invalid user" |-2147482799| -|TSDB_CODE_MND_INVALID_USER_FORMAT| |0 |0x0352 |"Invalid user format" |-2147482798| +|TSDB_CODE_MND_INVALID_USER_FORMAT| 0 |0x0352 |"Invalid user format" |-2147482798| |TSDB_CODE_MND_INVALID_PASS_FORMAT| 0| 0x0353 | "Invalid password format"| -2147482797| |TSDB_CODE_MND_NO_USER_FROM_CONN| 0 | 0x0354 | "Can not get user from conn"| -2147482796| |TSDB_CODE_MND_TOO_MANY_USERS| 0 | 0x0355| "Too many users"| -2147482795| @@ -107,7 +107,7 @@ |TSDB_CODE_MND_DB_NOT_SELECTED| 0 | 0x0380 | "Database not specified or available"| -2147482752| |TSDB_CODE_MND_DB_ALREADY_EXIST| 0 | 0x0381 | "Database already exists"| -2147482751| |TSDB_CODE_MND_INVALID_DB_OPTION| 0 | 0x0382 | "Invalid database options"| -2147482750| -|TSDB_CODE_MND_INVALID_DB| |0 | 0x0383 | "Invalid database name"| -2147482749| +|TSDB_CODE_MND_INVALID_DB| 0 | 0x0383 | "Invalid database name"| -2147482749| |TSDB_CODE_MND_MONITOR_DB_FORBIDDEN| 0 | 0x0384 | "Cannot delete monitor database"| -2147482748| |TSDB_CODE_MND_TOO_MANY_DATABASES| 0| 0x0385 | "Too many databases for account"| -2147482747| |TSDB_CODE_MND_DB_IN_DROPPING| 0 | 0x0386| "Database not available" |-2147482746| From 9d0bb483f018c2ab4c5cb49261548b0e515d7c91 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 16 Sep 2020 16:08:31 +0800 Subject: [PATCH 090/105] [TD-1472] --- alert/cmd/alert/install_driver.sh | 35 ++++++++++++++----------------- alert/cmd/alert/main.go | 4 ++-- alert/release.sh | 27 +++++++++++++++++------- 3 files changed, 38 insertions(+), 28 deletions(-) mode change 100644 => 100755 alert/cmd/alert/install_driver.sh diff --git a/alert/cmd/alert/install_driver.sh b/alert/cmd/alert/install_driver.sh old mode 100644 new mode 100755 index 1171a99538..c7d44786ec --- a/alert/cmd/alert/install_driver.sh +++ b/alert/cmd/alert/install_driver.sh @@ -9,9 +9,7 @@ set -e script_dir=$(dirname $(readlink -f "$0")) # Dynamic directory lib_link_dir="/usr/lib" - -#install main path -install_main_dir="/usr/local/taos" +lib64_link_dir="/usr/lib64" # Color setting RED='\033[0;31m' @@ -25,24 +23,23 @@ if command -v sudo > /dev/null; then csudo="sudo" fi -function clean_driver() { - ${csudo} rm -f /usr/lib/libtaos.so || : -} - function install_driver() { - echo -e "${GREEN}Start to install TDengine client driver ...${NC}" - - #create install main dir and all sub dir - ${csudo} mkdir -p ${install_main_dir} - ${csudo} mkdir -p ${install_main_dir}/driver - - ${csudo} rm -f ${lib_link_dir}/libtaos.* || : - ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 - ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - echo - echo -e "\033[44;32;1mTDengine client driver is successfully installed!${NC}" + if [[ -d ${lib_link_dir} && ! -e ${lib_link_dir}/libtaos.so ]]; then + echo -e "${GREEN}Start to install TDengine client driver ...${NC}" + ${csudo} ln -s ${script_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so || : + + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo} ln -s ${script_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + + echo + echo -e "${GREEN}TDengine client driver is successfully installed!${NC}" + else + echo -e "${GREEN}TDengine client driver already exists, Please confirm whether the alert version matches the client driver version!${NC}" + fi } install_driver diff --git a/alert/cmd/alert/main.go b/alert/cmd/alert/main.go index f4c30e156a..1bd185b580 100644 --- a/alert/cmd/alert/main.go +++ b/alert/cmd/alert/main.go @@ -119,7 +119,7 @@ WantedBy=multi-user.target return nil } -const version = "TDengine alert v2.0.0.1" +var version = "2.0.0.1s" func main() { var ( @@ -133,7 +133,7 @@ func main() { flag.Parse() if showVersion { - fmt.Println(version) + fmt.Println("TDengine alert v" + version) return } diff --git a/alert/release.sh b/alert/release.sh index 3342b0e567..35eb4d677f 100755 --- a/alert/release.sh +++ b/alert/release.sh @@ -6,9 +6,9 @@ set -e # set parameters by default value cpuType=amd64 # [armv6l | arm64 | amd64 | 386] osType=linux # [linux | darwin | windows] - +version="" declare -A archMap=(["armv6l"]="arm" ["arm64"]="arm64" ["amd64"]="x64" ["386"]="x86") -while getopts "h:c:o:" arg +while getopts "h:c:o:n:" arg do case $arg in c) @@ -19,6 +19,10 @@ do #echo "osType=$OPTARG" osType=$(echo $OPTARG) ;; + n) + #echo "version=$OPTARG" + version=$(echo $OPTARG) + ;; h) echo "Usage: `basename $0` -c [armv6l | arm64 | amd64 | 386] -o [linux | darwin | windows]" exit 0 @@ -30,18 +34,27 @@ do esac done +if [ "$version" == "" ]; then + echo "Please input the correct version!" + exit 1 +fi startdir=$(pwd) scriptdir=$(dirname $(readlink -f $0)) cd ${scriptdir}/cmd/alert -version=$(grep 'const version =' main.go | awk '{print $NF}') -version=${version%\"} -version=${version:1} echo "cpuType=${cpuType}" echo "osType=${osType}" echo "version=${version}" -GOOS=${osType} GOARCH=${cpuType} go build +GOOS=${osType} GOARCH=${cpuType} go build -ldflags '-X main.version='${version} + +mkdir -p TDengine-alert/driver + +cp alert alert.cfg install_driver.sh ./TDengine-alert/. +cp ../../../debug/build/lib/libtaos.so.${version} ./TDengine-alert/driver/. +chmod 777 ./TDengine-alert/install_driver.sh + +tar -I 'gzip -9' -cf ${startdir}/TDengine-alert-${version}-${osType^}-${archMap[${cpuType}]}.tar.gz TDengine-alert/ +rm -rf ./TDengine-alert -tar -I 'gzip -9' -cf ${startdir}/TDengine-alert-${version}-${osType^}-${archMap[${cpuType}]}.tar.gz alert alert.cfg install_driver.sh driver/ From 6ea7e4ee84f8ab02f2b6a50b66fc1b525fc6aebc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Sep 2020 18:36:58 +0800 Subject: [PATCH 091/105] [td-1478] --- src/client/src/tscSubquery.c | 2 +- src/query/src/qExecutor.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 0706e10aa9..4246abf52d 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -92,7 +92,7 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); #ifdef _DEBUG_VIEW - tscInfo("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); + tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key); #endif int32_t res = tVariantCompare(&elem1.tag, &elem2.tag); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b5adeafd23..b873714c49 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2836,6 +2836,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { // all results have been return to client, try next group if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) { pGroupResInfo->numOfDataPages = 0; + pGroupResInfo->pos.pageId = 0; pGroupResInfo->pos.rowId = 0; // current results of group has been sent to client, try next group From 36fda719df7c28030e6ec32e416889a5ca5a42b2 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 16 Sep 2020 18:42:47 +0800 Subject: [PATCH 092/105] [TD-1478] add test case query groupby with interval --- tests/pytest/fulltest.sh | 1 + .../pytest/query/queryGroupbyWithInterval.py | 53 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 tests/pytest/query/queryGroupbyWithInterval.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index f6c68378ea..bfa546f92e 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -186,6 +186,7 @@ python3 ./test.py -f functions/function_sum.py python3 ./test.py -f functions/function_top.py #python3 ./test.py -f functions/function_twa.py python3 queryCount.py +python3 ./test.py -f query/queryGroupbyWithInterval.py # tools python3 test.py -f tools/taosdemo.py diff --git a/tests/pytest/query/queryGroupbyWithInterval.py b/tests/pytest/query/queryGroupbyWithInterval.py new file mode 100644 index 0000000000..14f6999021 --- /dev/null +++ b/tests/pytest/query/queryGroupbyWithInterval.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdSql.execute( + "create table stest(ts timestamp,size INT,filenum INT) tags (appname binary(500),tenant binary(500))") + tdSql.execute( + "insert into test1 using stest tags('test1','aaa') values ('2020-09-04 16:53:54.003',210,3)") + tdSql.execute( + "insert into test2 using stest tags('test1','aaa') values ('2020-09-04 16:53:56.003',210,3)") + tdSql.execute( + "insert into test11 using stest tags('test11','bbb') values ('2020-09-04 16:53:57.003',210,3)") + tdSql.execute( + "insert into test12 using stest tags('test11','bbb') values ('2020-09-04 16:53:58.003',210,3)") + tdSql.execute( + "insert into test21 using stest tags('test21','ccc') values ('2020-09-04 16:53:59.003',210,3)") + tdSql.execute( + "insert into test22 using stest tags('test21','ccc') values ('2020-09-04 16:54:54.003',210,3)") + + tdSql.query("select sum(size) from stest interval(1d) group by appname") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 22000cf6481f3486b3f1b88790104b5f4b780de1 Mon Sep 17 00:00:00 2001 From: Bo Xiao <69349626+boxiaobj@users.noreply.github.com> Date: Wed, 16 Sep 2020 19:24:20 +0800 Subject: [PATCH 093/105] TDengine cluster document modification Add DATA Node Many minor modifications relate to firstEP and etc. --- .../webdocs/markdowndocs/cluster-ch.md | 154 +++++++++++------- 1 file changed, 97 insertions(+), 57 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index a1ac1d6fd6..193de729ba 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -1,48 +1,69 @@ # TDengine 集群安装、管理 -多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。而且在安装集群之前，请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验过单节点功能。 +多个TDengine服务器，也就是多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。而且在安装集群之前，先请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验单节点功能。 -集群的每个节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令`hostname -f`获取,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。端口是这个节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。一个节点可能配置了多个hostname, TDengine会自动获取第一个，但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问，可以将参数fqdn设置为本节点的IP地址。 +集群的每个数据节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令`hostname -f`获取,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。端口是这个数据节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。一个物理节点可能配置了多个hostname, TDengine会自动获取第一个，但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问，可以将参数fqdn设置为本节点的IP地址。 TDengine的集群管理极其简单，除添加和删除节点需要人工干预之外，其他全部是自动完成，最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 ## 准备工作 -**第一步**：如果搭建集群的节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) +**第零步**：规划集群所有物理节点的FQDN，将规划好的FQDN分别添加到每个物理节点的/etc/hostname；修改每个物理节点的/etc/hosts，将所有集群物理节点的IP与FQDN的对应添加好【如部署了DNS，请联系网络管理员在DNS上做好相关配置】； + +**第一步**：如果搭建集群的物理节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) **注意1：**因为FQDN的信息会写进文件，如果之前没有配置或者更改FQDN，且启动了TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（rm -rf /var/lib/taos/）； **注意2：**客户端也需要配置，确保它可以正确解析每个节点的FQDN配置，不管是通过DNS服务，还是 Host 文件。 -**第二步**：建议关闭防火墙，至少保证端口：6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙，集群搭建完毕之后，再来配置端口； +**第二步**：建议关闭所有物理节点的防火墙，至少保证端口：6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙，集群搭建完毕之后，再来配置端口； -**第三步**：在所有节点安装TDengine，且版本必须是一致的，**但不要启动taosd**； +**第三步**：在所有节点安装TDengine，且版本必须是一致的，**但不要启动taosd**。安装时，提示输入是否要加入一个已经存在的TDengine集群时，第一个物理节点直接回车创建新集群，后续物理节点则输入该集群任何一个在线的物理节点的FQDN:端口号(默认6030)； -**第四步**：检查、配置所有节点的FQDN： +**第四步**：检查所有数据节点，以及应用所在物理节点的网络设置： -1. 每个节点上执行命令`hostname -f`，查看和确认所有节点的hostname是不相同的； -2. 每个节点上执行`ping host`, 其中host是其他节点的hostname, 看能否ping通其它节点; 如果不能ping通，需要检查网络设置, 或/etc/hosts文件，或DNS的配置。如果无法ping通，是无法组成集群的。 -3. 每个节点的FQDN就是输出的hostname外加端口号，比如h1.taosdata.com:6030 +1. 每个物理节点上执行命令`hostname -f`，查看和确认所有节点的hostname是不相同的(应用驱动所在节点无需做此项检查)； +2. 每个物理节点上执行`ping host`, 其中host是其他物理节点的hostname, 看能否ping通其它物理节点; 如果不能ping通，需要检查网络设置, 或/etc/hosts文件(Windows系统默认路径为C:\Windows\system32\drivers\etc\hosts)，或DNS的配置。如果无法ping通，是无法组成集群的； +3. 每个数据节点的End Point就是输出的hostname外加端口号，比如h1.taosdata.com:6030 -**第五步**：修改TDengine的配置文件（所有节点的文件/etc/taos/taos.cfg都需要修改）。假设准备启动的第一个节点End Point为 h1.taosdata.com:6030, 那么以下几个参数与集群相关： +**第五步**：修改TDengine的配置文件（所有节点的文件/etc/taos/taos.cfg都需要修改）。假设准备启动的第一个数据节点End Point为 h1.taosdata.com:6030, 其与集群配置相关参数如下： ``` -// firstEp 集群中所有节点的配置都是一致的，对其第一次访问后，就获得了整个集群的信息 +// firstEp 是每个数据节点首次启动后连接的第一个数据节点 firstEp h1.taosdata.com:6030 -// 配置本节点的FQDN，如果本机只有一个hostname, 无需配置 +// 配置本数据节点的FQDN，如果本机只有一个hostname, 无需配置 fqdn h1.taosdata.com -// 配置本节点的端口号，缺省是6030 +// 配置本数据节点的端口号，缺省是6030 serverPort 6030 // 服务端节点数为偶数的时候，需要配置，请参考《Arbitrator的使用》的部分 arbitrator ha.taosdata.com:6042 ``` -一定要修改的参数是firstEp, 其他参数可不做任何修改，除非你很清楚为什么要修改。 +一定要修改的参数是firstEp和fqdn, 其他参数可不做任何修改，除非你很清楚为什么要修改。 -## 启动第一个节点 +**加入到集群中的数据节点dnode，涉及集群相关的下表11项参数必须完全相同，否则不能成功加入到集群中。** + +| **#** | **配置参数名称** | **含义** | +| ----- | ------------------ | ---------------------------------------- | +| 1 | numOfMnodes | 系统中管理节点个数 | +| 2 | mnodeEqualVnodeNum | 一个mnode等同于vnode消耗的个数 | +| 3 | offlineThreshold | dnode离线阈值，超过该时间将导致Dnode离线 | +| 4 | statusInterval | dnode向mnode报告状态时长 | +| 5 | arbitrator | 系统中裁决器的end point | +| 6 | timezone | 时区 | +| 7 | locale | 系统区位信息及编码格式 | +| 8 | charset | 字符集编码 | +| 9 | balance | 是否启动负载均衡 | +| 10 | maxTablesPerVnode | 每个vnode中能够创建的最大表个数 | +| 11 | maxVgroupsPerDb | 每个DB中 能够使用的最大vnode个数 | + + + +## 启动第一个数据节点 + +按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)里的指示，启动第一个数据节点，例如h1.taosdata.com，然后执行taos, 启动taos shell，从shell里执行命令"show dnodes;"，如下所示： -按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)里的指示，启动第一个节点h1.taosdata.com，然后执行taos, 启动taos shell，从shell里执行命令"show dnodes;"，如下所示： ``` Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. @@ -55,74 +76,86 @@ Query OK, 1 row(s) in set (0.006385s) taos> ``` -上述命令里，可以看到这个刚启动的这个节点的End Point是：h1.taos.com:6030 -## 启动后续节点 +上述命令里，可以看到这个刚启动的这个数据节点的End Point是：h1.taos.com:6030，就是这个新集群的firstEP。 -将后续的节点添加到现有集群，具体有以下几步： +## 启动后续数据节点 -1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法在每个节点启动taosd。 +将后续的数据节点添加到现有集群，具体有以下几步： -2. 在第一个节点，使用CLI程序taos, 登录进TDengine系统, 执行命令: +1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法在每个物理节点启动taosd； - ``` - CREATE DNODE "h2.taos.com:6030"； - ``` +2. 在第一个数据节点，使用CLI程序taos, 登录进TDengine系统, 执行命令: - 将新节点的End Point (准备工作中第四步获知的) 添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。请注意将示例的“h2.taos.com:6030" 替换为这个新节点的End Point。 + ``` + CREATE DNODE "h2.taos.com:6030"； + ``` + + 将新数据节点的End Point (准备工作中第四步获知的) 添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。请注意将示例的“h2.taos.com:6030" 替换为这个新数据节点的End Point。 3. 然后执行命令 - ``` - SHOW DNODES； - ``` + ``` + SHOW DNODES； + ``` - 查看新节点是否被成功加入。如果该被加入的节点处于离线状态，请做两个检查 - - - 查看该节点的taosd是否正常工作，如果没有正常运行，需要先检查为什么 - - 查看该节点taosd日志文件taosdlog.0里前面几行日志(一般在/var/log/taos目录)，看日志里输出的该节点fqdn以及端口号是否为刚添加的End Point。如果不一致，需要将正确的End Point添加进去。 + 查看新节点是否被成功加入。如果该被加入的数据节点处于离线状态，请做两个检查 -按照上述步骤可以源源不断的将新的节点加入到集群。 + - 查看该数据节点的taosd是否正常工作，如果没有正常运行，需要先检查为什么 + - 查看该数据节点taosd日志文件taosdlog.0里前面几行日志(一般在/var/log/taos目录)，看日志里输出的该数据节点fqdn以及端口号是否为刚添加的End Point。如果不一致，需要将正确的End Point添加进去。 + +按照上述步骤可以源源不断的将新的数据节点加入到集群。 **提示：** -- firstEp这个参数仅仅在该节点第一次加入集群时有作用，加入集群后，该节点会保存最新的mnode的End Point列表，不再依赖这两个参数。 -- 两个没有配置firstEp参数的dnode启动后，会独立运行起来。这个时候，无法将其中一个节点加入到另外一个节点，形成集群。**无法将两个独立的集群合并成为新的集群**。 +- 任何已经加入集群在线的数据节点，都可以作为后续待加入节点的firstEP。 +- firstEp这个参数仅仅在该数据节点首次加入集群时有作用，加入集群后，该数据节点会保存最新的mnode的End Point列表，不再依赖这个参数。 +- 两个没有配置firstEp参数的数据节点dnode启动后，会独立运行起来。这个时候，无法将其中一个数据节点加入到另外一个数据节点，形成集群。**无法将两个独立的集群合并成为新的集群**。 -## 节点管理 +## 数据节点管理 + +### 添加数据节点 -### 添加节点 执行CLI程序taos, 使用root账号登录进系统, 执行: + ``` CREATE DNODE "fqdn:port"； ``` -将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。一个节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置，缺省是自动获取。 -### 删除节点 +将新数据节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。一个数据节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置，缺省是自动获取。【强烈不建议用自动获取方式来配置FQDN，可能导致生成的数据节点的End Point不是所期望的】 + +### 删除数据节点 + 执行CLI程序taos, 使用root账号登录进TDengine系统，执行： ``` DROP DNODE "fqdn:port"; ``` + 其中fqdn是被删除的节点的FQDN，port是其对外服务器的端口号 -### 查看节点 +### 查看数据节点 + 执行CLI程序taos,使用root账号登录进TDengine系统，执行： ``` SHOW DNODES; ``` -它将列出集群中所有的dnode,每个dnode的fqdn:port, 状态(ready, offline等），vnode数目，还未使用的vnode数目等信息。在添加或删除一个节点后，可以使用该命令查看。 + +它将列出集群中所有的dnode,每个dnode的fqdn:port, 状态(ready, offline等），vnode数目，还未使用的vnode数目等信息。在添加或删除一个数据节点后，可以使用该命令查看。 ### 查看虚拟节点组 -为充分利用多核技术，并提供scalability，数据需要分片处理。因此TDengine会将一个DB的数据切分成多份，存放在多个vnode里。这些vnode可能分布在多个dnode里，这样就实现了水平扩展。一个vnode仅仅属于一个DB，但一个DB可以有多个vnode。vnode的是mnode根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 +为充分利用多核技术，并提供scalability，数据需要分片处理。因此TDengine会将一个DB的数据切分成多份，存放在多个vnode里。这些vnode可能分布在多个数据节点dnode里，这样就实现了水平扩展。一个vnode仅仅属于一个DB，但一个DB可以有多个vnode。vnode的是mnode根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 执行CLI程序taos,使用root账号登录进TDengine系统，执行： + ``` SHOW VGROUPS; ``` + ## vnode的高可用性 + TDengine通过多副本的机制来提供系统的高可用性，包括vnode和mnode的高可用性。 vnode的副本数是与DB关联的，一个集群里可以有多个DB，根据运营的需求，每个DB可以配置不同的副本数。创建数据库时，通过参数replica 指定副本数（缺省为1）。如果副本数为1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed"。比如下面的命令将创建副本数为3的数据库demo： @@ -130,21 +163,25 @@ vnode的副本数是与DB关联的，一个集群里可以有多个DB，根据 ``` CREATE DATABASE demo replica 3; ``` -一个DB里的数据会被切片分到多个vnode group，vnode group里的vnode数目就是DB的副本数，同一个vnode group里各vnode的数据是完全一致的。为保证高可用性，vnode group里的vnode一定要分布在不同的dnode里（实际部署时，需要在不同的物理机上），只要一个vgroup里超过半数的vnode处于工作状态，这个vgroup就能正常的对外服务。 -一个dnode里可能有多个DB的数据，因此一个dnode离线时，可能会影响到多个DB。如果一个vnode group里的一半或一半以上的vnode不工作，那么该vnode group就无法对外服务，无法插入或读取数据，这样会影响到它所属的DB的一部分表的读写操作。 +一个DB里的数据会被切片分到多个vnode group，vnode group里的vnode数目就是DB的副本数，同一个vnode group里各vnode的数据是完全一致的。为保证高可用性，vnode group里的vnode一定要分布在不同的数据节点dnode里（实际部署时，需要在不同的物理机上），只要一个vgroup里超过半数的vnode处于工作状态，这个vgroup就能正常的对外服务。 -因为vnode的引入，无法简单的给出结论：“集群中过半dnode工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为3，只有三个dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个节点不工作，那整个集群就无法正常工作了。 +一个数据节点dnode里可能有多个DB的数据，因此一个dnode离线时，可能会影响到多个DB。如果一个vnode group里的一半或一半以上的vnode不工作，那么该vnode group就无法对外服务，无法插入或读取数据，这样会影响到它所属的DB的一部分表的读写操作。 + +因为vnode的引入，无法简单的给出结论：“集群中过半数据节点dnode工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为3，只有三个dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个数据节点不工作，那整个集群就无法正常工作了。 ## Mnode的高可用性 -TDengine集群是由mnode (taosd的一个模块，逻辑节点) 负责管理的，为保证mnode的高可用，可以配置多个mnode副本，副本数由系统配置参数numOfMnodes决定，有效范围为1-3。为保证元数据的强一致性，mnode副本之间是通过同步的方式进行数据复制的。 -一个集群有多个dnode, 但一个dnode至多运行一个mnode实例。多个dnode情况下，哪个dnode可以作为mnode呢？这是完全由系统根据整个系统资源情况，自动指定的。用户可通过CLI程序taos，在TDengine的console里，执行如下命令： +TDengine集群是由mnode (taosd的一个模块，管理节点) 负责管理的，为保证mnode的高可用，可以配置多个mnode副本，副本数由系统配置参数numOfMnodes决定，有效范围为1-3。为保证元数据的强一致性，mnode副本之间是通过同步的方式进行数据复制的。 + +一个集群有多个数据节点dnode, 但一个dnode至多运行一个mnode实例。多个dnode情况下，哪个dnode可以作为mnode呢？这是完全由系统根据整个系统资源情况，自动指定的。用户可通过CLI程序taos，在TDengine的console里，执行如下命令： + ``` SHOW MNODES; ``` + 来查看mnode列表，该列表将列出mnode所处的dnode的End Point和角色(master, slave, unsynced 或offline)。 -当集群中第一个节点启动时，该节点一定会运行一个mnode实例，否则该dnode无法正常工作，因为一个系统是必须有至少一个mnode的。如果numOfMnodes配置为2，启动第二个dnode时，该dnode也将运行一个mnode实例。 +当集群中第一个数据节点启动时，该数据节点一定会运行一个mnode实例，否则该数据节点dnode无法正常工作，因为一个系统是必须有至少一个mnode的。如果numOfMnodes配置为2，启动第二个dnode时，该dnode也将运行一个mnode实例。 为保证mnode服务的高可用性，numOfMnodes必须设置为2或更大。因为mnode保存的元数据必须是强一致的，如果numOfMnodes大于2，复制参数quorum自动设为2，也就是说，至少要保证有两个副本写入数据成功，才通知客户端应用写入成功。 @@ -154,22 +191,25 @@ SHOW MNODES; 有三种情况，将触发负载均衡，而且都无需人工干预。 -- 当一个新节点添加进集群时，系统将自动触发负载均衡，一些节点上的数据将被自动转移到新节点上，无需任何人工干预。 -- 当一个节点从集群中移除时，系统将自动把该节点上的数据转移到其他节点，无需任何人工干预。 -- 如果一个节点过热（数据量过大），系统将自动进行负载均衡，将该节点的一些vnode自动挪到其他节点。 +- 当一个新数据节点添加进集群时，系统将自动触发负载均衡，一些节点上的数据将被自动转移到新数据节点上，无需任何人工干预。 +- 当一个数据节点从集群中移除时，系统将自动把该数据节点上的数据转移到其他数据节点，无需任何人工干预。 +- 如果一个数据节点过热（数据量过大），系统将自动进行负载均衡，将该数据节点的一些vnode自动挪到其他节点。 -当上述三种情况发生时，系统将启动一各个节点的负载计算，从而决定如何挪动。 +当上述三种情况发生时，系统将启动一各个数据节点的负载计算，从而决定如何挪动。 -##节点离线处理 -如果一个节点离线，TDengine集群将自动检测到。有如下两种情况： -- 改节点离线超过一定时间（taos.cfg里配置参数offlineThreshold控制时长)，系统将自动把该节点删除，产生系统报警信息，触发负载均衡流程。如果该被删除的节点重现上线时，它将无法加入集群，需要系统管理员重新将其添加进集群才会开始工作。 +**【提示】负载均衡由参数balance控制，它决定是否启动自动负载均衡。** + +## 数据节点离线处理 + +如果一个数据节点离线，TDengine集群将自动检测到。有如下两种情况： + +- 该数据节点离线超过一定时间（taos.cfg里配置参数offlineThreshold控制时长)，系统将自动把该数据节点删除，产生系统报警信息，触发负载均衡流程。如果该被删除的数据节点重现上线时，它将无法加入集群，需要系统管理员重新将其添加进集群才会开始工作。 - 离线后，在offlineThreshold的时长内重新上线，系统将自动启动数据恢复流程，等数据完全恢复后，该节点将开始正常工作。 -**注意：**如果一个虚拟节点组（包括mnode组）里每个节点都处于离线或unsynced状态，必须等该虚拟节点组里的所有节点都上线、都能交换状态信息后，才能选出Master，该虚拟节点组才能对外提供服务。比如整个集群有3个节点，副本数为3，如果3个节点都宕机，然后2个节点重启，是无法工作的，只有等3个节点都重启成功，才能对外服务。 +**注意：**如果一个虚拟节点组（包括mnode组）里所归属的每个数据节点都处于离线或unsynced状态，必须等该虚拟节点组里的所有数据节点都上线、都能交换状态信息后，才能选出Master，该虚拟节点组才能对外提供服务。比如整个集群有3个数据节点，副本数为3，如果3个数据节点都宕机，然后2个数据节点重启，是无法工作的，只有等3个数据节点都重启成功，才能对外服务。 ## Arbitrator的使用 如果副本数为偶数，当一个vnode group里一半或超过一半的vnode不工作时，是无法从中选出master的。同理，一半或超过一半的mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。 TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。 - From 4456951e6665da090ee8017f76156107e55cfb0c Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 10 Sep 2020 17:40:22 +0800 Subject: [PATCH 094/105] td-1245: update sql grammar to support offset --- src/client/inc/tsclient.h | 3 +- src/client/src/tscSQLParser.c | 42 + src/inc/taosmsg.h | 3 +- src/query/inc/qSqlparser.h | 8 +- src/query/inc/sql.y | 7 +- src/query/src/qParserImpl.c | 5 +- src/query/src/sql.c | 2261 +++++++++++++++++++-------------- src/util/src/tarray.c | 0 8 files changed, 1334 insertions(+), 995 deletions(-) mode change 100755 => 100644 src/util/src/tarray.c diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 74a0e8c11c..d98826e339 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -228,10 +228,11 @@ typedef struct SQueryInfo { // TODO refactor char intervalTimeUnit; char slidingTimeUnit; + char offsetTimeUnit; STimeWindow window; // query time window int64_t intervalTime; // aggregation time window range int64_t slidingTime; // sliding window in mseconds - int64_t intervalOffset;// start offset of each time window + int64_t offsetTime; // start offset of each time window int32_t tz; // query client timezone SSqlGroupbyExpr groupbyExpr; // group by tags info diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 517fb32bb3..83d3bd35b2 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -81,6 +81,7 @@ static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd); static int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); +static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem); @@ -613,6 +614,10 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ // for top/bottom + interval query, we do not add additional timestamp column in the front if (isTopBottomQuery(pQueryInfo)) { + if (parseOffsetClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -662,6 +667,10 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL); + if (parseOffsetClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -669,6 +678,39 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ return TSDB_CODE_SUCCESS; } +int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + SStrToken* pOffset = &pQuerySql->offset; + if (pOffset->n == 0) { + pQueryInfo->offsetTimeUnit = pQueryInfo->offsetTimeUnit; + pQueryInfo->offsetTime = 0; + return TSDB_CODE_SUCCESS; + } + + getTimestampInUsFromStr(pOffset->z, pOffset->n, &pQueryInfo->offsetTime); + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { + pQueryInfo->offsetTime /= 1000; + } + +/* + if (pQueryInfo->offsetTime < 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); + } + + if (pQueryInfo->slidingTime >= pQueryInfo->intervalTime) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + if ((pQueryInfo->intervalTime != 0) && (pQueryInfo->intervalTime/pQueryInfo->slidingTime > INTERVAL_SLIDING_FACTOR)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + */ + + return TSDB_CODE_SUCCESS; +} + int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg0 = "sliding value too small"; const char* msg1 = "sliding value no larger than the interval value"; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index e2df886320..94ac5dc995 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -461,10 +461,11 @@ typedef struct { int16_t orderColId; int16_t numOfCols; // the number of columns will be load from vnode int64_t intervalTime; // time interval for aggregation, in million second - int64_t intervalOffset; // start offset for interval query int64_t slidingTime; // value for sliding window + int64_t offsetTime; // start offset for interval query char intervalTimeUnit; char slidingTimeUnit; // time interval type, for revisement of interval(1d) + char offsetTimeUnit; uint16_t tagCondLen; // tag length in current query int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 69fc0bc7ef..d6664577a3 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -65,6 +65,11 @@ typedef struct tVariantList { tVariantListItem *a; /* One entry for each expression */ } tVariantList; +typedef struct SIntervalVal { + SStrToken interval; + SStrToken offset; +} SIntervalVal; + typedef struct SQuerySQL { struct tSQLExprList *pSelection; // select clause tVariantList * from; // from clause @@ -72,6 +77,7 @@ typedef struct SQuerySQL { tVariantList * pGroupby; // groupby clause, only for tags[optional] tVariantList * pSortOrder; // orderby [optional] SStrToken interval; // interval [optional] + SStrToken offset; // offset window [optional] SStrToken sliding; // sliding window [optional] SLimitVal limit; // limit offset [optional] SLimitVal slimit; // group limit offset [optional] @@ -259,7 +265,7 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken void tSQLExprListDestroy(tSQLExprList *pList); SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + tVariantList *pGroupby, tVariantList *pSortOrder, SIntervalVal *pInterval, SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pMetricName, diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index e0dd9d98f0..8a5b9d61e2 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -458,9 +458,10 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { %type tmvar {SStrToken} tmvar(A) ::= VARIABLE(X). {A = X;} -%type interval_opt {SStrToken} -interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N = E; } -interval_opt(N) ::= . {N.n = 0; N.z = NULL; N.type = 0; } +%type interval_opt {SIntervalVal} +interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N.interval = E; N.offset.n = 0; N.offset.z = NULL; N.offset.type = 0;} +interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(O) RP. {N.interval = E; N.offset = O;} +interval_opt(N) ::= . {memset(&N, 0, sizeof(N));} %type fill_opt {tVariantList*} %destructor fill_opt {tVariantListDestroy($$);} diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index c1b0bd5706..f8c60c9e18 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -535,7 +535,7 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { * extract the select info out of sql string */ SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + tVariantList *pGroupby, tVariantList *pSortOrder, SIntervalVal *pInterval, SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { assert(pSelection != NULL); @@ -558,7 +558,8 @@ SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, } if (pInterval != NULL) { - pQuery->interval = *pInterval; + pQuery->interval = pInterval->interval; + pQuery->offset = pInterval->offset; } if (pSliding != NULL) { diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 35e36032d4..3145f39a5e 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -23,6 +23,7 @@ ** input grammar file: */ #include +#include /************ Begin %include sections from the grammar ************************/ #include @@ -76,8 +77,10 @@ ** zero the stack is dynamically sized using realloc() ** ParseARG_SDECL A static variable declaration for the %extra_argument ** ParseARG_PDECL A parameter declaration for the %extra_argument +** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter ** ParseARG_STORE Code to store %extra_argument into yypParser ** ParseARG_FETCH Code to extract %extra_argument from yypParser +** ParseCTX_* As ParseARG_ except for %extra_context ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -97,47 +100,56 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 272 +#define YYNOCODE 270 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SSubclauseInfo* yy25; - tSQLExpr* yy66; - SCreateAcctSQL yy73; - int yy82; - SQuerySQL* yy150; - SCreateDBInfo yy158; - TAOS_FIELD yy181; - SLimitVal yy188; - tSQLExprList* yy224; - int64_t yy271; - tVariant yy312; - SCreateTableSQL* yy374; - tFieldList* yy449; - tVariantList* yy494; + int yy112; + SCreateDBInfo yy118; + tVariantList* yy156; + tSQLExprList* yy158; + tSQLExpr* yy190; + SSubclauseInfo* yy333; + SIntervalVal yy340; + TAOS_FIELD yy343; + int64_t yy369; + SCreateTableSQL* yy398; + SLimitVal yy414; + SQuerySQL* yy444; + SCreateAcctSQL yy479; + tVariant yy506; + tFieldList* yy511; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 #endif #define ParseARG_SDECL SSqlInfo* pInfo; #define ParseARG_PDECL ,SSqlInfo* pInfo -#define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo -#define ParseARG_STORE yypParser->pInfo = pInfo +#define ParseARG_PARAM ,pInfo +#define ParseARG_FETCH SSqlInfo* pInfo=yypParser->pInfo; +#define ParseARG_STORE yypParser->pInfo=pInfo; +#define ParseCTX_SDECL +#define ParseCTX_PDECL +#define ParseCTX_PARAM +#define ParseCTX_FETCH +#define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 246 -#define YYNRULE 227 +#define YYNSTATE 248 +#define YYNRULE 228 +#define YYNRULE_WITH_ACTION 228 #define YYNTOKEN 206 -#define YY_MAX_SHIFT 245 -#define YY_MIN_SHIFTREDUCE 407 -#define YY_MAX_SHIFTREDUCE 633 -#define YY_ERROR_ACTION 634 -#define YY_ACCEPT_ACTION 635 -#define YY_NO_ACTION 636 -#define YY_MIN_REDUCE 637 -#define YY_MAX_REDUCE 863 +#define YY_MAX_SHIFT 247 +#define YY_MIN_SHIFTREDUCE 410 +#define YY_MAX_SHIFTREDUCE 637 +#define YY_ERROR_ACTION 638 +#define YY_ACCEPT_ACTION 639 +#define YY_NO_ACTION 640 +#define YY_MIN_REDUCE 641 +#define YY_MAX_REDUCE 868 /************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -202,151 +214,151 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (556) +#define YY_ACTTAB_COUNT (560) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 737, 448, 11, 735, 736, 635, 245, 448, 738, 449, - /* 10 */ 740, 741, 739, 35, 36, 449, 37, 38, 155, 243, - /* 20 */ 165, 29, 137, 136, 200, 41, 39, 43, 40, 105, - /* 30 */ 514, 160, 851, 34, 33, 778, 137, 32, 31, 30, - /* 40 */ 35, 36, 767, 37, 38, 159, 851, 165, 29, 767, - /* 50 */ 105, 200, 41, 39, 43, 40, 185, 157, 221, 220, - /* 60 */ 34, 33, 137, 156, 32, 31, 30, 35, 36, 448, - /* 70 */ 37, 38, 850, 141, 165, 29, 756, 449, 200, 41, - /* 80 */ 39, 43, 40, 197, 78, 60, 775, 34, 33, 232, - /* 90 */ 232, 32, 31, 30, 21, 41, 39, 43, 40, 32, - /* 100 */ 31, 30, 56, 34, 33, 847, 803, 32, 31, 30, - /* 110 */ 21, 21, 105, 408, 409, 410, 411, 412, 413, 414, - /* 120 */ 415, 416, 417, 418, 419, 244, 587, 169, 36, 753, - /* 130 */ 37, 38, 223, 50, 165, 29, 21, 62, 200, 41, - /* 140 */ 39, 43, 40, 170, 219, 753, 753, 34, 33, 27, - /* 150 */ 51, 32, 31, 30, 8, 37, 38, 63, 115, 165, - /* 160 */ 29, 101, 754, 200, 41, 39, 43, 40, 804, 224, - /* 170 */ 195, 753, 34, 33, 168, 846, 32, 31, 30, 16, - /* 180 */ 212, 238, 237, 211, 210, 209, 236, 208, 235, 234, - /* 190 */ 233, 207, 733, 756, 721, 722, 723, 724, 725, 726, - /* 200 */ 727, 728, 729, 730, 731, 732, 164, 600, 12, 239, - /* 210 */ 591, 17, 594, 188, 597, 105, 164, 600, 26, 559, - /* 220 */ 591, 845, 594, 46, 597, 34, 33, 150, 756, 32, - /* 230 */ 31, 30, 21, 90, 89, 144, 568, 569, 161, 162, - /* 240 */ 171, 149, 199, 76, 80, 85, 88, 79, 161, 162, - /* 250 */ 164, 600, 546, 82, 591, 589, 594, 100, 597, 242, - /* 260 */ 241, 97, 17, 16, 26, 238, 237, 752, 201, 26, - /* 270 */ 236, 61, 235, 234, 233, 118, 119, 70, 66, 69, - /* 280 */ 538, 676, 161, 162, 128, 530, 178, 187, 527, 184, - /* 290 */ 528, 590, 529, 182, 181, 593, 152, 596, 132, 130, - /* 300 */ 93, 92, 91, 42, 172, 543, 685, 218, 217, 128, - /* 310 */ 18, 163, 677, 42, 599, 128, 173, 174, 560, 619, - /* 320 */ 153, 47, 14, 13, 599, 592, 601, 595, 520, 598, - /* 330 */ 13, 519, 46, 154, 205, 22, 75, 74, 22, 598, - /* 340 */ 48, 10, 9, 534, 532, 535, 533, 42, 87, 86, - /* 350 */ 3, 139, 860, 140, 755, 142, 143, 603, 599, 147, - /* 360 */ 148, 146, 135, 145, 138, 814, 813, 166, 810, 809, - /* 370 */ 167, 777, 747, 598, 222, 769, 782, 784, 102, 796, - /* 380 */ 114, 116, 795, 117, 687, 206, 133, 531, 186, 26, - /* 390 */ 24, 95, 215, 684, 216, 859, 72, 858, 856, 120, - /* 400 */ 705, 25, 555, 23, 134, 674, 81, 672, 83, 189, - /* 410 */ 84, 670, 193, 669, 175, 52, 129, 667, 49, 666, - /* 420 */ 665, 106, 664, 44, 663, 107, 655, 131, 198, 766, - /* 430 */ 196, 661, 659, 657, 194, 57, 58, 797, 192, 190, - /* 440 */ 28, 214, 77, 225, 226, 227, 228, 229, 230, 203, - /* 450 */ 53, 231, 240, 633, 151, 177, 64, 67, 176, 668, - /* 460 */ 632, 179, 180, 631, 624, 187, 123, 183, 122, 706, - /* 470 */ 121, 125, 124, 94, 127, 662, 126, 96, 1, 2, - /* 480 */ 540, 112, 108, 109, 751, 110, 113, 111, 59, 55, - /* 490 */ 556, 103, 158, 19, 191, 20, 561, 104, 5, 602, - /* 500 */ 6, 4, 604, 15, 202, 7, 204, 65, 489, 485, - /* 510 */ 483, 482, 481, 478, 452, 213, 68, 45, 71, 22, - /* 520 */ 516, 515, 513, 54, 473, 471, 463, 469, 465, 467, - /* 530 */ 73, 461, 459, 488, 487, 486, 484, 480, 479, 46, - /* 540 */ 450, 423, 421, 637, 636, 636, 98, 636, 636, 636, - /* 550 */ 636, 636, 636, 636, 636, 99, + /* 0 */ 741, 451, 11, 739, 740, 639, 247, 451, 742, 452, + /* 10 */ 744, 745, 743, 35, 36, 452, 37, 38, 156, 245, + /* 20 */ 167, 29, 138, 137, 202, 41, 39, 43, 40, 106, + /* 30 */ 517, 162, 856, 34, 33, 782, 138, 32, 31, 30, + /* 40 */ 35, 36, 771, 37, 38, 161, 856, 167, 29, 771, + /* 50 */ 106, 202, 41, 39, 43, 40, 187, 159, 223, 222, + /* 60 */ 34, 33, 138, 157, 32, 31, 30, 35, 36, 451, + /* 70 */ 37, 38, 855, 142, 167, 29, 760, 452, 202, 41, + /* 80 */ 39, 43, 40, 199, 78, 60, 779, 34, 33, 234, + /* 90 */ 234, 32, 31, 30, 21, 41, 39, 43, 40, 32, + /* 100 */ 31, 30, 56, 34, 33, 852, 808, 32, 31, 30, + /* 110 */ 21, 21, 106, 411, 412, 413, 414, 415, 416, 417, + /* 120 */ 418, 419, 420, 421, 422, 246, 591, 171, 36, 757, + /* 130 */ 37, 38, 225, 50, 167, 29, 21, 62, 202, 41, + /* 140 */ 39, 43, 40, 172, 221, 757, 757, 34, 33, 27, + /* 150 */ 51, 32, 31, 30, 8, 37, 38, 63, 116, 167, + /* 160 */ 29, 101, 758, 202, 41, 39, 43, 40, 809, 226, + /* 170 */ 197, 757, 34, 33, 170, 851, 32, 31, 30, 16, + /* 180 */ 214, 240, 239, 213, 212, 211, 238, 210, 237, 236, + /* 190 */ 235, 209, 737, 760, 725, 726, 727, 728, 729, 730, + /* 200 */ 731, 732, 733, 734, 735, 736, 166, 604, 12, 241, + /* 210 */ 595, 17, 598, 190, 601, 559, 166, 604, 26, 103, + /* 220 */ 595, 597, 598, 600, 601, 34, 33, 151, 760, 32, + /* 230 */ 31, 30, 21, 90, 89, 145, 572, 573, 163, 164, + /* 240 */ 173, 150, 201, 76, 80, 85, 88, 79, 163, 164, + /* 250 */ 166, 604, 549, 82, 595, 106, 598, 100, 601, 244, + /* 260 */ 243, 97, 17, 16, 26, 240, 239, 756, 680, 26, + /* 270 */ 238, 129, 237, 236, 235, 119, 120, 70, 66, 69, + /* 280 */ 203, 165, 163, 164, 689, 533, 180, 129, 530, 186, + /* 290 */ 531, 850, 532, 184, 183, 596, 153, 599, 133, 131, + /* 300 */ 93, 92, 91, 42, 174, 546, 681, 220, 219, 129, + /* 310 */ 18, 61, 541, 42, 603, 593, 175, 176, 563, 189, + /* 320 */ 3, 47, 46, 537, 603, 538, 564, 623, 605, 602, + /* 330 */ 14, 13, 13, 154, 523, 75, 74, 522, 46, 602, + /* 340 */ 48, 22, 207, 535, 155, 536, 22, 42, 10, 9, + /* 350 */ 140, 594, 87, 86, 141, 143, 144, 148, 603, 149, + /* 360 */ 147, 136, 146, 139, 865, 759, 819, 818, 168, 607, + /* 370 */ 815, 814, 169, 602, 751, 224, 781, 773, 786, 788, + /* 380 */ 102, 801, 117, 800, 115, 118, 26, 534, 188, 691, + /* 390 */ 208, 134, 24, 217, 688, 218, 864, 72, 863, 861, + /* 400 */ 121, 95, 709, 25, 23, 135, 678, 81, 558, 676, + /* 410 */ 83, 191, 84, 674, 158, 673, 195, 177, 130, 671, + /* 420 */ 52, 670, 770, 669, 49, 44, 668, 107, 108, 200, + /* 430 */ 667, 194, 659, 132, 665, 663, 198, 196, 192, 661, + /* 440 */ 28, 57, 58, 802, 216, 77, 227, 228, 229, 230, + /* 450 */ 231, 232, 205, 233, 242, 53, 637, 178, 179, 636, + /* 460 */ 152, 64, 67, 182, 181, 672, 635, 628, 94, 96, + /* 470 */ 185, 666, 124, 55, 123, 710, 122, 125, 126, 128, + /* 480 */ 127, 1, 2, 189, 755, 543, 59, 560, 111, 109, + /* 490 */ 112, 110, 104, 113, 114, 160, 19, 193, 5, 565, + /* 500 */ 105, 6, 606, 4, 20, 15, 204, 7, 608, 65, + /* 510 */ 206, 492, 488, 486, 485, 484, 481, 455, 215, 68, + /* 520 */ 45, 71, 22, 519, 73, 518, 516, 54, 476, 474, + /* 530 */ 466, 472, 468, 470, 464, 462, 491, 490, 489, 487, + /* 540 */ 483, 482, 46, 453, 426, 424, 641, 640, 640, 640, + /* 550 */ 640, 640, 640, 640, 640, 640, 640, 640, 98, 99, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 226, 1, 260, 229, 230, 207, 208, 1, 234, 9, - /* 10 */ 236, 237, 238, 13, 14, 9, 16, 17, 209, 210, - /* 20 */ 20, 21, 260, 260, 24, 25, 26, 27, 28, 210, - /* 30 */ 5, 269, 270, 33, 34, 210, 260, 37, 38, 39, - /* 40 */ 13, 14, 244, 16, 17, 269, 270, 20, 21, 244, - /* 50 */ 210, 24, 25, 26, 27, 28, 258, 227, 33, 34, - /* 60 */ 33, 34, 260, 258, 37, 38, 39, 13, 14, 1, - /* 70 */ 16, 17, 270, 260, 20, 21, 246, 9, 24, 25, - /* 80 */ 26, 27, 28, 264, 72, 266, 261, 33, 34, 78, - /* 90 */ 78, 37, 38, 39, 210, 25, 26, 27, 28, 37, - /* 100 */ 38, 39, 102, 33, 34, 260, 266, 37, 38, 39, - /* 110 */ 210, 210, 210, 45, 46, 47, 48, 49, 50, 51, - /* 120 */ 52, 53, 54, 55, 56, 57, 99, 243, 14, 245, - /* 130 */ 16, 17, 210, 103, 20, 21, 210, 247, 24, 25, - /* 140 */ 26, 27, 28, 243, 243, 245, 245, 33, 34, 259, + /* 0 */ 225, 1, 259, 228, 229, 206, 207, 1, 233, 9, + /* 10 */ 235, 236, 237, 13, 14, 9, 16, 17, 208, 209, + /* 20 */ 20, 21, 259, 259, 24, 25, 26, 27, 28, 209, + /* 30 */ 5, 268, 269, 33, 34, 209, 259, 37, 38, 39, + /* 40 */ 13, 14, 243, 16, 17, 268, 269, 20, 21, 243, + /* 50 */ 209, 24, 25, 26, 27, 28, 257, 226, 33, 34, + /* 60 */ 33, 34, 259, 257, 37, 38, 39, 13, 14, 1, + /* 70 */ 16, 17, 269, 259, 20, 21, 245, 9, 24, 25, + /* 80 */ 26, 27, 28, 263, 72, 265, 260, 33, 34, 78, + /* 90 */ 78, 37, 38, 39, 209, 25, 26, 27, 28, 37, + /* 100 */ 38, 39, 102, 33, 34, 259, 265, 37, 38, 39, + /* 110 */ 209, 209, 209, 45, 46, 47, 48, 49, 50, 51, + /* 120 */ 52, 53, 54, 55, 56, 57, 99, 242, 14, 244, + /* 130 */ 16, 17, 209, 103, 20, 21, 209, 246, 24, 25, + /* 140 */ 26, 27, 28, 242, 242, 244, 244, 33, 34, 258, /* 150 */ 120, 37, 38, 39, 98, 16, 17, 101, 102, 20, - /* 160 */ 21, 210, 240, 24, 25, 26, 27, 28, 266, 243, - /* 170 */ 268, 245, 33, 34, 227, 260, 37, 38, 39, 85, + /* 160 */ 21, 209, 239, 24, 25, 26, 27, 28, 265, 242, + /* 170 */ 267, 244, 33, 34, 226, 259, 37, 38, 39, 85, /* 180 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 190 */ 96, 97, 226, 246, 228, 229, 230, 231, 232, 233, - /* 200 */ 234, 235, 236, 237, 238, 239, 1, 2, 44, 227, - /* 210 */ 5, 98, 7, 262, 9, 210, 1, 2, 105, 99, - /* 220 */ 5, 260, 7, 103, 9, 33, 34, 63, 246, 37, - /* 230 */ 38, 39, 210, 69, 70, 71, 115, 116, 33, 34, + /* 190 */ 96, 97, 225, 245, 227, 228, 229, 230, 231, 232, + /* 200 */ 233, 234, 235, 236, 237, 238, 1, 2, 44, 226, + /* 210 */ 5, 98, 7, 261, 9, 99, 1, 2, 105, 103, + /* 220 */ 5, 5, 7, 7, 9, 33, 34, 63, 245, 37, + /* 230 */ 38, 39, 209, 69, 70, 71, 115, 116, 33, 34, /* 240 */ 63, 77, 37, 64, 65, 66, 67, 68, 33, 34, - /* 250 */ 1, 2, 37, 74, 5, 1, 7, 98, 9, 60, - /* 260 */ 61, 62, 98, 85, 105, 87, 88, 245, 15, 105, - /* 270 */ 92, 266, 94, 95, 96, 64, 65, 66, 67, 68, - /* 280 */ 99, 214, 33, 34, 217, 2, 126, 106, 5, 125, - /* 290 */ 7, 37, 9, 133, 134, 5, 132, 7, 64, 65, - /* 300 */ 66, 67, 68, 98, 127, 103, 214, 130, 131, 217, - /* 310 */ 108, 59, 214, 98, 109, 217, 33, 34, 99, 99, - /* 320 */ 260, 103, 103, 103, 109, 5, 99, 7, 99, 124, - /* 330 */ 103, 99, 103, 260, 99, 103, 128, 129, 103, 124, - /* 340 */ 122, 128, 129, 5, 5, 7, 7, 98, 72, 73, - /* 350 */ 98, 260, 246, 260, 246, 260, 260, 104, 109, 260, - /* 360 */ 260, 260, 260, 260, 260, 241, 241, 241, 241, 241, - /* 370 */ 241, 210, 242, 124, 241, 244, 210, 210, 210, 267, - /* 380 */ 248, 210, 267, 210, 210, 210, 210, 104, 244, 105, - /* 390 */ 210, 59, 210, 210, 210, 210, 210, 210, 210, 210, - /* 400 */ 210, 210, 109, 210, 210, 210, 210, 210, 210, 263, - /* 410 */ 210, 210, 263, 210, 210, 119, 210, 210, 121, 210, - /* 420 */ 210, 256, 210, 118, 210, 255, 210, 210, 113, 257, - /* 430 */ 117, 210, 210, 210, 112, 211, 211, 211, 111, 110, - /* 440 */ 123, 75, 84, 83, 49, 80, 82, 53, 81, 211, - /* 450 */ 211, 79, 75, 5, 211, 5, 215, 215, 135, 211, - /* 460 */ 5, 135, 5, 5, 86, 106, 219, 126, 223, 225, - /* 470 */ 224, 220, 222, 212, 218, 211, 221, 212, 216, 213, - /* 480 */ 99, 250, 254, 253, 244, 252, 249, 251, 103, 107, - /* 490 */ 99, 98, 1, 103, 98, 103, 99, 98, 114, 99, - /* 500 */ 114, 98, 104, 98, 100, 98, 100, 72, 9, 5, - /* 510 */ 5, 5, 5, 5, 76, 15, 72, 16, 129, 103, - /* 520 */ 5, 5, 99, 98, 5, 5, 5, 5, 5, 5, - /* 530 */ 129, 5, 5, 5, 5, 5, 5, 5, 5, 103, - /* 540 */ 76, 59, 58, 0, 271, 271, 21, 271, 271, 271, - /* 550 */ 271, 271, 271, 271, 271, 21, 271, 271, 271, 271, - /* 560 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 570 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 580 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 590 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 600 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 610 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 620 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 630 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 640 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 650 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 660 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 670 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 680 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 690 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 700 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 710 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 720 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 730 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 740 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 750 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 760 */ 271, 271, + /* 250 */ 1, 2, 37, 74, 5, 209, 7, 98, 9, 60, + /* 260 */ 61, 62, 98, 85, 105, 87, 88, 244, 213, 105, + /* 270 */ 92, 216, 94, 95, 96, 64, 65, 66, 67, 68, + /* 280 */ 15, 59, 33, 34, 213, 2, 126, 216, 5, 125, + /* 290 */ 7, 259, 9, 133, 134, 5, 132, 7, 64, 65, + /* 300 */ 66, 67, 68, 98, 127, 103, 213, 130, 131, 216, + /* 310 */ 108, 265, 99, 98, 109, 1, 33, 34, 99, 106, + /* 320 */ 98, 103, 103, 5, 109, 7, 99, 99, 99, 124, + /* 330 */ 103, 103, 103, 259, 99, 128, 129, 99, 103, 124, + /* 340 */ 122, 103, 99, 5, 259, 7, 103, 98, 128, 129, + /* 350 */ 259, 37, 72, 73, 259, 259, 259, 259, 109, 259, + /* 360 */ 259, 259, 259, 259, 245, 245, 240, 240, 240, 104, + /* 370 */ 240, 240, 240, 124, 241, 240, 209, 243, 209, 209, + /* 380 */ 209, 266, 209, 266, 247, 209, 105, 104, 243, 209, + /* 390 */ 209, 209, 209, 209, 209, 209, 209, 209, 209, 209, + /* 400 */ 209, 59, 209, 209, 209, 209, 209, 209, 109, 209, + /* 410 */ 209, 262, 209, 209, 262, 209, 262, 209, 209, 209, + /* 420 */ 119, 209, 256, 209, 121, 118, 209, 255, 254, 113, + /* 430 */ 209, 111, 209, 209, 209, 209, 117, 112, 110, 209, + /* 440 */ 123, 210, 210, 210, 75, 84, 83, 49, 80, 82, + /* 450 */ 53, 81, 210, 79, 75, 210, 5, 135, 5, 5, + /* 460 */ 210, 214, 214, 5, 135, 210, 5, 86, 211, 211, + /* 470 */ 126, 210, 218, 107, 222, 224, 223, 221, 219, 217, + /* 480 */ 220, 215, 212, 106, 243, 99, 103, 99, 251, 253, + /* 490 */ 250, 252, 98, 249, 248, 1, 103, 98, 114, 99, + /* 500 */ 98, 114, 99, 98, 103, 98, 100, 98, 104, 72, + /* 510 */ 100, 9, 5, 5, 5, 5, 5, 76, 15, 72, + /* 520 */ 16, 129, 103, 5, 129, 5, 99, 98, 5, 5, + /* 530 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + /* 540 */ 5, 5, 103, 76, 59, 58, 0, 270, 270, 270, + /* 550 */ 270, 270, 270, 270, 270, 270, 270, 270, 21, 21, + /* 560 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 570 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 580 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 590 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 600 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 610 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 620 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 630 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 640 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 650 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 660 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 670 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 680 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 690 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 700 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 710 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 720 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 730 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 740 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 750 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, + /* 760 */ 270, 270, 270, 270, 270, 270, }; -#define YY_SHIFT_COUNT (245) +#define YY_SHIFT_COUNT (247) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (543) +#define YY_SHIFT_MAX (546) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 164, 94, 178, 205, 249, 6, 6, 6, 6, 6, /* 10 */ 6, 0, 68, 249, 283, 283, 283, 113, 6, 6, - /* 20 */ 6, 6, 6, 12, 11, 11, 556, 215, 249, 249, + /* 20 */ 6, 6, 6, 12, 11, 11, 560, 215, 249, 249, /* 30 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, /* 40 */ 249, 249, 249, 249, 249, 283, 283, 25, 25, 25, /* 50 */ 25, 25, 25, 56, 25, 159, 6, 6, 6, 6, @@ -354,67 +366,67 @@ static const unsigned short int yy_shift_ofst[] = { /* 70 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, /* 80 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, /* 90 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 100 */ 284, 332, 332, 293, 293, 332, 296, 297, 305, 315, - /* 110 */ 313, 322, 327, 329, 317, 284, 332, 332, 366, 366, - /* 120 */ 332, 358, 360, 395, 365, 364, 394, 367, 372, 332, - /* 130 */ 377, 332, 377, 556, 556, 27, 54, 54, 54, 114, - /* 140 */ 139, 70, 70, 70, 179, 192, 192, 192, 192, 211, - /* 150 */ 234, 177, 160, 62, 62, 199, 181, 120, 219, 220, - /* 160 */ 227, 290, 320, 254, 252, 253, 218, 30, 229, 232, - /* 170 */ 235, 208, 213, 338, 339, 276, 448, 323, 450, 455, - /* 180 */ 326, 457, 458, 378, 341, 359, 381, 382, 385, 391, - /* 190 */ 393, 491, 396, 397, 399, 390, 384, 392, 386, 400, - /* 200 */ 403, 398, 405, 404, 407, 406, 435, 499, 504, 505, - /* 210 */ 506, 507, 508, 438, 500, 444, 501, 389, 401, 416, - /* 220 */ 515, 516, 423, 425, 416, 519, 520, 521, 522, 523, - /* 230 */ 524, 526, 527, 528, 529, 530, 531, 532, 533, 436, - /* 240 */ 464, 525, 534, 482, 484, 543, + /* 100 */ 281, 342, 342, 299, 299, 299, 342, 301, 303, 307, + /* 110 */ 316, 319, 325, 320, 328, 317, 281, 342, 342, 369, + /* 120 */ 369, 342, 361, 363, 398, 368, 367, 397, 370, 374, + /* 130 */ 342, 379, 342, 379, 560, 560, 27, 54, 54, 54, + /* 140 */ 114, 139, 70, 70, 70, 179, 192, 192, 192, 192, + /* 150 */ 211, 234, 177, 160, 62, 62, 199, 213, 116, 219, + /* 160 */ 227, 228, 229, 216, 290, 314, 222, 265, 218, 30, + /* 170 */ 235, 238, 243, 207, 220, 318, 338, 280, 451, 322, + /* 180 */ 453, 454, 329, 458, 461, 381, 344, 377, 386, 366, + /* 190 */ 383, 388, 394, 494, 399, 400, 402, 393, 384, 401, + /* 200 */ 387, 403, 405, 404, 407, 406, 409, 410, 437, 502, + /* 210 */ 507, 508, 509, 510, 511, 441, 503, 447, 504, 392, + /* 220 */ 395, 419, 518, 520, 427, 429, 419, 523, 524, 525, + /* 230 */ 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, + /* 240 */ 536, 439, 467, 537, 538, 485, 487, 546, }; -#define YY_REDUCE_COUNT (134) -#define YY_REDUCE_MIN (-258) -#define YY_REDUCE_MAX (266) +#define YY_REDUCE_COUNT (135) +#define YY_REDUCE_MIN (-257) +#define YY_REDUCE_MAX (270) static const short yy_reduce_ofst[] = { - /* 0 */ -202, -34, -226, -238, -224, -98, -181, -116, -100, -99, - /* 10 */ -74, -175, -191, -198, -170, -53, -18, -195, -49, -160, - /* 20 */ 5, -78, 22, 67, 92, 98, -110, -258, -237, -187, - /* 30 */ -155, -85, -39, 60, 73, 91, 93, 95, 96, 99, - /* 40 */ 100, 101, 102, 103, 104, 106, 108, 124, 125, 126, - /* 50 */ 127, 128, 129, 130, 133, 131, 161, 166, 167, 168, - /* 60 */ 112, 115, 132, 171, 173, 174, 175, 176, 180, 182, - /* 70 */ 183, 184, 185, 186, 187, 188, 189, 190, 191, 193, - /* 80 */ 194, 195, 196, 197, 198, 200, 201, 203, 204, 206, - /* 90 */ 207, 209, 210, 212, 214, 216, 217, 221, 222, 223, - /* 100 */ 144, 224, 225, 146, 149, 226, 172, 165, 170, 228, - /* 110 */ 230, 233, 236, 231, 237, 240, 238, 239, 241, 242, - /* 120 */ 243, 244, 246, 245, 247, 250, 251, 255, 256, 248, - /* 130 */ 261, 264, 265, 262, 266, + /* 0 */ -201, -33, -225, -237, -223, -97, -180, -115, -99, -98, + /* 10 */ -73, -174, -190, -197, -169, -52, -17, -194, -48, -159, + /* 20 */ 46, -77, 23, 55, 71, 93, -109, -257, -236, -186, + /* 30 */ -154, -84, 32, 74, 85, 91, 95, 96, 97, 98, + /* 40 */ 100, 101, 102, 103, 104, 119, 120, 126, 127, 128, + /* 50 */ 130, 131, 132, 133, 135, 134, 167, 169, 170, 171, + /* 60 */ 115, 117, 137, 173, 176, 180, 181, 182, 183, 184, + /* 70 */ 185, 186, 187, 188, 189, 190, 191, 193, 194, 195, + /* 80 */ 196, 197, 198, 200, 201, 203, 204, 206, 208, 209, + /* 90 */ 210, 212, 214, 217, 221, 223, 224, 225, 226, 230, + /* 100 */ 145, 231, 232, 149, 152, 154, 233, 166, 172, 174, + /* 110 */ 236, 239, 237, 240, 244, 246, 241, 242, 245, 247, + /* 120 */ 248, 250, 251, 253, 252, 254, 256, 259, 260, 262, + /* 130 */ 255, 257, 261, 258, 266, 270, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 634, 686, 675, 853, 853, 634, 634, 634, 634, 634, - /* 10 */ 634, 779, 652, 853, 634, 634, 634, 634, 634, 634, - /* 20 */ 634, 634, 634, 688, 688, 688, 774, 634, 634, 634, - /* 30 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, - /* 40 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, - /* 50 */ 634, 634, 634, 634, 634, 634, 634, 781, 783, 634, - /* 60 */ 800, 800, 772, 634, 634, 634, 634, 634, 634, 634, - /* 70 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, - /* 80 */ 634, 673, 634, 671, 634, 634, 634, 634, 634, 634, - /* 90 */ 634, 634, 634, 634, 634, 634, 634, 660, 634, 634, - /* 100 */ 634, 654, 654, 634, 634, 654, 807, 811, 805, 793, - /* 110 */ 801, 792, 788, 787, 815, 634, 654, 654, 683, 683, - /* 120 */ 654, 704, 702, 700, 692, 698, 694, 696, 690, 654, - /* 130 */ 681, 654, 681, 720, 734, 634, 816, 852, 806, 842, - /* 140 */ 841, 848, 840, 839, 634, 835, 836, 838, 837, 634, - /* 150 */ 634, 634, 634, 844, 843, 634, 634, 634, 634, 634, - /* 160 */ 634, 634, 634, 634, 818, 634, 812, 808, 634, 634, - /* 170 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, - /* 180 */ 634, 634, 634, 634, 634, 771, 634, 634, 780, 634, - /* 190 */ 634, 634, 634, 634, 634, 802, 634, 794, 634, 634, - /* 200 */ 634, 634, 634, 634, 634, 748, 634, 634, 634, 634, - /* 210 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 857, - /* 220 */ 634, 634, 634, 742, 855, 634, 634, 634, 634, 634, - /* 230 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 707, - /* 240 */ 634, 658, 656, 634, 650, 634, + /* 0 */ 638, 690, 679, 858, 858, 638, 638, 638, 638, 638, + /* 10 */ 638, 783, 656, 858, 638, 638, 638, 638, 638, 638, + /* 20 */ 638, 638, 638, 692, 692, 692, 778, 638, 638, 638, + /* 30 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, + /* 40 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, + /* 50 */ 638, 638, 638, 638, 638, 638, 638, 785, 787, 638, + /* 60 */ 805, 805, 776, 638, 638, 638, 638, 638, 638, 638, + /* 70 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, + /* 80 */ 638, 677, 638, 675, 638, 638, 638, 638, 638, 638, + /* 90 */ 638, 638, 638, 638, 638, 638, 638, 664, 638, 638, + /* 100 */ 638, 658, 658, 638, 638, 638, 658, 812, 816, 810, + /* 110 */ 798, 806, 797, 793, 792, 820, 638, 658, 658, 687, + /* 120 */ 687, 658, 708, 706, 704, 696, 702, 698, 700, 694, + /* 130 */ 658, 685, 658, 685, 724, 738, 638, 821, 857, 811, + /* 140 */ 847, 846, 853, 845, 844, 638, 840, 841, 843, 842, + /* 150 */ 638, 638, 638, 638, 849, 848, 638, 638, 638, 638, + /* 160 */ 638, 638, 638, 638, 638, 638, 823, 638, 817, 813, + /* 170 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, + /* 180 */ 638, 638, 638, 638, 638, 638, 638, 775, 638, 638, + /* 190 */ 784, 638, 638, 638, 638, 638, 638, 807, 638, 799, + /* 200 */ 638, 638, 638, 638, 638, 638, 638, 752, 638, 638, + /* 210 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, + /* 220 */ 638, 862, 638, 638, 638, 746, 860, 638, 638, 638, + /* 230 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, + /* 240 */ 638, 711, 638, 662, 660, 638, 654, 638, }; /********** End of lemon-generated parsing tables *****************************/ @@ -679,6 +691,7 @@ struct yyParser { int yyerrcnt; /* Shifts left before out of the error */ #endif ParseARG_SDECL /* A place to hold %extra_argument */ + ParseCTX_SDECL /* A place to hold %extra_context */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ yyStackEntry *yystack; /* The parser's stack */ @@ -932,71 +945,70 @@ static const char *const yyTokenName[] = { /* 203 */ "INSERT", /* 204 */ "INTO", /* 205 */ "VALUES", - /* 206 */ "error", - /* 207 */ "program", - /* 208 */ "cmd", - /* 209 */ "dbPrefix", - /* 210 */ "ids", - /* 211 */ "cpxName", - /* 212 */ "ifexists", - /* 213 */ "alter_db_optr", - /* 214 */ "acct_optr", - /* 215 */ "ifnotexists", - /* 216 */ "db_optr", - /* 217 */ "pps", - /* 218 */ "tseries", - /* 219 */ "dbs", - /* 220 */ "streams", - /* 221 */ "storage", - /* 222 */ "qtime", - /* 223 */ "users", - /* 224 */ "conns", - /* 225 */ "state", - /* 226 */ "keep", - /* 227 */ "tagitemlist", - /* 228 */ "cache", - /* 229 */ "replica", - /* 230 */ "quorum", - /* 231 */ "days", - /* 232 */ "minrows", - /* 233 */ "maxrows", - /* 234 */ "blocks", - /* 235 */ "ctime", - /* 236 */ "wal", - /* 237 */ "fsync", - /* 238 */ "comp", - /* 239 */ "prec", - /* 240 */ "typename", - /* 241 */ "signed", - /* 242 */ "create_table_args", - /* 243 */ "columnlist", - /* 244 */ "select", - /* 245 */ "column", - /* 246 */ "tagitem", - /* 247 */ "selcollist", - /* 248 */ "from", - /* 249 */ "where_opt", - /* 250 */ "interval_opt", - /* 251 */ "fill_opt", - /* 252 */ "sliding_opt", - /* 253 */ "groupby_opt", - /* 254 */ "orderby_opt", - /* 255 */ "having_opt", - /* 256 */ "slimit_opt", - /* 257 */ "limit_opt", - /* 258 */ "union", - /* 259 */ "sclp", - /* 260 */ "expr", - /* 261 */ "as", - /* 262 */ "tablelist", - /* 263 */ "tmvar", - /* 264 */ "sortlist", - /* 265 */ "sortitem", - /* 266 */ "item", - /* 267 */ "sortorder", - /* 268 */ "grouplist", - /* 269 */ "exprlist", - /* 270 */ "expritem", + /* 206 */ "program", + /* 207 */ "cmd", + /* 208 */ "dbPrefix", + /* 209 */ "ids", + /* 210 */ "cpxName", + /* 211 */ "ifexists", + /* 212 */ "alter_db_optr", + /* 213 */ "acct_optr", + /* 214 */ "ifnotexists", + /* 215 */ "db_optr", + /* 216 */ "pps", + /* 217 */ "tseries", + /* 218 */ "dbs", + /* 219 */ "streams", + /* 220 */ "storage", + /* 221 */ "qtime", + /* 222 */ "users", + /* 223 */ "conns", + /* 224 */ "state", + /* 225 */ "keep", + /* 226 */ "tagitemlist", + /* 227 */ "cache", + /* 228 */ "replica", + /* 229 */ "quorum", + /* 230 */ "days", + /* 231 */ "minrows", + /* 232 */ "maxrows", + /* 233 */ "blocks", + /* 234 */ "ctime", + /* 235 */ "wal", + /* 236 */ "fsync", + /* 237 */ "comp", + /* 238 */ "prec", + /* 239 */ "typename", + /* 240 */ "signed", + /* 241 */ "create_table_args", + /* 242 */ "columnlist", + /* 243 */ "select", + /* 244 */ "column", + /* 245 */ "tagitem", + /* 246 */ "selcollist", + /* 247 */ "from", + /* 248 */ "where_opt", + /* 249 */ "interval_opt", + /* 250 */ "fill_opt", + /* 251 */ "sliding_opt", + /* 252 */ "groupby_opt", + /* 253 */ "orderby_opt", + /* 254 */ "having_opt", + /* 255 */ "slimit_opt", + /* 256 */ "limit_opt", + /* 257 */ "union", + /* 258 */ "sclp", + /* 259 */ "expr", + /* 260 */ "as", + /* 261 */ "tablelist", + /* 262 */ "tmvar", + /* 263 */ "sortlist", + /* 264 */ "sortitem", + /* 265 */ "item", + /* 266 */ "sortorder", + /* 267 */ "grouplist", + /* 268 */ "exprlist", + /* 269 */ "expritem", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1154,83 +1166,84 @@ static const char *const yyRuleName[] = { /* 147 */ "tablelist ::= tablelist COMMA ids cpxName ids", /* 148 */ "tmvar ::= VARIABLE", /* 149 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 150 */ "interval_opt ::=", - /* 151 */ "fill_opt ::=", - /* 152 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 153 */ "fill_opt ::= FILL LP ID RP", - /* 154 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 155 */ "sliding_opt ::=", - /* 156 */ "orderby_opt ::=", - /* 157 */ "orderby_opt ::= ORDER BY sortlist", - /* 158 */ "sortlist ::= sortlist COMMA item sortorder", - /* 159 */ "sortlist ::= item sortorder", - /* 160 */ "item ::= ids cpxName", - /* 161 */ "sortorder ::= ASC", - /* 162 */ "sortorder ::= DESC", - /* 163 */ "sortorder ::=", - /* 164 */ "groupby_opt ::=", - /* 165 */ "groupby_opt ::= GROUP BY grouplist", - /* 166 */ "grouplist ::= grouplist COMMA item", - /* 167 */ "grouplist ::= item", - /* 168 */ "having_opt ::=", - /* 169 */ "having_opt ::= HAVING expr", - /* 170 */ "limit_opt ::=", - /* 171 */ "limit_opt ::= LIMIT signed", - /* 172 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 173 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 174 */ "slimit_opt ::=", - /* 175 */ "slimit_opt ::= SLIMIT signed", - /* 176 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 177 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 178 */ "where_opt ::=", - /* 179 */ "where_opt ::= WHERE expr", - /* 180 */ "expr ::= LP expr RP", - /* 181 */ "expr ::= ID", - /* 182 */ "expr ::= ID DOT ID", - /* 183 */ "expr ::= ID DOT STAR", - /* 184 */ "expr ::= INTEGER", - /* 185 */ "expr ::= MINUS INTEGER", - /* 186 */ "expr ::= PLUS INTEGER", - /* 187 */ "expr ::= FLOAT", - /* 188 */ "expr ::= MINUS FLOAT", - /* 189 */ "expr ::= PLUS FLOAT", - /* 190 */ "expr ::= STRING", - /* 191 */ "expr ::= NOW", - /* 192 */ "expr ::= VARIABLE", - /* 193 */ "expr ::= BOOL", - /* 194 */ "expr ::= ID LP exprlist RP", - /* 195 */ "expr ::= ID LP STAR RP", - /* 196 */ "expr ::= expr IS NULL", - /* 197 */ "expr ::= expr IS NOT NULL", - /* 198 */ "expr ::= expr LT expr", - /* 199 */ "expr ::= expr GT expr", - /* 200 */ "expr ::= expr LE expr", - /* 201 */ "expr ::= expr GE expr", - /* 202 */ "expr ::= expr NE expr", - /* 203 */ "expr ::= expr EQ expr", - /* 204 */ "expr ::= expr AND expr", - /* 205 */ "expr ::= expr OR expr", - /* 206 */ "expr ::= expr PLUS expr", - /* 207 */ "expr ::= expr MINUS expr", - /* 208 */ "expr ::= expr STAR expr", - /* 209 */ "expr ::= expr SLASH expr", - /* 210 */ "expr ::= expr REM expr", - /* 211 */ "expr ::= expr LIKE expr", - /* 212 */ "expr ::= expr IN LP exprlist RP", - /* 213 */ "exprlist ::= exprlist COMMA expritem", - /* 214 */ "exprlist ::= expritem", - /* 215 */ "expritem ::= expr", - /* 216 */ "expritem ::=", - /* 217 */ "cmd ::= RESET QUERY CACHE", - /* 218 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 219 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 220 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 221 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 222 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 223 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 224 */ "cmd ::= KILL CONNECTION INTEGER", - /* 225 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 226 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 150 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", + /* 151 */ "interval_opt ::=", + /* 152 */ "fill_opt ::=", + /* 153 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 154 */ "fill_opt ::= FILL LP ID RP", + /* 155 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 156 */ "sliding_opt ::=", + /* 157 */ "orderby_opt ::=", + /* 158 */ "orderby_opt ::= ORDER BY sortlist", + /* 159 */ "sortlist ::= sortlist COMMA item sortorder", + /* 160 */ "sortlist ::= item sortorder", + /* 161 */ "item ::= ids cpxName", + /* 162 */ "sortorder ::= ASC", + /* 163 */ "sortorder ::= DESC", + /* 164 */ "sortorder ::=", + /* 165 */ "groupby_opt ::=", + /* 166 */ "groupby_opt ::= GROUP BY grouplist", + /* 167 */ "grouplist ::= grouplist COMMA item", + /* 168 */ "grouplist ::= item", + /* 169 */ "having_opt ::=", + /* 170 */ "having_opt ::= HAVING expr", + /* 171 */ "limit_opt ::=", + /* 172 */ "limit_opt ::= LIMIT signed", + /* 173 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 174 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 175 */ "slimit_opt ::=", + /* 176 */ "slimit_opt ::= SLIMIT signed", + /* 177 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 178 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 179 */ "where_opt ::=", + /* 180 */ "where_opt ::= WHERE expr", + /* 181 */ "expr ::= LP expr RP", + /* 182 */ "expr ::= ID", + /* 183 */ "expr ::= ID DOT ID", + /* 184 */ "expr ::= ID DOT STAR", + /* 185 */ "expr ::= INTEGER", + /* 186 */ "expr ::= MINUS INTEGER", + /* 187 */ "expr ::= PLUS INTEGER", + /* 188 */ "expr ::= FLOAT", + /* 189 */ "expr ::= MINUS FLOAT", + /* 190 */ "expr ::= PLUS FLOAT", + /* 191 */ "expr ::= STRING", + /* 192 */ "expr ::= NOW", + /* 193 */ "expr ::= VARIABLE", + /* 194 */ "expr ::= BOOL", + /* 195 */ "expr ::= ID LP exprlist RP", + /* 196 */ "expr ::= ID LP STAR RP", + /* 197 */ "expr ::= expr IS NULL", + /* 198 */ "expr ::= expr IS NOT NULL", + /* 199 */ "expr ::= expr LT expr", + /* 200 */ "expr ::= expr GT expr", + /* 201 */ "expr ::= expr LE expr", + /* 202 */ "expr ::= expr GE expr", + /* 203 */ "expr ::= expr NE expr", + /* 204 */ "expr ::= expr EQ expr", + /* 205 */ "expr ::= expr AND expr", + /* 206 */ "expr ::= expr OR expr", + /* 207 */ "expr ::= expr PLUS expr", + /* 208 */ "expr ::= expr MINUS expr", + /* 209 */ "expr ::= expr STAR expr", + /* 210 */ "expr ::= expr SLASH expr", + /* 211 */ "expr ::= expr REM expr", + /* 212 */ "expr ::= expr LIKE expr", + /* 213 */ "expr ::= expr IN LP exprlist RP", + /* 214 */ "exprlist ::= exprlist COMMA expritem", + /* 215 */ "exprlist ::= expritem", + /* 216 */ "expritem ::= expr", + /* 217 */ "expritem ::=", + /* 218 */ "cmd ::= RESET QUERY CACHE", + /* 219 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 220 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 221 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 222 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 223 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 224 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 225 */ "cmd ::= KILL CONNECTION INTEGER", + /* 226 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 227 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1279,28 +1292,29 @@ static int yyGrowStack(yyParser *p){ /* Initialize a new parser that has already been allocated. */ -void ParseInit(void *yypParser){ - yyParser *pParser = (yyParser*)yypParser; +void ParseInit(void *yypRawParser ParseCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + ParseCTX_STORE #ifdef YYTRACKMAXSTACKDEPTH - pParser->yyhwm = 0; + yypParser->yyhwm = 0; #endif #if YYSTACKDEPTH<=0 - pParser->yytos = NULL; - pParser->yystack = NULL; - pParser->yystksz = 0; - if( yyGrowStack(pParser) ){ - pParser->yystack = &pParser->yystk0; - pParser->yystksz = 1; + yypParser->yytos = NULL; + yypParser->yystack = NULL; + yypParser->yystksz = 0; + if( yyGrowStack(yypParser) ){ + yypParser->yystack = &yypParser->yystk0; + yypParser->yystksz = 1; } #endif #ifndef YYNOERRORRECOVERY - pParser->yyerrcnt = -1; + yypParser->yyerrcnt = -1; #endif - pParser->yytos = pParser->yystack; - pParser->yystack[0].stateno = 0; - pParser->yystack[0].major = 0; + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; #if YYSTACKDEPTH>0 - pParser->yystackEnd = &pParser->yystack[YYSTACKDEPTH-1]; + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; #endif } @@ -1317,11 +1331,14 @@ void ParseInit(void *yypParser){ ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. */ -void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ - yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( pParser ) ParseInit(pParser); - return pParser; +void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + ParseCTX_STORE + ParseInit(yypParser ParseCTX_PARAM); + } + return (void*)yypParser; } #endif /* Parse_ENGINEALWAYSONSTACK */ @@ -1338,7 +1355,8 @@ static void yy_destructor( YYCODETYPE yymajor, /* Type code for object to destroy */ YYMINORTYPE *yypminor /* The object to be destroyed */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen @@ -1351,50 +1369,50 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 226: /* keep */ - case 227: /* tagitemlist */ - case 251: /* fill_opt */ - case 253: /* groupby_opt */ - case 254: /* orderby_opt */ - case 264: /* sortlist */ - case 268: /* grouplist */ + case 225: /* keep */ + case 226: /* tagitemlist */ + case 250: /* fill_opt */ + case 252: /* groupby_opt */ + case 253: /* orderby_opt */ + case 263: /* sortlist */ + case 267: /* grouplist */ { -tVariantListDestroy((yypminor->yy494)); +tVariantListDestroy((yypminor->yy156)); } break; - case 243: /* columnlist */ + case 242: /* columnlist */ { -tFieldListDestroy((yypminor->yy449)); +tFieldListDestroy((yypminor->yy511)); } break; - case 244: /* select */ + case 243: /* select */ { -doDestroyQuerySql((yypminor->yy150)); +doDestroyQuerySql((yypminor->yy444)); } break; - case 247: /* selcollist */ - case 259: /* sclp */ - case 269: /* exprlist */ + case 246: /* selcollist */ + case 258: /* sclp */ + case 268: /* exprlist */ { -tSQLExprListDestroy((yypminor->yy224)); +tSQLExprListDestroy((yypminor->yy158)); } break; - case 249: /* where_opt */ - case 255: /* having_opt */ - case 260: /* expr */ - case 270: /* expritem */ + case 248: /* where_opt */ + case 254: /* having_opt */ + case 259: /* expr */ + case 269: /* expritem */ { -tSQLExprDestroy((yypminor->yy66)); +tSQLExprDestroy((yypminor->yy190)); } break; - case 258: /* union */ + case 257: /* union */ { -destroyAllSelectClause((yypminor->yy25)); +destroyAllSelectClause((yypminor->yy333)); } break; - case 265: /* sortitem */ + case 264: /* sortitem */ { -tVariantDestroy(&(yypminor->yy312)); +tVariantDestroy(&(yypminor->yy506)); } break; /********* End destructor definitions *****************************************/ @@ -1506,13 +1524,12 @@ int ParseCoverage(FILE *out){ ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. */ -static unsigned int yy_find_shift_action( - yyParser *pParser, /* The parser */ - YYCODETYPE iLookAhead /* The look-ahead token */ +static YYACTIONTYPE yy_find_shift_action( + YYCODETYPE iLookAhead, /* The look-ahead token */ + YYACTIONTYPE stateno /* Current state number */ ){ int i; - int stateno = pParser->yytos->stateno; - + if( stateno>YY_MAX_SHIFT ) return stateno; assert( stateno <= YY_SHIFT_COUNT ); #if defined(YYCOVERAGE) @@ -1520,15 +1537,19 @@ static unsigned int yy_find_shift_action( #endif do{ i = yy_shift_ofst[stateno]; - assert( i>=0 && i+YYNTOKEN<=sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); + assert( i>=0 ); + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; + assert( i<(int)YY_NLOOKAHEAD ); if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", @@ -1543,15 +1564,8 @@ static unsigned int yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 - ){ + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -1565,6 +1579,7 @@ static unsigned int yy_find_shift_action( #endif /* YYWILDCARD */ return yy_default[stateno]; }else{ + assert( i>=0 && iyytos; - yytos->stateno = (YYACTIONTYPE)yyNewState; - yytos->major = (YYCODETYPE)yyMajor; + yytos->stateno = yyNewState; + yytos->major = yyMajor; yytos->minor.yy0 = yyMinor; yyTraceShift(yypParser, yyNewState, "Shift"); } -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - signed char nrhs; /* Negative of the number of RHS symbols in the rule */ -} yyRuleInfo[] = { - { 207, -1 }, /* (0) program ::= cmd */ - { 208, -2 }, /* (1) cmd ::= SHOW DATABASES */ - { 208, -2 }, /* (2) cmd ::= SHOW MNODES */ - { 208, -2 }, /* (3) cmd ::= SHOW DNODES */ - { 208, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */ - { 208, -2 }, /* (5) cmd ::= SHOW USERS */ - { 208, -2 }, /* (6) cmd ::= SHOW MODULES */ - { 208, -2 }, /* (7) cmd ::= SHOW QUERIES */ - { 208, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ - { 208, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 208, -2 }, /* (10) cmd ::= SHOW VARIABLES */ - { 208, -2 }, /* (11) cmd ::= SHOW SCORES */ - { 208, -2 }, /* (12) cmd ::= SHOW GRANTS */ - { 208, -2 }, /* (13) cmd ::= SHOW VNODES */ - { 208, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */ - { 209, 0 }, /* (15) dbPrefix ::= */ - { 209, -2 }, /* (16) dbPrefix ::= ids DOT */ - { 211, 0 }, /* (17) cpxName ::= */ - { 211, -2 }, /* (18) cpxName ::= DOT ids */ - { 208, -3 }, /* (19) cmd ::= SHOW dbPrefix TABLES */ - { 208, -5 }, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - { 208, -3 }, /* (21) cmd ::= SHOW dbPrefix STABLES */ - { 208, -5 }, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - { 208, -3 }, /* (23) cmd ::= SHOW dbPrefix VGROUPS */ - { 208, -4 }, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */ - { 208, -5 }, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */ - { 208, -4 }, /* (26) cmd ::= DROP DATABASE ifexists ids */ - { 208, -3 }, /* (27) cmd ::= DROP DNODE ids */ - { 208, -3 }, /* (28) cmd ::= DROP USER ids */ - { 208, -3 }, /* (29) cmd ::= DROP ACCOUNT ids */ - { 208, -2 }, /* (30) cmd ::= USE ids */ - { 208, -3 }, /* (31) cmd ::= DESCRIBE ids cpxName */ - { 208, -5 }, /* (32) cmd ::= ALTER USER ids PASS ids */ - { 208, -5 }, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */ - { 208, -4 }, /* (34) cmd ::= ALTER DNODE ids ids */ - { 208, -5 }, /* (35) cmd ::= ALTER DNODE ids ids ids */ - { 208, -3 }, /* (36) cmd ::= ALTER LOCAL ids */ - { 208, -4 }, /* (37) cmd ::= ALTER LOCAL ids ids */ - { 208, -4 }, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */ - { 208, -4 }, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */ - { 208, -6 }, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - { 210, -1 }, /* (41) ids ::= ID */ - { 210, -1 }, /* (42) ids ::= STRING */ - { 212, -2 }, /* (43) ifexists ::= IF EXISTS */ - { 212, 0 }, /* (44) ifexists ::= */ - { 215, -3 }, /* (45) ifnotexists ::= IF NOT EXISTS */ - { 215, 0 }, /* (46) ifnotexists ::= */ - { 208, -3 }, /* (47) cmd ::= CREATE DNODE ids */ - { 208, -6 }, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - { 208, -5 }, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - { 208, -5 }, /* (50) cmd ::= CREATE USER ids PASS ids */ - { 217, 0 }, /* (51) pps ::= */ - { 217, -2 }, /* (52) pps ::= PPS INTEGER */ - { 218, 0 }, /* (53) tseries ::= */ - { 218, -2 }, /* (54) tseries ::= TSERIES INTEGER */ - { 219, 0 }, /* (55) dbs ::= */ - { 219, -2 }, /* (56) dbs ::= DBS INTEGER */ - { 220, 0 }, /* (57) streams ::= */ - { 220, -2 }, /* (58) streams ::= STREAMS INTEGER */ - { 221, 0 }, /* (59) storage ::= */ - { 221, -2 }, /* (60) storage ::= STORAGE INTEGER */ - { 222, 0 }, /* (61) qtime ::= */ - { 222, -2 }, /* (62) qtime ::= QTIME INTEGER */ - { 223, 0 }, /* (63) users ::= */ - { 223, -2 }, /* (64) users ::= USERS INTEGER */ - { 224, 0 }, /* (65) conns ::= */ - { 224, -2 }, /* (66) conns ::= CONNS INTEGER */ - { 225, 0 }, /* (67) state ::= */ - { 225, -2 }, /* (68) state ::= STATE ids */ - { 214, -9 }, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - { 226, -2 }, /* (70) keep ::= KEEP tagitemlist */ - { 228, -2 }, /* (71) cache ::= CACHE INTEGER */ - { 229, -2 }, /* (72) replica ::= REPLICA INTEGER */ - { 230, -2 }, /* (73) quorum ::= QUORUM INTEGER */ - { 231, -2 }, /* (74) days ::= DAYS INTEGER */ - { 232, -2 }, /* (75) minrows ::= MINROWS INTEGER */ - { 233, -2 }, /* (76) maxrows ::= MAXROWS INTEGER */ - { 234, -2 }, /* (77) blocks ::= BLOCKS INTEGER */ - { 235, -2 }, /* (78) ctime ::= CTIME INTEGER */ - { 236, -2 }, /* (79) wal ::= WAL INTEGER */ - { 237, -2 }, /* (80) fsync ::= FSYNC INTEGER */ - { 238, -2 }, /* (81) comp ::= COMP INTEGER */ - { 239, -2 }, /* (82) prec ::= PRECISION STRING */ - { 216, 0 }, /* (83) db_optr ::= */ - { 216, -2 }, /* (84) db_optr ::= db_optr cache */ - { 216, -2 }, /* (85) db_optr ::= db_optr replica */ - { 216, -2 }, /* (86) db_optr ::= db_optr quorum */ - { 216, -2 }, /* (87) db_optr ::= db_optr days */ - { 216, -2 }, /* (88) db_optr ::= db_optr minrows */ - { 216, -2 }, /* (89) db_optr ::= db_optr maxrows */ - { 216, -2 }, /* (90) db_optr ::= db_optr blocks */ - { 216, -2 }, /* (91) db_optr ::= db_optr ctime */ - { 216, -2 }, /* (92) db_optr ::= db_optr wal */ - { 216, -2 }, /* (93) db_optr ::= db_optr fsync */ - { 216, -2 }, /* (94) db_optr ::= db_optr comp */ - { 216, -2 }, /* (95) db_optr ::= db_optr prec */ - { 216, -2 }, /* (96) db_optr ::= db_optr keep */ - { 213, 0 }, /* (97) alter_db_optr ::= */ - { 213, -2 }, /* (98) alter_db_optr ::= alter_db_optr replica */ - { 213, -2 }, /* (99) alter_db_optr ::= alter_db_optr quorum */ - { 213, -2 }, /* (100) alter_db_optr ::= alter_db_optr keep */ - { 213, -2 }, /* (101) alter_db_optr ::= alter_db_optr blocks */ - { 213, -2 }, /* (102) alter_db_optr ::= alter_db_optr comp */ - { 213, -2 }, /* (103) alter_db_optr ::= alter_db_optr wal */ - { 213, -2 }, /* (104) alter_db_optr ::= alter_db_optr fsync */ - { 240, -1 }, /* (105) typename ::= ids */ - { 240, -4 }, /* (106) typename ::= ids LP signed RP */ - { 241, -1 }, /* (107) signed ::= INTEGER */ - { 241, -2 }, /* (108) signed ::= PLUS INTEGER */ - { 241, -2 }, /* (109) signed ::= MINUS INTEGER */ - { 208, -6 }, /* (110) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ - { 242, -3 }, /* (111) create_table_args ::= LP columnlist RP */ - { 242, -7 }, /* (112) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ - { 242, -7 }, /* (113) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ - { 242, -2 }, /* (114) create_table_args ::= AS select */ - { 243, -3 }, /* (115) columnlist ::= columnlist COMMA column */ - { 243, -1 }, /* (116) columnlist ::= column */ - { 245, -2 }, /* (117) column ::= ids typename */ - { 227, -3 }, /* (118) tagitemlist ::= tagitemlist COMMA tagitem */ - { 227, -1 }, /* (119) tagitemlist ::= tagitem */ - { 246, -1 }, /* (120) tagitem ::= INTEGER */ - { 246, -1 }, /* (121) tagitem ::= FLOAT */ - { 246, -1 }, /* (122) tagitem ::= STRING */ - { 246, -1 }, /* (123) tagitem ::= BOOL */ - { 246, -1 }, /* (124) tagitem ::= NULL */ - { 246, -2 }, /* (125) tagitem ::= MINUS INTEGER */ - { 246, -2 }, /* (126) tagitem ::= MINUS FLOAT */ - { 246, -2 }, /* (127) tagitem ::= PLUS INTEGER */ - { 246, -2 }, /* (128) tagitem ::= PLUS FLOAT */ - { 244, -12 }, /* (129) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - { 258, -1 }, /* (130) union ::= select */ - { 258, -3 }, /* (131) union ::= LP union RP */ - { 258, -4 }, /* (132) union ::= union UNION ALL select */ - { 258, -6 }, /* (133) union ::= union UNION ALL LP select RP */ - { 208, -1 }, /* (134) cmd ::= union */ - { 244, -2 }, /* (135) select ::= SELECT selcollist */ - { 259, -2 }, /* (136) sclp ::= selcollist COMMA */ - { 259, 0 }, /* (137) sclp ::= */ - { 247, -3 }, /* (138) selcollist ::= sclp expr as */ - { 247, -2 }, /* (139) selcollist ::= sclp STAR */ - { 261, -2 }, /* (140) as ::= AS ids */ - { 261, -1 }, /* (141) as ::= ids */ - { 261, 0 }, /* (142) as ::= */ - { 248, -2 }, /* (143) from ::= FROM tablelist */ - { 262, -2 }, /* (144) tablelist ::= ids cpxName */ - { 262, -3 }, /* (145) tablelist ::= ids cpxName ids */ - { 262, -4 }, /* (146) tablelist ::= tablelist COMMA ids cpxName */ - { 262, -5 }, /* (147) tablelist ::= tablelist COMMA ids cpxName ids */ - { 263, -1 }, /* (148) tmvar ::= VARIABLE */ - { 250, -4 }, /* (149) interval_opt ::= INTERVAL LP tmvar RP */ - { 250, 0 }, /* (150) interval_opt ::= */ - { 251, 0 }, /* (151) fill_opt ::= */ - { 251, -6 }, /* (152) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 251, -4 }, /* (153) fill_opt ::= FILL LP ID RP */ - { 252, -4 }, /* (154) sliding_opt ::= SLIDING LP tmvar RP */ - { 252, 0 }, /* (155) sliding_opt ::= */ - { 254, 0 }, /* (156) orderby_opt ::= */ - { 254, -3 }, /* (157) orderby_opt ::= ORDER BY sortlist */ - { 264, -4 }, /* (158) sortlist ::= sortlist COMMA item sortorder */ - { 264, -2 }, /* (159) sortlist ::= item sortorder */ - { 266, -2 }, /* (160) item ::= ids cpxName */ - { 267, -1 }, /* (161) sortorder ::= ASC */ - { 267, -1 }, /* (162) sortorder ::= DESC */ - { 267, 0 }, /* (163) sortorder ::= */ - { 253, 0 }, /* (164) groupby_opt ::= */ - { 253, -3 }, /* (165) groupby_opt ::= GROUP BY grouplist */ - { 268, -3 }, /* (166) grouplist ::= grouplist COMMA item */ - { 268, -1 }, /* (167) grouplist ::= item */ - { 255, 0 }, /* (168) having_opt ::= */ - { 255, -2 }, /* (169) having_opt ::= HAVING expr */ - { 257, 0 }, /* (170) limit_opt ::= */ - { 257, -2 }, /* (171) limit_opt ::= LIMIT signed */ - { 257, -4 }, /* (172) limit_opt ::= LIMIT signed OFFSET signed */ - { 257, -4 }, /* (173) limit_opt ::= LIMIT signed COMMA signed */ - { 256, 0 }, /* (174) slimit_opt ::= */ - { 256, -2 }, /* (175) slimit_opt ::= SLIMIT signed */ - { 256, -4 }, /* (176) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 256, -4 }, /* (177) slimit_opt ::= SLIMIT signed COMMA signed */ - { 249, 0 }, /* (178) where_opt ::= */ - { 249, -2 }, /* (179) where_opt ::= WHERE expr */ - { 260, -3 }, /* (180) expr ::= LP expr RP */ - { 260, -1 }, /* (181) expr ::= ID */ - { 260, -3 }, /* (182) expr ::= ID DOT ID */ - { 260, -3 }, /* (183) expr ::= ID DOT STAR */ - { 260, -1 }, /* (184) expr ::= INTEGER */ - { 260, -2 }, /* (185) expr ::= MINUS INTEGER */ - { 260, -2 }, /* (186) expr ::= PLUS INTEGER */ - { 260, -1 }, /* (187) expr ::= FLOAT */ - { 260, -2 }, /* (188) expr ::= MINUS FLOAT */ - { 260, -2 }, /* (189) expr ::= PLUS FLOAT */ - { 260, -1 }, /* (190) expr ::= STRING */ - { 260, -1 }, /* (191) expr ::= NOW */ - { 260, -1 }, /* (192) expr ::= VARIABLE */ - { 260, -1 }, /* (193) expr ::= BOOL */ - { 260, -4 }, /* (194) expr ::= ID LP exprlist RP */ - { 260, -4 }, /* (195) expr ::= ID LP STAR RP */ - { 260, -3 }, /* (196) expr ::= expr IS NULL */ - { 260, -4 }, /* (197) expr ::= expr IS NOT NULL */ - { 260, -3 }, /* (198) expr ::= expr LT expr */ - { 260, -3 }, /* (199) expr ::= expr GT expr */ - { 260, -3 }, /* (200) expr ::= expr LE expr */ - { 260, -3 }, /* (201) expr ::= expr GE expr */ - { 260, -3 }, /* (202) expr ::= expr NE expr */ - { 260, -3 }, /* (203) expr ::= expr EQ expr */ - { 260, -3 }, /* (204) expr ::= expr AND expr */ - { 260, -3 }, /* (205) expr ::= expr OR expr */ - { 260, -3 }, /* (206) expr ::= expr PLUS expr */ - { 260, -3 }, /* (207) expr ::= expr MINUS expr */ - { 260, -3 }, /* (208) expr ::= expr STAR expr */ - { 260, -3 }, /* (209) expr ::= expr SLASH expr */ - { 260, -3 }, /* (210) expr ::= expr REM expr */ - { 260, -3 }, /* (211) expr ::= expr LIKE expr */ - { 260, -5 }, /* (212) expr ::= expr IN LP exprlist RP */ - { 269, -3 }, /* (213) exprlist ::= exprlist COMMA expritem */ - { 269, -1 }, /* (214) exprlist ::= expritem */ - { 270, -1 }, /* (215) expritem ::= expr */ - { 270, 0 }, /* (216) expritem ::= */ - { 208, -3 }, /* (217) cmd ::= RESET QUERY CACHE */ - { 208, -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 208, -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 208, -7 }, /* (220) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 208, -7 }, /* (221) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 208, -8 }, /* (222) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 208, -9 }, /* (223) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 208, -3 }, /* (224) cmd ::= KILL CONNECTION INTEGER */ - { 208, -5 }, /* (225) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 208, -5 }, /* (226) cmd ::= KILL QUERY INTEGER COLON INTEGER */ +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { + 206, /* (0) program ::= cmd */ + 207, /* (1) cmd ::= SHOW DATABASES */ + 207, /* (2) cmd ::= SHOW MNODES */ + 207, /* (3) cmd ::= SHOW DNODES */ + 207, /* (4) cmd ::= SHOW ACCOUNTS */ + 207, /* (5) cmd ::= SHOW USERS */ + 207, /* (6) cmd ::= SHOW MODULES */ + 207, /* (7) cmd ::= SHOW QUERIES */ + 207, /* (8) cmd ::= SHOW CONNECTIONS */ + 207, /* (9) cmd ::= SHOW STREAMS */ + 207, /* (10) cmd ::= SHOW VARIABLES */ + 207, /* (11) cmd ::= SHOW SCORES */ + 207, /* (12) cmd ::= SHOW GRANTS */ + 207, /* (13) cmd ::= SHOW VNODES */ + 207, /* (14) cmd ::= SHOW VNODES IPTOKEN */ + 208, /* (15) dbPrefix ::= */ + 208, /* (16) dbPrefix ::= ids DOT */ + 210, /* (17) cpxName ::= */ + 210, /* (18) cpxName ::= DOT ids */ + 207, /* (19) cmd ::= SHOW dbPrefix TABLES */ + 207, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + 207, /* (21) cmd ::= SHOW dbPrefix STABLES */ + 207, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + 207, /* (23) cmd ::= SHOW dbPrefix VGROUPS */ + 207, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */ + 207, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */ + 207, /* (26) cmd ::= DROP DATABASE ifexists ids */ + 207, /* (27) cmd ::= DROP DNODE ids */ + 207, /* (28) cmd ::= DROP USER ids */ + 207, /* (29) cmd ::= DROP ACCOUNT ids */ + 207, /* (30) cmd ::= USE ids */ + 207, /* (31) cmd ::= DESCRIBE ids cpxName */ + 207, /* (32) cmd ::= ALTER USER ids PASS ids */ + 207, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */ + 207, /* (34) cmd ::= ALTER DNODE ids ids */ + 207, /* (35) cmd ::= ALTER DNODE ids ids ids */ + 207, /* (36) cmd ::= ALTER LOCAL ids */ + 207, /* (37) cmd ::= ALTER LOCAL ids ids */ + 207, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */ + 207, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */ + 207, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + 209, /* (41) ids ::= ID */ + 209, /* (42) ids ::= STRING */ + 211, /* (43) ifexists ::= IF EXISTS */ + 211, /* (44) ifexists ::= */ + 214, /* (45) ifnotexists ::= IF NOT EXISTS */ + 214, /* (46) ifnotexists ::= */ + 207, /* (47) cmd ::= CREATE DNODE ids */ + 207, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + 207, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + 207, /* (50) cmd ::= CREATE USER ids PASS ids */ + 216, /* (51) pps ::= */ + 216, /* (52) pps ::= PPS INTEGER */ + 217, /* (53) tseries ::= */ + 217, /* (54) tseries ::= TSERIES INTEGER */ + 218, /* (55) dbs ::= */ + 218, /* (56) dbs ::= DBS INTEGER */ + 219, /* (57) streams ::= */ + 219, /* (58) streams ::= STREAMS INTEGER */ + 220, /* (59) storage ::= */ + 220, /* (60) storage ::= STORAGE INTEGER */ + 221, /* (61) qtime ::= */ + 221, /* (62) qtime ::= QTIME INTEGER */ + 222, /* (63) users ::= */ + 222, /* (64) users ::= USERS INTEGER */ + 223, /* (65) conns ::= */ + 223, /* (66) conns ::= CONNS INTEGER */ + 224, /* (67) state ::= */ + 224, /* (68) state ::= STATE ids */ + 213, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + 225, /* (70) keep ::= KEEP tagitemlist */ + 227, /* (71) cache ::= CACHE INTEGER */ + 228, /* (72) replica ::= REPLICA INTEGER */ + 229, /* (73) quorum ::= QUORUM INTEGER */ + 230, /* (74) days ::= DAYS INTEGER */ + 231, /* (75) minrows ::= MINROWS INTEGER */ + 232, /* (76) maxrows ::= MAXROWS INTEGER */ + 233, /* (77) blocks ::= BLOCKS INTEGER */ + 234, /* (78) ctime ::= CTIME INTEGER */ + 235, /* (79) wal ::= WAL INTEGER */ + 236, /* (80) fsync ::= FSYNC INTEGER */ + 237, /* (81) comp ::= COMP INTEGER */ + 238, /* (82) prec ::= PRECISION STRING */ + 215, /* (83) db_optr ::= */ + 215, /* (84) db_optr ::= db_optr cache */ + 215, /* (85) db_optr ::= db_optr replica */ + 215, /* (86) db_optr ::= db_optr quorum */ + 215, /* (87) db_optr ::= db_optr days */ + 215, /* (88) db_optr ::= db_optr minrows */ + 215, /* (89) db_optr ::= db_optr maxrows */ + 215, /* (90) db_optr ::= db_optr blocks */ + 215, /* (91) db_optr ::= db_optr ctime */ + 215, /* (92) db_optr ::= db_optr wal */ + 215, /* (93) db_optr ::= db_optr fsync */ + 215, /* (94) db_optr ::= db_optr comp */ + 215, /* (95) db_optr ::= db_optr prec */ + 215, /* (96) db_optr ::= db_optr keep */ + 212, /* (97) alter_db_optr ::= */ + 212, /* (98) alter_db_optr ::= alter_db_optr replica */ + 212, /* (99) alter_db_optr ::= alter_db_optr quorum */ + 212, /* (100) alter_db_optr ::= alter_db_optr keep */ + 212, /* (101) alter_db_optr ::= alter_db_optr blocks */ + 212, /* (102) alter_db_optr ::= alter_db_optr comp */ + 212, /* (103) alter_db_optr ::= alter_db_optr wal */ + 212, /* (104) alter_db_optr ::= alter_db_optr fsync */ + 239, /* (105) typename ::= ids */ + 239, /* (106) typename ::= ids LP signed RP */ + 240, /* (107) signed ::= INTEGER */ + 240, /* (108) signed ::= PLUS INTEGER */ + 240, /* (109) signed ::= MINUS INTEGER */ + 207, /* (110) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ + 241, /* (111) create_table_args ::= LP columnlist RP */ + 241, /* (112) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ + 241, /* (113) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ + 241, /* (114) create_table_args ::= AS select */ + 242, /* (115) columnlist ::= columnlist COMMA column */ + 242, /* (116) columnlist ::= column */ + 244, /* (117) column ::= ids typename */ + 226, /* (118) tagitemlist ::= tagitemlist COMMA tagitem */ + 226, /* (119) tagitemlist ::= tagitem */ + 245, /* (120) tagitem ::= INTEGER */ + 245, /* (121) tagitem ::= FLOAT */ + 245, /* (122) tagitem ::= STRING */ + 245, /* (123) tagitem ::= BOOL */ + 245, /* (124) tagitem ::= NULL */ + 245, /* (125) tagitem ::= MINUS INTEGER */ + 245, /* (126) tagitem ::= MINUS FLOAT */ + 245, /* (127) tagitem ::= PLUS INTEGER */ + 245, /* (128) tagitem ::= PLUS FLOAT */ + 243, /* (129) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + 257, /* (130) union ::= select */ + 257, /* (131) union ::= LP union RP */ + 257, /* (132) union ::= union UNION ALL select */ + 257, /* (133) union ::= union UNION ALL LP select RP */ + 207, /* (134) cmd ::= union */ + 243, /* (135) select ::= SELECT selcollist */ + 258, /* (136) sclp ::= selcollist COMMA */ + 258, /* (137) sclp ::= */ + 246, /* (138) selcollist ::= sclp expr as */ + 246, /* (139) selcollist ::= sclp STAR */ + 260, /* (140) as ::= AS ids */ + 260, /* (141) as ::= ids */ + 260, /* (142) as ::= */ + 247, /* (143) from ::= FROM tablelist */ + 261, /* (144) tablelist ::= ids cpxName */ + 261, /* (145) tablelist ::= ids cpxName ids */ + 261, /* (146) tablelist ::= tablelist COMMA ids cpxName */ + 261, /* (147) tablelist ::= tablelist COMMA ids cpxName ids */ + 262, /* (148) tmvar ::= VARIABLE */ + 249, /* (149) interval_opt ::= INTERVAL LP tmvar RP */ + 249, /* (150) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ + 249, /* (151) interval_opt ::= */ + 250, /* (152) fill_opt ::= */ + 250, /* (153) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + 250, /* (154) fill_opt ::= FILL LP ID RP */ + 251, /* (155) sliding_opt ::= SLIDING LP tmvar RP */ + 251, /* (156) sliding_opt ::= */ + 253, /* (157) orderby_opt ::= */ + 253, /* (158) orderby_opt ::= ORDER BY sortlist */ + 263, /* (159) sortlist ::= sortlist COMMA item sortorder */ + 263, /* (160) sortlist ::= item sortorder */ + 265, /* (161) item ::= ids cpxName */ + 266, /* (162) sortorder ::= ASC */ + 266, /* (163) sortorder ::= DESC */ + 266, /* (164) sortorder ::= */ + 252, /* (165) groupby_opt ::= */ + 252, /* (166) groupby_opt ::= GROUP BY grouplist */ + 267, /* (167) grouplist ::= grouplist COMMA item */ + 267, /* (168) grouplist ::= item */ + 254, /* (169) having_opt ::= */ + 254, /* (170) having_opt ::= HAVING expr */ + 256, /* (171) limit_opt ::= */ + 256, /* (172) limit_opt ::= LIMIT signed */ + 256, /* (173) limit_opt ::= LIMIT signed OFFSET signed */ + 256, /* (174) limit_opt ::= LIMIT signed COMMA signed */ + 255, /* (175) slimit_opt ::= */ + 255, /* (176) slimit_opt ::= SLIMIT signed */ + 255, /* (177) slimit_opt ::= SLIMIT signed SOFFSET signed */ + 255, /* (178) slimit_opt ::= SLIMIT signed COMMA signed */ + 248, /* (179) where_opt ::= */ + 248, /* (180) where_opt ::= WHERE expr */ + 259, /* (181) expr ::= LP expr RP */ + 259, /* (182) expr ::= ID */ + 259, /* (183) expr ::= ID DOT ID */ + 259, /* (184) expr ::= ID DOT STAR */ + 259, /* (185) expr ::= INTEGER */ + 259, /* (186) expr ::= MINUS INTEGER */ + 259, /* (187) expr ::= PLUS INTEGER */ + 259, /* (188) expr ::= FLOAT */ + 259, /* (189) expr ::= MINUS FLOAT */ + 259, /* (190) expr ::= PLUS FLOAT */ + 259, /* (191) expr ::= STRING */ + 259, /* (192) expr ::= NOW */ + 259, /* (193) expr ::= VARIABLE */ + 259, /* (194) expr ::= BOOL */ + 259, /* (195) expr ::= ID LP exprlist RP */ + 259, /* (196) expr ::= ID LP STAR RP */ + 259, /* (197) expr ::= expr IS NULL */ + 259, /* (198) expr ::= expr IS NOT NULL */ + 259, /* (199) expr ::= expr LT expr */ + 259, /* (200) expr ::= expr GT expr */ + 259, /* (201) expr ::= expr LE expr */ + 259, /* (202) expr ::= expr GE expr */ + 259, /* (203) expr ::= expr NE expr */ + 259, /* (204) expr ::= expr EQ expr */ + 259, /* (205) expr ::= expr AND expr */ + 259, /* (206) expr ::= expr OR expr */ + 259, /* (207) expr ::= expr PLUS expr */ + 259, /* (208) expr ::= expr MINUS expr */ + 259, /* (209) expr ::= expr STAR expr */ + 259, /* (210) expr ::= expr SLASH expr */ + 259, /* (211) expr ::= expr REM expr */ + 259, /* (212) expr ::= expr LIKE expr */ + 259, /* (213) expr ::= expr IN LP exprlist RP */ + 268, /* (214) exprlist ::= exprlist COMMA expritem */ + 268, /* (215) exprlist ::= expritem */ + 269, /* (216) expritem ::= expr */ + 269, /* (217) expritem ::= */ + 207, /* (218) cmd ::= RESET QUERY CACHE */ + 207, /* (219) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + 207, /* (220) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + 207, /* (221) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + 207, /* (222) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + 207, /* (223) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + 207, /* (224) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + 207, /* (225) cmd ::= KILL CONNECTION INTEGER */ + 207, /* (226) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + 207, /* (227) cmd ::= KILL QUERY INTEGER COLON INTEGER */ +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { + -1, /* (0) program ::= cmd */ + -2, /* (1) cmd ::= SHOW DATABASES */ + -2, /* (2) cmd ::= SHOW MNODES */ + -2, /* (3) cmd ::= SHOW DNODES */ + -2, /* (4) cmd ::= SHOW ACCOUNTS */ + -2, /* (5) cmd ::= SHOW USERS */ + -2, /* (6) cmd ::= SHOW MODULES */ + -2, /* (7) cmd ::= SHOW QUERIES */ + -2, /* (8) cmd ::= SHOW CONNECTIONS */ + -2, /* (9) cmd ::= SHOW STREAMS */ + -2, /* (10) cmd ::= SHOW VARIABLES */ + -2, /* (11) cmd ::= SHOW SCORES */ + -2, /* (12) cmd ::= SHOW GRANTS */ + -2, /* (13) cmd ::= SHOW VNODES */ + -3, /* (14) cmd ::= SHOW VNODES IPTOKEN */ + 0, /* (15) dbPrefix ::= */ + -2, /* (16) dbPrefix ::= ids DOT */ + 0, /* (17) cpxName ::= */ + -2, /* (18) cpxName ::= DOT ids */ + -3, /* (19) cmd ::= SHOW dbPrefix TABLES */ + -5, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + -3, /* (21) cmd ::= SHOW dbPrefix STABLES */ + -5, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + -3, /* (23) cmd ::= SHOW dbPrefix VGROUPS */ + -4, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */ + -5, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */ + -4, /* (26) cmd ::= DROP DATABASE ifexists ids */ + -3, /* (27) cmd ::= DROP DNODE ids */ + -3, /* (28) cmd ::= DROP USER ids */ + -3, /* (29) cmd ::= DROP ACCOUNT ids */ + -2, /* (30) cmd ::= USE ids */ + -3, /* (31) cmd ::= DESCRIBE ids cpxName */ + -5, /* (32) cmd ::= ALTER USER ids PASS ids */ + -5, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */ + -4, /* (34) cmd ::= ALTER DNODE ids ids */ + -5, /* (35) cmd ::= ALTER DNODE ids ids ids */ + -3, /* (36) cmd ::= ALTER LOCAL ids */ + -4, /* (37) cmd ::= ALTER LOCAL ids ids */ + -4, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */ + -4, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */ + -6, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + -1, /* (41) ids ::= ID */ + -1, /* (42) ids ::= STRING */ + -2, /* (43) ifexists ::= IF EXISTS */ + 0, /* (44) ifexists ::= */ + -3, /* (45) ifnotexists ::= IF NOT EXISTS */ + 0, /* (46) ifnotexists ::= */ + -3, /* (47) cmd ::= CREATE DNODE ids */ + -6, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + -5, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + -5, /* (50) cmd ::= CREATE USER ids PASS ids */ + 0, /* (51) pps ::= */ + -2, /* (52) pps ::= PPS INTEGER */ + 0, /* (53) tseries ::= */ + -2, /* (54) tseries ::= TSERIES INTEGER */ + 0, /* (55) dbs ::= */ + -2, /* (56) dbs ::= DBS INTEGER */ + 0, /* (57) streams ::= */ + -2, /* (58) streams ::= STREAMS INTEGER */ + 0, /* (59) storage ::= */ + -2, /* (60) storage ::= STORAGE INTEGER */ + 0, /* (61) qtime ::= */ + -2, /* (62) qtime ::= QTIME INTEGER */ + 0, /* (63) users ::= */ + -2, /* (64) users ::= USERS INTEGER */ + 0, /* (65) conns ::= */ + -2, /* (66) conns ::= CONNS INTEGER */ + 0, /* (67) state ::= */ + -2, /* (68) state ::= STATE ids */ + -9, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + -2, /* (70) keep ::= KEEP tagitemlist */ + -2, /* (71) cache ::= CACHE INTEGER */ + -2, /* (72) replica ::= REPLICA INTEGER */ + -2, /* (73) quorum ::= QUORUM INTEGER */ + -2, /* (74) days ::= DAYS INTEGER */ + -2, /* (75) minrows ::= MINROWS INTEGER */ + -2, /* (76) maxrows ::= MAXROWS INTEGER */ + -2, /* (77) blocks ::= BLOCKS INTEGER */ + -2, /* (78) ctime ::= CTIME INTEGER */ + -2, /* (79) wal ::= WAL INTEGER */ + -2, /* (80) fsync ::= FSYNC INTEGER */ + -2, /* (81) comp ::= COMP INTEGER */ + -2, /* (82) prec ::= PRECISION STRING */ + 0, /* (83) db_optr ::= */ + -2, /* (84) db_optr ::= db_optr cache */ + -2, /* (85) db_optr ::= db_optr replica */ + -2, /* (86) db_optr ::= db_optr quorum */ + -2, /* (87) db_optr ::= db_optr days */ + -2, /* (88) db_optr ::= db_optr minrows */ + -2, /* (89) db_optr ::= db_optr maxrows */ + -2, /* (90) db_optr ::= db_optr blocks */ + -2, /* (91) db_optr ::= db_optr ctime */ + -2, /* (92) db_optr ::= db_optr wal */ + -2, /* (93) db_optr ::= db_optr fsync */ + -2, /* (94) db_optr ::= db_optr comp */ + -2, /* (95) db_optr ::= db_optr prec */ + -2, /* (96) db_optr ::= db_optr keep */ + 0, /* (97) alter_db_optr ::= */ + -2, /* (98) alter_db_optr ::= alter_db_optr replica */ + -2, /* (99) alter_db_optr ::= alter_db_optr quorum */ + -2, /* (100) alter_db_optr ::= alter_db_optr keep */ + -2, /* (101) alter_db_optr ::= alter_db_optr blocks */ + -2, /* (102) alter_db_optr ::= alter_db_optr comp */ + -2, /* (103) alter_db_optr ::= alter_db_optr wal */ + -2, /* (104) alter_db_optr ::= alter_db_optr fsync */ + -1, /* (105) typename ::= ids */ + -4, /* (106) typename ::= ids LP signed RP */ + -1, /* (107) signed ::= INTEGER */ + -2, /* (108) signed ::= PLUS INTEGER */ + -2, /* (109) signed ::= MINUS INTEGER */ + -6, /* (110) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ + -3, /* (111) create_table_args ::= LP columnlist RP */ + -7, /* (112) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ + -7, /* (113) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ + -2, /* (114) create_table_args ::= AS select */ + -3, /* (115) columnlist ::= columnlist COMMA column */ + -1, /* (116) columnlist ::= column */ + -2, /* (117) column ::= ids typename */ + -3, /* (118) tagitemlist ::= tagitemlist COMMA tagitem */ + -1, /* (119) tagitemlist ::= tagitem */ + -1, /* (120) tagitem ::= INTEGER */ + -1, /* (121) tagitem ::= FLOAT */ + -1, /* (122) tagitem ::= STRING */ + -1, /* (123) tagitem ::= BOOL */ + -1, /* (124) tagitem ::= NULL */ + -2, /* (125) tagitem ::= MINUS INTEGER */ + -2, /* (126) tagitem ::= MINUS FLOAT */ + -2, /* (127) tagitem ::= PLUS INTEGER */ + -2, /* (128) tagitem ::= PLUS FLOAT */ + -12, /* (129) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + -1, /* (130) union ::= select */ + -3, /* (131) union ::= LP union RP */ + -4, /* (132) union ::= union UNION ALL select */ + -6, /* (133) union ::= union UNION ALL LP select RP */ + -1, /* (134) cmd ::= union */ + -2, /* (135) select ::= SELECT selcollist */ + -2, /* (136) sclp ::= selcollist COMMA */ + 0, /* (137) sclp ::= */ + -3, /* (138) selcollist ::= sclp expr as */ + -2, /* (139) selcollist ::= sclp STAR */ + -2, /* (140) as ::= AS ids */ + -1, /* (141) as ::= ids */ + 0, /* (142) as ::= */ + -2, /* (143) from ::= FROM tablelist */ + -2, /* (144) tablelist ::= ids cpxName */ + -3, /* (145) tablelist ::= ids cpxName ids */ + -4, /* (146) tablelist ::= tablelist COMMA ids cpxName */ + -5, /* (147) tablelist ::= tablelist COMMA ids cpxName ids */ + -1, /* (148) tmvar ::= VARIABLE */ + -4, /* (149) interval_opt ::= INTERVAL LP tmvar RP */ + -6, /* (150) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ + 0, /* (151) interval_opt ::= */ + 0, /* (152) fill_opt ::= */ + -6, /* (153) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + -4, /* (154) fill_opt ::= FILL LP ID RP */ + -4, /* (155) sliding_opt ::= SLIDING LP tmvar RP */ + 0, /* (156) sliding_opt ::= */ + 0, /* (157) orderby_opt ::= */ + -3, /* (158) orderby_opt ::= ORDER BY sortlist */ + -4, /* (159) sortlist ::= sortlist COMMA item sortorder */ + -2, /* (160) sortlist ::= item sortorder */ + -2, /* (161) item ::= ids cpxName */ + -1, /* (162) sortorder ::= ASC */ + -1, /* (163) sortorder ::= DESC */ + 0, /* (164) sortorder ::= */ + 0, /* (165) groupby_opt ::= */ + -3, /* (166) groupby_opt ::= GROUP BY grouplist */ + -3, /* (167) grouplist ::= grouplist COMMA item */ + -1, /* (168) grouplist ::= item */ + 0, /* (169) having_opt ::= */ + -2, /* (170) having_opt ::= HAVING expr */ + 0, /* (171) limit_opt ::= */ + -2, /* (172) limit_opt ::= LIMIT signed */ + -4, /* (173) limit_opt ::= LIMIT signed OFFSET signed */ + -4, /* (174) limit_opt ::= LIMIT signed COMMA signed */ + 0, /* (175) slimit_opt ::= */ + -2, /* (176) slimit_opt ::= SLIMIT signed */ + -4, /* (177) slimit_opt ::= SLIMIT signed SOFFSET signed */ + -4, /* (178) slimit_opt ::= SLIMIT signed COMMA signed */ + 0, /* (179) where_opt ::= */ + -2, /* (180) where_opt ::= WHERE expr */ + -3, /* (181) expr ::= LP expr RP */ + -1, /* (182) expr ::= ID */ + -3, /* (183) expr ::= ID DOT ID */ + -3, /* (184) expr ::= ID DOT STAR */ + -1, /* (185) expr ::= INTEGER */ + -2, /* (186) expr ::= MINUS INTEGER */ + -2, /* (187) expr ::= PLUS INTEGER */ + -1, /* (188) expr ::= FLOAT */ + -2, /* (189) expr ::= MINUS FLOAT */ + -2, /* (190) expr ::= PLUS FLOAT */ + -1, /* (191) expr ::= STRING */ + -1, /* (192) expr ::= NOW */ + -1, /* (193) expr ::= VARIABLE */ + -1, /* (194) expr ::= BOOL */ + -4, /* (195) expr ::= ID LP exprlist RP */ + -4, /* (196) expr ::= ID LP STAR RP */ + -3, /* (197) expr ::= expr IS NULL */ + -4, /* (198) expr ::= expr IS NOT NULL */ + -3, /* (199) expr ::= expr LT expr */ + -3, /* (200) expr ::= expr GT expr */ + -3, /* (201) expr ::= expr LE expr */ + -3, /* (202) expr ::= expr GE expr */ + -3, /* (203) expr ::= expr NE expr */ + -3, /* (204) expr ::= expr EQ expr */ + -3, /* (205) expr ::= expr AND expr */ + -3, /* (206) expr ::= expr OR expr */ + -3, /* (207) expr ::= expr PLUS expr */ + -3, /* (208) expr ::= expr MINUS expr */ + -3, /* (209) expr ::= expr STAR expr */ + -3, /* (210) expr ::= expr SLASH expr */ + -3, /* (211) expr ::= expr REM expr */ + -3, /* (212) expr ::= expr LIKE expr */ + -5, /* (213) expr ::= expr IN LP exprlist RP */ + -3, /* (214) exprlist ::= exprlist COMMA expritem */ + -1, /* (215) exprlist ::= expritem */ + -1, /* (216) expritem ::= expr */ + 0, /* (217) expritem ::= */ + -3, /* (218) cmd ::= RESET QUERY CACHE */ + -7, /* (219) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + -7, /* (220) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + -7, /* (221) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + -7, /* (222) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + -8, /* (223) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + -9, /* (224) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + -3, /* (225) cmd ::= KILL CONNECTION INTEGER */ + -5, /* (226) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + -5, /* (227) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1929,30 +2176,34 @@ static void yy_accept(yyParser*); /* Forward Declaration */ ** only called from one place, optimizing compilers will in-line it, which ** means that the extra parameters have no performance impact. */ -static void yy_reduce( +static YYACTIONTYPE yy_reduce( yyParser *yypParser, /* The parser */ unsigned int yyruleno, /* Number of the rule by which to reduce */ int yyLookahead, /* Lookahead token, or YYNOCODE if none */ ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + ParseCTX_PDECL /* %extra_context */ ){ int yygoto; /* The next state */ - int yyact; /* The next action */ + YYACTIONTYPE yyact; /* The next action */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ - ParseARG_FETCH; + ParseARG_FETCH (void)yyLookahead; (void)yyLookaheadToken; yymsp = yypParser->yytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; + yysize = yyRuleInfoNRhs[yyruleno]; if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", yyTracePrompt, - yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; @@ -1970,13 +2221,19 @@ static void yy_reduce( #if YYSTACKDEPTH>0 if( yypParser->yytos>=yypParser->yystackEnd ){ yyStackOverflow(yypParser); - return; + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; } #else if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ if( yyGrowStack(yypParser) ){ yyStackOverflow(yypParser); - return; + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; } yymsp = yypParser->yytos; } @@ -2134,13 +2391,13 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 38: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy118, &t);} break; case 39: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy73);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy479);} break; case 40: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy479);} break; case 41: /* ids ::= ID */ case 42: /* ids ::= STRING */ yytestcase(yyruleno==42); @@ -2161,10 +2418,10 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; case 48: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy479);} break; case 49: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ -{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &yymsp[-2].minor.yy0);} +{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy118, &yymsp[-2].minor.yy0);} break; case 50: /* cmd ::= CREATE USER ids PASS ids */ { setCreateUserSQL(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} @@ -2193,20 +2450,20 @@ static void yy_reduce( break; case 69: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy73.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy73.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy73.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy73.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy73.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy73.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy73.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy73.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy73.stat = yymsp[0].minor.yy0; + yylhsminor.yy479.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yylhsminor.yy479.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yylhsminor.yy479.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yylhsminor.yy479.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yylhsminor.yy479.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yylhsminor.yy479.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy479.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy479.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yylhsminor.yy479.stat = yymsp[0].minor.yy0; } - yymsp[-8].minor.yy73 = yylhsminor.yy73; + yymsp[-8].minor.yy479 = yylhsminor.yy479; break; case 70: /* keep ::= KEEP tagitemlist */ -{ yymsp[-1].minor.yy494 = yymsp[0].minor.yy494; } +{ yymsp[-1].minor.yy156 = yymsp[0].minor.yy156; } break; case 71: /* cache ::= CACHE INTEGER */ case 72: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==72); @@ -2223,98 +2480,98 @@ static void yy_reduce( { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; case 83: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy158);} +{setDefaultCreateDbOption(&yymsp[1].minor.yy118);} break; case 84: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 85: /* db_optr ::= db_optr replica */ case 98: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==98); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 86: /* db_optr ::= db_optr quorum */ case 99: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==99); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 87: /* db_optr ::= db_optr days */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 88: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 89: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 90: /* db_optr ::= db_optr blocks */ case 101: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==101); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 91: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 92: /* db_optr ::= db_optr wal */ case 103: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==103); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 93: /* db_optr ::= db_optr fsync */ case 104: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==104); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 94: /* db_optr ::= db_optr comp */ case 102: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==102); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 95: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.precision = yymsp[0].minor.yy0; } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 96: /* db_optr ::= db_optr keep */ case 100: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==100); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.keep = yymsp[0].minor.yy494; } - yymsp[-1].minor.yy158 = yylhsminor.yy158; +{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.keep = yymsp[0].minor.yy156; } + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; case 97: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy158);} +{ setDefaultCreateDbOption(&yymsp[1].minor.yy118);} break; case 105: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSQLSetColumnType (&yylhsminor.yy181, &yymsp[0].minor.yy0); + tSQLSetColumnType (&yylhsminor.yy343, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy181 = yylhsminor.yy181; + yymsp[0].minor.yy343 = yylhsminor.yy343; break; case 106: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy271 <= 0) { + if (yymsp[-1].minor.yy369 <= 0) { yymsp[-3].minor.yy0.type = 0; - tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0); + tSQLSetColumnType(&yylhsminor.yy343, &yymsp[-3].minor.yy0); } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy271; // negative value of name length - tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0); + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy369; // negative value of name length + tSQLSetColumnType(&yylhsminor.yy343, &yymsp[-3].minor.yy0); } } - yymsp[-3].minor.yy181 = yylhsminor.yy181; + yymsp[-3].minor.yy343 = yylhsminor.yy343; break; case 107: /* signed ::= INTEGER */ -{ yylhsminor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy271 = yylhsminor.yy271; +{ yylhsminor.yy369 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[0].minor.yy369 = yylhsminor.yy369; break; case 108: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy369 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; case 109: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy271 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} +{ yymsp[-1].minor.yy369 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; case 110: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ { @@ -2324,61 +2581,61 @@ static void yy_reduce( break; case 111: /* create_table_args ::= LP columnlist RP */ { - yymsp[-2].minor.yy374 = tSetCreateSQLElems(yymsp[-1].minor.yy449, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); - setSQLInfo(pInfo, yymsp[-2].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-2].minor.yy398 = tSetCreateSQLElems(yymsp[-1].minor.yy511, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); + setSQLInfo(pInfo, yymsp[-2].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); } break; case 112: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ { - yymsp[-6].minor.yy374 = tSetCreateSQLElems(yymsp[-5].minor.yy449, yymsp[-1].minor.yy449, NULL, NULL, NULL, TSQL_CREATE_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-6].minor.yy398 = tSetCreateSQLElems(yymsp[-5].minor.yy511, yymsp[-1].minor.yy511, NULL, NULL, NULL, TSQL_CREATE_STABLE); + setSQLInfo(pInfo, yymsp[-6].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); } break; case 113: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - yymsp[-6].minor.yy374 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy494, NULL, TSQL_CREATE_TABLE_FROM_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-6].minor.yy398 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy156, NULL, TSQL_CREATE_TABLE_FROM_STABLE); + setSQLInfo(pInfo, yymsp[-6].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); } break; case 114: /* create_table_args ::= AS select */ { - yymsp[-1].minor.yy374 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy150, TSQL_CREATE_STREAM); - setSQLInfo(pInfo, yymsp[-1].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-1].minor.yy398 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy444, TSQL_CREATE_STREAM); + setSQLInfo(pInfo, yymsp[-1].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); } break; case 115: /* columnlist ::= columnlist COMMA column */ -{yylhsminor.yy449 = tFieldListAppend(yymsp[-2].minor.yy449, &yymsp[0].minor.yy181); } - yymsp[-2].minor.yy449 = yylhsminor.yy449; +{yylhsminor.yy511 = tFieldListAppend(yymsp[-2].minor.yy511, &yymsp[0].minor.yy343); } + yymsp[-2].minor.yy511 = yylhsminor.yy511; break; case 116: /* columnlist ::= column */ -{yylhsminor.yy449 = tFieldListAppend(NULL, &yymsp[0].minor.yy181);} - yymsp[0].minor.yy449 = yylhsminor.yy449; +{yylhsminor.yy511 = tFieldListAppend(NULL, &yymsp[0].minor.yy343);} + yymsp[0].minor.yy511 = yylhsminor.yy511; break; case 117: /* column ::= ids typename */ { - tSQLSetColumnInfo(&yylhsminor.yy181, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy181); + tSQLSetColumnInfo(&yylhsminor.yy343, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy343); } - yymsp[-1].minor.yy181 = yylhsminor.yy181; + yymsp[-1].minor.yy343 = yylhsminor.yy343; break; case 118: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1); } - yymsp[-2].minor.yy494 = yylhsminor.yy494; +{ yylhsminor.yy156 = tVariantListAppend(yymsp[-2].minor.yy156, &yymsp[0].minor.yy506, -1); } + yymsp[-2].minor.yy156 = yylhsminor.yy156; break; case 119: /* tagitemlist ::= tagitem */ -{ yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1); } - yymsp[0].minor.yy494 = yylhsminor.yy494; +{ yylhsminor.yy156 = tVariantListAppend(NULL, &yymsp[0].minor.yy506, -1); } + yymsp[0].minor.yy156 = yylhsminor.yy156; break; case 120: /* tagitem ::= INTEGER */ case 121: /* tagitem ::= FLOAT */ yytestcase(yyruleno==121); case 122: /* tagitem ::= STRING */ yytestcase(yyruleno==122); case 123: /* tagitem ::= BOOL */ yytestcase(yyruleno==123); -{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy312 = yylhsminor.yy312; +{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy506, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy506 = yylhsminor.yy506; break; case 124: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy312 = yylhsminor.yy312; +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy506, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy506 = yylhsminor.yy506; break; case 125: /* tagitem ::= MINUS INTEGER */ case 126: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==126); @@ -2388,59 +2645,59 @@ static void yy_reduce( yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy506, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy312 = yylhsminor.yy312; + yymsp[-1].minor.yy506 = yylhsminor.yy506; break; case 129: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy224, yymsp[-9].minor.yy494, yymsp[-8].minor.yy66, yymsp[-4].minor.yy494, yymsp[-3].minor.yy494, &yymsp[-7].minor.yy0, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy494, &yymsp[0].minor.yy188, &yymsp[-1].minor.yy188); + yylhsminor.yy444 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy158, yymsp[-9].minor.yy156, yymsp[-8].minor.yy190, yymsp[-4].minor.yy156, yymsp[-3].minor.yy156, &yymsp[-7].minor.yy340, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy156, &yymsp[0].minor.yy414, &yymsp[-1].minor.yy414); } - yymsp[-11].minor.yy150 = yylhsminor.yy150; + yymsp[-11].minor.yy444 = yylhsminor.yy444; break; case 130: /* union ::= select */ -{ yylhsminor.yy25 = setSubclause(NULL, yymsp[0].minor.yy150); } - yymsp[0].minor.yy25 = yylhsminor.yy25; +{ yylhsminor.yy333 = setSubclause(NULL, yymsp[0].minor.yy444); } + yymsp[0].minor.yy333 = yylhsminor.yy333; break; case 131: /* union ::= LP union RP */ -{ yymsp[-2].minor.yy25 = yymsp[-1].minor.yy25; } +{ yymsp[-2].minor.yy333 = yymsp[-1].minor.yy333; } break; case 132: /* union ::= union UNION ALL select */ -{ yylhsminor.yy25 = appendSelectClause(yymsp[-3].minor.yy25, yymsp[0].minor.yy150); } - yymsp[-3].minor.yy25 = yylhsminor.yy25; +{ yylhsminor.yy333 = appendSelectClause(yymsp[-3].minor.yy333, yymsp[0].minor.yy444); } + yymsp[-3].minor.yy333 = yylhsminor.yy333; break; case 133: /* union ::= union UNION ALL LP select RP */ -{ yylhsminor.yy25 = appendSelectClause(yymsp[-5].minor.yy25, yymsp[-1].minor.yy150); } - yymsp[-5].minor.yy25 = yylhsminor.yy25; +{ yylhsminor.yy333 = appendSelectClause(yymsp[-5].minor.yy333, yymsp[-1].minor.yy444); } + yymsp[-5].minor.yy333 = yylhsminor.yy333; break; case 134: /* cmd ::= union */ -{ setSQLInfo(pInfo, yymsp[0].minor.yy25, NULL, TSDB_SQL_SELECT); } +{ setSQLInfo(pInfo, yymsp[0].minor.yy333, NULL, TSDB_SQL_SELECT); } break; case 135: /* select ::= SELECT selcollist */ { - yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy224, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + yylhsminor.yy444 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy158, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy150 = yylhsminor.yy150; + yymsp[-1].minor.yy444 = yylhsminor.yy444; break; case 136: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy224 = yymsp[-1].minor.yy224;} - yymsp[-1].minor.yy224 = yylhsminor.yy224; +{yylhsminor.yy158 = yymsp[-1].minor.yy158;} + yymsp[-1].minor.yy158 = yylhsminor.yy158; break; case 137: /* sclp ::= */ -{yymsp[1].minor.yy224 = 0;} +{yymsp[1].minor.yy158 = 0;} break; case 138: /* selcollist ::= sclp expr as */ { - yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224, yymsp[-1].minor.yy66, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yylhsminor.yy158 = tSQLExprListAppend(yymsp[-2].minor.yy158, yymsp[-1].minor.yy190, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-2].minor.yy224 = yylhsminor.yy224; + yymsp[-2].minor.yy158 = yylhsminor.yy158; break; case 139: /* selcollist ::= sclp STAR */ { tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL); - yylhsminor.yy224 = tSQLExprListAppend(yymsp[-1].minor.yy224, pNode, 0); + yylhsminor.yy158 = tSQLExprListAppend(yymsp[-1].minor.yy158, pNode, 0); } - yymsp[-1].minor.yy224 = yylhsminor.yy224; + yymsp[-1].minor.yy158 = yylhsminor.yy158; break; case 140: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } @@ -2453,300 +2710,307 @@ static void yy_reduce( { yymsp[1].minor.yy0.n = 0; } break; case 143: /* from ::= FROM tablelist */ -{yymsp[-1].minor.yy494 = yymsp[0].minor.yy494;} +{yymsp[-1].minor.yy156 = yymsp[0].minor.yy156;} break; case 144: /* tablelist ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[-1].minor.yy0, -1); // table alias name + yylhsminor.yy156 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[-1].minor.yy0, -1); // table alias name } - yymsp[-1].minor.yy494 = yylhsminor.yy494; + yymsp[-1].minor.yy156 = yylhsminor.yy156; break; case 145: /* tablelist ::= ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy494 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[0].minor.yy0, -1); + yylhsminor.yy156 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[0].minor.yy0, -1); } - yymsp[-2].minor.yy494 = yylhsminor.yy494; + yymsp[-2].minor.yy156 = yylhsminor.yy156; break; case 146: /* tablelist ::= tablelist COMMA ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy494 = tVariantListAppendToken(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy156 = tVariantListAppendToken(yymsp[-3].minor.yy156, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[-1].minor.yy0, -1); } - yymsp[-3].minor.yy494 = yylhsminor.yy494; + yymsp[-3].minor.yy156 = yylhsminor.yy156; break; case 147: /* tablelist ::= tablelist COMMA ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy494 = tVariantListAppendToken(yymsp[-4].minor.yy494, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[0].minor.yy0, -1); + yylhsminor.yy156 = tVariantListAppendToken(yymsp[-4].minor.yy156, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[0].minor.yy0, -1); } - yymsp[-4].minor.yy494 = yylhsminor.yy494; + yymsp[-4].minor.yy156 = yylhsminor.yy156; break; case 148: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; case 149: /* interval_opt ::= INTERVAL LP tmvar RP */ - case 154: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==154); -{yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } +{yymsp[-3].minor.yy340.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy340.offset.n = 0; yymsp[-3].minor.yy340.offset.z = NULL; yymsp[-3].minor.yy340.offset.type = 0;} break; - case 150: /* interval_opt ::= */ - case 155: /* sliding_opt ::= */ yytestcase(yyruleno==155); -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } + case 150: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ +{yymsp[-5].minor.yy340.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy340.offset = yymsp[-1].minor.yy0;} break; - case 151: /* fill_opt ::= */ -{yymsp[1].minor.yy494 = 0; } + case 151: /* interval_opt ::= */ +{memset(&yymsp[1].minor.yy340, 0, sizeof(yymsp[1].minor.yy340));} break; - case 152: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 152: /* fill_opt ::= */ +{yymsp[1].minor.yy156 = 0; } + break; + case 153: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy494, &A, -1, 0); - yymsp[-5].minor.yy494 = yymsp[-1].minor.yy494; + tVariantListInsert(yymsp[-1].minor.yy156, &A, -1, 0); + yymsp[-5].minor.yy156 = yymsp[-1].minor.yy156; } break; - case 153: /* fill_opt ::= FILL LP ID RP */ + case 154: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yymsp[-3].minor.yy156 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 156: /* orderby_opt ::= */ - case 164: /* groupby_opt ::= */ yytestcase(yyruleno==164); -{yymsp[1].minor.yy494 = 0;} + case 155: /* sliding_opt ::= SLIDING LP tmvar RP */ +{yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 157: /* orderby_opt ::= ORDER BY sortlist */ - case 165: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==165); -{yymsp[-2].minor.yy494 = yymsp[0].minor.yy494;} + case 156: /* sliding_opt ::= */ +{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 158: /* sortlist ::= sortlist COMMA item sortorder */ + case 157: /* orderby_opt ::= */ + case 165: /* groupby_opt ::= */ yytestcase(yyruleno==165); +{yymsp[1].minor.yy156 = 0;} + break; + case 158: /* orderby_opt ::= ORDER BY sortlist */ + case 166: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==166); +{yymsp[-2].minor.yy156 = yymsp[0].minor.yy156;} + break; + case 159: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy494 = tVariantListAppend(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82); + yylhsminor.yy156 = tVariantListAppend(yymsp[-3].minor.yy156, &yymsp[-1].minor.yy506, yymsp[0].minor.yy112); } - yymsp[-3].minor.yy494 = yylhsminor.yy494; + yymsp[-3].minor.yy156 = yylhsminor.yy156; break; - case 159: /* sortlist ::= item sortorder */ + case 160: /* sortlist ::= item sortorder */ { - yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82); + yylhsminor.yy156 = tVariantListAppend(NULL, &yymsp[-1].minor.yy506, yymsp[0].minor.yy112); } - yymsp[-1].minor.yy494 = yylhsminor.yy494; + yymsp[-1].minor.yy156 = yylhsminor.yy156; break; - case 160: /* item ::= ids cpxName */ + case 161: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy506, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy312 = yylhsminor.yy312; + yymsp[-1].minor.yy506 = yylhsminor.yy506; break; - case 161: /* sortorder ::= ASC */ -{yymsp[0].minor.yy82 = TSDB_ORDER_ASC; } + case 162: /* sortorder ::= ASC */ +{yymsp[0].minor.yy112 = TSDB_ORDER_ASC; } break; - case 162: /* sortorder ::= DESC */ -{yymsp[0].minor.yy82 = TSDB_ORDER_DESC;} + case 163: /* sortorder ::= DESC */ +{yymsp[0].minor.yy112 = TSDB_ORDER_DESC;} break; - case 163: /* sortorder ::= */ -{yymsp[1].minor.yy82 = TSDB_ORDER_ASC;} + case 164: /* sortorder ::= */ +{yymsp[1].minor.yy112 = TSDB_ORDER_ASC;} break; - case 166: /* grouplist ::= grouplist COMMA item */ + case 167: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1); + yylhsminor.yy156 = tVariantListAppend(yymsp[-2].minor.yy156, &yymsp[0].minor.yy506, -1); } - yymsp[-2].minor.yy494 = yylhsminor.yy494; + yymsp[-2].minor.yy156 = yylhsminor.yy156; break; - case 167: /* grouplist ::= item */ + case 168: /* grouplist ::= item */ { - yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1); + yylhsminor.yy156 = tVariantListAppend(NULL, &yymsp[0].minor.yy506, -1); } - yymsp[0].minor.yy494 = yylhsminor.yy494; + yymsp[0].minor.yy156 = yylhsminor.yy156; break; - case 168: /* having_opt ::= */ - case 178: /* where_opt ::= */ yytestcase(yyruleno==178); - case 216: /* expritem ::= */ yytestcase(yyruleno==216); -{yymsp[1].minor.yy66 = 0;} + case 169: /* having_opt ::= */ + case 179: /* where_opt ::= */ yytestcase(yyruleno==179); + case 217: /* expritem ::= */ yytestcase(yyruleno==217); +{yymsp[1].minor.yy190 = 0;} break; - case 169: /* having_opt ::= HAVING expr */ - case 179: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==179); -{yymsp[-1].minor.yy66 = yymsp[0].minor.yy66;} + case 170: /* having_opt ::= HAVING expr */ + case 180: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==180); +{yymsp[-1].minor.yy190 = yymsp[0].minor.yy190;} break; - case 170: /* limit_opt ::= */ - case 174: /* slimit_opt ::= */ yytestcase(yyruleno==174); -{yymsp[1].minor.yy188.limit = -1; yymsp[1].minor.yy188.offset = 0;} + case 171: /* limit_opt ::= */ + case 175: /* slimit_opt ::= */ yytestcase(yyruleno==175); +{yymsp[1].minor.yy414.limit = -1; yymsp[1].minor.yy414.offset = 0;} break; - case 171: /* limit_opt ::= LIMIT signed */ - case 175: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==175); -{yymsp[-1].minor.yy188.limit = yymsp[0].minor.yy271; yymsp[-1].minor.yy188.offset = 0;} + case 172: /* limit_opt ::= LIMIT signed */ + case 176: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==176); +{yymsp[-1].minor.yy414.limit = yymsp[0].minor.yy369; yymsp[-1].minor.yy414.offset = 0;} break; - case 172: /* limit_opt ::= LIMIT signed OFFSET signed */ - case 176: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==176); -{yymsp[-3].minor.yy188.limit = yymsp[-2].minor.yy271; yymsp[-3].minor.yy188.offset = yymsp[0].minor.yy271;} + case 173: /* limit_opt ::= LIMIT signed OFFSET signed */ + case 177: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==177); +{yymsp[-3].minor.yy414.limit = yymsp[-2].minor.yy369; yymsp[-3].minor.yy414.offset = yymsp[0].minor.yy369;} break; - case 173: /* limit_opt ::= LIMIT signed COMMA signed */ - case 177: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==177); -{yymsp[-3].minor.yy188.limit = yymsp[0].minor.yy271; yymsp[-3].minor.yy188.offset = yymsp[-2].minor.yy271;} + case 174: /* limit_opt ::= LIMIT signed COMMA signed */ + case 178: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==178); +{yymsp[-3].minor.yy414.limit = yymsp[0].minor.yy369; yymsp[-3].minor.yy414.offset = yymsp[-2].minor.yy369;} break; - case 180: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy66 = yymsp[-1].minor.yy66; } + case 181: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy190 = yymsp[-1].minor.yy190; } break; - case 181: /* expr ::= ID */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 182: /* expr ::= ID */ +{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} + yymsp[0].minor.yy190 = yylhsminor.yy190; break; - case 182: /* expr ::= ID DOT ID */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 183: /* expr ::= ID DOT ID */ +{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 183: /* expr ::= ID DOT STAR */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 184: /* expr ::= ID DOT STAR */ +{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 184: /* expr ::= INTEGER */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 185: /* expr ::= INTEGER */ +{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} + yymsp[0].minor.yy190 = yylhsminor.yy190; break; - case 185: /* expr ::= MINUS INTEGER */ - case 186: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==186); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy66 = yylhsminor.yy66; + case 186: /* expr ::= MINUS INTEGER */ + case 187: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==187); +{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} + yymsp[-1].minor.yy190 = yylhsminor.yy190; break; - case 187: /* expr ::= FLOAT */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 188: /* expr ::= FLOAT */ +{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} + yymsp[0].minor.yy190 = yylhsminor.yy190; break; - case 188: /* expr ::= MINUS FLOAT */ - case 189: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==189); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy66 = yylhsminor.yy66; + case 189: /* expr ::= MINUS FLOAT */ + case 190: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==190); +{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} + yymsp[-1].minor.yy190 = yylhsminor.yy190; break; - case 190: /* expr ::= STRING */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 191: /* expr ::= STRING */ +{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} + yymsp[0].minor.yy190 = yylhsminor.yy190; break; - case 191: /* expr ::= NOW */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 192: /* expr ::= NOW */ +{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } + yymsp[0].minor.yy190 = yylhsminor.yy190; break; - case 192: /* expr ::= VARIABLE */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 193: /* expr ::= VARIABLE */ +{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} + yymsp[0].minor.yy190 = yylhsminor.yy190; break; - case 193: /* expr ::= BOOL */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 194: /* expr ::= BOOL */ +{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} + yymsp[0].minor.yy190 = yylhsminor.yy190; break; - case 194: /* expr ::= ID LP exprlist RP */ -{ yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy66 = yylhsminor.yy66; + case 195: /* expr ::= ID LP exprlist RP */ +{ yylhsminor.yy190 = tSQLExprCreateFunction(yymsp[-1].minor.yy158, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy190 = yylhsminor.yy190; break; - case 195: /* expr ::= ID LP STAR RP */ -{ yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy66 = yylhsminor.yy66; + case 196: /* expr ::= ID LP STAR RP */ +{ yylhsminor.yy190 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy190 = yylhsminor.yy190; break; - case 196: /* expr ::= expr IS NULL */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, NULL, TK_ISNULL);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 197: /* expr ::= expr IS NULL */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, NULL, TK_ISNULL);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 197: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-3].minor.yy66, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy66 = yylhsminor.yy66; + case 198: /* expr ::= expr IS NOT NULL */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-3].minor.yy190, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy190 = yylhsminor.yy190; break; - case 198: /* expr ::= expr LT expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LT);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 199: /* expr ::= expr LT expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_LT);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 199: /* expr ::= expr GT expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GT);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 200: /* expr ::= expr GT expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_GT);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 200: /* expr ::= expr LE expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LE);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 201: /* expr ::= expr LE expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_LE);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 201: /* expr ::= expr GE expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GE);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 202: /* expr ::= expr GE expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_GE);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 202: /* expr ::= expr NE expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_NE);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 203: /* expr ::= expr NE expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_NE);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 203: /* expr ::= expr EQ expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_EQ);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 204: /* expr ::= expr EQ expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_EQ);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 204: /* expr ::= expr AND expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 205: /* expr ::= expr AND expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_AND);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 205: /* expr ::= expr OR expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 206: /* expr ::= expr OR expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_OR); } + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 206: /* expr ::= expr PLUS expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_PLUS); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 207: /* expr ::= expr PLUS expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_PLUS); } + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 207: /* expr ::= expr MINUS expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_MINUS); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 208: /* expr ::= expr MINUS expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_MINUS); } + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 208: /* expr ::= expr STAR expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_STAR); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 209: /* expr ::= expr STAR expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_STAR); } + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 209: /* expr ::= expr SLASH expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_DIVIDE);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 210: /* expr ::= expr SLASH expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_DIVIDE);} + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 210: /* expr ::= expr REM expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_REM); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 211: /* expr ::= expr REM expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_REM); } + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 211: /* expr ::= expr LIKE expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LIKE); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 212: /* expr ::= expr LIKE expr */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_LIKE); } + yymsp[-2].minor.yy190 = yylhsminor.yy190; break; - case 212: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-4].minor.yy66, (tSQLExpr*)yymsp[-1].minor.yy224, TK_IN); } - yymsp[-4].minor.yy66 = yylhsminor.yy66; + case 213: /* expr ::= expr IN LP exprlist RP */ +{yylhsminor.yy190 = tSQLExprCreate(yymsp[-4].minor.yy190, (tSQLExpr*)yymsp[-1].minor.yy158, TK_IN); } + yymsp[-4].minor.yy190 = yylhsminor.yy190; break; - case 213: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224,yymsp[0].minor.yy66,0);} - yymsp[-2].minor.yy224 = yylhsminor.yy224; + case 214: /* exprlist ::= exprlist COMMA expritem */ +{yylhsminor.yy158 = tSQLExprListAppend(yymsp[-2].minor.yy158,yymsp[0].minor.yy190,0);} + yymsp[-2].minor.yy158 = yylhsminor.yy158; break; - case 214: /* exprlist ::= expritem */ -{yylhsminor.yy224 = tSQLExprListAppend(0,yymsp[0].minor.yy66,0);} - yymsp[0].minor.yy224 = yylhsminor.yy224; + case 215: /* exprlist ::= expritem */ +{yylhsminor.yy158 = tSQLExprListAppend(0,yymsp[0].minor.yy190,0);} + yymsp[0].minor.yy158 = yylhsminor.yy158; break; - case 215: /* expritem ::= expr */ -{yylhsminor.yy66 = yymsp[0].minor.yy66;} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 216: /* expritem ::= expr */ +{yylhsminor.yy190 = yymsp[0].minor.yy190;} + yymsp[0].minor.yy190 = yylhsminor.yy190; break; - case 217: /* cmd ::= RESET QUERY CACHE */ + case 218: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 218: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 219: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy511, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 219: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 220: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2757,14 +3021,14 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 220: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 221: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy511, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 221: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 222: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2775,7 +3039,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 222: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 223: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2789,34 +3053,34 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 223: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 224: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy312, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy506, -1); SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 224: /* cmd ::= KILL CONNECTION INTEGER */ + case 225: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 225: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 226: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 226: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 227: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: break; /********** End reduce actions ************************************************/ }; - assert( yyrulenostateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; } /* @@ -2840,7 +3105,8 @@ static void yy_reduce( static void yy_parse_failed( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); @@ -2851,7 +3117,8 @@ static void yy_parse_failed( ** parser fails */ /************ Begin %parse_failure code ***************************************/ /************ End %parse_failure code *****************************************/ - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } #endif /* YYNOERRORRECOVERY */ @@ -2863,7 +3130,8 @@ static void yy_syntax_error( int yymajor, /* The major type of the error token */ ParseTOKENTYPE yyminor /* The minor type of the error token */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ @@ -2889,7 +3157,8 @@ static void yy_syntax_error( assert(len <= outputBufLen); /************ End %syntax_error code ******************************************/ - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } /* @@ -2898,7 +3167,8 @@ static void yy_syntax_error( static void yy_accept( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH; + ParseARG_FETCH + ParseCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); @@ -2913,7 +3183,8 @@ static void yy_accept( /*********** Begin %parse_accept code *****************************************/ /*********** End %parse_accept code *******************************************/ - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE } /* The main parser program. @@ -2942,45 +3213,47 @@ void Parse( ParseARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; - unsigned int yyact; /* The parser action. */ + YYACTIONTYPE yyact; /* The parser action. */ #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) int yyendofinput; /* True if we are at the end of input */ #endif #ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif - yyParser *yypParser; /* The parser */ + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + ParseCTX_FETCH + ParseARG_STORE - yypParser = (yyParser*)yyp; assert( yypParser->yytos!=0 ); #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); #endif - ParseARG_STORE; + yyact = yypParser->yytos->stateno; #ifndef NDEBUG if( yyTraceFILE ){ - int stateno = yypParser->yytos->stateno; - if( stateno < YY_MIN_REDUCE ){ + if( yyact < YY_MIN_REDUCE ){ fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", - yyTracePrompt,yyTokenName[yymajor],stateno); + yyTracePrompt,yyTokenName[yymajor],yyact); }else{ fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", - yyTracePrompt,yyTokenName[yymajor],stateno-YY_MIN_REDUCE); + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); } } #endif do{ - yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); if( yyact >= YY_MIN_REDUCE ){ - yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor); + yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, + yyminor ParseCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,yymajor,yyminor); + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; #endif - yymajor = YYNOCODE; + break; }else if( yyact==YY_ACCEPT_ACTION ){ yypParser->yytos--; yy_accept(yypParser); @@ -3031,10 +3304,9 @@ void Parse( yymajor = YYNOCODE; }else{ while( yypParser->yytos >= yypParser->yystack - && yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yytos->stateno, - YYERRORSYMBOL)) >= YY_MIN_REDUCE + YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE ){ yy_pop_parser_stack(yypParser); } @@ -3051,6 +3323,8 @@ void Parse( } yypParser->yyerrcnt = 3; yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; #elif defined(YYNOERRORRECOVERY) /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to ** do any kind of error recovery. Instead, simply invoke the syntax @@ -3061,8 +3335,7 @@ void Parse( */ yy_syntax_error(yypParser,yymajor, yyminor); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yymajor = YYNOCODE; - + break; #else /* YYERRORSYMBOL is not defined */ /* This is what we do if the grammar does not define ERROR: ** @@ -3084,10 +3357,10 @@ void Parse( yypParser->yyerrcnt = -1; #endif } - yymajor = YYNOCODE; + break; #endif } - }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack ); + }while( yypParser->yytos>yypParser->yystack ); #ifndef NDEBUG if( yyTraceFILE ){ yyStackEntry *i; @@ -3102,3 +3375,17 @@ void Parse( #endif return; } + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +int ParseFallback(int iToken){ +#ifdef YYFALLBACK + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c old mode 100755 new mode 100644 From 76d407395ff917e2aee084ef3b2be59626f26eff Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 11 Sep 2020 15:40:26 +0800 Subject: [PATCH 095/105] td-1245: refactor interval/sliding/offset to one struct --- src/client/inc/tscUtil.h | 1 + src/client/inc/tsclient.h | 7 +-- src/client/src/tscLocalMerge.c | 25 ++++---- src/client/src/tscSQLParser.c | 98 +++++++++++++++++++------------- src/client/src/tscServer.c | 14 +++-- src/client/src/tscStream.c | 36 ++++++------ src/client/src/tscSubquery.c | 10 +++- src/client/src/tscUtil.c | 5 +- src/inc/taosmsg.h | 7 +-- src/os/inc/osTime.h | 13 +++++ src/os/src/detail/osTime.c | 96 +++++++++++++++++++++++++++++++ src/query/inc/qExecutor.h | 5 +- src/query/inc/qFill.h | 3 +- src/query/src/qExecutor.c | 101 +++++++++++++++++---------------- src/query/src/qFill.c | 32 +++++------ src/query/src/qUtil.c | 2 +- 16 files changed, 284 insertions(+), 171 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 748b19069b..c4fba06426 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -69,6 +69,7 @@ typedef struct SJoinSupporter { SSubqueryState* pState; SSqlObj* pObj; // parent SqlObj int32_t subqueryIndex; // index of sub query + SInterval interval; SLimitVal limit; // limit info uint64_t uid; // query table uid SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index d98826e339..17c4e7fc55 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -226,13 +226,8 @@ typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. uint32_t type; // query/insert type // TODO refactor - char intervalTimeUnit; - char slidingTimeUnit; - char offsetTimeUnit; STimeWindow window; // query time window - int64_t intervalTime; // aggregation time window range - int64_t slidingTime; // sliding window in mseconds - int64_t offsetTime; // start offset of each time window + SInterval interval; int32_t tz; // query client timezone SSqlGroupbyExpr groupbyExpr; // group by tags info diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index af6a546ff4..19abbe32e5 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -368,13 +368,12 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); TSKEY stime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey : pQueryInfo->window.ekey; - int64_t revisedSTime = - taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision); + int64_t revisedSTime = taosTimeTruncate(stime, &pQueryInfo->interval, tinfo.precision); if (pQueryInfo->fillType != TSDB_FILL_NONE) { SFillColInfo* pFillCol = createFillColInfo(pQueryInfo); pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, - 4096, (int32_t)numOfCols, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit, + 4096, (int32_t)numOfCols, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit, tinfo.precision, pQueryInfo->fillType, pFillCol); } } @@ -551,7 +550,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } // primary timestamp column is involved in final result - if (pQueryInfo->intervalTime != 0 || tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + if (pQueryInfo->interval.interval != 0 || tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { numOfGroupByCols++; } @@ -568,7 +567,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm orderIdx[i] = startCols++; } - if (pQueryInfo->intervalTime != 0) { + if (pQueryInfo->interval.interval != 0) { // the first column is the timestamp, handles queries like "interval(10m) group by tags" orderIdx[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX; } @@ -612,12 +611,12 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage * super table interval query * if the order columns is the primary timestamp, all result data belongs to one group */ - assert(pQueryInfo->intervalTime > 0); + assert(pQueryInfo->interval.interval > 0); if (numOfCols == 1) { return true; } } else { // simple group by query - assert(pQueryInfo->intervalTime == 0); + assert(pQueryInfo->interval.interval == 0); } // only one row exists @@ -825,8 +824,7 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ if (pFillInfo != NULL) { int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey; - int64_t revisedSTime = - taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision); + int64_t revisedSTime = taosTimeTruncate(stime, &pQueryInfo->interval, tinfo.precision); taosResetFillInfo(pFillInfo, revisedSTime); } @@ -839,7 +837,7 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ } static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, SQueryInfo* pQueryInfo) { - assert(pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE); + assert(pQueryInfo->interval.interval == 0 || pQueryInfo->fillType == TSDB_FILL_NONE); tFilePage * pBeforeFillData = pLocalReducer->pResultBuf; @@ -1220,7 +1218,7 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur #endif // no interval query, no fill operation - if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { + if (pQueryInfo->interval.interval == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { genFinalResWithoutFill(pRes, pLocalReducer, pQueryInfo); } else { SFillInfo* pFillInfo = pLocalReducer->pFillInfo; @@ -1258,13 +1256,10 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - int8_t precision = tinfo.precision; - // for group result interpolation, do not return if not data is generated if (pQueryInfo->fillType != TSDB_FILL_NONE) { TSKEY skey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey:pQueryInfo->window.ekey;//MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); - int64_t newTime = - taosGetIntervalStartTimestamp(skey, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); + int64_t newTime = taosTimeTruncate(skey, &pQueryInfo->interval, tinfo.precision); taosResetFillInfo(pLocalReducer->pFillInfo, newTime); } } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 83d3bd35b2..7ed2e7dbde 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -596,18 +596,18 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ // interval is not null SStrToken* t = &pQuerySql->interval; - if (parseDuration(t->z, t->n, &pQueryInfo->intervalTime, &pQueryInfo->intervalTimeUnit) != TSDB_CODE_SUCCESS) { + if (parseDuration(t->z, t->n, &pQueryInfo->interval.interval, &pQueryInfo->interval.intervalUnit) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } - if (pQueryInfo->intervalTimeUnit != 'n' && pQueryInfo->intervalTimeUnit != 'y') { + if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { // if the unit of time window value is millisecond, change the value from microsecond if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { - pQueryInfo->intervalTime = pQueryInfo->intervalTime / 1000; + pQueryInfo->interval.interval = pQueryInfo->interval.interval / 1000; } // interval cannot be less than 10 milliseconds - if (pQueryInfo->intervalTime < tsMinIntervalTime) { + if (pQueryInfo->interval.interval < tsMinIntervalTime) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } } @@ -641,7 +641,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ * check invalid SQL: * select tbname, tags_fields from super_table_name interval(1s) */ - if (tscQueryTags(pQueryInfo) && pQueryInfo->intervalTime > 0) { + if (tscQueryTags(pQueryInfo) && pQueryInfo->interval.interval > 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -679,34 +679,52 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ } int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { + const char* msg1 = "interval offset cannot be negative"; + const char* msg2 = "interval offset should be shorter than interval"; + const char* msg3 = "cannot use 'year' as offset when interval is 'month'"; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SStrToken* pOffset = &pQuerySql->offset; - if (pOffset->n == 0) { - pQueryInfo->offsetTimeUnit = pQueryInfo->offsetTimeUnit; - pQueryInfo->offsetTime = 0; + SStrToken* t = &pQuerySql->offset; + if (t->n == 0) { + pQueryInfo->interval.offsetUnit = pQueryInfo->interval.intervalUnit; + pQueryInfo->interval.offset = 0; return TSDB_CODE_SUCCESS; } - getTimestampInUsFromStr(pOffset->z, pOffset->n, &pQueryInfo->offsetTime); - if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { - pQueryInfo->offsetTime /= 1000; + if (parseDuration(t->z, t->n, &pQueryInfo->interval.offset, &pQueryInfo->interval.offsetUnit) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; } -/* - if (pQueryInfo->offsetTime < 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); - } - - if (pQueryInfo->slidingTime >= pQueryInfo->intervalTime) { + if (pQueryInfo->interval.offset < 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if ((pQueryInfo->intervalTime != 0) && (pQueryInfo->intervalTime/pQueryInfo->slidingTime > INTERVAL_SLIDING_FACTOR)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + if (pQueryInfo->interval.offsetUnit != 'n' && pQueryInfo->interval.offsetUnit != 'y') { + // if the unit of time window value is millisecond, change the value from microsecond + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { + pQueryInfo->interval.offset = pQueryInfo->interval.offset / 1000; + } + if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { + if (pQueryInfo->interval.offset >= pQueryInfo->interval.interval) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + } + } else if (pQueryInfo->interval.offsetUnit == pQueryInfo->interval.intervalUnit) { + if (pQueryInfo->interval.offset >= pQueryInfo->interval.interval) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + } else if (pQueryInfo->interval.intervalUnit == 'n' && pQueryInfo->interval.offsetUnit == 'y') { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } else if (pQueryInfo->interval.intervalUnit == 'y' && pQueryInfo->interval.offsetUnit == 'n') { + if (pQueryInfo->interval.interval * 12 <= pQueryInfo->interval.offset) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + } else { + // TODO: offset should be shorter than interval, but how to check + // conflicts like 30days offset and 1 month interval } - */ return TSDB_CODE_SUCCESS; } @@ -724,29 +742,29 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu SStrToken* pSliding = &pQuerySql->sliding; if (pSliding->n == 0) { - pQueryInfo->slidingTimeUnit = pQueryInfo->intervalTimeUnit; - pQueryInfo->slidingTime = pQueryInfo->intervalTime; + pQueryInfo->interval.slidingUnit = pQueryInfo->interval.intervalUnit; + pQueryInfo->interval.sliding = pQueryInfo->interval.interval; return TSDB_CODE_SUCCESS; } - if (pQueryInfo->intervalTimeUnit == 'n' || pQueryInfo->intervalTimeUnit == 'y') { + if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->slidingTime); + getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->interval.sliding); if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { - pQueryInfo->slidingTime /= 1000; + pQueryInfo->interval.sliding /= 1000; } - if (pQueryInfo->slidingTime < tsMinSlidingTime) { + if (pQueryInfo->interval.sliding < tsMinSlidingTime) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } - if (pQueryInfo->slidingTime > pQueryInfo->intervalTime) { + if (pQueryInfo->interval.sliding > pQueryInfo->interval.interval) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if ((pQueryInfo->intervalTime != 0) && (pQueryInfo->intervalTime/pQueryInfo->slidingTime > INTERVAL_SLIDING_FACTOR)) { + if ((pQueryInfo->interval.interval != 0) && (pQueryInfo->interval.interval/pQueryInfo->interval.sliding > INTERVAL_SLIDING_FACTOR)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -4758,9 +4776,9 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { const char* msg0 = "sample interval can not be less than 10ms."; const char* msg1 = "functions not allowed in select clause"; - if (pQueryInfo->intervalTime != 0 && pQueryInfo->intervalTime < 10 && - pQueryInfo->intervalTimeUnit != 'n' && - pQueryInfo->intervalTimeUnit != 'y') { + if (pQueryInfo->interval.interval != 0 && pQueryInfo->interval.interval < 10 && + pQueryInfo->interval.intervalUnit != 'n' && + pQueryInfo->interval.intervalUnit != 'y') { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } @@ -5545,7 +5563,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo insertResultField(pQueryInfo, (int32_t)size, &ids, bytes, (int8_t)type, name, pExpr); } else { // if this query is "group by" normal column, interval is not allowed - if (pQueryInfo->intervalTime > 0) { + if (pQueryInfo->interval.interval > 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -5578,7 +5596,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { // only retrieve tags, group by is not supportted if (tscQueryTags(pQueryInfo)) { - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || pQueryInfo->intervalTime > 0) { + if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || pQueryInfo->interval.interval > 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } else { return TSDB_CODE_SUCCESS; @@ -6030,7 +6048,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } else { - if ((pQueryInfo->intervalTime > 0) && + if ((pQueryInfo->interval.interval > 0) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6060,7 +6078,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { * not here. */ if (pQuerySql->fillType != NULL) { - if (pQueryInfo->intervalTime == 0) { + if (pQueryInfo->interval.interval == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -6228,7 +6246,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } else { - if ((pQueryInfo->intervalTime > 0) && + if ((pQueryInfo->interval.interval > 0) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6279,14 +6297,14 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { * the columns may be increased due to group by operation */ if (pQuerySql->fillType != NULL) { - if (pQueryInfo->intervalTime == 0 && (!tscIsPointInterpQuery(pQueryInfo))) { + if (pQueryInfo->interval.interval == 0 && (!tscIsPointInterpQuery(pQueryInfo))) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - if (pQueryInfo->intervalTime > 0 && pQueryInfo->intervalTimeUnit != 'n' && pQueryInfo->intervalTimeUnit != 'y') { + if (pQueryInfo->interval.interval > 0 && pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_INTERVAL_TIME_WINDOW) { + if ((timeRange == 0) || (timeRange / pQueryInfo->interval.interval) > MAX_INTERVAL_TIME_WINDOW) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 5639719c15..c6e9cbafd7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -647,8 +647,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_INVALID_SQL; } - if (pQueryInfo->intervalTime < 0) { - tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); + if (pQueryInfo->interval.interval < 0) { + tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->interval.interval); return TSDB_CODE_TSC_INVALID_SQL; } @@ -675,10 +675,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); pQueryMsg->numOfCols = htons((int16_t)taosArrayGetSize(pQueryInfo->colList)); - pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); - pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); - pQueryMsg->intervalTimeUnit = pQueryInfo->intervalTimeUnit; - pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; + pQueryMsg->interval.interval = htobe64(pQueryInfo->interval.interval); + pQueryMsg->interval.sliding = htobe64(pQueryInfo->interval.sliding); + pQueryMsg->interval.offset = htobe64(pQueryInfo->interval.offset); + pQueryMsg->interval.intervalUnit = pQueryInfo->interval.intervalUnit; + pQueryMsg->interval.slidingUnit = pQueryInfo->interval.slidingUnit; + pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 71ba76dc6c..99669ecd89 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -400,43 +400,43 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - if (pQueryInfo->intervalTimeUnit != 'n' && pQueryInfo->intervalTimeUnit != 'y' && pQueryInfo->intervalTime < minIntervalTime) { + if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.interval < minIntervalTime) { tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64, pSql, pStream, - pQueryInfo->intervalTime, minIntervalTime); - pQueryInfo->intervalTime = minIntervalTime; + pQueryInfo->interval.interval, minIntervalTime); + pQueryInfo->interval.interval = minIntervalTime; } - pStream->intervalTimeUnit = pQueryInfo->intervalTimeUnit; - pStream->intervalTime = pQueryInfo->intervalTime; // it shall be derived from sql string + pStream->intervalTimeUnit = pQueryInfo->interval.intervalUnit; + pStream->intervalTime = pQueryInfo->interval.interval; // it shall be derived from sql string - if (pQueryInfo->slidingTime <= 0) { - pQueryInfo->slidingTime = pQueryInfo->intervalTime; - pQueryInfo->slidingTimeUnit = pQueryInfo->intervalTimeUnit; + if (pQueryInfo->interval.sliding <= 0) { + pQueryInfo->interval.sliding = pQueryInfo->interval.interval; + pQueryInfo->interval.slidingUnit = pQueryInfo->interval.intervalUnit; } int64_t minSlidingTime = (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMinSlidingTime * 1000L : tsMinSlidingTime; - if (pQueryInfo->intervalTimeUnit != 'n' && pQueryInfo->intervalTimeUnit != 'y' && pQueryInfo->slidingTime < minSlidingTime) { + if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.sliding < minSlidingTime) { tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql, pStream, - pQueryInfo->slidingTime, minSlidingTime); + pQueryInfo->interval.sliding, minSlidingTime); - pQueryInfo->slidingTime = minSlidingTime; + pQueryInfo->interval.sliding = minSlidingTime; } - if (pQueryInfo->slidingTime > pQueryInfo->intervalTime) { + if (pQueryInfo->interval.sliding > pQueryInfo->interval.interval) { tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql, pStream, - pQueryInfo->slidingTime, pQueryInfo->intervalTime); + pQueryInfo->interval.sliding, pQueryInfo->interval.interval); - pQueryInfo->slidingTime = pQueryInfo->intervalTime; + pQueryInfo->interval.sliding = pQueryInfo->interval.interval; } - pStream->slidingTimeUnit = pQueryInfo->slidingTimeUnit; - pStream->slidingTime = pQueryInfo->slidingTime; + pStream->slidingTimeUnit = pQueryInfo->interval.slidingUnit; + pStream->slidingTime = pQueryInfo->interval.sliding; if (pStream->isProject) { - pQueryInfo->intervalTime = 0; // clear the interval value to avoid the force time window split by query processor - pQueryInfo->slidingTime = 0; + pQueryInfo->interval.interval = 0; // clear the interval value to avoid the force time window split by query processor + pQueryInfo->interval.sliding = 0; } } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 4246abf52d..c13e864293 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -113,7 +113,7 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the * final results which is acquired after the secondry merge of in the client. */ - if (pLimit->offset == 0 || pQueryInfo->intervalTime > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { + if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { if (win->skey > elem1.ts) { win->skey = elem1.ts; } @@ -178,6 +178,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in pSupporter->subqueryIndex = index; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); + memcpy(&pSupporter->interval, &pQueryInfo->interval, sizeof(pSupporter->interval)); pSupporter->limit = pQueryInfo->limit; STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index); @@ -307,6 +308,9 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { // set the second stage sub query for join process TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE); + memcpy(&pQueryInfo->interval, &pSupporter->interval, sizeof(pQueryInfo->interval)); + pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; + tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); pQueryInfo->colList = pSupporter->colList; @@ -1204,7 +1208,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter } pNew->cmd.numOfCols = 0; - pNewQueryInfo->intervalTime = 0; + pNewQueryInfo->interval.interval = 0; pSupporter->limit = pNewQueryInfo->limit; pNewQueryInfo->limit.limit = -1; @@ -2185,7 +2189,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { } // primary key column cannot be null in interval query, no need to check - if (i == 0 && pQueryInfo->intervalTime > 0) { + if (i == 0 && pQueryInfo->interval.interval > 0) { continue; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9b3f5927ab..cc0b844a8a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1899,10 +1899,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); pNewQueryInfo->command = pQueryInfo->command; - pNewQueryInfo->intervalTimeUnit = pQueryInfo->intervalTimeUnit; - pNewQueryInfo->slidingTimeUnit = pQueryInfo->slidingTimeUnit; - pNewQueryInfo->intervalTime = pQueryInfo->intervalTime; - pNewQueryInfo->slidingTime = pQueryInfo->slidingTime; + memcpy(&pNewQueryInfo->interval, &pQueryInfo->interval, sizeof(pNewQueryInfo->interval)); pNewQueryInfo->type = pQueryInfo->type; pNewQueryInfo->window = pQueryInfo->window; pNewQueryInfo->limit = pQueryInfo->limit; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 94ac5dc995..e49e2caca1 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -460,12 +460,7 @@ typedef struct { int16_t order; int16_t orderColId; int16_t numOfCols; // the number of columns will be load from vnode - int64_t intervalTime; // time interval for aggregation, in million second - int64_t slidingTime; // value for sliding window - int64_t offsetTime; // start offset for interval query - char intervalTimeUnit; - char slidingTimeUnit; // time interval type, for revisement of interval(1d) - char offsetTimeUnit; + SInterval interval; uint16_t tagCondLen; // tag length in current query int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; diff --git a/src/os/inc/osTime.h b/src/os/inc/osTime.h index 97432ca241..c03fc1c36f 100644 --- a/src/os/inc/osTime.h +++ b/src/os/inc/osTime.h @@ -63,6 +63,19 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) { } } + +typedef struct SInterval { + char intervalUnit; + char slidingUnit; + char offsetUnit; + int64_t interval; + int64_t sliding; + int64_t offset; +} SInterval; + +int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); +int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); + int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts); int32_t parseDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit); diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index 9d8328a71b..c9c6130498 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -403,6 +403,102 @@ int32_t parseDuration(const char* token, int32_t tokenLen, int64_t* duration, ch return getTimestampInUsFromStrImpl(*duration, *unit, duration); } +int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { + if (duration == 0) { + return t; + } + if (unit == 'y') { + duration *= 12; + } else if (unit != 'n') { + return t + duration; + } + + t /= 1000; + if (precision == TSDB_TIME_PRECISION_MICRO) { + t /= 1000; + } + + struct tm tm; + time_t tt = (time_t)t; + localtime_r(&tt, &tm); + int mon = tm.tm_year * 12 + tm.tm_mon + (int)duration; + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + + t = mktime(&tm) * 1000L; + if (precision == TSDB_TIME_PRECISION_MICRO) { + t *= 1000L; + } + + return t; +} + +int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) { + if (pInterval->sliding == 0) { + assert(pInterval->interval == 0); + return t; + } + + int64_t start = t; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + start /= 1000; + if (precision == TSDB_TIME_PRECISION_MICRO) { + start /= 1000; + } + struct tm tm; + time_t t = (time_t)start; + localtime_r(&t, &tm); + tm.tm_sec = 0; + tm.tm_min = 0; + tm.tm_hour = 0; + tm.tm_mday = 1; + + if (pInterval->slidingUnit == 'y') { + tm.tm_mon = 0; + tm.tm_year = (int)(tm.tm_year / pInterval->sliding * pInterval->sliding); + } else { + int mon = tm.tm_year * 12 + tm.tm_mon; + mon = (int)(mon / pInterval->sliding * pInterval->sliding); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + } + + start = mktime(&tm) * 1000L; + if (precision == TSDB_TIME_PRECISION_MICRO) { + start *= 1000L; + } + } else { + int64_t delta = t - pInterval->interval; + int32_t factor = delta > 0 ? 1 : -1; + + start = (delta / pInterval->sliding + factor) * pInterval->sliding; + + if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { + /* + * here we revised the start time of day according to the local time zone, + * but in case of DST, the start time of one day need to be dynamically decided. + */ + // todo refactor to extract function that is available for Linux/Windows/Mac platform + #if defined(WINDOWS) && _MSC_VER >= 1900 + // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char** tzname = _tzname; + #endif + + int64_t t = (precision == TSDB_TIME_PRECISION_MILLI) ? MILLISECOND_PER_SECOND : MILLISECOND_PER_SECOND * 1000L; + start += timezone * t; + } + + int64_t end = start + pInterval->interval - 1; + if (end < t) { + start += pInterval->sliding; + } + } + + return taosTimeAdd(start, pInterval->offset, pInterval->intervalUnit, precision); +} + // internal function, when program is paused in debugger, // one can call this function from debugger to print a // timestamp as human readable string, for example (gdb): diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 169bf907c6..5d570821cb 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -132,12 +132,9 @@ typedef struct SQueryCostInfo { typedef struct SQuery { int16_t numOfCols; int16_t numOfTags; - char intervalTimeUnit; - char slidingTimeUnit; // interval data type, used for daytime revise SOrderVal order; STimeWindow window; - int64_t intervalTime; - int64_t slidingTime; // sliding time for sliding window query + SInterval interval; int16_t precision; int16_t numOfOutput; int16_t fillType; diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h index 6b8dcb0bf9..6d44fee095 100644 --- a/src/query/inc/qFill.h +++ b/src/query/inc/qFill.h @@ -51,12 +51,11 @@ typedef struct SFillInfo { int32_t rowSize; // size of each row // char ** pTags; // tags value for current interpolation SFillTagColInfo* pTags; // tags value for filling gap - int64_t slidingTime; // sliding value to determine the number of result for a given time window + SInterval interval; char * prevValues; // previous row of data, to generate the interpolation results char * nextValues; // next row of data char** pData; // original result data block involved in filling data int32_t capacityInRows; // data buffer size in rows - int8_t slidingUnit; // sliding time unit int8_t precision; // time resoluation SFillColInfo* pFillCol; // column info for fill operations } SFillInfo; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b873714c49..64d05f2205 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -131,21 +131,21 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { static void setQueryStatus(SQuery *pQuery, int8_t status); static void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv); -#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->intervalTime > 0) +#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - if (pQuery->intervalTimeUnit != 'n' && pQuery->intervalTimeUnit != 'y') { - tw->skey += pQuery->slidingTime * factor; - tw->ekey = tw->skey + pQuery->intervalTime - 1; + if (pQuery->interval.intervalUnit != 'n' && pQuery->interval.intervalUnit != 'y') { + tw->skey += pQuery->interval.sliding * factor; + tw->ekey = tw->skey + pQuery->interval.interval - 1; return; } - int64_t key = tw->skey / 1000, interval = pQuery->intervalTime; + int64_t key = tw->skey / 1000, interval = pQuery->interval.interval; if (pQuery->precision == TSDB_TIME_PRECISION_MICRO) { key /= 1000; } - if (pQuery->intervalTimeUnit == 'y') { + if (pQuery->interval.intervalUnit == 'y') { interval *= 12; } @@ -510,10 +510,10 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t if (pWindowResInfo->curIndex == -1) { // the first window, from the previous stored value w.skey = pWindowResInfo->prevSKey; - if (pQuery->intervalTimeUnit == 'n' || pQuery->intervalTimeUnit == 'y') { - w.ekey = taosAddNatualInterval(w.skey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision) - 1; + if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') { + w.ekey = taosTimeAdd(w.skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision); } else { - w.ekey = w.skey + pQuery->intervalTime - 1; + w.ekey = w.skey + pQuery->interval.interval - 1; } } else { int32_t slot = curTimeWindowIndex(pWindowResInfo); @@ -522,23 +522,23 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t } if (w.skey > ts || w.ekey < ts) { - if (pQuery->intervalTimeUnit == 'n' || pQuery->intervalTimeUnit == 'y') { - w.skey = taosGetIntervalStartTimestamp(ts, pQuery->slidingTime, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision); - w.ekey = taosAddNatualInterval(w.skey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision) - 1; + if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, &pQuery->interval, pQuery->precision); + w.ekey = taosTimeAdd(w.skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1; } else { int64_t st = w.skey; if (st > ts) { - st -= ((st - ts + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime; + st -= ((st - ts + pQuery->interval.sliding - 1) / pQuery->interval.sliding) * pQuery->interval.sliding; } - int64_t et = st + pQuery->intervalTime - 1; + int64_t et = st + pQuery->interval.interval - 1; if (et < ts) { - st += ((ts - et + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime; + st += ((ts - et + pQuery->interval.sliding - 1) / pQuery->interval.sliding) * pQuery->interval.sliding; } w.skey = st; - w.ekey = w.skey + pQuery->intervalTime - 1; + w.ekey = w.skey + pQuery->interval.interval - 1; } } @@ -848,7 +848,7 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow int32_t startPos = 0; // tumbling time window query, a special case of sliding time window query - if (pQuery->slidingTime == pQuery->intervalTime && prevPosition != -1) { + if (pQuery->interval.sliding == pQuery->interval.interval && prevPosition != -1) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); startPos = prevPosition + factor; } else { @@ -861,21 +861,21 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow */ if (QUERY_IS_ASC_QUERY(pQuery) && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; - if (pQuery->intervalTimeUnit == 'n' || pQuery->intervalTimeUnit == 'y') { - pNext->skey = taosGetIntervalStartTimestamp(next, pQuery->slidingTime, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision); - pNext->ekey = taosAddNatualInterval(pNext->skey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision) - 1; + if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, &pQuery->interval, pQuery->precision); + pNext->ekey = taosTimeAdd(pNext->skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1; } else { - pNext->ekey += ((next - pNext->ekey + pQuery->slidingTime - 1)/pQuery->slidingTime) * pQuery->slidingTime; - pNext->skey = pNext->ekey - pQuery->intervalTime + 1; + pNext->ekey += ((next - pNext->ekey + pQuery->interval.sliding - 1)/pQuery->interval.sliding) * pQuery->interval.sliding; + pNext->skey = pNext->ekey - pQuery->interval.interval + 1; } } else if ((!QUERY_IS_ASC_QUERY(pQuery)) && primaryKeys[startPos] < pNext->skey) { TSKEY next = primaryKeys[startPos]; - if (pQuery->intervalTimeUnit == 'n' || pQuery->intervalTimeUnit == 'y') { - pNext->skey = taosGetIntervalStartTimestamp(next, pQuery->slidingTime, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision); - pNext->ekey = taosAddNatualInterval(pNext->skey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision) - 1; + if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, &pQuery->interval, pQuery->precision); + pNext->ekey = taosTimeAdd(pNext->skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1; } else { - pNext->skey -= ((pNext->skey - next + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime; - pNext->ekey = pNext->skey + pQuery->intervalTime - 1; + pNext->skey -= ((pNext->skey - next + pQuery->interval.sliding - 1) / pQuery->interval.sliding) * pQuery->interval.sliding; + pNext->ekey = pNext->skey + pQuery->interval.interval - 1; } } @@ -1871,20 +1871,20 @@ static bool onlyQueryTags(SQuery* pQuery) { ///////////////////////////////////////////////////////////////////////////////////////////// void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { - assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime); - win->skey = taosGetIntervalStartTimestamp(key, pQuery->slidingTime, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision); + assert(key >= keyFirst && key <= keyLast && pQuery->interval.sliding <= pQuery->interval.interval); + win->skey = taosTimeTruncate(key, &pQuery->interval, pQuery->precision); /* - * if the realSkey > INT64_MAX - pQuery->intervalTime, the query duration between + * if the realSkey > INT64_MAX - pQuery->interval.interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - if (keyFirst > (INT64_MAX - pQuery->intervalTime)) { - assert(keyLast - keyFirst < pQuery->intervalTime); + if (keyFirst > (INT64_MAX - pQuery->interval.interval)) { + assert(keyLast - keyFirst < pQuery->interval.interval); win->ekey = INT64_MAX; - } else if (pQuery->intervalTimeUnit == 'n' || pQuery->intervalTimeUnit == 'y') { - win->ekey = taosAddNatualInterval(win->skey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision) - 1; + } else if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') { + win->ekey = taosTimeAdd(win->skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1; } else { - win->ekey = win->skey + pQuery->intervalTime - 1; + win->ekey = win->skey + pQuery->interval.interval - 1; } } @@ -1998,7 +1998,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo return; } - if (isPointInterpoQuery(pQuery) && pQuery->intervalTime == 0) { + if (isPointInterpoQuery(pQuery) && pQuery->interval.interval == 0) { if (!QUERY_IS_ASC_QUERY(pQuery)) { qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); @@ -2009,7 +2009,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo return; } - if (pQuery->intervalTime == 0) { + if (pQuery->interval.interval == 0) { if (onlyFirstQuery(pQuery)) { if (!QUERY_IS_ASC_QUERY(pQuery)) { qDebug(msg, GET_QINFO_ADDR(pQuery), "only-first", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, @@ -4269,8 +4269,8 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { } /* - * 1. for interval without interpolation query we forward pQuery->intervalTime at a time for - * pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is + * 1. for interval without interpolation query we forward pQuery->interval.interval at a time for + * pQuery->limit.offset times. Since hole exists, pQuery->interval.interval*pQuery->limit.offset value is * not valid. otherwise, we only forward pQuery->limit.offset number of points */ assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); @@ -4569,7 +4569,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo getAlignQueryTimeWindow(pQuery, pQuery->window.skey, sk, ek, &w); pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pQuery->rec.capacity, pQuery->numOfOutput, - pQuery->slidingTime, pQuery->slidingTimeUnit, (int8_t)pQuery->precision, + pQuery->interval.sliding, pQuery->interval.slidingUnit, (int8_t)pQuery->precision, pQuery->fillType, pColInfo); } @@ -5430,7 +5430,7 @@ static void stableQueryImpl(SQInfo *pQInfo) { (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) { multiTableQueryProcess(pQInfo); } else { - assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || + assert((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); sequentialTableProcess(pQInfo); @@ -5476,8 +5476,8 @@ bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SC } static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { - if (pQueryMsg->intervalTime < 0) { - qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->intervalTime); + if (pQueryMsg->interval.interval < 0) { + qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->interval.interval); return false; } @@ -5556,8 +5556,12 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); - pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime); - pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); + pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); + pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); + pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); + pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit; + pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit; + pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit; pQueryMsg->limit = htobe64(pQueryMsg->limit); pQueryMsg->offset = htobe64(pQueryMsg->offset); @@ -5770,7 +5774,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64, pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, - pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime, + pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); return TSDB_CODE_SUCCESS; @@ -6109,10 +6113,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->order.orderColId = pQueryMsg->orderColId; pQuery->pSelectExpr = pExprs; pQuery->pGroupbyExpr = pGroupbyExpr; - pQuery->intervalTime = pQueryMsg->intervalTime; - pQuery->slidingTime = pQueryMsg->slidingTime; - pQuery->intervalTimeUnit = pQueryMsg->intervalTimeUnit; - pQuery->slidingTimeUnit = pQueryMsg->slidingTimeUnit; + memcpy(&pQuery->interval, &pQueryMsg->interval, sizeof(pQuery->interval)); pQuery->fillType = pQueryMsg->fillType; pQuery->numOfTags = pQueryMsg->numOfTags; pQuery->tagColList = pTagCols; diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index ddb63c5012..6ae36a160d 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -38,8 +38,8 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_ pFillInfo->numOfTags = numOfTags; pFillInfo->numOfCols = numOfCols; pFillInfo->precision = precision; - pFillInfo->slidingTime = slidingTime; - pFillInfo->slidingUnit = slidingUnit; + pFillInfo->interval.sliding = slidingTime; + pFillInfo->interval.slidingUnit = slidingUnit; pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); if (numOfTags > 0) { @@ -121,7 +121,7 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) return; } - pFillInfo->endKey = taosGetRevisedEndKey(endKey, pFillInfo->order, pFillInfo->slidingTime, pFillInfo->slidingUnit, + pFillInfo->endKey = taosGetRevisedEndKey(endKey, pFillInfo->order, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->precision); pFillInfo->rowIdx = 0; @@ -172,17 +172,17 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows int32_t numOfRows = taosNumOfRemainRows(pFillInfo); - TSKEY ekey1 = taosGetRevisedEndKey(ekey, pFillInfo->order, pFillInfo->slidingTime, pFillInfo->slidingUnit, + TSKEY ekey1 = taosGetRevisedEndKey(ekey, pFillInfo->order, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->precision); int64_t numOfRes = -1; if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; - if (pFillInfo->slidingUnit != 'y' && pFillInfo->slidingUnit != 'n') { - numOfRes = (int64_t)(ABS(lastKey - pFillInfo->start) / pFillInfo->slidingTime) + 1; + if (pFillInfo->interval.slidingUnit != 'y' && pFillInfo->interval.slidingUnit != 'n') { + numOfRes = (int64_t)(ABS(lastKey - pFillInfo->start) / pFillInfo->interval.sliding) + 1; } else { - numOfRes = taosCountNatualInterval(lastKey, pFillInfo->start, pFillInfo->slidingTime, pFillInfo->slidingUnit, pFillInfo->precision) + 1; + numOfRes = taosCountNatualInterval(lastKey, pFillInfo->start, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->precision) + 1; } assert(numOfRes >= numOfRows); } else { // reach the end of data @@ -191,10 +191,10 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows return 0; } // the numOfRes rows are all filled with specified policy - if (pFillInfo->slidingUnit != 'y' && pFillInfo->slidingUnit != 'n') { - numOfRes = (ABS(ekey1 - pFillInfo->start) / pFillInfo->slidingTime) + 1; + if (pFillInfo->interval.slidingUnit != 'y' && pFillInfo->interval.slidingUnit != 'n') { + numOfRes = (ABS(ekey1 - pFillInfo->start) / pFillInfo->interval.sliding) + 1; } else { - numOfRes = taosCountNatualInterval(ekey1, pFillInfo->start, pFillInfo->slidingTime, pFillInfo->slidingUnit, pFillInfo->precision) + 1; + numOfRes = taosCountNatualInterval(ekey1, pFillInfo->start, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->precision) + 1; } } @@ -375,10 +375,10 @@ static void doFillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* nu } // TODO natual sliding time - if (pFillInfo->slidingUnit != 'n' && pFillInfo->slidingUnit != 'y') { - pFillInfo->start += (pFillInfo->slidingTime * step); + if (pFillInfo->interval.slidingUnit != 'n' && pFillInfo->interval.slidingUnit != 'y') { + pFillInfo->start += (pFillInfo->interval.sliding * step); } else { - pFillInfo->start = taosAddNatualInterval(pFillInfo->start, pFillInfo->slidingTime*step, pFillInfo->slidingUnit, pFillInfo->precision); + pFillInfo->start = taosAddNatualInterval(pFillInfo->start, pFillInfo->interval.sliding*step, pFillInfo->interval.slidingUnit, pFillInfo->precision); } pFillInfo->numOfCurrent++; @@ -487,10 +487,10 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu setTagsValue(pFillInfo, data, num); // TODO natual sliding time - if (pFillInfo->slidingUnit != 'n' && pFillInfo->slidingUnit != 'y') { - pFillInfo->start += (pFillInfo->slidingTime * step); + if (pFillInfo->interval.slidingUnit != 'n' && pFillInfo->interval.slidingUnit != 'y') { + pFillInfo->start += (pFillInfo->interval.sliding * step); } else { - pFillInfo->start = taosAddNatualInterval(pFillInfo->start, pFillInfo->slidingTime*step, pFillInfo->slidingUnit, pFillInfo->precision); + pFillInfo->start = taosAddNatualInterval(pFillInfo->start, pFillInfo->interval.sliding*step, pFillInfo->interval.slidingUnit, pFillInfo->precision); } pFillInfo->rowIdx += 1; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index c195a0b76c..2bd92c74a4 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -54,7 +54,7 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pWindowResInfo->interval = pRuntimeEnv->pQuery->intervalTime; + pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval; pSummary->internalSupSize += sizeof(SWindowResult) * threshold; pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; From 85aca0c276ae4359431535574ee7d6253b6b6280 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 11 Sep 2020 18:14:38 +0800 Subject: [PATCH 096/105] td-1245: replace taosAddNatualInterval with taosTimeAdd --- src/client/src/tscStream.c | 6 +----- src/common/inc/tname.h | 1 - src/common/src/tname.c | 27 --------------------------- src/os/src/detail/osTime.c | 2 +- src/query/src/qFill.c | 34 ++++++++++------------------------ 5 files changed, 12 insertions(+), 58 deletions(-) diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 99669ecd89..a1b4ba0bf4 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -246,11 +246,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf } if (!pStream->isProject) { - if (pStream->intervalTimeUnit == 'y' || pStream->intervalTimeUnit == 'n') { - pStream->stime = taosAddNatualInterval(pStream->stime, pStream->slidingTime, pStream->slidingTimeUnit, pStream->precision); - } else { - pStream->stime += pStream->slidingTime; - } + pStream->stime = taosTimeAdd(pStream->stime, pStream->slidingTime, pStream->slidingTimeUnit, pStream->precision); } // actually only one row is returned. this following is not necessary taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream); diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index beef9ff375..b21d93fe8d 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -35,7 +35,6 @@ bool tscValidateTableNameLength(size_t len); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); -int64_t taosAddNatualInterval(int64_t key, int64_t intervalTime, char timeUnit, int16_t precision); int32_t taosCountNatualInterval(int64_t skey, int64_t ekey, int64_t intervalTime, char timeUnit, int16_t precision); int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index bb18cb6847..4193b770d2 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -99,33 +99,6 @@ SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numO return pFilter; } -int64_t taosAddNatualInterval(int64_t key, int64_t intervalTime, char timeUnit, int16_t precision) { - key /= 1000; - if (precision == TSDB_TIME_PRECISION_MICRO) { - key /= 1000; - } - - struct tm tm; - time_t t = (time_t)key; - localtime_r(&t, &tm); - - if (timeUnit == 'y') { - intervalTime *= 12; - } - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + intervalTime); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - - key = mktime(&tm) * 1000L; - - if (precision == TSDB_TIME_PRECISION_MICRO) { - key *= 1000L; - } - - return key; -} - int32_t taosCountNatualInterval(int64_t skey, int64_t ekey, int64_t intervalTime, char timeUnit, int16_t precision) { skey /= 1000; ekey /= 1000; diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index c9c6130498..21b972c7c8 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -496,7 +496,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio } } - return taosTimeAdd(start, pInterval->offset, pInterval->intervalUnit, precision); + return taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision); } // internal function, when program is paused in debugger, diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index 6ae36a160d..d0ecd22739 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -108,21 +108,15 @@ void* taosDestoryFillInfo(SFillInfo* pFillInfo) { return NULL; } -static TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision) { - if (order == TSDB_ORDER_ASC) { - return ekey; - } else { - return taosGetIntervalStartTimestamp(ekey, timeInterval, timeInterval, slidingTimeUnit, precision); - } -} - void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) { if (pFillInfo->fillType == TSDB_FILL_NONE) { return; } - pFillInfo->endKey = taosGetRevisedEndKey(endKey, pFillInfo->order, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, - pFillInfo->precision); + pFillInfo->endKey = endKey; + if (pFillInfo->order != TSDB_ORDER_ASC) { + pFillInfo->endKey = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision); + } pFillInfo->rowIdx = 0; pFillInfo->numOfRows = numOfRows; @@ -172,8 +166,10 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows int32_t numOfRows = taosNumOfRemainRows(pFillInfo); - TSKEY ekey1 = taosGetRevisedEndKey(ekey, pFillInfo->order, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, - pFillInfo->precision); + TSKEY ekey1 = ekey; + if (pFillInfo->order != TSDB_ORDER_ASC) { + pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); + } int64_t numOfRes = -1; if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. @@ -374,12 +370,7 @@ static void doFillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* nu setTagsValue(pFillInfo, data, *num); } -// TODO natual sliding time - if (pFillInfo->interval.slidingUnit != 'n' && pFillInfo->interval.slidingUnit != 'y') { - pFillInfo->start += (pFillInfo->interval.sliding * step); - } else { - pFillInfo->start = taosAddNatualInterval(pFillInfo->start, pFillInfo->interval.sliding*step, pFillInfo->interval.slidingUnit, pFillInfo->precision); - } + pFillInfo->start = taosTimeAdd(pFillInfo->start, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision); pFillInfo->numOfCurrent++; (*num) += 1; @@ -486,12 +477,7 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu // set the tag value for final result setTagsValue(pFillInfo, data, num); - // TODO natual sliding time - if (pFillInfo->interval.slidingUnit != 'n' && pFillInfo->interval.slidingUnit != 'y') { - pFillInfo->start += (pFillInfo->interval.sliding * step); - } else { - pFillInfo->start = taosAddNatualInterval(pFillInfo->start, pFillInfo->interval.sliding*step, pFillInfo->interval.slidingUnit, pFillInfo->precision); - } + pFillInfo->start = taosTimeAdd(pFillInfo->start, pFillInfo->interval.sliding*step, pFillInfo->interval.slidingUnit, pFillInfo->precision); pFillInfo->rowIdx += 1; pFillInfo->numOfCurrent +=1; From 6fa86bfb8f31b70d8ddb85844a7f0b061f6223dd Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sat, 12 Sep 2020 17:29:18 +0800 Subject: [PATCH 097/105] td-1245: fix crash and refactor --- src/client/inc/tsclient.h | 5 +- src/client/src/tscProfile.c | 6 +- src/client/src/tscStream.c | 61 +++++++++++---------- src/common/inc/tname.h | 3 +- src/common/src/tname.c | 32 +---------- src/inc/taosdef.h | 1 + src/os/inc/osTime.h | 1 + src/os/src/detail/osTime.c | 106 +++++++++++++++++++++++------------- src/query/src/qExecutor.c | 2 - src/query/src/qFill.c | 26 +++++---- 10 files changed, 126 insertions(+), 117 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 17c4e7fc55..023594d865 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -366,8 +366,6 @@ typedef struct SSqlStream { uint32_t streamId; char listed; bool isProject; - char intervalTimeUnit; - char slidingTimeUnit; int16_t precision; int64_t num; // number of computing count @@ -381,8 +379,7 @@ typedef struct SSqlStream { int64_t ctime; // stream created time int64_t stime; // stream next executed time int64_t etime; // stream end query time, when time is larger then etime, the stream will be closed - int64_t intervalTime; - int64_t slidingTime; + SInterval interval; void * pTimer; void (*fp)(); diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index b8c3830204..f2f997a594 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -259,11 +259,11 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { pSdesc->num = htobe64(pStream->num); pSdesc->useconds = htobe64(pStream->useconds); - pSdesc->stime = htobe64(pStream->stime - pStream->intervalTime); + pSdesc->stime = htobe64(pStream->stime - pStream->interval.interval); pSdesc->ctime = htobe64(pStream->ctime); - pSdesc->slidingTime = htobe64(pStream->slidingTime); - pSdesc->interval = htobe64(pStream->intervalTime); + pSdesc->slidingTime = htobe64(pStream->interval.sliding); + pSdesc->interval = htobe64(pStream->interval.interval); pHeartbeat->numOfStreams++; pSdesc++; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index a1b4ba0bf4..4a1f4d9d87 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -51,7 +51,7 @@ static int64_t tscGetRetryDelayTime(SSqlStream* pStream, int64_t slidingTime, in int64_t retryDelta = (int64_t)(tsStreamCompRetryDelay * retryRangeFactor); retryDelta = ((rand() % retryDelta) + tsStreamCompRetryDelay) * 1000L; - if (pStream->intervalTimeUnit != 'n' && pStream->intervalTimeUnit != 'y') { + if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') { // change to ms if (prec == TSDB_TIME_PRECISION_MICRO) { slidingTime = slidingTime / 1000; @@ -87,7 +87,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { // failed to get meter/metric meta, retry in 10sec. if (code != TSDB_CODE_SUCCESS) { - int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->slidingTime, pStream->precision); + int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); tscDebug("%p stream:%p,get metermeta failed, retry in %" PRId64 "ms", pStream->pSql, pStream, retryDelayTime); tscSetRetryTimer(pStream, pSql, retryDelayTime); @@ -132,15 +132,16 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { } if (etime > pStream->etime) { etime = pStream->etime; - } else if (pStream->intervalTimeUnit != 'y' && pStream->intervalTimeUnit != 'n') { - etime = pStream->stime + (etime - pStream->stime) / pStream->intervalTime * pStream->intervalTime; + } else if (pStream->interval.intervalUnit != 'y' && pStream->interval.intervalUnit != 'n') { + etime = pStream->stime + (etime - pStream->stime) / pStream->interval.interval * pStream->interval.interval; } else { - etime = taosGetIntervalStartTimestamp(etime, pStream->slidingTime, pStream->intervalTime, pStream->slidingTimeUnit, pStream->precision); + etime = taosTimeTruncate(etime, &pStream->interval, pStream->precision); + //etime = taosGetIntervalStartTimestamp(etime, pStream->interval.sliding, pStream->interval.sliding, pStream->interval.slidingUnit, pStream->precision); } pQueryInfo->window.ekey = etime; if (pQueryInfo->window.skey >= pQueryInfo->window.ekey) { - int64_t timer = pStream->slidingTime; - if (pStream->intervalTimeUnit == 'y' || pStream->intervalTimeUnit == 'n') { + int64_t timer = pStream->interval.sliding; + if (pStream->interval.intervalUnit == 'y' || pStream->interval.intervalUnit == 'n') { timer = 86400 * 1000l; } else if (pStream->precision == TSDB_TIME_PRECISION_MICRO) { timer /= 1000l; @@ -162,7 +163,7 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows) { SSqlStream *pStream = (SSqlStream *)param; if (tres == NULL || numOfRows < 0) { - int64_t retryDelay = tscGetRetryDelayTime(pStream, pStream->slidingTime, pStream->precision); + int64_t retryDelay = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); tscError("%p stream:%p, query data failed, code:0x%08x, retry in %" PRId64 "ms", pStream->pSql, pStream, numOfRows, retryDelay); @@ -223,7 +224,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf SSqlObj * pSql = (SSqlObj *)res; if (pSql == NULL || numOfRows < 0) { - int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->slidingTime, pStream->precision); + int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision); tscError("%p stream:%p, retrieve data failed, code:0x%08x, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retryDelayTime); tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); @@ -246,7 +247,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf } if (!pStream->isProject) { - pStream->stime = taosTimeAdd(pStream->stime, pStream->slidingTime, pStream->slidingTimeUnit, pStream->precision); + pStream->stime = taosTimeAdd(pStream->stime, pStream->interval.sliding, pStream->interval.slidingUnit, pStream->precision); } // actually only one row is returned. this following is not necessary taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream); @@ -306,7 +307,7 @@ static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer) now + timer, timer, delay, pStream->stime, etime); } else { tscDebug("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql, pStream, - pStream->stime, timer, delay, pStream->stime - pStream->intervalTime, pStream->stime - 1); + pStream->stime, timer, delay, pStream->stime - pStream->interval.interval, pStream->stime - 1); } pSql->cmd.command = TSDB_SQL_SELECT; @@ -320,12 +321,12 @@ static int64_t getLaunchTimeDelay(const SSqlStream* pStream) { (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMaxStreamComputDelay * 1000L : tsMaxStreamComputDelay; int64_t delayDelta = maxDelay; - if (pStream->intervalTimeUnit != 'n' && pStream->intervalTimeUnit != 'y') { - delayDelta = (int64_t)(pStream->slidingTime * tsStreamComputDelayRatio); + if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') { + delayDelta = (int64_t)(pStream->interval.sliding * tsStreamComputDelayRatio); if (delayDelta > maxDelay) { delayDelta = maxDelay; } - int64_t remainTimeWindow = pStream->slidingTime - delayDelta; + int64_t remainTimeWindow = pStream->interval.sliding - delayDelta; if (maxDelay > remainTimeWindow) { maxDelay = (int64_t)(remainTimeWindow / 1.5f); } @@ -333,8 +334,8 @@ static int64_t getLaunchTimeDelay(const SSqlStream* pStream) { int64_t currentDelay = (rand() % maxDelay); // a random number currentDelay += delayDelta; - if (pStream->intervalTimeUnit != 'n' && pStream->intervalTimeUnit != 'y') { - assert(currentDelay < pStream->slidingTime); + if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') { + assert(currentDelay < pStream->interval.sliding); } return currentDelay; @@ -349,7 +350,7 @@ static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { * for project query, no mater fetch data successfully or not, next launch will issue * more than the sliding time window */ - timer = pStream->slidingTime; + timer = pStream->interval.sliding; if (pStream->stime > pStream->etime) { tscDebug("%p stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql, pStream, pStream->stime, pStream->etime); @@ -362,7 +363,8 @@ static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { return; } } else { - int64_t stime = taosGetIntervalStartTimestamp(pStream->stime - 1, pStream->intervalTime, pStream->intervalTime, pStream->intervalTimeUnit, pStream->precision); + int64_t stime = taosTimeTruncate(pStream->stime - 1, &pStream->interval, pStream->precision); + //int64_t stime = taosGetIntervalStartTimestamp(pStream->stime - 1, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision); if (stime >= pStream->etime) { tscDebug("%p stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql, pStream, pStream->stime, pStream->etime); @@ -402,8 +404,8 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { pQueryInfo->interval.interval = minIntervalTime; } - pStream->intervalTimeUnit = pQueryInfo->interval.intervalUnit; - pStream->intervalTime = pQueryInfo->interval.interval; // it shall be derived from sql string + pStream->interval.intervalUnit = pQueryInfo->interval.intervalUnit; + pStream->interval.interval = pQueryInfo->interval.interval; // it shall be derived from sql string if (pQueryInfo->interval.sliding <= 0) { pQueryInfo->interval.sliding = pQueryInfo->interval.interval; @@ -427,8 +429,8 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { pQueryInfo->interval.sliding = pQueryInfo->interval.interval; } - pStream->slidingTimeUnit = pQueryInfo->interval.slidingUnit; - pStream->slidingTime = pQueryInfo->interval.sliding; + pStream->interval.slidingUnit = pQueryInfo->interval.slidingUnit; + pStream->interval.sliding = pQueryInfo->interval.sliding; if (pStream->isProject) { pQueryInfo->interval.interval = 0; // clear the interval value to avoid the force time window split by query processor @@ -441,8 +443,8 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in if (pStream->isProject) { // no data in table, flush all data till now to destination meter, 10sec delay - pStream->intervalTime = tsProjectExecInterval; - pStream->slidingTime = tsProjectExecInterval; + pStream->interval.interval = tsProjectExecInterval; + pStream->interval.sliding = tsProjectExecInterval; if (stime != 0) { // first projection start from the latest event timestamp assert(stime >= pQueryInfo->window.skey); @@ -455,12 +457,15 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in stime = pQueryInfo->window.skey; if (stime == INT64_MIN) { stime = (int64_t)taosGetTimestamp(pStream->precision); - stime = taosGetIntervalStartTimestamp(stime, pStream->intervalTime, pStream->intervalTime, pStream->intervalTimeUnit, pStream->precision); - stime = taosGetIntervalStartTimestamp(stime - 1, pStream->intervalTime, pStream->intervalTime, pStream->intervalTimeUnit, pStream->precision); + stime = taosTimeTruncate(stime, &pStream->interval, pStream->precision); + stime = taosTimeTruncate(stime - 1, &pStream->interval, pStream->precision); + //stime = taosGetIntervalStartTimestamp(stime, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision); + //stime = taosGetIntervalStartTimestamp(stime - 1, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision); tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64, pSql, pStream, stime); } } else { - int64_t newStime = taosGetIntervalStartTimestamp(stime, pStream->intervalTime, pStream->intervalTime, pStream->intervalTimeUnit, pStream->precision); + //int64_t newStime = taosGetIntervalStartTimestamp(stime, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision); + int64_t newStime = taosTimeTruncate(stime, &pStream->interval, pStream->precision); if (newStime != stime) { tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql, pStream, stime, newStime); stime = newStime; @@ -530,7 +535,7 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { taosTmrReset(tscProcessStreamTimer, (int32_t)starttime, pStream, tscTmr, &pStream->pTimer); tscDebug("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql, - pStream, pTableMetaInfo->name, pStream->intervalTime, pStream->slidingTime, starttime, pSql->sqlstr); + pStream, pTableMetaInfo->name, pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr); } TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index b21d93fe8d..6b73d98b81 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -35,7 +35,6 @@ bool tscValidateTableNameLength(size_t len); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); -int32_t taosCountNatualInterval(int64_t skey, int64_t ekey, int64_t intervalTime, char timeUnit, int16_t precision); -int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision); +// int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision); #endif // TDENGINE_NAME_H diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 4193b770d2..8879e9e797 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -99,35 +99,7 @@ SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numO return pFilter; } -int32_t taosCountNatualInterval(int64_t skey, int64_t ekey, int64_t intervalTime, char timeUnit, int16_t precision) { - skey /= 1000; - ekey /= 1000; - if (precision == TSDB_TIME_PRECISION_MICRO) { - skey /= 1000; - ekey /= 1000; - } - if (ekey < skey) { - int64_t tmp = ekey; - ekey = skey; - skey = tmp; - } - - struct tm tm; - time_t t = (time_t)skey; - localtime_r(&t, &tm); - int smon = tm.tm_year * 12 + tm.tm_mon; - - t = (time_t)ekey; - localtime_r(&t, &tm); - int emon = tm.tm_year * 12 + tm.tm_mon; - - if (timeUnit == 'y') { - intervalTime *= 12; - } - - return (emon - smon) / (int32_t)intervalTime; -} - +#if 0 int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision) { if (slidingTime == 0) { return startTime; @@ -192,6 +164,8 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in return start; } +#endif + /* * tablePrefix.columnName * extract table name and save it in pTable, with only column name in pToken diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 17cbbca834..9a411d8f85 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -101,6 +101,7 @@ extern const int32_t TYPE_BYTES[11]; #define TSDB_TIME_PRECISION_MILLI 0 #define TSDB_TIME_PRECISION_MICRO 1 #define TSDB_TIME_PRECISION_NANO 2 +#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L)) #define TSDB_TIME_PRECISION_MILLI_STR "ms" #define TSDB_TIME_PRECISION_MICRO_STR "us" diff --git a/src/os/inc/osTime.h b/src/os/inc/osTime.h index c03fc1c36f..2bc24caa14 100644 --- a/src/os/inc/osTime.h +++ b/src/os/inc/osTime.h @@ -75,6 +75,7 @@ typedef struct SInterval { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); +int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts); int32_t parseDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit); diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index 21b972c7c8..5e6922cd51 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -413,24 +413,43 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { return t + duration; } - t /= 1000; - if (precision == TSDB_TIME_PRECISION_MICRO) { - t /= 1000; - } - struct tm tm; - time_t tt = (time_t)t; + time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); localtime_r(&tt, &tm); int mon = tm.tm_year * 12 + tm.tm_mon + (int)duration; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - t = mktime(&tm) * 1000L; - if (precision == TSDB_TIME_PRECISION_MICRO) { - t *= 1000L; + return mktime(&tm) * TSDB_TICK_PER_SECOND(precision); +} + +int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision) { + if (ekey < skey) { + int64_t tmp = ekey; + ekey = skey; + skey = tmp; + } + if (unit != 'n' && unit != 'y') { + return (int32_t)((ekey - skey) / interval); } - return t; + skey /= TSDB_TICK_PER_SECOND(precision); + ekey /= TSDB_TICK_PER_SECOND(precision); + + struct tm tm; + time_t t = (time_t)skey; + localtime_r(&t, &tm); + int smon = tm.tm_year * 12 + tm.tm_mon; + + t = (time_t)ekey; + localtime_r(&t, &tm); + int emon = tm.tm_year * 12 + tm.tm_mon; + + if (unit == 'y') { + interval *= 12; + } + + return (emon - smon) / (int32_t)interval; } int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) { @@ -440,14 +459,11 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio } int64_t start = t; - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - start /= 1000; - if (precision == TSDB_TIME_PRECISION_MICRO) { - start /= 1000; - } + if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') { + start /= TSDB_TICK_PER_SECOND(precision); struct tm tm; - time_t t = (time_t)start; - localtime_r(&t, &tm); + time_t tt = (time_t)start; + localtime_r(&tt, &tm); tm.tm_sec = 0; tm.tm_min = 0; tm.tm_hour = 0; @@ -463,10 +479,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio tm.tm_mon = mon % 12; } - start = mktime(&tm) * 1000L; - if (precision == TSDB_TIME_PRECISION_MICRO) { - start *= 1000L; - } + start = mktime(&tm) * TSDB_TICK_PER_SECOND(precision); } else { int64_t delta = t - pInterval->interval; int32_t factor = delta > 0 ? 1 : -1; @@ -486,8 +499,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio char** tzname = _tzname; #endif - int64_t t = (precision == TSDB_TIME_PRECISION_MILLI) ? MILLISECOND_PER_SECOND : MILLISECOND_PER_SECOND * 1000L; - start += timezone * t; + start += timezone * TSDB_TICK_PER_SECOND(precision); } int64_t end = start + pInterval->interval - 1; @@ -496,7 +508,13 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio } } - return taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision); + if (pInterval->offset > 0) { + start = taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision); + if (start > t) { + start = taosTimeAdd(start, -pInterval->interval, pInterval->intervalUnit, precision); + } + } + return start; } // internal function, when program is paused in debugger, @@ -507,24 +525,38 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // 2020-07-03 17:48:42 // and the parameter can also be a variable. const char* fmtts(int64_t ts) { - static char buf[32]; + static char buf[96]; + size_t pos = 0; + struct tm tm; - time_t tt; if (ts > -62135625943 && ts < 32503651200) { - tt = ts; - } else if (ts > -62135625943000 && ts < 32503651200000) { - tt = ts / 1000; - } else { - tt = ts / 1000000; + time_t t = (time_t)ts; + localtime_r(&t, &tm); + pos += strftime(buf + pos, sizeof(buf), "s=%Y-%m-%d %H:%M:%S", &tm); } - struct tm* ptm = localtime(&tt); - size_t pos = strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ptm); + if (ts > -62135625943000 && ts < 32503651200000) { + time_t t = (time_t)(ts / 1000); + localtime_r(&t, &tm); + if (pos > 0) { + buf[pos++] = ' '; + buf[pos++] = '|'; + buf[pos++] = ' '; + } + pos += strftime(buf + pos, sizeof(buf), "ms=%Y-%m-%d %H:%M:%S", &tm); + pos += sprintf(buf + pos, ".%03d", (int)(ts % 1000)); + } - if (ts <= -62135625943000 || ts >= 32503651200000) { - sprintf(buf + pos, ".%06d", (int)(ts % 1000000)); - } else if (ts <= -62135625943 || ts >= 32503651200) { - sprintf(buf + pos, ".%03d", (int)(ts % 1000)); + { + time_t t = (time_t)(ts / 1000000); + localtime_r(&t, &tm); + if (pos > 0) { + buf[pos++] = ' '; + buf[pos++] = '|'; + buf[pos++] = ' '; + } + pos += strftime(buf + pos, sizeof(buf), "us=%Y-%m-%d %H:%M:%S", &tm); + pos += sprintf(buf + pos, ".%06d", (int)(ts % 1000000)); } return buf; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 64d05f2205..e26eeefb6a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -550,8 +550,6 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t w.ekey = pQuery->window.ekey; } - assert(ts >= w.skey && ts <= w.ekey); - return w; } diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index d0ecd22739..8660d9f4fe 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -174,24 +174,26 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows int64_t numOfRes = -1; if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; - - if (pFillInfo->interval.slidingUnit != 'y' && pFillInfo->interval.slidingUnit != 'n') { - numOfRes = (int64_t)(ABS(lastKey - pFillInfo->start) / pFillInfo->interval.sliding) + 1; - } else { - numOfRes = taosCountNatualInterval(lastKey, pFillInfo->start, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->precision) + 1; - } + numOfRes = taosTimeCountInterval( + lastKey, + pFillInfo->start, + pFillInfo->interval.sliding, + pFillInfo->interval.slidingUnit, + pFillInfo->precision); + numOfRes += 1; assert(numOfRes >= numOfRows); } else { // reach the end of data if ((ekey1 < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) || (ekey1 > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) { return 0; } - // the numOfRes rows are all filled with specified policy - if (pFillInfo->interval.slidingUnit != 'y' && pFillInfo->interval.slidingUnit != 'n') { - numOfRes = (ABS(ekey1 - pFillInfo->start) / pFillInfo->interval.sliding) + 1; - } else { - numOfRes = taosCountNatualInterval(ekey1, pFillInfo->start, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->precision) + 1; - } + numOfRes = taosTimeCountInterval( + ekey1, + pFillInfo->start, + pFillInfo->interval.sliding, + pFillInfo->interval.slidingUnit, + pFillInfo->precision); + numOfRes += 1; } return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; From 9c219df20607fe33aafee87c99884b032b08de90 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 16 Sep 2020 16:39:32 +0800 Subject: [PATCH 098/105] td-1245: add test cases for interval offset --- tests/pytest/query/intervalOffset.py | 225 +++++++++++++++++++++++++++ tests/pytest/query/natualInterval.py | 8 +- 2 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 tests/pytest/query/intervalOffset.py diff --git a/tests/pytest/query/intervalOffset.py b/tests/pytest/query/intervalOffset.py new file mode 100644 index 0000000000..6d4dd720ee --- /dev/null +++ b/tests/pytest/query/intervalOffset.py @@ -0,0 +1,225 @@ +################################################################### +# Copyright (c) 2020 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def general(self): + tdSql.execute("create table meters(ts timestamp, s int) tags(id int)") + tdSql.execute("create table t0 using meters tags(0)") + tdSql.execute("create table t1 using meters tags(1)") + tdSql.execute("create table t2 using meters tags(2)") + tdSql.execute("create table t3 using meters tags(3)") + tdSql.execute("create table t4 using meters tags(4)") + + tdSql.execute("insert into t0 values('2019-01-01 00:00:00', 1)") + tdSql.execute("insert into t1 values('2019-01-01 00:00:01', 1)") + tdSql.execute("insert into t2 values('2019-01-01 00:01:00', 1)") + tdSql.execute("insert into t1 values('2019-01-01 00:01:01', 1)") + tdSql.execute("insert into t1 values('2019-01-01 00:01:02', 1)") + tdSql.execute("insert into t1 values('2019-01-01 00:01:03', 1)") + tdSql.execute("insert into t1 values('2019-01-01 00:01:30', 1)") + tdSql.execute("insert into t1 values('2019-01-01 00:01:50', 1)") + tdSql.execute("insert into t2 values('2019-01-01 00:02:00', 1)") + tdSql.execute("insert into t3 values('2019-01-01 00:02:02', 1)") + tdSql.execute("insert into t3 values('2019-01-01 00:02:59', 1)") + tdSql.execute("insert into t4 values('2019-01-01 00:02:59', 1)") + tdSql.execute("insert into t1 values('2019-01-01 00:03:10', 1)") + tdSql.execute("insert into t2 values('2019-01-01 00:08:00', 1)") + tdSql.execute("insert into t1 values('2019-01-01 00:08:00', 1)") + + tdSql.query("select count(*) from meters interval(1m, 1s)") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + tdSql.checkData(2, 1, 6) + tdSql.checkData(3, 1, 3) + tdSql.checkData(4, 1, 1) + tdSql.checkData(5, 1, 2) + + tdSql.query("select count(*) from meters interval(1m, 2s)") + tdSql.checkData(0, 1, 2) + tdSql.checkData(1, 1, 2) + tdSql.checkData(2, 1, 5) + tdSql.checkData(3, 1, 3) + tdSql.checkData(4, 1, 1) + tdSql.checkData(5, 1, 2) + + tdSql.query("select count(*) from meters interval(90s, 1500a)") + tdSql.checkData(0, 1, 2) + tdSql.checkData(1, 1, 5) + tdSql.checkData(2, 1, 5) + tdSql.checkData(3, 1, 1) + tdSql.checkData(4, 1, 2) + + def singleTable(self): + tdSql.execute("create table car(ts timestamp, s int)") + tdSql.execute("insert into car values('2019-01-01 00:00:00', 1)") + tdSql.execute("insert into car values('2019-05-13 12:00:00', 1)") + tdSql.execute("insert into car values('2019-12-31 23:59:59', 1)") + tdSql.execute("insert into car values('2020-01-01 12:00:00', 1)") + tdSql.execute("insert into car values('2020-01-02 12:00:00', 1)") + tdSql.execute("insert into car values('2020-01-03 12:00:00', 1)") + tdSql.execute("insert into car values('2020-01-04 12:00:00', 1)") + tdSql.execute("insert into car values('2020-01-05 12:00:00', 1)") + tdSql.execute("insert into car values('2020-01-31 12:00:00', 1)") + tdSql.execute("insert into car values('2020-02-01 12:00:00', 1)") + tdSql.execute("insert into car values('2020-02-02 12:00:00', 1)") + tdSql.execute("insert into car values('2020-02-29 12:00:00', 1)") + tdSql.execute("insert into car values('2020-03-01 12:00:00', 1)") + tdSql.execute("insert into car values('2020-03-02 12:00:00', 1)") + tdSql.execute("insert into car values('2020-03-15 12:00:00', 1)") + tdSql.execute("insert into car values('2020-03-31 12:00:00', 1)") + tdSql.execute("insert into car values('2020-05-01 12:00:00', 1)") + + tdSql.query("select count(*) from car interval(1n, 10d)") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 6) + tdSql.checkData(3, 1, 3) + tdSql.checkData(4, 1, 3) + tdSql.checkData(5, 1, 2) + tdSql.checkData(6, 1, 1) + + tdSql.query("select count(*) from car interval(1n, 10d) order by ts desc") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + tdSql.checkData(2, 1, 3) + tdSql.checkData(3, 1, 3) + tdSql.checkData(4, 1, 6) + tdSql.checkData(5, 1, 1) + tdSql.checkData(6, 1, 1) + + tdSql.query("select count(*) from car interval(2n, 5d)") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 6) + tdSql.checkData(3, 1, 6) + tdSql.checkData(4, 1, 3) + + tdSql.query("select count(*) from car interval(2n) order by ts desc") + tdSql.checkData(0, 1, 3) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 6) + tdSql.checkData(3, 1, 1) + tdSql.checkData(4, 1, 1) + + tdSql.query("select count(*) from car interval(1y, 1n)") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 8) + tdSql.checkData(2, 1, 8) + + tdSql.query("select count(*) from car interval(1y, 2n)") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 11) + tdSql.checkData(2, 1, 5) + + tdSql.query("select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d)") + tdSql.checkData(0, 1, 6) + tdSql.checkData(1, 1, 9) + + def superTable(self): + tdSql.execute("create table cars(ts timestamp, s int) tags(id int)") + tdSql.execute("create table car0 using cars tags(0)") + tdSql.execute("create table car1 using cars tags(1)") + tdSql.execute("create table car2 using cars tags(2)") + tdSql.execute("create table car3 using cars tags(3)") + tdSql.execute("create table car4 using cars tags(4)") + + tdSql.execute("insert into car0 values('2019-01-01 00:00:00', 1)") + tdSql.execute("insert into car1 values('2019-05-13 12:00:00', 1)") + tdSql.execute("insert into car2 values('2019-12-31 23:59:59', 1)") + tdSql.execute("insert into car1 values('2020-01-01 12:00:00', 1)") + tdSql.execute("insert into car1 values('2020-01-02 12:00:00', 1)") + tdSql.execute("insert into car1 values('2020-01-03 12:00:00', 1)") + tdSql.execute("insert into car1 values('2020-01-04 12:00:00', 1)") + tdSql.execute("insert into car1 values('2020-01-05 12:00:00', 1)") + tdSql.execute("insert into car1 values('2020-01-31 12:00:00', 1)") + tdSql.execute("insert into car1 values('2020-02-01 12:00:00', 1)") + tdSql.execute("insert into car2 values('2020-02-02 12:00:00', 1)") + tdSql.execute("insert into car2 values('2020-02-29 12:00:00', 1)") + tdSql.execute("insert into car3 values('2020-03-01 12:00:00', 1)") + tdSql.execute("insert into car3 values('2020-03-02 12:00:00', 1)") + tdSql.execute("insert into car3 values('2020-03-15 12:00:00', 1)") + tdSql.execute("insert into car4 values('2020-03-31 12:00:00', 1)") + tdSql.execute("insert into car3 values('2020-05-01 12:00:00', 1)") + + tdSql.query("select count(*) from cars interval(1n, 10d)") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 6) + tdSql.checkData(3, 1, 3) + tdSql.checkData(4, 1, 3) + tdSql.checkData(5, 1, 2) + tdSql.checkData(6, 1, 1) + + tdSql.query("select count(*) from cars interval(1n, 10d) order by ts desc") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + tdSql.checkData(2, 1, 3) + tdSql.checkData(3, 1, 3) + tdSql.checkData(4, 1, 6) + tdSql.checkData(5, 1, 1) + tdSql.checkData(6, 1, 1) + + tdSql.query("select count(*) from cars interval(2n, 5d)") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 6) + tdSql.checkData(3, 1, 6) + tdSql.checkData(4, 1, 3) + + tdSql.query("select count(*) from cars interval(2n) order by ts desc") + tdSql.checkData(0, 1, 3) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 6) + tdSql.checkData(3, 1, 1) + tdSql.checkData(4, 1, 1) + + tdSql.query("select count(*) from cars interval(1y, 1n)") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 8) + tdSql.checkData(2, 1, 8) + + tdSql.query("select count(*) from cars interval(1y, 2n)") + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 11) + tdSql.checkData(2, 1, 5) + + tdSql.query("select count(*) from cars where ts > '2019-05-14 00:00:00' interval(1y, 5d)") + tdSql.checkData(0, 1, 6) + tdSql.checkData(1, 1, 9) + + def run(self): + tdSql.prepare() + self.general() + self.singleTable() + self.superTable() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + diff --git a/tests/pytest/query/natualInterval.py b/tests/pytest/query/natualInterval.py index 1ed91e1c68..434cd392bb 100644 --- a/tests/pytest/query/natualInterval.py +++ b/tests/pytest/query/natualInterval.py @@ -89,10 +89,10 @@ class TDTestCase: def superTable(self): tdSql.execute("create table cars(ts timestamp, s int) tags(id int)") tdSql.execute("create table car0 using cars tags(0)") - tdSql.execute("create table car1 using cars tags(0)") - tdSql.execute("create table car2 using cars tags(0)") - tdSql.execute("create table car3 using cars tags(0)") - tdSql.execute("create table car4 using cars tags(0)") + tdSql.execute("create table car1 using cars tags(1)") + tdSql.execute("create table car2 using cars tags(2)") + tdSql.execute("create table car3 using cars tags(3)") + tdSql.execute("create table car4 using cars tags(4)") tdSql.execute("insert into car0 values('2019-01-01 00:00:00', 1)") tdSql.execute("insert into car1 values('2019-05-13 12:00:00', 1)") From 7cdb6c3f2418294b485d2d4a6b79709884c2b5e3 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 16 Sep 2020 18:05:48 +0800 Subject: [PATCH 099/105] td-1245: disable natual duration in 'insert'&'where' --- src/client/src/tscParseInsert.c | 2 +- src/client/src/tscSQLParser.c | 6 ++--- src/os/inc/osTime.h | 6 ++--- src/os/src/detail/osTime.c | 23 +++++++++---------- src/query/src/qParserImpl.c | 40 +-------------------------------- 5 files changed, 18 insertions(+), 59 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 18926a84b9..327aac22d1 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -142,7 +142,7 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 return tscInvalidSQLErrMsg(error, "value expected in timestamp", sToken.z); } - if (getTimestampInUsFromStr(valueToken.z, valueToken.n, &interval) != TSDB_CODE_SUCCESS) { + if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 7ed2e7dbde..ceb5271962 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -596,7 +596,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ // interval is not null SStrToken* t = &pQuerySql->interval; - if (parseDuration(t->z, t->n, &pQueryInfo->interval.interval, &pQueryInfo->interval.intervalUnit) != TSDB_CODE_SUCCESS) { + if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, &pQueryInfo->interval.intervalUnit) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -693,7 +693,7 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue return TSDB_CODE_SUCCESS; } - if (parseDuration(t->z, t->n, &pQueryInfo->interval.offset, &pQueryInfo->interval.offsetUnit) != TSDB_CODE_SUCCESS) { + if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.offset, &pQueryInfo->interval.offsetUnit) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -751,7 +751,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->interval.sliding); + parseAbsoluteDuration(pSliding->z, pSliding->n, &pQueryInfo->interval.sliding); if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { pQueryInfo->interval.sliding /= 1000; } diff --git a/src/os/inc/osTime.h b/src/os/inc/osTime.h index 2bc24caa14..99f7586f72 100644 --- a/src/os/inc/osTime.h +++ b/src/os/inc/osTime.h @@ -30,8 +30,6 @@ extern "C" { #define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60) #define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) #define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) -#define MILLISECOND_PER_MONTH (MILLISECOND_PER_DAY * 30) -#define MILLISECOND_PER_YEAR (MILLISECOND_PER_DAY * 365) //@return timestamp in second int32_t taosGetTimestampSec(); @@ -77,8 +75,8 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); -int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts); -int32_t parseDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit); +int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* ts); +int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit); int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); void deltaToUtcInitOnce(); diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index 5e6922cd51..012c107371 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -321,7 +321,7 @@ int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec) { } -static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* result) { +static int32_t getDurationInUs(int64_t val, char unit, int64_t* result) { *result = val; int64_t factor = 1000L; @@ -342,19 +342,12 @@ static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* resu case 'w': (*result) *= MILLISECOND_PER_WEEK*factor; break; - case 'n': - (*result) *= MILLISECOND_PER_MONTH*factor; - break; - case 'y': - (*result) *= MILLISECOND_PER_YEAR*factor; - break; case 'a': (*result) *= factor; break; case 'u': break; default: { - ; return -1; } } @@ -373,7 +366,7 @@ static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* resu * n - Months (30 days) * y - Years (365 days) */ -int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts) { +int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* duration) { errno = 0; char* endPtr = NULL; @@ -383,10 +376,16 @@ int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts) { return -1; } - return getTimestampInUsFromStrImpl(timestamp, token[tokenlen - 1], ts); + /* natual month/year are not allowed in absolute duration */ + char unit = token[tokenlen - 1]; + if (unit == 'n' || unit == 'y') { + return -1; + } + + return getDurationInUs(timestamp, unit, duration); } -int32_t parseDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit) { +int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit) { errno = 0; /* get the basic numeric value */ @@ -400,7 +399,7 @@ int32_t parseDuration(const char* token, int32_t tokenLen, int64_t* duration, ch return 0; } - return getTimestampInUsFromStrImpl(*duration, *unit, duration); + return getDurationInUs(*duration, *unit, duration); } int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index f8c60c9e18..33237a58c2 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -135,7 +135,7 @@ tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; pSQLExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond } else if (optrType == TK_VARIABLE) { - int32_t ret = getTimestampInUsFromStr(pToken->z, pToken->n, &pSQLExpr->val.i64Key); + int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSQLExpr->val.i64Key); UNUSED(ret); pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; @@ -443,44 +443,6 @@ void setDBName(SStrToken *pCpxName, SStrToken *pDB) { pCpxName->n = pDB->n; } -int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) { - *result = val; - - switch (unit) { - case 's': - (*result) *= MILLISECOND_PER_SECOND; - break; - case 'm': - (*result) *= MILLISECOND_PER_MINUTE; - break; - case 'h': - (*result) *= MILLISECOND_PER_HOUR; - break; - case 'd': - (*result) *= MILLISECOND_PER_DAY; - break; - case 'w': - (*result) *= MILLISECOND_PER_WEEK; - break; - case 'n': - (*result) *= MILLISECOND_PER_MONTH; - break; - case 'y': - (*result) *= MILLISECOND_PER_YEAR; - break; - case 'a': - break; - default: { - ; - return -1; - } - } - - /* get the value in microsecond */ - (*result) *= 1000L; - return 0; -} - void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); From 01d62e6edc07c637ef45e349fb90884782ee05ef Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 17 Sep 2020 09:34:26 +0800 Subject: [PATCH 100/105] td-1245: update documentation for offset --- documentation20/webdocs/markdowndocs/Queries-ch.md | 12 ++++++++++++ .../webdocs/markdowndocs/Super Table-ch.md | 2 +- documentation20/webdocs/markdowndocs/TAOS SQL-ch.md | 11 +++++------ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/Queries-ch.md b/documentation20/webdocs/markdowndocs/Queries-ch.md index ed8342c88d..1394338f54 100644 --- a/documentation20/webdocs/markdowndocs/Queries-ch.md +++ b/documentation20/webdocs/markdowndocs/Queries-ch.md @@ -78,6 +78,18 @@ taos> SELECT SUM(current) FROM meters INTERVAL(1s); 2018-10-03 14:38:16.000 | 36.000000000 | Query OK, 5 row(s) in set (0.001538s) ``` +降采样操作也支持时间偏移，比如：将所有智能电表采集的电流值每秒钟求和，但要求每个时间窗口从 500 毫秒开始 +```mysql +taos> SELECT SUM(current) FROM meters INTERVAL(1s, 500a); + ts | sum(current) | +====================================================== + 2018-10-03 14:38:04.500 | 11.189999809 | + 2018-10-03 14:38:05.500 | 31.900000572 | + 2018-10-03 14:38:06.500 | 11.600000000 | + 2018-10-03 14:38:15.500 | 12.300000381 | + 2018-10-03 14:38:16.500 | 35.000000000 | +Query OK, 5 row(s) in set (0.001521s) +``` 物联网场景里，每个数据采集点采集数据的时间是难同步的，但很多分析算法(比如FFT)需要把采集的数据严格按照时间等间隔的对齐，在很多系统里，需要应用自己写程序来处理，但使用TDengine的降采样操作就轻松解决。如果一个时间间隔里，没有采集的数据，TDengine还提供插值计算的功能。 diff --git a/documentation20/webdocs/markdowndocs/Super Table-ch.md b/documentation20/webdocs/markdowndocs/Super Table-ch.md index 14145cbb70..96e7104ab7 100644 --- a/documentation20/webdocs/markdowndocs/Super Table-ch.md +++ b/documentation20/webdocs/markdowndocs/Super Table-ch.md @@ -148,7 +148,7 @@ INSERT INTO USING TAGS (, ...) VALUES (,… FROM WHERE <[=|<=|>=|<>] values..> ([AND|OR] …) - INTERVAL (