diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 09eb8f167e..18926a84b9 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -526,7 +526,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe int32_t index = 0; SStrToken sToken; - int16_t numOfRows = 0; + int32_t numOfRows = 0; SSchema *pSchema = tscGetTableSchema(pTableMeta); STableComInfo tinfo = tscGetTableInfo(pTableMeta); diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index cad5dd8f91..0c3204687a 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -238,6 +238,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive); } +#if 0 else if (0 == strcasecmp(key, "Content-Encoding")) { if (0 == strcmp(val, "gzip")) { parser->contentChunked = 1; @@ -245,8 +246,9 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } return 0; } + #endif - else if (0 == strcasecmp(key, "Transfer-Encoding")) { + else if (0 == strcasecmp(key, "Transfer-Encoding") || 0 == strcasecmp(key, "Content-Encoding")) { if (strstr(val, "gzip")) { parser->transferGzip = 1; ehttp_gzip_conf_t conf = {0}; diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 2d95a0ac72..d85a236cb1 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -317,10 +317,10 @@ static bool httpReadData(HttpContext *pContext) { pContext->lastAccessTime = taosGetTimestampSec(); char buf[HTTP_STEP_SIZE + 1] = {0}; - int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, sizeof(buf)); + int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE); if (nread > 0) { buf[nread] = '\0'; - httpTrace("context:%p, fd:%d, nread:%d", pContext, pContext->fd, nread); + httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf); int32_t ok = httpParseBuf(pParser, buf, nread); if (ok) { @@ -341,7 +341,7 @@ static bool httpReadData(HttpContext *pContext) { httpTrace("context:%p, fd:%d, read not over yet, len:%d", pContext, pContext->fd, pParser->body.pos); return false; } else { - httpTraceL("context:%p, fd:%d, len:%d, body:%s", pContext, pContext->fd, pParser->body.pos, pParser->body.str); + httpDebug("context:%p, fd:%d, totalLen:%d", pContext, pContext->fd, pParser->body.pos); return true; } } else if (nread < 0) { diff --git a/tests/script/general/http/autocreate.sim b/tests/script/general/http/autocreate.sim index 6a005b028a..98d64ab839 100644 --- a/tests/script/general/http/autocreate.sim +++ b/tests/script/general/http/autocreate.sim @@ -24,5 +24,10 @@ print curl 127.0.0.1:7111/rest/sql -----> $system_content # return -1 #endi +sql select * from db.win_cpu_windows_1_processor +print rows: $rows +if $rows != 1 then + return -1 +endi #system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/chunked.sim b/tests/script/general/http/chunked.sim new file mode 100644 index 0000000000..8673655d96 --- /dev/null +++ b/tests/script/general/http/chunked.sim @@ -0,0 +1,37 @@ +system sh/stop_dnodes.sh +sleep 3000 +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 0 +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print ============================ dnode1 start + +print =============== step1 - prepare data +sql create database d1 +sql use d1 + +sql create table table_rest (ts timestamp, i int) +print sql length is 270KB +restful d1 table_rest 1591072800 10000 +restful d1 table_rest 1591172800 10000 +restful d1 table_rest 1591272800 10000 +restful d1 table_rest 1591372800 10000 +restful d1 table_rest 1591472800 10000 +restful d1 table_rest 1591572800 10000 +restful d1 table_rest 1591672800 10000 +restful d1 table_rest 1591772800 10000 +restful d1 table_rest 1591872800 10000 +restful d1 table_rest 1591972800 10000 + +sql select * from table_rest; +print rows: $rows +if $rows != 100000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/gzip.sim b/tests/script/general/http/gzip.sim new file mode 100644 index 0000000000..0289e337a6 --- /dev/null +++ b/tests/script/general/http/gzip.sim @@ -0,0 +1,27 @@ +system sh/stop_dnodes.sh +sleep 3000 +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 0 +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print ============================ dnode1 start + +print =============== step1 - prepare data +sql create database d1 +sql use d1 + +sql create table table_rest (ts timestamp, i int) +print sql length is 270KB +restful d1 table_rest 1591072800 10000 gzip +sql select * from table_rest; +print rows: $rows +if $rows != 10000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim index d91e9f452d..f35362bf07 100644 --- a/tests/script/general/http/testSuite.sim +++ b/tests/script/general/http/testSuite.sim @@ -1,3 +1,5 @@ +run general/http/autocreate.sim +run general/http/chunked.sim run general/http/restful.sim run general/http/restful_insert.sim run general/http/restful_limit.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 063d11bd9d..adb22aa265 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -77,6 +77,8 @@ cd ../../../debug; make ./test.sh -f general/field/smallint.sim ./test.sh -f general/field/tinyint.sim +./test.sh -f general/http/autocreate.sim +./test.sh -f general/http/chunked.sim ./test.sh -f general/http/restful.sim ./test.sh -f general/http/restful_insert.sim ./test.sh -f general/http/restful_limit.sim diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 5339203ec2..ffab39d41c 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -39,5 +39,8 @@ IF (TD_LINUX) add_executable(httpTest httpTest.c) target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) + + add_executable(cacheTest cacheTest.c) + target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) ENDIF() diff --git a/tests/test/c/cacheTest.c b/tests/test/c/cacheTest.c new file mode 100644 index 0000000000..54aca0038e --- /dev/null +++ b/tests/test/c/cacheTest.c @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include +#include "os.h" +#include "taos.h" +#include "tcache.h" +#include "tulog.h" +#include "tutil.h" + +#define MAX_REFRESH_TIME_SEC 2 +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +int32_t tsKeepTimeInSec = 3; +int32_t tsNumOfRows = 1000000; +int32_t tsSizeOfRow = 64 * 1024; +void * tsCacheObj = NULL; +int32_t destroyTimes = 0; + +typedef int64_t CacheTestKey; +typedef struct CacheTestRow { + int32_t index; + void ** ppRow; + void * data; +} CacheTestRow; + +CacheTestRow *initRow(int32_t index) { + CacheTestRow *row = calloc(sizeof(CacheTestRow), 1); + row->index = index; + row->data = malloc(tsSizeOfRow * sizeof(int8_t)); + return row; +} + +void detroyRow(void *data) { + CacheTestRow *row = *(CacheTestRow **)data; + free(row->data); + free(row); + destroyTimes++; + if (destroyTimes % 50000 == 0) { + pPrint("%s ===> destroyTimes:%d %s", GREEN, destroyTimes, NC); + } +} + +void initCache() { + tsCacheObj = taosCacheInit(TSDB_DATA_TYPE_BIGINT, MAX_REFRESH_TIME_SEC, true, detroyRow, "cachetest"); +} + +void putRowInCache() { + for (int index = 0; index < tsNumOfRows; ++index) { + CacheTestRow *row = initRow(index); + uint64_t key = (uint64_t)row; + void **ppRow = taosCachePut(tsCacheObj, &key, sizeof(int64_t), &row, sizeof(int64_t), tsKeepTimeInSec * 1000); + row->ppRow = ppRow; + taosCacheRelease(tsCacheObj, (void **)&ppRow, false); + } +} + +void cleanupCache() { + taosCacheCleanup(tsCacheObj); +} + +void initGetMemory() { + osInit(); + taos_init(); +} + +float getProcMemory() { + float procMemoryUsedMB = 0; + taosGetProcMemory(&procMemoryUsedMB); + return procMemoryUsedMB; +} + +void doTest() { + initCache(); + pPrint("%s initialize procMemory %f MB %s", GREEN, getProcMemory(), NC); + + putRowInCache(); + pPrint("%s insert %d rows, procMemory %f MB %s", GREEN, tsNumOfRows, getProcMemory(), NC); + + int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3) * 1000 + tsKeepTimeInSec * 1000; + taosMsleep(sleepMs); + pPrint("%s after sleep %d ms, procMemory %f MB %s", GREEN, sleepMs, getProcMemory(), NC); + + cleanupCache(); + taosMsleep(sleepMs); + pPrint("%s after cleanup cache, procMemory %f MB %s", GREEN, getProcMemory(), NC); + + malloc_trim(0); + taosMsleep(sleepMs); + pPrint("%s after malloc_trim, procMemory %f MB %s", GREEN, getProcMemory(), NC); +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance of cache\n"); + + printf("%s%s\n", indent, "-k"); + printf("%s%s%s%d\n", indent, indent, "KeepTimeInSec, default is ", tsKeepTimeInSec); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%d\n", indent, indent, "NumOfRows, default is ", tsNumOfRows); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%d\n", indent, indent, "SizeOfData, default is ", tsSizeOfRow); + + exit(EXIT_SUCCESS); +} + +void parseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-k") == 0) { + tsKeepTimeInSec = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + tsNumOfRows = atoi(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + tsSizeOfRow = atoi(argv[++i]); + } else { + } + } + + pPrint("%s KeepTimeInSec:%d %s", GREEN, tsKeepTimeInSec, NC); + pPrint("%s NumOfRows:%d %s", GREEN, tsNumOfRows, NC); + pPrint("%s SizeOfData:%d %s", GREEN, tsSizeOfRow, NC); +} + +int main(int argc, char *argv[]) { + initGetMemory(); + parseArgument(argc, argv); + doTest(); +} diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 6f3bc7099d..58e58a442c 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -84,6 +84,7 @@ enum { SIM_CMD_SQL, SIM_CMD_SQL_ERROR, SIM_CMD_SQL_SLOW, + SIM_CMD_RESTFUL, SIM_CMD_TEST, SIM_CMD_RETURN, SIM_CMD_END @@ -172,6 +173,7 @@ bool simExecuteReturnCmd(SScript *script, char *option); bool simExecuteSqlCmd(SScript *script, char *option); bool simExecuteSqlErrorCmd(SScript *script, char *rest); bool simExecuteSqlSlowCmd(SScript *script, char *option); +bool simExecuteRestfulCmd(SScript *script, char *rest); void simVisuallizeOption(SScript *script, char *src, char *dst); #endif \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index adc2fd0b9d..463dc33c7c 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -915,6 +915,47 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) { return simExecuteSqlImpCmd(script, rest, isSlow); } +bool simExecuteRestfulCmd(SScript *script, char *rest) { + FILE *fp = NULL; + char filename[256]; + sprintf(filename, "%s/tmp.sql", tsScriptDir); + fp = fopen(filename, "w"); + if (fp == NULL) { + fprintf(stderr, "ERROR: failed to open file: %s\n", filename); + return false; + } + + char db[64] = {0}; + char tb[64] = {0}; + char gzip[32] = {0}; + int32_t ts; + int32_t times; + sscanf(rest, "%s %s %d %d %s", db, tb, &ts, ×, gzip); + + fprintf(fp, "insert into %s.%s values ", db, tb); + for (int i = 0; i < times; ++i) { + fprintf(fp, "(%d000, %d)", ts + i, ts); + } + fprintf(fp, " \n"); + fflush(fp); + fclose(fp); + + char cmd[1024] = {0}; + if (strcmp(gzip, "gzip") == 0) { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "--compressed --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } else { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "'Transfer-Encoding: chunked' --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } + + return simExecuteSystemCmd(script, cmd); +} + bool simExecuteSqlErrorCmd(SScript *script, char *rest) { char buf[3000]; SCmdLine *line = &script->lines[script->linePos]; diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 8dcf83806f..2e6121304f 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -721,6 +721,12 @@ bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } +bool simParseRestfulCmd(char *rest, SCommand *pCmd, int lineNum) { + simParseSqlCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; + return true; +} + bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) { int expLen; @@ -1020,6 +1026,14 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = simExecuteSqlSlowCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); + cmdno = SIM_CMD_RESTFUL; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "restful"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseRestfulCmd; + simCmdList[cmdno].executeCmd = simExecuteRestfulCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + /* test is only an internal command */ cmdno = SIM_CMD_TEST; simCmdList[cmdno].cmdno = cmdno;