From 41a21ca29ebefe2bd69b9c5a96c53498730efed0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 15 Sep 2020 16:23:37 +0800 Subject: [PATCH] TD-1225 --- src/client/src/tscParseInsert.c | 2 +- src/plugins/http/src/httpServer.c | 6 ++-- tests/script/general/http/autocreate.sim | 5 +++ tests/script/general/http/chunked.sim | 37 +++++++++++++++++++++ tests/script/general/http/gzip.sim | 27 ++++++++++++++++ tests/script/general/http/testSuite.sim | 2 ++ tests/script/jenkins/basic.txt | 2 ++ tests/test/c/cacheTest.c | 11 ++++--- tests/tsim/inc/sim.h | 2 ++ tests/tsim/src/simExe.c | 41 ++++++++++++++++++++++++ tests/tsim/src/simParse.c | 14 ++++++++ 11 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 tests/script/general/http/chunked.sim create mode 100644 tests/script/general/http/gzip.sim diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 09eb8f167e..18926a84b9 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -526,7 +526,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe int32_t index = 0; SStrToken sToken; - int16_t numOfRows = 0; + int32_t numOfRows = 0; SSchema *pSchema = tscGetTableSchema(pTableMeta); STableComInfo tinfo = tscGetTableInfo(pTableMeta); diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 2d95a0ac72..d85a236cb1 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -317,10 +317,10 @@ static bool httpReadData(HttpContext *pContext) { pContext->lastAccessTime = taosGetTimestampSec(); char buf[HTTP_STEP_SIZE + 1] = {0}; - int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, sizeof(buf)); + int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE); if (nread > 0) { buf[nread] = '\0'; - httpTrace("context:%p, fd:%d, nread:%d", pContext, pContext->fd, nread); + httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf); int32_t ok = httpParseBuf(pParser, buf, nread); if (ok) { @@ -341,7 +341,7 @@ static bool httpReadData(HttpContext *pContext) { httpTrace("context:%p, fd:%d, read not over yet, len:%d", pContext, pContext->fd, pParser->body.pos); return false; } else { - httpTraceL("context:%p, fd:%d, len:%d, body:%s", pContext, pContext->fd, pParser->body.pos, pParser->body.str); + httpDebug("context:%p, fd:%d, totalLen:%d", pContext, pContext->fd, pParser->body.pos); return true; } } else if (nread < 0) { diff --git a/tests/script/general/http/autocreate.sim b/tests/script/general/http/autocreate.sim index 6a005b028a..98d64ab839 100644 --- a/tests/script/general/http/autocreate.sim +++ b/tests/script/general/http/autocreate.sim @@ -24,5 +24,10 @@ print curl 127.0.0.1:7111/rest/sql -----> $system_content # return -1 #endi +sql select * from db.win_cpu_windows_1_processor +print rows: $rows +if $rows != 1 then + return -1 +endi #system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/chunked.sim b/tests/script/general/http/chunked.sim new file mode 100644 index 0000000000..8673655d96 --- /dev/null +++ b/tests/script/general/http/chunked.sim @@ -0,0 +1,37 @@ +system sh/stop_dnodes.sh +sleep 3000 +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 0 +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print ============================ dnode1 start + +print =============== step1 - prepare data +sql create database d1 +sql use d1 + +sql create table table_rest (ts timestamp, i int) +print sql length is 270KB +restful d1 table_rest 1591072800 10000 +restful d1 table_rest 1591172800 10000 +restful d1 table_rest 1591272800 10000 +restful d1 table_rest 1591372800 10000 +restful d1 table_rest 1591472800 10000 +restful d1 table_rest 1591572800 10000 +restful d1 table_rest 1591672800 10000 +restful d1 table_rest 1591772800 10000 +restful d1 table_rest 1591872800 10000 +restful d1 table_rest 1591972800 10000 + +sql select * from table_rest; +print rows: $rows +if $rows != 100000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/gzip.sim b/tests/script/general/http/gzip.sim new file mode 100644 index 0000000000..0289e337a6 --- /dev/null +++ b/tests/script/general/http/gzip.sim @@ -0,0 +1,27 @@ +system sh/stop_dnodes.sh +sleep 3000 +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 0 +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print ============================ dnode1 start + +print =============== step1 - prepare data +sql create database d1 +sql use d1 + +sql create table table_rest (ts timestamp, i int) +print sql length is 270KB +restful d1 table_rest 1591072800 10000 gzip +sql select * from table_rest; +print rows: $rows +if $rows != 10000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim index d91e9f452d..f35362bf07 100644 --- a/tests/script/general/http/testSuite.sim +++ b/tests/script/general/http/testSuite.sim @@ -1,3 +1,5 @@ +run general/http/autocreate.sim +run general/http/chunked.sim run general/http/restful.sim run general/http/restful_insert.sim run general/http/restful_limit.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 063d11bd9d..adb22aa265 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -77,6 +77,8 @@ cd ../../../debug; make ./test.sh -f general/field/smallint.sim ./test.sh -f general/field/tinyint.sim +./test.sh -f general/http/autocreate.sim +./test.sh -f general/http/chunked.sim ./test.sh -f general/http/restful.sim ./test.sh -f general/http/restful_insert.sim ./test.sh -f general/http/restful_limit.sim diff --git a/tests/test/c/cacheTest.c b/tests/test/c/cacheTest.c index 341b64638e..54aca0038e 100644 --- a/tests/test/c/cacheTest.c +++ b/tests/test/c/cacheTest.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include #include "os.h" #include "taos.h" #include "tcache.h" @@ -91,15 +92,17 @@ void doTest() { putRowInCache(); pPrint("%s insert %d rows, procMemory %f MB %s", GREEN, tsNumOfRows, getProcMemory(), NC); - int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3 + 10) * 1000 + tsKeepTimeInSec * 1000; + int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3) * 1000 + tsKeepTimeInSec * 1000; taosMsleep(sleepMs); pPrint("%s after sleep %d ms, procMemory %f MB %s", GREEN, sleepMs, getProcMemory(), NC); - //cleanupCache(); + cleanupCache(); taosMsleep(sleepMs); pPrint("%s after cleanup cache, procMemory %f MB %s", GREEN, getProcMemory(), NC); - - pPrint("%s finally destroyTimes:%d %s", GREEN, destroyTimes, NC); + + malloc_trim(0); + taosMsleep(sleepMs); + pPrint("%s after malloc_trim, procMemory %f MB %s", GREEN, getProcMemory(), NC); } void printHelp() { diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 6f3bc7099d..58e58a442c 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -84,6 +84,7 @@ enum { SIM_CMD_SQL, SIM_CMD_SQL_ERROR, SIM_CMD_SQL_SLOW, + SIM_CMD_RESTFUL, SIM_CMD_TEST, SIM_CMD_RETURN, SIM_CMD_END @@ -172,6 +173,7 @@ bool simExecuteReturnCmd(SScript *script, char *option); bool simExecuteSqlCmd(SScript *script, char *option); bool simExecuteSqlErrorCmd(SScript *script, char *rest); bool simExecuteSqlSlowCmd(SScript *script, char *option); +bool simExecuteRestfulCmd(SScript *script, char *rest); void simVisuallizeOption(SScript *script, char *src, char *dst); #endif \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index adc2fd0b9d..463dc33c7c 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -915,6 +915,47 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) { return simExecuteSqlImpCmd(script, rest, isSlow); } +bool simExecuteRestfulCmd(SScript *script, char *rest) { + FILE *fp = NULL; + char filename[256]; + sprintf(filename, "%s/tmp.sql", tsScriptDir); + fp = fopen(filename, "w"); + if (fp == NULL) { + fprintf(stderr, "ERROR: failed to open file: %s\n", filename); + return false; + } + + char db[64] = {0}; + char tb[64] = {0}; + char gzip[32] = {0}; + int32_t ts; + int32_t times; + sscanf(rest, "%s %s %d %d %s", db, tb, &ts, ×, gzip); + + fprintf(fp, "insert into %s.%s values ", db, tb); + for (int i = 0; i < times; ++i) { + fprintf(fp, "(%d000, %d)", ts + i, ts); + } + fprintf(fp, " \n"); + fflush(fp); + fclose(fp); + + char cmd[1024] = {0}; + if (strcmp(gzip, "gzip") == 0) { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "--compressed --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } else { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "'Transfer-Encoding: chunked' --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } + + return simExecuteSystemCmd(script, cmd); +} + bool simExecuteSqlErrorCmd(SScript *script, char *rest) { char buf[3000]; SCmdLine *line = &script->lines[script->linePos]; diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 8dcf83806f..2e6121304f 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -721,6 +721,12 @@ bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } +bool simParseRestfulCmd(char *rest, SCommand *pCmd, int lineNum) { + simParseSqlCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; + return true; +} + bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) { int expLen; @@ -1020,6 +1026,14 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = simExecuteSqlSlowCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); + cmdno = SIM_CMD_RESTFUL; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "restful"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseRestfulCmd; + simCmdList[cmdno].executeCmd = simExecuteRestfulCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + /* test is only an internal command */ cmdno = SIM_CMD_TEST; simCmdList[cmdno].cmdno = cmdno;