From 087f88a617976744ca6270b2221391df906d67f2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 3 Jan 2023 18:18:46 +0800 Subject: [PATCH 01/10] fix:malloc too large --- source/client/src/clientSml.c | 1 + source/client/src/clientSmlJson.c | 9 +++++---- utils/test/c/sml_test.c | 3 +-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index ca6395b52b..71349e61ec 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1067,6 +1067,7 @@ void smlDestroyInfo(SSmlHandle *info) { taosMemoryFree(info->lines); } + cJSON_Delete(info->root); taosMemoryFreeClear(info); } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 60e29638f5..27175420a9 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -335,6 +335,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset){ (*start)++; } } + if(*(*start) == '\0'){ + break; + } if(*(*start) == '}'){ (*start)++; break; @@ -923,9 +926,6 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo cJSON *tsJson = NULL; cJSON *valueJson = NULL; cJSON *tagsJson = NULL; - char* rootStr = cJSON_PrintUnformatted(root); - uError("rootStr:%s", rootStr); - taosMemoryFree(rootStr); int32_t size = cJSON_GetArraySize(root); // outmost json fields has to be exactly 4 @@ -956,6 +956,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo } // Parse tags + bool needFree = info->dataFormat; elements->tags = cJSON_PrintUnformatted(tagsJson); elements->tagsLen = strlen(elements->tags); if(is_same_child_table_telnet(elements, &info->preLine) != 0) { @@ -968,7 +969,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo } } - if(info->dataFormat){ + if(needFree){ taosMemoryFree(elements->tags); elements->tags = NULL; } diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 30d0ab27d8..d592184836 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -211,8 +211,7 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { -// "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]" - "{\"metric\": \"dcxnmr\", \"timestamp\": {\"value\": 1626006833639000000, \"type\": \"ns\"}, \"value\": {\"value\": false, \"type\": \"bool\"}, \"tags\": {\"t0\": {\"value\": false, \"type\": \"bool\"}, \"t1\": {\"value\": 127, \"type\": \"tinyint\"}, \"t2\": {\"value\": 32767, \"type\": \"smallint\"}, \"t3\": {\"value\": 2147483647, \"type\": \"int\"}, \"t4\": {\"value\": 9223372036854775807, \"type\": \"bigint\"}, \"t5\": {\"value\": 11.12345027923584, \"type\": \"float\"}, \"t6\": {\"value\": 22.123456789, \"type\": \"double\"}, \"t7\": {\"value\": \"binaryTagValue\", \"type\": \"binary\"}, \"t8\": {\"value\": \"abc{aaa\", \"type\": \"nchar\"}}}" + "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]" }; char *sql1[1] = {0}; for(int i = 0; i < 1; i++){ From 21c62ea43716d2f17d8753da0570ea97bef29727 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 3 Jan 2023 18:36:08 +0800 Subject: [PATCH 02/10] fix:check table name length for json parser --- source/client/src/clientSmlJson.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 27175420a9..dbb77d9972 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -1096,6 +1096,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * if(unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; if (elements->colsLen == 0 || smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { uError("SML:cols invalidate:%s", elements->cols); From d4f69eb57582d938f959d070ed8d76c7538d8605 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 3 Jan 2023 18:40:06 +0800 Subject: [PATCH 03/10] fix:open test cases --- tests/parallel_test/cases.task | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 8e635999ce..44bf6cc6f1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -420,8 +420,8 @@ ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py -,,n,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -,,n,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py From af64ab26c8ae6346f657f82ea66f97fcfa3918c9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 4 Jan 2023 10:07:14 +0800 Subject: [PATCH 04/10] fix:memory leak --- include/libs/parser/parser.h | 1 + source/client/src/clientSml.c | 1 + source/libs/parser/inc/parInsertUtil.h | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 8f22745973..7f28bfc175 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -78,6 +78,7 @@ int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, void qDestroyParseContext(SParseContext* pCxt); void qDestroyQuery(SQuery* pQueryNode); +void insDestroyTableDataCxt(STableDataCxt *pTableCxt); int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 71349e61ec..dd754aba7a 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1021,6 +1021,7 @@ static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { taosMemoryFree(tag->key); taosArrayDestroy(tag->cols); taosArrayDestroy(tag->tags); + insDestroyTableDataCxt(tag->tableDataCtx); taosMemoryFree(tag); } diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 7b816359f9..9eec5cbaf0 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -55,7 +55,6 @@ void insDestroyTableDataCxtHashMap(SHashObj *pTableCxtHash); void insDestroyVgroupDataCxt(SVgroupDataCxt *pVgCxt); void insDestroyVgroupDataCxtList(SArray *pVgCxtList); void insDestroyVgroupDataCxtHashMap(SHashObj *pVgCxtHash); -void insDestroyTableDataCxt(STableDataCxt *pTableCxt); void insDestroyBoundColInfo(SBoundColInfo *pInfo); #endif // TDENGINE_PAR_INSERT_UTIL_H From 25643c4409446e5b90ab764516b53052827c047c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 4 Jan 2023 10:27:33 +0800 Subject: [PATCH 05/10] fix:memory leak --- include/libs/parser/parser.h | 1 - source/client/src/clientSml.c | 1 - source/libs/parser/inc/parInsertUtil.h | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 7f28bfc175..8f22745973 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -78,7 +78,6 @@ int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, void qDestroyParseContext(SParseContext* pCxt); void qDestroyQuery(SQuery* pQueryNode); -void insDestroyTableDataCxt(STableDataCxt *pTableCxt); int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index dd754aba7a..71349e61ec 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1021,7 +1021,6 @@ static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { taosMemoryFree(tag->key); taosArrayDestroy(tag->cols); taosArrayDestroy(tag->tags); - insDestroyTableDataCxt(tag->tableDataCtx); taosMemoryFree(tag); } diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 9eec5cbaf0..7b816359f9 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -55,6 +55,7 @@ void insDestroyTableDataCxtHashMap(SHashObj *pTableCxtHash); void insDestroyVgroupDataCxt(SVgroupDataCxt *pVgCxt); void insDestroyVgroupDataCxtList(SArray *pVgCxtList); void insDestroyVgroupDataCxtHashMap(SHashObj *pVgCxtHash); +void insDestroyTableDataCxt(STableDataCxt *pTableCxt); void insDestroyBoundColInfo(SBoundColInfo *pInfo); #endif // TDENGINE_PAR_INSERT_UTIL_H From 983b754198204f3666376ffa3d1355a95f3aae97 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 4 Jan 2023 11:45:09 +0800 Subject: [PATCH 06/10] fix:error in parse json --- source/client/src/clientSmlJson.c | 4 ++-- source/client/src/clientSmlLine.c | 8 ++++---- source/client/src/clientSmlTelnet.c | 4 ++-- source/libs/parser/src/parInsertSml.c | 2 ++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index dbb77d9972..a5b5abeda2 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -658,14 +658,14 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); if(unlikely(sMeta == NULL)){ - sMeta = smlBuildSTableMeta(info->dataFormat); STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); - sMeta->tableMeta = pTableMeta; if(pTableMeta == NULL){ info->dataFormat = false; info->reRun = true; return TSDB_CODE_SUCCESS; } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); } info->currSTableMeta = sMeta->tableMeta; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index d93fdf198f..80cb6e72c1 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -151,14 +151,14 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); if(unlikely(sMeta == NULL)){ - sMeta = smlBuildSTableMeta(info->dataFormat); STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); - sMeta->tableMeta = pTableMeta; if(pTableMeta == NULL){ info->dataFormat = false; info->reRun = true; return TSDB_CODE_SUCCESS; } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); } info->currSTableMeta = sMeta->tableMeta; @@ -353,14 +353,14 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); if(unlikely(sMeta == NULL)){ - sMeta = smlBuildSTableMeta(info->dataFormat); STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); - sMeta->tableMeta = pTableMeta; if(pTableMeta == NULL){ info->dataFormat = false; info->reRun = true; return TSDB_CODE_SUCCESS; } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); } info->currSTableMeta = sMeta->tableMeta; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 71d4e21400..b8c7675f67 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -86,14 +86,14 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); if(unlikely(sMeta == NULL)){ - sMeta = smlBuildSTableMeta(info->dataFormat); STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); - sMeta->tableMeta = pTableMeta; if(pTableMeta == NULL){ info->dataFormat = false; info->reRun = true; return TSDB_CODE_SUCCESS; } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); } info->currSTableMeta = sMeta->tableMeta; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index e613739f26..0feef4712d 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -359,6 +359,8 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc end: insDestroyBoundColInfo(&bindTags); taosMemoryFree(pCreateTblReq); + tdDestroySVCreateTbReq(pCreateTblReq); + taosMemoryFree(pCreateTblReq); taosArrayDestroy(tagName); return ret; } From e7e4d84627b26b6ed4355a9092fc88865c79a5fa Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 4 Jan 2023 11:47:14 +0800 Subject: [PATCH 07/10] fix:memory leak --- source/libs/parser/src/parInsertSml.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 0feef4712d..d2fd503dd5 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -358,7 +358,6 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc end: insDestroyBoundColInfo(&bindTags); - taosMemoryFree(pCreateTblReq); tdDestroySVCreateTbReq(pCreateTblReq); taosMemoryFree(pCreateTblReq); taosArrayDestroy(tagName); From d3fe415bef2790705c71ec465a277ba8a2482ffa Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 4 Jan 2023 15:45:38 +0800 Subject: [PATCH 08/10] fix:memory leak --- source/client/inc/clientSml.h | 1 + source/client/src/clientSml.c | 7 +++++++ source/client/src/clientSmlJson.c | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 11f72efeaf..99bb27fe32 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -182,6 +182,7 @@ typedef struct { int8_t offset[OTD_JSON_FIELDS_NUM]; SSmlLineInfo *lines; // element is SSmlLineInfo bool parseJsonByLib; + SArray *tagJsonArray; // SArray *preLineTagKV; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 71349e61ec..574af8d336 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1053,6 +1053,12 @@ void smlDestroyInfo(SSmlHandle *info) { // destroy info->pVgHash taosHashCleanup(info->pVgHash); + for(int i = 0; i< taosArrayGetSize(info->tagJsonArray); i++){ + cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i); + cJSON_Delete(tags); + } + taosArrayDestroy(info->tagJsonArray); + taosArrayDestroy(info->preLineTagKV); taosArrayDestroy(info->maxTagKVs); taosArrayDestroy(info->preLineColKV); @@ -1091,6 +1097,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->pQuery = smlInitHandle(); info->dataFormat = true; + info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); info->maxTagKVs = taosArrayInit(8, sizeof(SSmlKv)); info->preLineColKV = taosArrayInit(8, sizeof(SSmlKv)); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index a5b5abeda2..26b8e9e13c 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -1118,8 +1118,8 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_TSC_INVALID_JSON; } + taosArrayPush(info->tagJsonArray, &tagsJson); ret = smlParseTagsFromJSON(info, tagsJson, elements); - cJSON_free(tagsJson); if (unlikely(ret)) { uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); return ret; From 9c4cebb1b7fad90feae83383af1d108afef097d4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 4 Jan 2023 17:15:30 +0800 Subject: [PATCH 09/10] fix:memory leak --- source/client/inc/clientSml.h | 1 + source/client/src/clientSml.c | 10 ++++++++++ source/client/src/clientSmlJson.c | 2 ++ source/client/src/clientSmlLine.c | 1 + source/client/src/clientSmlTelnet.c | 1 + source/libs/parser/src/parInsertSml.c | 20 ++++++++++++++++++-- 6 files changed, 33 insertions(+), 2 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 99bb27fe32..074daa7441 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -231,6 +231,7 @@ int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32 int32_t smlClearForRerun(SSmlHandle *info); int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); uint8_t smlGetTimestampLen(int64_t num); +void clearColValArray(SArray* pCols); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 574af8d336..b751f57432 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1024,6 +1024,16 @@ static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { taosMemoryFree(tag); } +void clearColValArray(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (TSDB_DATA_TYPE_NCHAR == pCol->type) { + taosMemoryFreeClear(pCol->value.pData); + } + } +} + void smlDestroyInfo(SSmlHandle *info) { if (!info) return; qDestroyQuery(info->pQuery); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 26b8e9e13c..6d80ac29e7 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -995,6 +995,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo if(ret == TSDB_CODE_SUCCESS){ ret = smlBuildRow(info->currTableDataCtx); } + clearColValArray(info->currTableDataCtx->pValues); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; @@ -1147,6 +1148,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * if(ret == TSDB_CODE_SUCCESS){ ret = smlBuildRow(info->currTableDataCtx); } + clearColValArray(info->currTableDataCtx->pValues); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 80cb6e72c1..6853e6d9f3 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -646,6 +646,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine if(info->dataFormat){ smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); smlBuildRow(info->currTableDataCtx); + clearColValArray(info->currTableDataCtx->pValues); }else{ taosArraySet(elements->colArray, 0, &kv); } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index b8c7675f67..11a58040b3 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -324,6 +324,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine if(ret == TSDB_CODE_SUCCESS){ ret = smlBuildRow(info->currTableDataCtx); } + clearColValArray(info->currTableDataCtx->pValues); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index d2fd503dd5..4742921d08 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -18,6 +18,16 @@ #include "parToken.h" #include "ttime.h" +static void clearColValArray(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (TSDB_DATA_TYPE_NCHAR == pCol->type) { + taosMemoryFreeClear(pCol->value.pData); + } + } +} + int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen) { SMsgBuf msg = {.buf = msgBuf, .len = msgBufLen}; @@ -189,7 +199,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 SSchema* pColSchema = schema + index; SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); SSmlKv* kv = (SSmlKv*)data; - if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0){ + if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ ret = TSDB_CODE_SML_INVALID_DATA; goto end; } @@ -207,9 +217,11 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 } if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len)) { if (errno == E2BIG) { + taosMemoryFree(pUcs4); ret = TSDB_CODE_PAR_VALUE_TOO_LONG; goto end; } + taosMemoryFree(pUcs4); ret = TSDB_CODE_TSC_INVALID_VALUE; goto end; } @@ -316,7 +328,10 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc continue; } SSmlKv* kv = *(SSmlKv**)p; - + if(kv->type != pColSchema->type){ + ret = buildInvalidOperationMsg(&pBuf, "kv type not equal to col type"); + goto end; + } if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); } @@ -354,6 +369,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc goto end; } insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + clearColValArray(pTableCxt->pValues); } end: From 3645ae868fd976e6cbbc688dea4e616c2636331d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 4 Jan 2023 17:18:03 +0800 Subject: [PATCH 10/10] opti:test case for schemaless --- utils/test/c/sml_test.c | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index d592184836..dfa81ab180 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -740,26 +740,11 @@ int sml_dup_time_Test() { taos_free_result(pRes); const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -942,8 +927,10 @@ int sml_ttl_Test() { pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); TAOS_ROW row = taos_fetch_row(pRes); - int32_t ttl = *(int32_t*)row[0]; - ASSERT(ttl == 20); + if(row != NULL && *row != NULL){ + int32_t ttl = *(int32_t*)row[0]; + ASSERT(ttl == 20); + } int code = taos_errno(pRes); taos_free_result(pRes);