From 7f8479f9531266fe09fa64a96048ac161e0c22bd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Dec 2023 15:16:33 +0800 Subject: [PATCH] opti:logic in parse raw line & make log more specific --- source/client/inc/clientSml.h | 1 - source/client/src/clientSml.c | 69 ++++++++++++++++++--------- source/libs/parser/src/parInsertSml.c | 10 +++- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 9ae28dd55e..b732abffb1 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -171,7 +171,6 @@ typedef struct { int8_t precision; bool reRun; bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) - bool isRawLine; int32_t ttl; int32_t uid; // used for automatic create child table diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 67b23792ad..c392308195 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1640,6 +1640,37 @@ int32_t smlClearForRerun(SSmlHandle *info) { return TSDB_CODE_SUCCESS; } +static bool getLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, + int numLines, int i, char** tmp, int *len){ + if (lines) { + *tmp = lines[i]; + *len = strlen(*tmp); + } else if (rawLine) { + *tmp = rawLine; + while (rawLine < rawLineEnd) { + if (*(rawLine++) == '\n') { + break; + } + (*len)++; + } + if (info->protocol == TSDB_SML_LINE_PROTOCOL && (*tmp)[0] == '#') { // this line is comment + false; + } + } + + if(rawLine != NULL){ + char* print = taosMemoryCalloc(*len + 1, 1); + memcpy(print, *tmp, *len); + uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, + numLines, info->protocol, *len, print); + taosMemoryFree(print); + }else{ + uDebug("SML:0x%" PRIx64 " smlParseLine is not numLines:%d, protocol:%d, len:%d, data:%s", info->id, + numLines, info->protocol, *len, *tmp); + } + return true; +} + static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); int32_t code = TSDB_CODE_SUCCESS; @@ -1661,25 +1692,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char while (i < numLines) { char *tmp = NULL; int len = 0; - if (lines) { - tmp = lines[i]; - len = strlen(tmp); - } else if (rawLine) { - tmp = rawLine; - while (rawLine < rawLineEnd) { - if (*(rawLine++) == '\n') { - break; - } - len++; - } - if (info->protocol == TSDB_SML_LINE_PROTOCOL && tmp[0] == '#') { // this line is comment - continue; - } + if(!getLine(info, lines, rawLine, rawLineEnd, numLines, i, &tmp, &len)){ + continue; } - - uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, - info->isRawLine, numLines, info->protocol, len, info->isRawLine ? "rawdata" : tmp); - if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->dataFormat) { SSmlLineInfo element = {0}; @@ -1699,7 +1714,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, info->isRawLine ? "rawdata" : tmp); + if(rawLine != NULL){ + char* print = taosMemoryCalloc(len + 1, 1); + memcpy(print, tmp, len); + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); + taosMemoryFree(print); + }else{ + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); + } return code; } if (info->reRun) { @@ -1828,7 +1850,6 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, return (TAOS_RES *)request; } info->pRequest = request; - info->isRawLine = rawLine != NULL; info->ttl = ttl; info->precision = precision; info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; @@ -1934,8 +1955,7 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi reqid); } -TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision, int32_t ttl, int64_t reqid) { +static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol){ int numLines = 0; *totalRows = 0; char *tmp = lines; @@ -1948,6 +1968,11 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int tmp = lines + i + 1; } } +} + +TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int32_t ttl, int64_t reqid) { + getRawLineLen(lines, len, totalRows, protocol); return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); } diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 2dbba38212..13c4431b62 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -210,7 +210,15 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 SSmlKv* kv = (SSmlKv*)data; if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ ret = TSDB_CODE_SML_INVALID_DATA; - uInfo("SML smlBuildCol error col not same %s", pColSchema->name); + char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1); + if(tmp){ + memcpy(tmp, kv->key, kv->keyLen); + uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", + tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name); + taosMemoryFree(tmp); + }else{ + uError("SML smlBuildCol out of memory"); + } goto end; } if (kv->type == TSDB_DATA_TYPE_NCHAR) {