From 7f8479f9531266fe09fa64a96048ac161e0c22bd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Dec 2023 15:16:33 +0800 Subject: [PATCH 1/4] opti:logic in parse raw line & make log more specific --- source/client/inc/clientSml.h | 1 - source/client/src/clientSml.c | 69 ++++++++++++++++++--------- source/libs/parser/src/parInsertSml.c | 10 +++- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 9ae28dd55e..b732abffb1 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -171,7 +171,6 @@ typedef struct { int8_t precision; bool reRun; bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) - bool isRawLine; int32_t ttl; int32_t uid; // used for automatic create child table diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 67b23792ad..c392308195 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1640,6 +1640,37 @@ int32_t smlClearForRerun(SSmlHandle *info) { return TSDB_CODE_SUCCESS; } +static bool getLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, + int numLines, int i, char** tmp, int *len){ + if (lines) { + *tmp = lines[i]; + *len = strlen(*tmp); + } else if (rawLine) { + *tmp = rawLine; + while (rawLine < rawLineEnd) { + if (*(rawLine++) == '\n') { + break; + } + (*len)++; + } + if (info->protocol == TSDB_SML_LINE_PROTOCOL && (*tmp)[0] == '#') { // this line is comment + false; + } + } + + if(rawLine != NULL){ + char* print = taosMemoryCalloc(*len + 1, 1); + memcpy(print, *tmp, *len); + uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, + numLines, info->protocol, *len, print); + taosMemoryFree(print); + }else{ + uDebug("SML:0x%" PRIx64 " smlParseLine is not numLines:%d, protocol:%d, len:%d, data:%s", info->id, + numLines, info->protocol, *len, *tmp); + } + return true; +} + static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); int32_t code = TSDB_CODE_SUCCESS; @@ -1661,25 +1692,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char while (i < numLines) { char *tmp = NULL; int len = 0; - if (lines) { - tmp = lines[i]; - len = strlen(tmp); - } else if (rawLine) { - tmp = rawLine; - while (rawLine < rawLineEnd) { - if (*(rawLine++) == '\n') { - break; - } - len++; - } - if (info->protocol == TSDB_SML_LINE_PROTOCOL && tmp[0] == '#') { // this line is comment - continue; - } + if(!getLine(info, lines, rawLine, rawLineEnd, numLines, i, &tmp, &len)){ + continue; } - - uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, - info->isRawLine, numLines, info->protocol, len, info->isRawLine ? "rawdata" : tmp); - if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->dataFormat) { SSmlLineInfo element = {0}; @@ -1699,7 +1714,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, info->isRawLine ? "rawdata" : tmp); + if(rawLine != NULL){ + char* print = taosMemoryCalloc(len + 1, 1); + memcpy(print, tmp, len); + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); + taosMemoryFree(print); + }else{ + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); + } return code; } if (info->reRun) { @@ -1828,7 +1850,6 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, return (TAOS_RES *)request; } info->pRequest = request; - info->isRawLine = rawLine != NULL; info->ttl = ttl; info->precision = precision; info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; @@ -1934,8 +1955,7 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi reqid); } -TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision, int32_t ttl, int64_t reqid) { +static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol){ int numLines = 0; *totalRows = 0; char *tmp = lines; @@ -1948,6 +1968,11 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int tmp = lines + i + 1; } } +} + +TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int32_t ttl, int64_t reqid) { + getRawLineLen(lines, len, totalRows, protocol); return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); } diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 2dbba38212..13c4431b62 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -210,7 +210,15 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 SSmlKv* kv = (SSmlKv*)data; if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ ret = TSDB_CODE_SML_INVALID_DATA; - uInfo("SML smlBuildCol error col not same %s", pColSchema->name); + char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1); + if(tmp){ + memcpy(tmp, kv->key, kv->keyLen); + uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", + tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name); + taosMemoryFree(tmp); + }else{ + uError("SML smlBuildCol out of memory"); + } goto end; } if (kv->type == TSDB_DATA_TYPE_NCHAR) { From ef621a06a71d55d36c7050f51247471d7cea1f40 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Dec 2023 19:21:02 +0800 Subject: [PATCH 2/4] opti:logic in parse raw line & make log more specific --- source/client/src/clientSml.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c392308195..f973d165bb 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1640,25 +1640,25 @@ int32_t smlClearForRerun(SSmlHandle *info) { return TSDB_CODE_SUCCESS; } -static bool getLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, +static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLineEnd, int numLines, int i, char** tmp, int *len){ if (lines) { *tmp = lines[i]; *len = strlen(*tmp); - } else if (rawLine) { - *tmp = rawLine; - while (rawLine < rawLineEnd) { - if (*(rawLine++) == '\n') { + } else if (*rawLine) { + *tmp = *rawLine; + while (*rawLine < rawLineEnd) { + if (*((*rawLine)++) == '\n') { break; } (*len)++; } if (info->protocol == TSDB_SML_LINE_PROTOCOL && (*tmp)[0] == '#') { // this line is comment - false; + return false; } } - if(rawLine != NULL){ + if(*rawLine != NULL){ char* print = taosMemoryCalloc(*len + 1, 1); memcpy(print, *tmp, *len); uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, @@ -1692,7 +1692,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char while (i < numLines) { char *tmp = NULL; int len = 0; - if(!getLine(info, lines, rawLine, rawLineEnd, numLines, i, &tmp, &len)){ + if(!getLine(info, lines, &rawLine, rawLineEnd, numLines, i, &tmp, &len)){ continue; } if (info->protocol == TSDB_SML_LINE_PROTOCOL) { From 84d18005f20554ee243e6c287190ffc03595bcae Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 19 Dec 2023 11:34:15 +0800 Subject: [PATCH 3/4] fix:add control for raw data in schemaless --- source/client/src/clientSml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index f973d165bb..6bcdb4e973 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1658,7 +1658,7 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi } } - if(*rawLine != NULL){ + if(*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)){ char* print = taosMemoryCalloc(*len + 1, 1); memcpy(print, *tmp, *len); uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, From 1f70f51ac8e2897abdfc4942cf5765f5cf6e753f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 20 Dec 2023 11:20:41 +0800 Subject: [PATCH 4/4] fix:sdb maxId should use int64* instead of int32* --- source/dnode/mnode/sdb/src/sdbHash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 109a3ca211..1d2e2de17d 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -184,7 +184,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); } if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT64) { - pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); + pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int64_t *)pRow->pObj)); } pSdb->tableVer[pRow->type]++;