From efee9afad99c18dedab283ac35087d59055801e7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 12 Dec 2022 22:42:33 +0800 Subject: [PATCH 01/10] opti:parse value in schemaless --- source/client/src/clientSml.c | 175 ++++++++++++++++++++++++++++++ source/client/src/clientSmlLine.c | 70 ++++++++---- utils/test/c/sml_test.c | 39 +++++++ 3 files changed, 261 insertions(+), 23 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index d7a6459a0e..9f03ed3656 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -235,7 +235,182 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { return NULL; } +//uint16_t smlCalTypeSum(char* endptr, int32_t left){ +// uint16_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +//} + +#define RETURN_FALSE \ +smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ +return false; + +#define SET_DOUBLE kvVal->type = TSDB_DATA_TYPE_DOUBLE;\ + kvVal->d = result; + +#define SET_FLOAT \ + if (!IS_VALID_FLOAT(result)) {\ + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_FLOAT;\ + kvVal->f = (float)result; + +#define SET_BIGINT \ + if (smlDoubleToInt64OverFlow(result)) {\ + errno = 0;\ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10);\ + if (errno == ERANGE) {\ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_BIGINT;\ + kvVal->i = tmp;\ + return true;\ + }\ + kvVal->type = TSDB_DATA_TYPE_BIGINT;\ + kvVal->i = (int64_t)result; + +#define SET_INT \ + if (!IS_VALID_INT(result)) {\ + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_INT;\ + kvVal->i = result; + +#define SET_SMALL_INT \ + if (!IS_VALID_SMALLINT(result)) {\ + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_SMALLINT;\ + kvVal->i = result; + +#define SET_UBIGINT \ + if (result >= (double)UINT64_MAX || result < 0) {\ + errno = 0;\ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);\ + if (errno == ERANGE || result < 0) {\ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_UBIGINT;\ + kvVal->u = tmp;\ + return true;\ + }\ + kvVal->type = TSDB_DATA_TYPE_UBIGINT;\ + kvVal->u = result; + +#define SET_UINT \ + if (!IS_VALID_UINT(result)) {\ + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_UINT;\ + kvVal->u = result; + +#define SET_USMALL_INT \ + if (!IS_VALID_USMALLINT(result)) {\ + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_USMALLINT;\ + kvVal->u = result; + +#define SET_TINYINT \ + if (!IS_VALID_TINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_TINYINT;\ + kvVal->i = result; + +#define SET_UTINYINT \ + if (!IS_VALID_UTINYINT(result)) {\ + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_UTINYINT;\ + kvVal->u = result; + bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + char * endptr = NULL; + double result = taosStr2Double(pVal, &endptr); + if (pVal == endptr) { + RETURN_FALSE + } + + int32_t left = len - (endptr - pVal); + if (left == 0) { + SET_DOUBLE + } else if (left == 3) { + if(endptr[0] == 'f' || endptr[0] == 'F'){ + if(endptr[1] == '6' && endptr[2] == '4'){ + SET_DOUBLE + }else if(endptr[1] == '3' && endptr[2] == '2'){ + SET_FLOAT + }else{ + RETURN_FALSE + } + }else if(endptr[0] == 'i' || endptr[0] == 'I'){ + if(endptr[1] == '6' && endptr[2] == '4'){ + SET_BIGINT + }else if(endptr[1] == '3' && endptr[2] == '2'){ + SET_INT + }else if(endptr[1] == '1' && endptr[2] == '6'){ + SET_SMALL_INT + }else{ + RETURN_FALSE + } + }else if(endptr[0] == 'u' || endptr[0] == 'U'){ + if(endptr[1] == '6' && endptr[2] == '4'){ + SET_UBIGINT + }else if(endptr[1] == '3' && endptr[2] == '2'){ + SET_UINT + }else if(endptr[1] == '1' && endptr[2] == '6'){ + SET_USMALL_INT + }else{ + RETURN_FALSE + } + }else{ + RETURN_FALSE + } + } else if(left == 2){ + if(endptr[0] == 'i' || endptr[0] == 'I'){ + if(endptr[1] == '8') { + SET_TINYINT + }else{ + RETURN_FALSE + } + }else if(endptr[0] == 'u' || endptr[0] == 'U') { + if (endptr[1] == '8') { + SET_UTINYINT + } else { + RETURN_FALSE + } + }else{ + RETURN_FALSE + } + } else if(left == 1){ + if(endptr[0] == 'i' || endptr[0] == 'I'){ + SET_BIGINT + }else if(endptr[0] == 'u' || endptr[0] == 'U') { + SET_UBIGINT + }else{ + RETURN_FALSE + } + } else { + RETURN_FALSE; + } + return true; +} + +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 4b554bd623..aca0fe752b 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -121,33 +121,57 @@ static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t le } static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { - // binary - if (smlIsBinary(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length -= BINARY_ADD_LEN; - if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + if (pVal->value[0] == '"'){ // binary + if (pVal->length >= 2 && pVal->value[pVal->length - 1] == '"') { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->length -= BINARY_ADD_LEN; + if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (BINARY_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; } - pVal->value += (BINARY_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; - } - // nchar - if (smlIsNchar(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length -= NCHAR_ADD_LEN; - if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - pVal->value += (NCHAR_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_TSC_INVALID_VALUE; } - // bool - if (smlParseBool(pVal)) { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; + if(pVal->value[0] == 'l' || pVal->value[0] == 'L'){ // nchar + if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3){ + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length -= NCHAR_ADD_LEN; + if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (NCHAR_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; } + + if (pVal->value[0] == 't' || pVal->value[0] == 'T'){ + if(pVal->length == 1 || (pVal->length == 4 && (pVal->value[1] == 'u' || pVal->value[1] == 'U') + && (pVal->value[2] == 'r' || pVal->value[2] == 'R') + && (pVal->value[3] == 'e' || pVal->value[3] == 'E'))){ + pVal->i = TSDB_TRUE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 'f' || pVal->value[0] == 'F'){ + if(pVal->length == 1 || (pVal->length == 5 && (pVal->value[1] == 'a' || pVal->value[1] == 'A') + && (pVal->value[2] == 'l' || pVal->value[2] == 'L') + && (pVal->value[3] == 's' || pVal->value[3] == 'S') + && (pVal->value[4] == 'e' || pVal->value[4] == 'E'))){ + pVal->i = TSDB_FALSE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + // number if (smlParseNumber(pVal, msg)) { pVal->length = (int16_t)tDataTypes[pVal->type].bytes; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 2597a26243..d8d665073c 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1159,7 +1159,46 @@ int sml_ttl_Test() { return code; } +//char *str[] ={ +// "", +// "f64", +// "F64", +// "f32", +// "F32", +// "i", +// "I", +// "i64", +// "I64", +// "u", +// "U", +// "u64", +// "U64", +// "i32", +// "I32", +// "u32", +// "U32", +// "i16", +// "I16", +// "u16", +// "U16", +// "i8", +// "I8", +// "u8", +// "U8", +//}; +//uint8_t smlCalTypeSum(char* endptr, int32_t left){ +// uint8_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +//} + int main(int argc, char *argv[]) { + +// for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ +// printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i]))); +// } int ret = 0; ret = sml_ttl_Test(); ASSERT(!ret); From bc8eb40f5f6afecae37898800ada4cd2384b818a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 12 Dec 2022 22:43:22 +0800 Subject: [PATCH 02/10] opti:parse value in schemaless --- source/client/src/clientSmlLine.c | 5 +++-- source/client/test/smlTest.cpp | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index aca0fe752b..cb57d85375 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -148,8 +148,8 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 't' || pVal->value[0] == 'T'){ - if(pVal->length == 1 || (pVal->length == 4 && (pVal->value[1] == 'u' || pVal->value[1] == 'U') - && (pVal->value[2] == 'r' || pVal->value[2] == 'R') + if(pVal->length == 1 || (pVal->length == 4 && (pVal->value[1] == 'r' || pVal->value[1] == 'R') + && (pVal->value[2] == 'u' || pVal->value[2] == 'U') && (pVal->value[3] == 'e' || pVal->value[3] == 'E'))){ pVal->i = TSDB_TRUE; pVal->type = TSDB_DATA_TYPE_BOOL; @@ -492,6 +492,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; int32_t ret = smlParseValue(&kv, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); return ret; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index d55783fd4c..b7a50f1cc8 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -239,6 +239,7 @@ TEST(testCase, smlParseCols_Test) { info->protocol = TSDB_SML_LINE_PROTOCOL; info->dataFormat = false; SSmlLineInfo elements = {0}; + info->msgBuf = msgBuf; const char *data = "st,t=1 cb\\=in=\"pass\\,it " From ac16bfc46d724ae65970f0626bfba43b3f76c321 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 13 Dec 2022 22:30:06 +0800 Subject: [PATCH 03/10] opti:performance for json in schemaless --- cmake/taostools_CMakeLists.txt.in | 2 +- source/client/src/clientSml.c | 5 +- source/client/src/clientSmlJson.c | 167 ++++++++++++++++++++---------- source/client/test/smlTest.cpp | 28 +++++ 4 files changed, 144 insertions(+), 58 deletions(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 7247200fe7..b05ed20d7e 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 4a4027c + GIT_TAG sml/json SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 9f03ed3656..2f0e4683b0 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1376,7 +1376,7 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL return code; } - info->cost.lineNum = numLines; + info->cost.lineNum = info->lineNum; info->cost.numOfSTables = nodeListSize(info->superTables); info->cost.numOfCTables = nodeListSize(info->childTables); @@ -1463,10 +1463,9 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, request->code = code; info->cost.endTime = taosGetTimestampUs(); info->cost.code = code; - smlPrintStatisticInfo(info); +// smlPrintStatisticInfo(info); end: - uDebug("resultend:%s", request->msgBuf); smlDestroyInfo(info); return (TAOS_RES *)request; } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 1b4231df4e..106d4e98ca 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -28,12 +28,7 @@ int32_t is_same_child_table_json(const void *a, const void *b){ return (cJSON_Compare((const cJSON *)a, (const cJSON *)b, true)) ? 0 : 1; } -static int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { - cJSON *metric = cJSON_GetObjectItem(root, "metric"); - if (!cJSON_IsString(metric)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - +static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { elements->measureLen = strlen(metric->valuestring); if (IS_INVALID_TABLE_LEN(elements->measureLen)) { uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); @@ -110,7 +105,7 @@ static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPr } } -static uint8_t smlGetTimestampLen(int64_t num) { +static inline uint8_t smlGetTimestampLen(int64_t num) { uint8_t len = 0; while ((num /= 10) != 0) { len++; @@ -119,10 +114,9 @@ static uint8_t smlGetTimestampLen(int64_t num) { return len; } -static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root) { +static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { // Timestamp must be the first KV to parse int32_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; - cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); if (cJSON_IsNumber(timestamp)) { // timestamp value 0 indicates current system time double timeDouble = timestamp->valuedouble; @@ -140,6 +134,7 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root) { } uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); + int8_t fromPrecision = smlGetTsTypeByLen(tsLen); if (unlikely(fromPrecision == -1)) { smlBuildInvalidDataMsg(&info->msgBuf, @@ -357,39 +352,20 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseColsFromJSON(cJSON *root, SSmlKv *kv) { - cJSON *metricVal = cJSON_GetObjectItem(root, "value"); - if (metricVal == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - int32_t ret = smlParseValueFromJSON(metricVal, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { +static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { int32_t ret = TSDB_CODE_SUCCESS; - cJSON *tags = cJSON_GetObjectItem(root, "tags"); - if (unlikely(tags == NULL || tags->type != cJSON_Object)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - // add measure to tags to identify one child table - cJSON *cMeasure = cJSON_AddStringToObject(tags, JSON_METERS_NAME, elements->measure); - if(unlikely(cMeasure == NULL)){ - return TSDB_CODE_TSC_INVALID_JSON; - } +// cJSON *cMeasure = cJSON_AddStringToObject(tags, JSON_METERS_NAME, elements->measure); +// if(unlikely(cMeasure == NULL)){ +// return TSDB_CODE_TSC_INVALID_JSON; +// } elements->tags = (char*)tags; if(is_same_child_table_json(elements->tags, info->preLine.tags) == 0){ - cJSON_DeleteItemFromObjectCaseSensitive(tags, JSON_METERS_NAME); +// cJSON_DeleteItemFromObjectCaseSensitive(tags, JSON_METERS_NAME); return TSDB_CODE_SUCCESS; } - cJSON_DeleteItemFromObjectCaseSensitive(tags, JSON_METERS_NAME); +// cJSON_DeleteItemFromObjectCaseSensitive(tags, JSON_METERS_NAME); bool isSameMeasure = IS_SAME_SUPER_TABLE; @@ -529,40 +505,62 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *root, SSmlLineInfo return ret; } +const char *jsonName[OTD_JSON_FIELDS_NUM] = {"metric", "timestamp", "value", "tags"}; +static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks){ + cJSON *child = root->child; + + for (int i = 0; i < OTD_JSON_FIELDS_NUM; ++i) { + while(child != NULL) + { + if(strcasecmp(child->string, jsonName[i]) == 0){ + *marks[i] = child; + break; + } + child = child->next; + } + if(*marks[i] == NULL){ + uError("smlGetJsonElements error, not find mark:%s", jsonName[i]); + return -1; + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { int32_t ret = TSDB_CODE_SUCCESS; - int32_t size = cJSON_GetArraySize(root); - // outmost json fields has to be exactly 4 - if (unlikely(size != OTD_JSON_FIELDS_NUM)) { - uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); - return TSDB_CODE_TSC_INVALID_JSON; + cJSON *metricJson = NULL; + cJSON *tsJson = NULL; + cJSON *valueJson = NULL; + cJSON *tagsJson = NULL; + + cJSON **marks[OTD_JSON_FIELDS_NUM] = {&metricJson, &tsJson, &valueJson, &tagsJson}; + ret = smlGetJsonElements(root, marks); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; } // Parse metric - ret = smlParseMetricFromJSON(info, root, elements); + ret = smlParseMetricFromJSON(info, metricJson, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); return ret; } - uDebug("OTD:0x%" PRIx64 " Parse metric from JSON payload finished", info->id); // Parse metric value SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN}; - ret = smlParseColsFromJSON(root, &kv); + ret = smlParseValueFromJSON(valueJson, &kv); if (unlikely(ret)) { uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); return ret; } - uDebug("OTD:0x%" PRIx64 " Parse metric value from JSON payload finished", info->id); // Parse tags - ret = smlParseTagsFromJSON(info, root, elements); + ret = smlParseTagsFromJSON(info, tagsJson, elements); if (unlikely(ret)) { uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); return ret; } - uDebug("OTD:0x%" PRIx64 " Parse tags from JSON payload finished", info->id); if(unlikely(info->reRun)){ return TSDB_CODE_SUCCESS; @@ -570,12 +568,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlLineInfo *e // Parse timestamp // notice!!! put ts back to tag to ensure get meta->precision - int64_t ts = smlParseTSFromJSON(info, root); + int64_t ts = smlParseTSFromJSON(info, tsJson); if (unlikely(ts < 0)) { uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); return TSDB_CODE_INVALID_TIMESTAMP; } - uDebug("OTD:0x%" PRIx64 " Parse timestamp from JSON payload finished", info->id); SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; if(info->dataFormat){ @@ -627,14 +624,16 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { return TSDB_CODE_TSC_INVALID_JSON; } - int32_t i = 0; - while (i < payloadNum) { - cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : cJSON_GetArrayItem(info->root, i); + cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child; + + int cnt = 0; + cJSON *dataPoint = head; + while (dataPoint) { if(info->dataFormat) { SSmlLineInfo element = {0}; ret = smlParseJSONString(info, dataPoint, &element); }else{ - ret = smlParseJSONString(info, dataPoint, info->lines + i); + ret = smlParseJSONString(info, dataPoint, info->lines + cnt); } if (unlikely(ret != TSDB_CODE_SUCCESS)) { uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); @@ -642,7 +641,8 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { } if(unlikely(info->reRun)){ - i = 0; + cnt = 0; + dataPoint = head; info->lineNum = payloadNum; ret = smlClearForRerun(info); if(ret != TSDB_CODE_SUCCESS){ @@ -650,7 +650,66 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { } continue; } - i++; + cnt++; + dataPoint = dataPoint->next; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseJSONOld(SSmlHandle *info, char *payload) { + int32_t payloadNum = 0; + int32_t ret = TSDB_CODE_SUCCESS; + + if (unlikely(payload == NULL)) { + uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + info->root = cJSON_Parse(payload); + if (unlikely(info->root == NULL)) { + uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); + return TSDB_CODE_TSC_INVALID_JSON; + } + + // multiple data points must be sent in JSON array + if (cJSON_IsArray(info->root)) { + payloadNum = cJSON_GetArraySize(info->root); + } else if (cJSON_IsObject(info->root)) { + payloadNum = 1; + } else { + uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child; + + int cnt = 0; + cJSON *dataPoint = head; + while (dataPoint) { + if(info->dataFormat) { + SSmlLineInfo element = {0}; + ret = smlParseJSONString(info, dataPoint, &element); + }else{ + ret = smlParseJSONString(info, dataPoint, info->lines + cnt); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); + return ret; + } + + if(unlikely(info->reRun)){ + cnt = 0; + dataPoint = head; + info->lineNum = payloadNum; + ret = smlClearForRerun(info); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + continue; + } + cnt++; + dataPoint = dataPoint->next; } return TSDB_CODE_SUCCESS; diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index b7a50f1cc8..3914e2689e 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -622,3 +622,31 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { ASSERT_NE(ret, 0); smlDestroyInfo(info); } + +TEST(testCase, smlParseNumber_performance_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + SSmlKv kv; + + char* str[3] = {"2893f64", "2323u32", "93u8"}; + for (int i = 0; i < 3; ++i) { + int64_t t1 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumber(&kv, &msgBuf); + } + printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); + printf("\n"); + int64_t t2 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumberOld(&kv, &msgBuf); + } + printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); + printf("\n\n"); + } + + +} \ No newline at end of file From 097a2b928c51f7e441f8c3fc55c6b0881e841cbb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 15 Dec 2022 00:35:58 +0800 Subject: [PATCH 04/10] opti:modify parse json --- source/client/inc/clientSml.h | 6 +- source/client/src/clientSml.c | 3 +- source/client/src/clientSmlJson.c | 698 +++++++++------------------- source/client/src/clientSmlLine.c | 54 +-- source/client/src/clientSmlTelnet.c | 2 +- source/client/test/smlTest.cpp | 187 ++------ utils/test/c/sml_test.c | 287 +----------- 7 files changed, 288 insertions(+), 949 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 69de767ec8..6a445432f0 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -113,7 +113,7 @@ typedef struct { int32_t sTableNameLen; char childTableName[TSDB_TABLE_NAME_LEN]; uint64_t uid; - void *key; // for openTsdb telnet + void *key; // for openTsdb SArray *tags; @@ -177,7 +177,7 @@ typedef struct { int32_t lineNum; SSmlMsgBuf msgBuf; - cJSON *root; // for parse json +// cJSON *root; // for parse json SSmlLineInfo *lines; // element is SSmlLineInfo // @@ -216,7 +216,7 @@ SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); int32_t smlSetCTableName(SSmlTableInfo *oneTable); STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); int32_t is_same_child_table_telnet(const void *a, const void *b); -int32_t is_same_child_table_json(const void *a, const void *b); +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); int32_t smlClearForRerun(SSmlHandle *info); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 2f0e4683b0..6d251544f6 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1060,7 +1060,6 @@ static void smlDestroyInfo(SSmlHandle *info) { taosMemoryFree(info->lines); } - cJSON_Delete(info->root); taosMemoryFreeClear(info); } @@ -1125,7 +1124,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { }else if(info->protocol == TSDB_SML_TELNET_PROTOCOL){ tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); }else{ - tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->tags, POINTER_BYTES, is_same_child_table_json); + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); } if(tinfo == NULL){ diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 106d4e98ca..6faee925d0 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -19,353 +19,65 @@ #include #include "clientSml.h" -#define OTD_JSON_SUB_FIELDS_NUM 2 -#define OTD_JSON_FIELDS_NUM 4 -#define JSON_METERS_NAME "__JM" - - -int32_t is_same_child_table_json(const void *a, const void *b){ - return (cJSON_Compare((const cJSON *)a, (const cJSON *)b, true)) ? 0 : 1; -} - -static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { - elements->measureLen = strlen(metric->valuestring); - if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; +#define JUMP_JSON_SPACE(start,end) \ +while(start < end){\ + if(unlikely(isspace(*start) == 0))\ + break;\ + else\ + start++;\ } - elements->measure = metric->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { - int32_t size = cJSON_GetArraySize(root); - if (unlikely(size != OTD_JSON_SUB_FIELDS_NUM)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (unlikely(!cJSON_IsNumber(value))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (unlikely(!cJSON_IsString(type))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); - return -1; - } - - double timeDouble = value->valuedouble; - if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return -1; - } - - if (timeDouble == 0) { - return taosGetTimestampNs()/smlFactorNS[toPrecision]; - } - - if (timeDouble < 0) { - return timeDouble; - } - - int64_t tsInt64 = timeDouble; - size_t typeLen = strlen(type->valuestring); - if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { - // seconds - int8_t fromPrecision = TSDB_TIME_PRECISION_SECONDS; - if(smlFactorS[toPrecision] < INT64_MAX / tsInt64){ - return tsInt64 * smlFactorS[toPrecision]; - } - return -1; - } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { - switch (type->valuestring[0]) { - case 'm': - case 'M': - // milliseconds - return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MILLI, toPrecision); +static SArray *smlJsonParseTags(char *start, char *end){ + SArray *tags = taosArrayInit(4, sizeof(SSmlKv)); + while(start < end){ + SSmlKv kv = {0}; + kv.type = TSDB_DATA_TYPE_NCHAR; + bool isInQuote = false; + while(start < end){ + if(!isInQuote && *start == '"'){ + start++; + kv.key = start; + isInQuote = true; + continue; + } + if(isInQuote && *start == '"'){ + kv.keyLen = start - kv.key; + start++; break; - case 'u': - case 'U': - // microseconds - return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MICRO, toPrecision); + } + start++; + } + bool hasColon = false; + while(start < end){ + if(!hasColon && *start == ':'){ + start++; + hasColon = true; + continue; + } + if(hasColon && kv.value == NULL && (isspace(*start) == 0 && *start != '"')){ + kv.value = start; + start++; + continue; + } + + if(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}')){ + kv.length = start - kv.value; + taosArrayPush(tags, &kv); + start++; break; - case 'n': - case 'N': - return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_NANO, toPrecision); - break; - default: - return -1; - } - } else { - return -1; - } -} - -static inline uint8_t smlGetTimestampLen(int64_t num) { - uint8_t len = 0; - while ((num /= 10) != 0) { - len++; - } - len++; - return len; -} - -static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { - // Timestamp must be the first KV to parse - int32_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; - if (cJSON_IsNumber(timestamp)) { - // timestamp value 0 indicates current system time - double timeDouble = timestamp->valuedouble; - if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return -1; - } - - if (unlikely(timeDouble < 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, - "timestamp is negative", NULL); - return timeDouble; - }else if (unlikely(timeDouble == 0)) { - return taosGetTimestampNs()/smlFactorNS[toPrecision]; - } - - uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); - - int8_t fromPrecision = smlGetTsTypeByLen(tsLen); - if (unlikely(fromPrecision == -1)) { - smlBuildInvalidDataMsg(&info->msgBuf, - "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", NULL); - return -1; - } - int64_t tsInt64 = timeDouble; - if(fromPrecision == TSDB_TIME_PRECISION_SECONDS){ - if(smlFactorS[toPrecision] < INT64_MAX / tsInt64){ - return tsInt64 * smlFactorS[toPrecision]; } - return -1; - }else{ - return convertTimePrecision(timeDouble, fromPrecision, toPrecision); + start++; } - } else if (cJSON_IsObject(timestamp)) { - return smlParseTSFromJSONObj(info, timestamp, toPrecision); - } else { - smlBuildInvalidDataMsg(&info->msgBuf, - "invalidate json", NULL); - return -1; } + return tags; } -static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "bool") != 0) { - uError("OTD:invalid type(%s) for JSON Bool", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valueint; - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { - // tinyint - if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { - if (!IS_VALID_TINYINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // smallint - if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { - if (!IS_VALID_SMALLINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // int - if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { - if (!IS_VALID_INT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // bigint - if (strcasecmp(typeStr, "i64") == 0 || strcasecmp(typeStr, "bigint") == 0) { - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if (value->valuedouble >= (double)INT64_MAX) { - pVal->i = INT64_MAX; - } else if (value->valuedouble <= (double)INT64_MIN) { - pVal->i = INT64_MIN; - } else { - pVal->i = value->valuedouble; - } - return TSDB_CODE_SUCCESS; - } - // float - if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { - if (!IS_VALID_FLOAT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->f = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // double - if (strcasecmp(typeStr, "f64") == 0 || strcasecmp(typeStr, "double") == 0) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->d = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - - // if reach here means type is unsupported - uError("OTD:invalid type(%s) for JSON Number", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; -} - -static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "binary") == 0) { - pVal->type = TSDB_DATA_TYPE_BINARY; - } else if (strcasecmp(typeStr, "nchar") == 0) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - } else { - uError("OTD:invalid type(%s) for JSON String", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->length = (int16_t)strlen(value->valuestring); - - if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - if (pVal->type == TSDB_DATA_TYPE_NCHAR && - pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - - pVal->value = value->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) { - int32_t ret = TSDB_CODE_SUCCESS; - int32_t size = cJSON_GetArraySize(root); - - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (value == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - switch (value->type) { - case cJSON_True: - case cJSON_False: { - ret = smlConvertJSONBool(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_Number: { - ret = smlConvertJSONNumber(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_String: { - ret = smlConvertJSONString(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { - switch (root->type) { - case cJSON_True: - case cJSON_False: { - kv->type = TSDB_DATA_TYPE_BOOL; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->i = root->valueint; - break; - } - case cJSON_Number: { - kv->type = TSDB_DATA_TYPE_DOUBLE; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->d = root->valuedouble; - break; - } - case cJSON_String: { - /* set default JSON type to binary/nchar according to - * user configured parameter tsDefaultJSONStrType - */ - - char *tsDefaultJSONStrType = "nchar"; // todo - smlConvertJSONString(kv, tsDefaultJSONStrType, root); - break; - } - case cJSON_Object: { - int32_t ret = smlParseValueFromJSONObj(root, kv); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse value from JSON Obj"); - return ret; - } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { +static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) { int32_t ret = TSDB_CODE_SUCCESS; - // add measure to tags to identify one child table -// cJSON *cMeasure = cJSON_AddStringToObject(tags, JSON_METERS_NAME, elements->measure); -// if(unlikely(cMeasure == NULL)){ -// return TSDB_CODE_TSC_INVALID_JSON; -// } - elements->tags = (char*)tags; - if(is_same_child_table_json(elements->tags, info->preLine.tags) == 0){ -// cJSON_DeleteItemFromObjectCaseSensitive(tags, JSON_METERS_NAME); + if(is_same_child_table_telnet(elements, &info->preLine) == 0){ return TSDB_CODE_SUCCESS; } -// cJSON_DeleteItemFromObjectCaseSensitive(tags, JSON_METERS_NAME); bool isSameMeasure = IS_SAME_SUPER_TABLE; @@ -400,31 +112,16 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo taosArraySetSize(preLineKV, 0); } - int32_t tagNum = cJSON_GetArraySize(tags); + SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen); + int32_t tagNum = taosArrayGetSize(tags); for (int32_t i = 0; i < tagNum; ++i) { - cJSON *tag = cJSON_GetArrayItem(tags, i); - if (unlikely(tag == NULL)) { - return TSDB_CODE_TSC_INVALID_JSON; - } -// if(unlikely(tag == cMeasure)) continue; - size_t keyLen = strlen(tag->string); - if (unlikely(IS_INVALID_COL_LEN(keyLen))) { - uError("OTD:Tag key length is 0 or too large than 64"); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - - // add kv to SSmlKv - SSmlKv kv ={.key = tag->string, .keyLen = keyLen}; - // value - ret = smlParseValueFromJSON(tag, &kv); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - return ret; - } + SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i); if(info->dataFormat){ if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ info->dataFormat = false; info->reRun = true; + taosArrayDestroy(tags); return TSDB_CODE_SUCCESS; } @@ -432,6 +129,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { info->dataFormat = false; info->reRun = true; + taosArrayDestroy(tags); return TSDB_CODE_SUCCESS; } SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); @@ -447,6 +145,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo if(unlikely(!IS_SAME_KEY)){ info->dataFormat = false; info->reRun = true; + taosArrayDestroy(tags); return TSDB_CODE_SUCCESS; } }else{ @@ -454,6 +153,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo if(unlikely(cnt >= taosArrayGetSize(superKV))) { info->dataFormat = false; info->reRun = true; + taosArrayDestroy(tags); return TSDB_CODE_SUCCESS; } SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); @@ -467,6 +167,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo if(unlikely(!IS_SAME_KEY)){ info->dataFormat = false; info->reRun = true; + taosArrayDestroy(tags); return TSDB_CODE_SUCCESS; } }else{ @@ -479,8 +180,9 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo } cnt++; } + taosArrayDestroy(tags); - SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->tags, POINTER_BYTES, is_same_child_table_json); + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); if (unlikely(tinfo == NULL)) { tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); if (unlikely(!tinfo)) { @@ -498,65 +200,152 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo } } - nodeListSet(&info->childTables, tags, POINTER_BYTES, tinfo, is_same_child_table_json); + SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + *key = *elements; + tinfo->key = key; + nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); } if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; return ret; } -const char *jsonName[OTD_JSON_FIELDS_NUM] = {"metric", "timestamp", "value", "tags"}; -static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks){ - cJSON *child = root->child; - - for (int i = 0; i < OTD_JSON_FIELDS_NUM; ++i) { - while(child != NULL) - { - if(strcasecmp(child->string, jsonName[i]) == 0){ - *marks[i] = child; - break; +static char* smlJsonGetObj(char *payload){ + int leftBracketCnt = 0; + while(*payload) { + if (*payload == '{') { + leftBracketCnt++; + payload++; + continue; + } + if (*payload == '}') { + leftBracketCnt--; + payload++; + if (leftBracketCnt == 0) { + return payload; + } else if (leftBracketCnt < 0) { + return NULL; } - child = child->next; - } - if(*marks[i] == NULL){ - uError("smlGetJsonElements error, not find mark:%s", jsonName[i]); - return -1; + continue; } + payload++; } - return TSDB_CODE_SUCCESS; + return NULL; } -static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { +static void smlJsonParseObj(char *start, char *end, SSmlLineInfo *element){ + while(start < end){ + if(start[0]== '"' && start[1] == 'm' && start[2] == 'e' && start[3] == 't' + && start[4] == 'r' && start[5] == 'i' && start[6] == 'c' && start[7] == '"'){ + + start += 8; + bool isInQuote = false; + while(start < end){ + if(!isInQuote && *start == '"'){ + start++; + element->measure = start; + isInQuote = true; + continue; + } + if(isInQuote && *start == '"'){ + element->measureLen = start - element->measure; + start++; + break; + } + start++; + } + }else if(start[0] == '"' && start[1] == 't' && start[2] == 'i' && start[3] == 'm' + && start[4] == 'e' && start[5] == 's' && start[6] == 't' + && start[7] == 'a' && start[8] == 'm' && start[9] == 'p' && start[10] == '"'){ + + start += 11; + bool hasColon = false; + while(start < end){ + if(!hasColon && *start == ':'){ + start++; + JUMP_JSON_SPACE(start,end) + element->timestamp = start; + hasColon = true; + continue; + } + if(hasColon && (*start == ',' || *start == '}' || isspace(*start) != 0)){ + element->timestampLen = start - element->timestamp; + start++; + break; + } + start++; + } + }else if(start[0]== '"' && start[1] == 'v' && start[2] == 'a' && start[3] == 'l' + && start[4] == 'u' && start[5] == 'e' && start[6] == '"'){ + + start += 7; + + bool hasColon = false; + while(start < end){ + if(!hasColon && *start == ':'){ + start++; + JUMP_JSON_SPACE(start,end) + element->cols = start; + hasColon = true; + continue; + } + if(hasColon && (*start == ',' || *start == '}' || isspace(*start) != 0)){ + element->colsLen = start - element->cols; + start++; + break; + } + start++; + } + }else if(start[0] == '"' && start[1] == 't' && start[2] == 'a' && start[3] == 'g' + && start[4] == 's' && start[5] == '"'){ + start += 6; + + while(start < end){ + if(*start == ':'){ + start++; + JUMP_JSON_SPACE(start,end) + element->tags = start; + element->tagsLen = smlJsonGetObj(start) - start; + break; + } + start++; + } + }else{ + start++; + } + } +} + +static int32_t smlParseJSONString(SSmlHandle *info, char *start, char *end, SSmlLineInfo *elements) { int32_t ret = TSDB_CODE_SUCCESS; - cJSON *metricJson = NULL; - cJSON *tsJson = NULL; - cJSON *valueJson = NULL; - cJSON *tagsJson = NULL; - - cJSON **marks[OTD_JSON_FIELDS_NUM] = {&metricJson, &tsJson, &valueJson, &tagsJson}; - ret = smlGetJsonElements(root, marks); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - return ret; + smlJsonParseObj(start, end, elements); + if(unlikely(elements->measure == NULL || elements->measureLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid measure data", start); + return TSDB_CODE_SML_INVALID_DATA; + } + if(unlikely(elements->tags == NULL || elements->tagsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid tags data", start); + return TSDB_CODE_SML_INVALID_DATA; + } + if(unlikely(elements->cols == NULL || elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid cols data", start); + return TSDB_CODE_SML_INVALID_DATA; + } + if(unlikely(elements->timestamp == NULL || elements->timestampLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp data", start); + return TSDB_CODE_SML_INVALID_DATA; } - // Parse metric - ret = smlParseMetricFromJSON(info, metricJson, elements); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); - return ret; - } - - // Parse metric value - SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN}; - ret = smlParseValueFromJSON(valueJson, &kv); - if (unlikely(ret)) { - uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); - return ret; + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + if (smlParseNumber(&kv, &info->msgBuf)) { + kv.length = (int16_t)tDataTypes[kv.type].bytes; + }else{ + return TSDB_CODE_TSC_INVALID_VALUE; } // Parse tags - ret = smlParseTagsFromJSON(info, tagsJson, elements); + ret = smlParseTagsFromJSON(info, elements); if (unlikely(ret)) { uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); return ret; @@ -568,7 +357,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlLineInfo *e // Parse timestamp // notice!!! put ts back to tag to ensure get meta->precision - int64_t ts = smlParseTSFromJSON(info, tsJson); + int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); if (unlikely(ts < 0)) { uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); return TSDB_CODE_INVALID_TIMESTAMP; @@ -599,41 +388,39 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlLineInfo *e return TSDB_CODE_SUCCESS; } +//#define JUMP_TO_QUOTE(sql) \ +// while (sql){ \ +// if (unlikely(isspace(*(sql))) != 0) \ +// (sql)++; \ +// else \ +// break; \ +// } +// + + int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 0; + int32_t payloadNum = 1 << 15; int32_t ret = TSDB_CODE_SUCCESS; - if (unlikely(payload == NULL)) { - uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - info->root = cJSON_Parse(payload); - if (unlikely(info->root == NULL)) { - uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); - return TSDB_CODE_TSC_INVALID_JSON; - } - - // multiple data points must be sent in JSON array - if (cJSON_IsArray(info->root)) { - payloadNum = cJSON_GetArraySize(info->root); - } else if (cJSON_IsObject(info->root)) { - payloadNum = 1; - } else { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child; - int cnt = 0; - cJSON *dataPoint = head; - while (dataPoint) { + char *dataPointStart = payload; + char *dataPointEnd = NULL; + while (1) { + dataPointEnd = smlJsonGetObj(dataPointStart); + if(dataPointEnd == NULL) break; + if(info->dataFormat) { SSmlLineInfo element = {0}; - ret = smlParseJSONString(info, dataPoint, &element); + ret = smlParseJSONString(info, dataPointStart, dataPointEnd, &element); }else{ - ret = smlParseJSONString(info, dataPoint, info->lines + cnt); + if(cnt >= payloadNum){ + payloadNum = payloadNum << 1; + void* tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); + if(tmp != NULL){ + info->lines = (SSmlLineInfo*)tmp; + } + } + ret = smlParseJSONString(info, dataPointStart, dataPointEnd, info->lines + cnt); } if (unlikely(ret != TSDB_CODE_SUCCESS)) { uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); @@ -642,7 +429,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { if(unlikely(info->reRun)){ cnt = 0; - dataPoint = head; + dataPointStart = payload; info->lineNum = payloadNum; ret = smlClearForRerun(info); if(ret != TSDB_CODE_SUCCESS){ @@ -651,66 +438,9 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { continue; } cnt++; - dataPoint = dataPoint->next; + dataPointStart = dataPointEnd; } + info->lineNum = cnt; return TSDB_CODE_SUCCESS; } - -int32_t smlParseJSONOld(SSmlHandle *info, char *payload) { - int32_t payloadNum = 0; - int32_t ret = TSDB_CODE_SUCCESS; - - if (unlikely(payload == NULL)) { - uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - info->root = cJSON_Parse(payload); - if (unlikely(info->root == NULL)) { - uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); - return TSDB_CODE_TSC_INVALID_JSON; - } - - // multiple data points must be sent in JSON array - if (cJSON_IsArray(info->root)) { - payloadNum = cJSON_GetArraySize(info->root); - } else if (cJSON_IsObject(info->root)) { - payloadNum = 1; - } else { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child; - - int cnt = 0; - cJSON *dataPoint = head; - while (dataPoint) { - if(info->dataFormat) { - SSmlLineInfo element = {0}; - ret = smlParseJSONString(info, dataPoint, &element); - }else{ - ret = smlParseJSONString(info, dataPoint, info->lines + cnt); - } - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - return ret; - } - - if(unlikely(info->reRun)){ - cnt = 0; - dataPoint = head; - info->lineNum = payloadNum; - ret = smlClearForRerun(info); - if(ret != TSDB_CODE_SUCCESS){ - return ret; - } - continue; - } - cnt++; - dataPoint = dataPoint->next; - } - - return TSDB_CODE_SUCCESS; -} \ No newline at end of file diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 085f16b12c..a75c9752bb 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -456,7 +456,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, bool isInQuote = false; while (*sql < sqlEnd) { // parse value - if (IS_QUOTE(*sql)) { + if (unlikely(IS_QUOTE(*sql))) { isInQuote = !isInQuote; (*sql)++; continue; @@ -513,32 +513,32 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, } if(isSameMeasure){ - if(cnt >= taosArrayGetSize(preLineKV)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); - if(kv.type != preKV->type){ - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - - if(unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > preKV->length)){ - preKV->length = kv.length; - SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); - ASSERT(tableMeta != NULL); - - SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->cols, cnt); - oldKV->length = kv.length; - info->needModifySchema = true; - } - if(unlikely(!IS_SAME_KEY)){ - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } +// if(cnt >= taosArrayGetSize(preLineKV)) { +// info->dataFormat = false; +// info->reRun = true; +// return TSDB_CODE_SUCCESS; +// } +// SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); +// if(kv.type != preKV->type){ +// info->dataFormat = false; +// info->reRun = true; +// return TSDB_CODE_SUCCESS; +// } +// +// if(unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > preKV->length)){ +// preKV->length = kv.length; +// SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); +// ASSERT(tableMeta != NULL); +// +// SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->cols, cnt); +// oldKV->length = kv.length; +// info->needModifySchema = true; +// } +// if(unlikely(!IS_SAME_KEY)){ +// info->dataFormat = false; +// info->reRun = true; +// return TSDB_CODE_SUCCESS; +// } }else{ if(isSuperKVInit){ if(unlikely(cnt >= taosArrayGetSize(superKV))) { diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 4549fe8c8c..7862fe54f6 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -27,7 +27,7 @@ int32_t is_same_child_table_telnet(const void *a, const void *b){ && ((t1->tagsLen == t2->tagsLen) && memcmp(t1->tags, t2->tags, t1->tagsLen) == 0)) ? 0 : 1; } -static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; if (unlikely(!data)) { diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 3914e2689e..dbe7bed5a4 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -414,28 +414,28 @@ TEST(testCase, smlParseCols_Test) { smlDestroyInfo(info); } -TEST(testCase, smlGetTimestampLen_Test) { - uint8_t len = smlGetTimestampLen(0); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(1); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(10); - ASSERT_EQ(len, 2); - - len = smlGetTimestampLen(390); - ASSERT_EQ(len, 3); - - len = smlGetTimestampLen(-1); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(-10); - ASSERT_EQ(len, 2); - - len = smlGetTimestampLen(-390); - ASSERT_EQ(len, 3); -} +//TEST(testCase, smlGetTimestampLen_Test) { +// uint8_t len = smlGetTimestampLen(0); +// ASSERT_EQ(len, 1); +// +// len = smlGetTimestampLen(1); +// ASSERT_EQ(len, 1); +// +// len = smlGetTimestampLen(10); +// ASSERT_EQ(len, 2); +// +// len = smlGetTimestampLen(390); +// ASSERT_EQ(len, 3); +// +// len = smlGetTimestampLen(-1); +// ASSERT_EQ(len, 1); +// +// len = smlGetTimestampLen(-10); +// ASSERT_EQ(len, 2); +// +// len = smlGetTimestampLen(-390); +// ASSERT_EQ(len, 3); +//} TEST(testCase, smlParseNumber_Test) { SSmlKv kv = {0}; @@ -487,139 +487,32 @@ TEST(testCase, smlParseTelnetLine_error_Test) { smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_json_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL); - info->protocol = TSDB_SML_JSON_PROTOCOL; - info->dataFormat = false; - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 13468464009999333322222223,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400i,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"groupid\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"nchar\"\n" - " },\n" - " \"location\": { \n" - " \"value\" : \"北京\",\n" - " \"type\" : \"binary\"\n" - " },\n" - " \"id\": \"d1001\"\n" - " }\n" - " },\n" - "]", - }; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - SSmlLineInfo elements = {0}; - ret = smlParseTelnetString(info, (char *)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); - ASSERT_NE(ret, 0); - } - - smlDestroyInfo(info); -} - -TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL); - info->protocol = TSDB_SML_JSON_PROTOCOL; - info->dataFormat = false; - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": 8\n" - " }\n" - " },\n" - "]", - }; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - SSmlLineInfo elements = {0}; - ret = smlParseTelnetString(info, (char *)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { SSmlHandle *info = smlBuildSmlInfo(NULL); info->protocol = TSDB_SML_JSON_PROTOCOL; - info->dataFormat = false; ASSERT_NE(info, nullptr); const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": \"18\",\n" - " \"tags\": {\n" - " \"host\": \"fff\"\n" - " }\n" - " },\n" - "]", + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\": 1346846400,\"value\": 18,\"tags\": {\"host\": \"lga\"}},{\"metric\": \"sys.sdfa\",\"timestamp\": 1346846400,\"value\": \"18\",\"tags\": {\"host\": 8932}},]", }; - int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - SSmlLineInfo elements = {0}; - ret = smlParseTelnetString(info, (char *)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); - if (ret != TSDB_CODE_SUCCESS) break; + char *dataPointStart = (char *)sql[i]; + char *dataPointEnd = NULL; + while (1) { + dataPointEnd = smlJsonGetObj(dataPointStart); + if (dataPointEnd == NULL) break; + + SSmlLineInfo elements = {0}; + smlJsonParseObj(dataPointStart, dataPointEnd, &elements); + + SArray *tags = smlJsonParseTags(elements.tags, elements.tags + elements.tagsLen); + size_t num = taosArrayGetSize(tags); + ASSERT_EQ(num, 1); + + taosArrayDestroy(tags); + dataPointStart = dataPointEnd; + } } - ASSERT_NE(ret, 0); smlDestroyInfo(info); } @@ -647,6 +540,4 @@ TEST(testCase, smlParseNumber_performance_Test) { printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); printf("\n\n"); } - - } \ No newline at end of file diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index d8d665073c..9ffb7efd63 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -156,28 +156,7 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { - "{\"metric\":\"meter_current1\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"t1\":{\"value\":2,\"type\":\"bigint\"},\"t2\":{\"value\":2,\"type\":\"int\"},\"t3\":{\"value\":2,\"type\":\"i16\"},\"t4\":{\"value\":2,\"type\":\"i8\"},\"t5\":{\"value\":2,\"type\":\"f32\"},\"t6\":{\"value\":2,\"type\":\"double\"},\"t7\":{\"value\":\"8323\",\"type\":\"binary\"},\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":{\"value\":true,\"type\":\"bool\"},\"id\":\"d1001\"}}"}; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - -int smlProcess_json4_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "{\"metric\":\"meter_current2\",\"timestamp\":{\"value\":1662344042000,\"type\":\"ms\"},\"value\":\"ni\",\"tags\":{\"t1\":{\"value\":20,\"type\":\"i64\"},\"t2\":{\"value\":25,\"type\":\"i32\"},\"t3\":{\"value\":2,\"type\":\"smallint\"},\"t4\":{\"value\":2,\"type\":\"tinyint\"},\"t5\":{\"value\":2,\"type\":\"float\"},\"t6\":{\"value\":0.2,\"type\":\"f64\"},\"t7\":\"nsj\",\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":false,\"id\":\"d1001\"}}" + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]" }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -686,52 +665,6 @@ int sml_oom_Test() { return code; } -int sml_16368_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, " - "\"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": " - "\"binary\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, " - "\"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}}," - "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": " - "{\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, " - "\"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": " - "\"nchar\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"ste2\", \"type\": \"nchar\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": " - "{\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, " - "\"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": " - "{\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]"}; - pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_dup_time_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -771,214 +704,6 @@ int sml_dup_time_Test() { return code; } -int sml_16960_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[" - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955000, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": 830525384, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955001, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -588348364, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955002, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -370310823, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955003, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -811250191, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955004, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -330340558, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}" - "]"}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_MILLI_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_add_tag_col_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1211,11 +936,9 @@ int main(int argc, char *argv[]) { ret = smlProcess_json1_Test(); ASSERT(!ret); ret = smlProcess_json2_Test(); - ASSERT(!ret); + ASSERT(ret); ret = smlProcess_json3_Test(); - ASSERT(!ret); - ret = smlProcess_json4_Test(); - ASSERT(!ret); + ASSERT(ret); ret = sml_TD15662_Test(); ASSERT(!ret); ret = sml_TD15742_Test(); @@ -1224,12 +947,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_oom_Test(); ASSERT(!ret); - ret = sml_16368_Test(); - ASSERT(!ret); ret = sml_dup_time_Test(); ASSERT(!ret); - ret = sml_16960_Test(); - ASSERT(!ret); ret = sml_add_tag_col_Test(); ASSERT(!ret); ret = smlProcess_18784_Test(); From 1b2a0473975fb4dac28060f17fa69ceabd393a1b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 15 Dec 2022 11:25:03 +0800 Subject: [PATCH 05/10] opti:json parser --- source/client/src/clientSmlJson.c | 141 +++++++++++++++--------------- source/client/src/clientSmlLine.c | 52 +++++------ source/client/test/smlTest.cpp | 8 +- utils/test/c/sml_test.c | 4 +- 4 files changed, 102 insertions(+), 103 deletions(-) diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 6faee925d0..6c073a370b 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -19,12 +19,12 @@ #include #include "clientSml.h" -#define JUMP_JSON_SPACE(start,end) \ -while(start < end){\ - if(unlikely(isspace(*start) == 0))\ +#define JUMP_JSON_SPACE(start) \ +while(*(start)){\ + if(unlikely(isspace(*(start)) == 0))\ break;\ else\ - start++;\ + (start)++;\ } static SArray *smlJsonParseTags(char *start, char *end){ @@ -213,12 +213,12 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) { static char* smlJsonGetObj(char *payload){ int leftBracketCnt = 0; while(*payload) { - if (*payload == '{') { + if (unlikely(*payload == '{')) { leftBracketCnt++; payload++; continue; } - if (*payload == '}') { + if (unlikely(*payload == '}')) { leftBracketCnt--; payload++; if (leftBracketCnt == 0) { @@ -233,107 +233,113 @@ static char* smlJsonGetObj(char *payload){ return NULL; } -static void smlJsonParseObj(char *start, char *end, SSmlLineInfo *element){ - while(start < end){ - if(start[0]== '"' && start[1] == 'm' && start[2] == 'e' && start[3] == 't' - && start[4] == 'r' && start[5] == 'i' && start[6] == 'c' && start[7] == '"'){ +static void smlJsonParseObj(char **start, SSmlLineInfo *element){ + while(*(*start)){ + if((*start)[0]== '"' && (*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' + && (*start)[4] == 'r' && (*start)[5] == 'i' && (*start)[6] == 'c' && (*start)[7] == '"'){ - start += 8; + (*start) += 8; bool isInQuote = false; - while(start < end){ - if(!isInQuote && *start == '"'){ - start++; - element->measure = start; + while(*(*start)){ + if(unlikely(!isInQuote && *(*start) == '"')){ + (*start)++; + element->measure = (*start); isInQuote = true; continue; } - if(isInQuote && *start == '"'){ - element->measureLen = start - element->measure; - start++; + if(unlikely(isInQuote && *(*start) == '"')){ + element->measureLen = (*start) - element->measure; break; } - start++; + (*start)++; } - }else if(start[0] == '"' && start[1] == 't' && start[2] == 'i' && start[3] == 'm' - && start[4] == 'e' && start[5] == 's' && start[6] == 't' - && start[7] == 'a' && start[8] == 'm' && start[9] == 'p' && start[10] == '"'){ + }else if((*start)[0] == '"' && (*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' + && (*start)[4] == 'e' && (*start)[5] == 's' && (*start)[6] == 't' + && (*start)[7] == 'a' && (*start)[8] == 'm' && (*start)[9] == 'p' && (*start)[10] == '"'){ - start += 11; + (*start) += 11; bool hasColon = false; - while(start < end){ - if(!hasColon && *start == ':'){ - start++; - JUMP_JSON_SPACE(start,end) - element->timestamp = start; + while(*(*start)){ + if(unlikely(!hasColon && *(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + element->timestamp = (*start); hasColon = true; continue; } - if(hasColon && (*start == ',' || *start == '}' || isspace(*start) != 0)){ - element->timestampLen = start - element->timestamp; - start++; + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || isspace(*(*start)) != 0))){ + element->timestampLen = (*start) - element->timestamp; break; } - start++; + (*start)++; } - }else if(start[0]== '"' && start[1] == 'v' && start[2] == 'a' && start[3] == 'l' - && start[4] == 'u' && start[5] == 'e' && start[6] == '"'){ + }else if((*start)[0]== '"' && (*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' + && (*start)[4] == 'u' && (*start)[5] == 'e' && (*start)[6] == '"'){ - start += 7; + (*start) += 7; bool hasColon = false; - while(start < end){ - if(!hasColon && *start == ':'){ - start++; - JUMP_JSON_SPACE(start,end) - element->cols = start; + while(*(*start)){ + if(unlikely(!hasColon && *(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + element->cols = (*start); hasColon = true; continue; } - if(hasColon && (*start == ',' || *start == '}' || isspace(*start) != 0)){ - element->colsLen = start - element->cols; - start++; + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || isspace(*(*start)) != 0))){ + element->colsLen = (*start) - element->cols; break; } - start++; + (*start)++; } - }else if(start[0] == '"' && start[1] == 't' && start[2] == 'a' && start[3] == 'g' - && start[4] == 's' && start[5] == '"'){ - start += 6; + }else if((*start)[0] == '"' && (*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' + && (*start)[4] == 's' && (*start)[5] == '"'){ + (*start) += 6; - while(start < end){ - if(*start == ':'){ - start++; - JUMP_JSON_SPACE(start,end) - element->tags = start; - element->tagsLen = smlJsonGetObj(start) - start; + while(*(*start)){ + if(unlikely(*(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + element->tags = (*start); + char* tmp = smlJsonGetObj((*start)); + if(tmp){ + element->tagsLen = tmp - (*start); + *start = tmp; + } break; } - start++; + (*start)++; } - }else{ - start++; } + if(*(*start) == '}'){ + (*start)++; + break; + } + (*start)++; } } -static int32_t smlParseJSONString(SSmlHandle *info, char *start, char *end, SSmlLineInfo *elements) { +static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) { int32_t ret = TSDB_CODE_SUCCESS; - smlJsonParseObj(start, end, elements); + smlJsonParseObj(start, elements); + if(**start == '\0') return TSDB_CODE_SUCCESS; + if(unlikely(elements->measure == NULL || elements->measureLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid measure data", start); + smlBuildInvalidDataMsg(&info->msgBuf, "invalid measure data", *start); return TSDB_CODE_SML_INVALID_DATA; } if(unlikely(elements->tags == NULL || elements->tagsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid tags data", start); + smlBuildInvalidDataMsg(&info->msgBuf, "invalid tags data", *start); return TSDB_CODE_SML_INVALID_DATA; } if(unlikely(elements->cols == NULL || elements->colsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid cols data", start); + smlBuildInvalidDataMsg(&info->msgBuf, "invalid cols data", *start); return TSDB_CODE_SML_INVALID_DATA; } if(unlikely(elements->timestamp == NULL || elements->timestampLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp data", start); + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp data", *start); return TSDB_CODE_SML_INVALID_DATA; } @@ -404,14 +410,10 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { int cnt = 0; char *dataPointStart = payload; - char *dataPointEnd = NULL; while (1) { - dataPointEnd = smlJsonGetObj(dataPointStart); - if(dataPointEnd == NULL) break; - if(info->dataFormat) { SSmlLineInfo element = {0}; - ret = smlParseJSONString(info, dataPointStart, dataPointEnd, &element); + ret = smlParseJSONString(info, &dataPointStart, &element); }else{ if(cnt >= payloadNum){ payloadNum = payloadNum << 1; @@ -420,13 +422,15 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { info->lines = (SSmlLineInfo*)tmp; } } - ret = smlParseJSONString(info, dataPointStart, dataPointEnd, info->lines + cnt); + ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); } if (unlikely(ret != TSDB_CODE_SUCCESS)) { uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); return ret; } + if(*dataPointStart == '\0') break; + if(unlikely(info->reRun)){ cnt = 0; dataPointStart = payload; @@ -438,7 +442,6 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { continue; } cnt++; - dataPointStart = dataPointEnd; } info->lineNum = cnt; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index a75c9752bb..dc05d7aa39 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -513,32 +513,32 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, } if(isSameMeasure){ -// if(cnt >= taosArrayGetSize(preLineKV)) { -// info->dataFormat = false; -// info->reRun = true; -// return TSDB_CODE_SUCCESS; -// } -// SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); -// if(kv.type != preKV->type){ -// info->dataFormat = false; -// info->reRun = true; -// return TSDB_CODE_SUCCESS; -// } -// -// if(unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > preKV->length)){ -// preKV->length = kv.length; -// SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); -// ASSERT(tableMeta != NULL); -// -// SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->cols, cnt); -// oldKV->length = kv.length; -// info->needModifySchema = true; -// } -// if(unlikely(!IS_SAME_KEY)){ -// info->dataFormat = false; -// info->reRun = true; -// return TSDB_CODE_SUCCESS; -// } + if(cnt >= taosArrayGetSize(preLineKV)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(kv.type != preKV->type){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->cols, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } }else{ if(isSuperKVInit){ if(unlikely(cnt >= taosArrayGetSize(superKV))) { diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index dbe7bed5a4..ecad51698c 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -497,20 +497,16 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { }; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { char *dataPointStart = (char *)sql[i]; - char *dataPointEnd = NULL; while (1) { - dataPointEnd = smlJsonGetObj(dataPointStart); - if (dataPointEnd == NULL) break; - SSmlLineInfo elements = {0}; - smlJsonParseObj(dataPointStart, dataPointEnd, &elements); + smlJsonParseObj(&dataPointStart, &elements); + if(*dataPointStart == '\0') break; SArray *tags = smlJsonParseTags(elements.tags, elements.tags + elements.tagsLen); size_t num = taosArrayGetSize(tags); ASSERT_EQ(num, 1); taosArrayDestroy(tags); - dataPointStart = dataPointEnd; } } smlDestroyInfo(info); diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 9ffb7efd63..49885c0aea 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -738,10 +738,10 @@ int sml_add_tag_col_Test() { int smlProcess_18784_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_18784 schemaless 1"); taos_free_result(pRes); - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use db_18784"); taos_free_result(pRes); const char *sql[] = { From 5ec92e7dc489aca72d56df36d941359d04faa831 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 15 Dec 2022 15:36:08 +0800 Subject: [PATCH 06/10] opti:parse json --- source/client/src/clientSmlJson.c | 57 +++++++++---------------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 6c073a370b..b36cad5ed9 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -21,7 +21,7 @@ #define JUMP_JSON_SPACE(start) \ while(*(start)){\ - if(unlikely(isspace(*(start)) == 0))\ + if(unlikely(*(start) > 32))\ break;\ else\ (start)++;\ @@ -34,13 +34,13 @@ static SArray *smlJsonParseTags(char *start, char *end){ kv.type = TSDB_DATA_TYPE_NCHAR; bool isInQuote = false; while(start < end){ - if(!isInQuote && *start == '"'){ + if(unlikely(!isInQuote && *start == '"')){ start++; kv.key = start; isInQuote = true; continue; } - if(isInQuote && *start == '"'){ + if(unlikely(isInQuote && *start == '"')){ kv.keyLen = start - kv.key; start++; break; @@ -49,18 +49,18 @@ static SArray *smlJsonParseTags(char *start, char *end){ } bool hasColon = false; while(start < end){ - if(!hasColon && *start == ':'){ + if(unlikely(!hasColon && *start == ':')){ start++; hasColon = true; continue; } - if(hasColon && kv.value == NULL && (isspace(*start) == 0 && *start != '"')){ + if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){ kv.value = start; start++; continue; } - if(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}')){ + if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){ kv.length = start - kv.value; taosArrayPush(tags, &kv); start++; @@ -233,9 +233,13 @@ static char* smlJsonGetObj(char *payload){ return NULL; } -static void smlJsonParseObj(char **start, SSmlLineInfo *element){ +static inline void smlJsonParseObj(char **start, SSmlLineInfo *element){ while(*(*start)){ - if((*start)[0]== '"' && (*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' + if((*start)[0] != '"'){ + (*start)++; + continue; + } + if((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' && (*start)[4] == 'r' && (*start)[5] == 'i' && (*start)[6] == 'c' && (*start)[7] == '"'){ (*start) += 8; @@ -253,7 +257,7 @@ static void smlJsonParseObj(char **start, SSmlLineInfo *element){ } (*start)++; } - }else if((*start)[0] == '"' && (*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' + }else if((*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' && (*start)[4] == 'e' && (*start)[5] == 's' && (*start)[6] == 't' && (*start)[7] == 'a' && (*start)[8] == 'm' && (*start)[9] == 'p' && (*start)[10] == '"'){ @@ -267,13 +271,13 @@ static void smlJsonParseObj(char **start, SSmlLineInfo *element){ hasColon = true; continue; } - if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || isspace(*(*start)) != 0))){ + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))){ element->timestampLen = (*start) - element->timestamp; break; } (*start)++; } - }else if((*start)[0]== '"' && (*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' + }else if((*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' && (*start)[4] == 'u' && (*start)[5] == 'e' && (*start)[6] == '"'){ (*start) += 7; @@ -287,13 +291,13 @@ static void smlJsonParseObj(char **start, SSmlLineInfo *element){ hasColon = true; continue; } - if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || isspace(*(*start)) != 0))){ + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))){ element->colsLen = (*start) - element->cols; break; } (*start)++; } - }else if((*start)[0] == '"' && (*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' + }else if((*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' && (*start)[4] == 's' && (*start)[5] == '"'){ (*start) += 6; @@ -326,23 +330,6 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * smlJsonParseObj(start, elements); if(**start == '\0') return TSDB_CODE_SUCCESS; - if(unlikely(elements->measure == NULL || elements->measureLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid measure data", *start); - return TSDB_CODE_SML_INVALID_DATA; - } - if(unlikely(elements->tags == NULL || elements->tagsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid tags data", *start); - return TSDB_CODE_SML_INVALID_DATA; - } - if(unlikely(elements->cols == NULL || elements->colsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid cols data", *start); - return TSDB_CODE_SML_INVALID_DATA; - } - if(unlikely(elements->timestamp == NULL || elements->timestampLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp data", *start); - return TSDB_CODE_SML_INVALID_DATA; - } - SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; if (smlParseNumber(&kv, &info->msgBuf)) { kv.length = (int16_t)tDataTypes[kv.type].bytes; @@ -394,16 +381,6 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_SUCCESS; } -//#define JUMP_TO_QUOTE(sql) \ -// while (sql){ \ -// if (unlikely(isspace(*(sql))) != 0) \ -// (sql)++; \ -// else \ -// break; \ -// } -// - - int32_t smlParseJSON(SSmlHandle *info, char *payload) { int32_t payloadNum = 1 << 15; int32_t ret = TSDB_CODE_SUCCESS; From 284acb1f3452a4cd7ce1d91b89d4fbb322f06b7c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 15 Dec 2022 17:17:18 +0800 Subject: [PATCH 07/10] opti:json parser --- source/client/inc/clientSml.h | 1 + source/client/src/clientSmlJson.c | 79 ++++++++++++++++++++++++++++++- source/client/test/smlTest.cpp | 7 ++- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 6a445432f0..e43d2a9177 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -178,6 +178,7 @@ typedef struct { SSmlMsgBuf msgBuf; // cJSON *root; // for parse json + int8_t offset[4]; SSmlLineInfo *lines; // element is SSmlLineInfo // diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index b36cad5ed9..1c2f62abf6 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -233,12 +233,19 @@ static char* smlJsonGetObj(char *payload){ return NULL; } -static inline void smlJsonParseObj(char **start, SSmlLineInfo *element){ +static inline void smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset){ + int index = 0; while(*(*start)){ if((*start)[0] != '"'){ (*start)++; continue; } + + if(unlikely(index >= 4)) { + uError("index >= 4, %s", *start) + break; + } + char *sTmp = *start; if((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' && (*start)[4] == 'r' && (*start)[5] == 'i' && (*start)[6] == 'c' && (*start)[7] == '"'){ @@ -247,6 +254,7 @@ static inline void smlJsonParseObj(char **start, SSmlLineInfo *element){ while(*(*start)){ if(unlikely(!isInQuote && *(*start) == '"')){ (*start)++; + offset[index++] = *start - sTmp; element->measure = (*start); isInQuote = true; continue; @@ -267,6 +275,7 @@ static inline void smlJsonParseObj(char **start, SSmlLineInfo *element){ if(unlikely(!hasColon && *(*start) == ':')){ (*start)++; JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; element->timestamp = (*start); hasColon = true; continue; @@ -287,6 +296,7 @@ static inline void smlJsonParseObj(char **start, SSmlLineInfo *element){ if(unlikely(!hasColon && *(*start) == ':')){ (*start)++; JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; element->cols = (*start); hasColon = true; continue; @@ -305,6 +315,7 @@ static inline void smlJsonParseObj(char **start, SSmlLineInfo *element){ if(unlikely(*(*start) == ':')){ (*start)++; JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; element->tags = (*start); char* tmp = smlJsonGetObj((*start)); if(tmp){ @@ -324,10 +335,74 @@ static inline void smlJsonParseObj(char **start, SSmlLineInfo *element){ } } +static inline void smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){ + int index = 0; + while(*(*start)){ + if((*start)[0] != '"'){ + (*start)++; + continue; + } + + if(unlikely(index >= 4)) { + uError("index >= 4, %s", *start) + break; + } + if((*start)[1] == 'm'){ + (*start) += offset[index++]; + element->measure = *start; + while(*(*start)){ + if(unlikely(*(*start) == '"')){ + element->measureLen = (*start) - element->measure; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'i'){ + (*start) += offset[index++]; + element->timestamp = *start; + while(*(*start)){ + if(unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){ + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + }else if((*start)[1] == 'v'){ + (*start) += offset[index++]; + element->cols = *start; + while(*(*start)){ + if(unlikely( *(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){ + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'a'){ + (*start) += offset[index++]; + element->tags = (*start); + char* tmp = smlJsonGetObj((*start)); + if(tmp){ + element->tagsLen = tmp - (*start); + *start = tmp; + } + break; + } + if(*(*start) == '}'){ + (*start)++; + break; + } + (*start)++; + } +} + static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) { int32_t ret = TSDB_CODE_SUCCESS; - smlJsonParseObj(start, elements); + if(info->offset[0] == 0){ + smlJsonParseObjFirst(start, elements, info->offset); + }else{ + smlJsonParseObj(start, elements, info->offset); + } if(**start == '\0') return TSDB_CODE_SUCCESS; SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index ecad51698c..91486aa96a 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -497,9 +497,14 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { }; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { char *dataPointStart = (char *)sql[i]; + int8_t offset[4] = {0}; while (1) { SSmlLineInfo elements = {0}; - smlJsonParseObj(&dataPointStart, &elements); + if(offset[0] == 0){ + smlJsonParseObjFirst(&dataPointStart, &elements, offset); + }else{ + smlJsonParseObj(&dataPointStart, &elements, offset); + } if(*dataPointStart == '\0') break; SArray *tags = smlJsonParseTags(elements.tags, elements.tags + elements.tagsLen); From ee88b16ad2c9e928b201e5c16a0f91328cceae15 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 19 Dec 2022 09:38:12 +0800 Subject: [PATCH 08/10] fix:enable multi type for telnet --- source/client/inc/clientSml.h | 1 + source/client/src/clientSmlLine.c | 2 +- source/client/src/clientSmlTelnet.c | 2 +- tests/system-test/2-query/sml.py | 13 +++++++------ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index e43d2a9177..8cae1fb1e8 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -219,6 +219,7 @@ STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t meas int32_t is_same_child_table_telnet(const void *a, const void *b); int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); int32_t smlClearForRerun(SSmlHandle *info); +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index dc05d7aa39..f56ad34fc5 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -120,7 +120,7 @@ static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t le return ts; } -static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { if (pVal->value[0] == '"'){ // binary if (pVal->length >= 2 && pVal->value[pVal->length - 1] == '"') { pVal->type = TSDB_DATA_TYPE_BINARY; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 7862fe54f6..e6574d030b 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -283,7 +283,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; - if (smlParseNumber(&kv, &info->msgBuf)) { + if (smlParseValue(&kv, &info->msgBuf)) { kv.length = (int16_t)tDataTypes[kv.type].bytes; }else{ return TSDB_CODE_TSC_INVALID_VALUE; diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index b764edebd7..60c29e8f22 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -76,13 +76,14 @@ class TDTestCase: tdSql.query(f"select * from {dbname}.`sys.cpu.nice` order by _ts") tdSql.checkRows(2) tdSql.checkData(0, 1, 9.000000000) - tdSql.checkData(0, 2, "lga") - tdSql.checkData(0, 3, "web02") - tdSql.checkData(0, 4, None) + tdSql.checkData(0, 2, "web02") + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4, "lga") + tdSql.checkData(1, 1, 18.000000000) - tdSql.checkData(1, 2, "lga") - tdSql.checkData(1, 3, "web01") - tdSql.checkData(1, 4, "t1") + tdSql.checkData(1, 2, "web01") + tdSql.checkData(1, 3, "t1") + tdSql.checkData(0, 4, "lga") tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) From 118252dff0eca397202c3c52a66fe7cb3805362d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 19 Dec 2022 09:53:51 +0800 Subject: [PATCH 09/10] fix:enable multi type for telnet --- source/client/src/clientSmlTelnet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index e6574d030b..84881df3f2 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -283,7 +283,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; - if (smlParseValue(&kv, &info->msgBuf)) { + if (smlParseValue(&kv, &info->msgBuf) == TSDB_CODE_SUCCESS) { kv.length = (int16_t)tDataTypes[kv.type].bytes; }else{ return TSDB_CODE_TSC_INVALID_VALUE; From cf8716f8896960eae4527aa6461970b15502d2a6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 19 Dec 2022 10:06:57 +0800 Subject: [PATCH 10/10] fix:enable multi type for telnet --- source/client/src/clientSmlTelnet.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 84881df3f2..53a7e3e81e 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -283,11 +283,10 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; - if (smlParseValue(&kv, &info->msgBuf) == TSDB_CODE_SUCCESS) { - kv.length = (int16_t)tDataTypes[kv.type].bytes; - }else{ + if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_VALUE; } + JUMP_SPACE(sql, sqlEnd) elements->tags = sql;