From 5b3087e48b5bc0627e7540c90252dbd6fb879fd1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Apr 2023 00:55:26 +0800 Subject: [PATCH 1/7] opti:escape logic in schemaless --- include/common/tcommon.h | 2 + source/client/inc/clientSml.h | 15 +++ source/client/src/clientSml.c | 25 +++-- source/client/src/clientSmlLine.c | 170 ++++++++++++++++++++---------- source/client/src/clientTmq.c | 6 +- 5 files changed, 151 insertions(+), 67 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a97c68be49..b83d91d85d 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -341,6 +341,8 @@ typedef struct { float f; }; size_t length; + bool keyEscaped; + bool valueEscaped; } SSmlKv; #define QUERY_ASC_FORWARD_STEP 1 diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 92896e6f23..bc342495be 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -107,6 +107,7 @@ typedef struct { int32_t colsLen; int32_t timestampLen; + bool measureEscaped; SArray *colArray; } SSmlLineInfo; @@ -206,6 +207,19 @@ typedef struct { #define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0) +#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + keyLen--; \ + } \ + } + extern int64_t smlFactorNS[3]; extern int64_t smlFactorS[3]; @@ -237,6 +251,7 @@ uint8_t smlGetTimestampLen(int64_t num); void clearColValArray(SArray* pCols); void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); +void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 17150286e1..855e48b88c 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -765,8 +765,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); + char* measure = taosMemoryMalloc(superTableLen); + memcpy(measure, superTable, superTableLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, superTable, superTableLen); + memcpy(pName.tname, measure, superTableLen); + taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1049,7 +1053,7 @@ void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { // } // taosMemoryFree(tag->key); taosArrayDestroy(tag->cols); - taosArrayDestroy(tag->tags); + taosArrayDestroyEx(tag->tags, freeSSmlKv); taosMemoryFree(tag); } @@ -1063,6 +1067,12 @@ void clearColValArray(SArray *pCols) { } } +void freeSSmlKv(void* data){ + SSmlKv *kv = (SSmlKv*)data; + if(kv->keyEscaped) taosMemoryFree((void*)(kv->key)); + if(kv->valueEscaped) taosMemoryFree((void*)(kv->value)); +} + void smlDestroyInfo(SSmlHandle *info) { if (!info) return; qDestroyQuery(info->pQuery); @@ -1098,11 +1108,11 @@ void smlDestroyInfo(SSmlHandle *info) { } taosArrayDestroy(info->valueJsonArray); - taosArrayDestroy(info->preLineTagKV); + taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { - taosArrayDestroy(info->lines[i].colArray); + taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); if (info->parseJsonByLib) { taosMemoryFree(info->lines[i].tags); } @@ -1420,14 +1430,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char char cTmp = 0; // for print tmp if is raw if (info->isRawLine) { - cTmp = tmp[len - 1]; - tmp[len - 1] = '\0'; + cTmp = tmp[len]; + tmp[len] = '\0'; } uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, info->isRawLine, numLines, info->protocol, len, tmp); if (info->isRawLine) { - tmp[len - 1] = cTmp; + tmp[len] = cTmp; } if (info->protocol == TSDB_SML_LINE_PROTOCOL) { @@ -1449,6 +1459,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { + tmp[len] = '\0'; uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 335e3a1dc7..f4f4a5b630 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -21,32 +21,33 @@ #include "clientSml.h" // comma , -// #define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) // space -// #define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) // equal = -// #define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) // quote " -// #define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) -#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) +//#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) // SLASH -// #define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) -#define IS_SLASH_LETTER(sql) \ - (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || \ - *(sql) == SLASH)) // (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || - // IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) +#define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) \ + (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH)) -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) +#define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL)) -#define PROCESS_SLASH(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER(key + i)) { \ +#define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_FIELD_VALUE(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + keyLen--; \ + } \ + } + +#define PROCESS_SLASH_IN_TAG_FIELD_KEY(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(key + i)) { \ MOVE_FORWARD_ONE(key + i, keyLen - i); \ - i--; \ keyLen--; \ } \ } @@ -151,7 +152,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + char* measure = currElement->measure; + int measureLen = currElement->measureLen; + if(currElement->measureEscaped){ + measure = taosMemoryMalloc(currElement->measureLen); + memcpy(measure, currElement->measure, currElement->measureLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + } + STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + if(currElement->measureEscaped){ + taosMemoryFree(measure); + } if (pTableMeta == NULL) { info->dataFormat = false; info->reRun = true; @@ -171,17 +182,18 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->maxTagKVs = (*tmp)->tags; } } - taosArrayClear(preLineKV); + taosArrayClearEx(preLineKV, freeSSmlKv); while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql))) { break; } - bool hasSlash = false; // parse key const char *key = *sql; size_t keyLen = 0; + bool keyEscaped = false; + size_t keyLenEscaped = 0; while (*sql < sqlEnd) { if (unlikely(IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); @@ -192,16 +204,14 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin (*sql)++; break; } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + keyLenEscaped++; + keyEscaped = true; } (*sql)++; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(key, keyLen) - } - if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -209,7 +219,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin // parse value const char *value = *sql; size_t valueLen = 0; - hasSlash = false; + bool valueEscaped = false; + size_t valueLenEscaped = 0; while (*sql < sqlEnd) { // parse value if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { @@ -219,8 +230,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SML_INVALID_DATA; } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(value)) { + valueLenEscaped++; + valueEscaped = true; } (*sql)++; @@ -232,15 +244,24 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SML_INVALID_DATA; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(value, valueLen) - } - - if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + if (keyEscaped){ + char *tmp = (char*)taosMemoryMalloc(keyLen); + memcpy(tmp, key, keyLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); + key = tmp; + } + if (valueEscaped){ + char *tmp = (char*)taosMemoryMalloc(valueLen); + memcpy(tmp, value, valueLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); + value = tmp; + } + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; + taosArrayPush(preLineKV, &kv); if (info->dataFormat) { if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { info->dataFormat = false; @@ -266,7 +287,6 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->needModifySchema = true; } } - taosArrayPush(preLineKV, &kv); cnt++; if (IS_SPACE(*sql)) { @@ -285,6 +305,11 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_OUT_OF_MEMORY; } tinfo->tags = taosArrayDup(preLineKV, NULL); + for(size_t i = 0; i < taosArrayGetSize(preLineKV); i++){ + SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i); + if(kv->keyEscaped)kv->key = NULL; + if(kv->valueEscaped)kv->value = NULL; + } smlSetCTableName(tinfo); tinfo->uid = info->uid++; @@ -321,7 +346,17 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + char* measure = currElement->measure; + int measureLen = currElement->measureLen; + if(currElement->measureEscaped){ + measure = taosMemoryMalloc(currElement->measureLen); + memcpy(measure, currElement->measure, currElement->measureLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + } + STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + if(currElement->measureEscaped){ + taosMemoryFree(measure); + } if (pTableMeta == NULL) { info->dataFormat = false; info->reRun = true; @@ -352,10 +387,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin break; } - bool hasSlash = false; // parse key const char *key = *sql; size_t keyLen = 0; + bool keyEscaped = false; + size_t keyLenEscaped = 0; while (*sql < sqlEnd) { if (unlikely(IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); @@ -366,16 +402,14 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin (*sql)++; break; } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + keyLenEscaped++; + keyEscaped = true; } (*sql)++; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(key, keyLen) - } - if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -383,11 +417,13 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin // parse value const char *value = *sql; size_t valueLen = 0; - hasSlash = false; - bool isInQuote = false; + bool valueEscaped = false; + size_t valueLenEscaped = 0; + bool isInQuote = false; + const char *escapeChar = NULL; while (*sql < sqlEnd) { // parse value - if (unlikely(IS_QUOTE(*sql))) { + if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) { isInQuote = !isInQuote; (*sql)++; continue; @@ -395,13 +431,12 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (!isInQuote) { if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { break; - } else if (unlikely(IS_EQUAL(*sql))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - return TSDB_CODE_SML_INVALID_DATA; } } - if (!hasSlash) { - hasSlash = (*(*sql) == SLASH); + if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) { + escapeChar = *sql; + valueEscaped = true; + valueLenEscaped++; } (*sql)++; @@ -416,14 +451,25 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); return TSDB_CODE_SML_INVALID_DATA; } - if (unlikely(hasSlash)) { - PROCESS_SLASH(value, valueLen) + + if (keyEscaped){ + char *tmp = (char*)taosMemoryMalloc(keyLen); + memcpy(tmp, key, keyLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); + key = tmp; + } + if (valueEscaped){ + char *tmp = (char*)taosMemoryMalloc(valueLen); + memcpy(tmp, value, valueLen); + PROCESS_SLASH_IN_FIELD_VALUE(tmp, valueLen); + value = tmp; } SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; int32_t ret = smlParseValue(&kv, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + freeSSmlKv(&kv); return ret; } @@ -432,6 +478,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } // bind data @@ -440,22 +487,26 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin uDebug("smlBuildCol error, retry"); info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } if (cnt >= taosArrayGetSize(info->masColKVs)) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt); if (kv.type != maxKV->type) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } if (unlikely(!IS_SAME_KEY)) { info->dataFormat = false; info->reRun = true; + freeSSmlKv(&kv); return TSDB_CODE_SUCCESS; } @@ -463,6 +514,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin maxKV->length = kv.length; info->needModifySchema = true; } + freeSSmlKv(&kv); } else { if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); @@ -487,10 +539,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->measure = sql; // parse measure + size_t measureLenEscaped = 0; while (sql < sqlEnd) { - if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) { - MOVE_FORWARD_ONE(sql, sqlEnd - sql); - sqlEnd--; + if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) { + elements->measureEscaped = true; + measureLenEscaped++; + sql++; continue; } if (unlikely(IS_COMMA(sql))) { @@ -503,7 +557,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine sql++; } elements->measureLen = sql - elements->measure; - if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -581,7 +635,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, + .keyEscaped = false, + .valueEscaped = false}; if (info->dataFormat) { uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts); ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ceca06e309..6642e716e6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1243,9 +1243,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pParam); if (code != 0) { - tscWarn("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, - vgId, epoch, tstrerror(code), requestId); - if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet); @@ -1267,6 +1264,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosWriteQitem(tmq->mqueue, pRspWrapper); } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert taosMsleep(500); + } else{ + tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, + vgId, epoch, tstrerror(code), requestId); } goto CREATE_MSG_FAIL; From b1561c95c33783be8478eb5b0fe1a8ddb0d5fe4e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 15:31:18 +0800 Subject: [PATCH 2/7] fix:add test cases for escape in schemaless --- source/client/src/clientSml.c | 2 + source/client/src/clientSmlLine.c | 2 +- source/client/src/clientTmq.c | 3 - source/client/test/smlTest.cpp | 36 ++++++++-- source/common/src/tglobal.c | 2 +- utils/test/c/sml_test.c | 107 ++++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 12 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 855e48b88c..c972575e59 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1175,6 +1175,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + terrno = 0; taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { return terrno; @@ -1250,6 +1251,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + terrno = 0; smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); if (terrno == TSDB_CODE_DUP_KEY) { return terrno; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index f4f4a5b630..da1bba19dc 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -465,7 +465,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin value = tmp; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; int32_t ret = smlParseValue(&kv, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 6642e716e6..16a4f55840 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -373,9 +373,6 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; char* topic = taosStrdup(src); - if (topic[0] != '`') { - strtolower(topic, src); - } if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 76911e229a..1578b8b607 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -50,8 +50,9 @@ TEST(testCase, smlParseInfluxString_Test) { int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); - ASSERT_EQ(elements.measureLen, strlen(",st")); - ASSERT_EQ(elements.measureTagsLen, strlen(",st,t1=3,t2=4,t3=t3")); + ASSERT_EQ(elements.measureLen, strlen("\\,st")); + ASSERT_EQ(elements.measureEscaped, true); + ASSERT_EQ(elements.measureTagsLen, strlen("\\,st,t1=3,t2=4,t3=t3")); ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3")); @@ -204,7 +205,26 @@ TEST(testCase, smlParseCols_Error_Test) { "st,t=1 c=-3.402823466e+39u64 1626006833639000000", "st,t=1 c=-339u64 1626006833639000000", "st,t=1 c=18446744073709551616u64 1626006833639000000", - "st,t=1 c=1=2 1626006833639000000"}; + "st,t=1 c=1=2 1626006833639000000,", + // escape error test + // measure comma,space + "s,t,t=1 c=1 1626006833639000000,", + "s t,t=1 c=1 1626006833639000000,", + //tag key comma,equal,space + "st,t,t=1 c=2 1626006833639000000,", + "st,t=t=1 c=2 1626006833639000000,", + "st,t t=1 c=2 1626006833639000000,", + //tag value comma,equal,space + "st,tt=a,a c=2 1626006833639000000,", + "st,t=t=a a c=2 1626006833639000000,", + "st,t t=a=a c=2 1626006833639000000,", + //field key comma,equal,space + "st,tt=aa c,1=2 1626006833639000000,", + "st,tt=aa c=1=2 1626006833639000000,", + "st,tt=aa c 1=2 1626006833639000000,", + //field value double quote,slash + "st,tt=aa c=\"a\"a\" 1626006833639000000,", + }; SSmlHandle *info = smlBuildSmlInfo(NULL); info->protocol = TSDB_SML_LINE_PROTOCOL; @@ -256,16 +276,18 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); - ASSERT_EQ(kv->length, 17); - ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0); + ASSERT_EQ(kv->length, 18); + ASSERT_EQ(kv->keyEscaped, true); + ASSERT_EQ(kv->valueEscaped, false); + ASSERT_EQ(strncasecmp(kv->value, "pass\\,it ", 9), 0); // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 8); - ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0); + ASSERT_EQ(kv->length, 9); + ASSERT_EQ(strncasecmp(kv->value, "ii\\=sd", 5), 0); // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index da4a912238..aa35b298e6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -198,7 +198,7 @@ int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; int32_t tsStreamCheckpointTickInterval = 1; int32_t tsTtlUnit = 86400; -int32_t tsTtlPushInterval = 86400; +int32_t tsTtlPushInterval = 3600; int32_t tsGrantHBInterval = 60; int32_t tsUptimeInterval = 300; // seconds char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 755ab55625..515da64081 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -895,6 +895,61 @@ int smlProcess_18784_Test() { return code; } +int sml_escape_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_escape"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use db_escape"); + taos_free_result(pRes); + + const char *sql[] = { + "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_used=176059i,total=1076048383523889174i 1661943960000000000", + "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_free=\"\\\"id,= ei\\\\\\f\" 1661943960000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); + int code = taos_errno(pRes); + ASSERT(!code); + ASSERT(taos_affected_rows(pRes) == 1); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from `d,i= s\\k\"`"); //check stable name + ASSERT(pRes); + int fieldNum = taos_field_count(pRes); + ASSERT(fieldNum == 5); + printf("fieldNum:%d\n", fieldNum); + + int numFields = taos_num_fields(pRes); + TAOS_FIELD *fields = taos_fetch_fields(pRes); + ASSERT(numFields == 5); + ASSERT(strcmp(fields[4].name, "dev\"i,= ce") == 0); + + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t *)row[0]; + int64_t used = *(int64_t *)row[1]; + int64_t total = *(int64_t *)row[2]; + int64_t freed = *(int64_t *)row[3]; + + if (rowIndex == 0) { + ASSERT(ts == 1661943960000); + ASSERT(used == 176059); + ASSERT(total == 1076048383523889174); + ASSERT(freed == 66932805); + ASSERT(strcmp(row[4], "s\"i,= dc") == 0); + + } + rowIndex++; + } + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_19221_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -961,6 +1016,55 @@ int sml_ts2164_Test() { return code; } + +int sml_ts3116_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = + taos_query(taos, "DROP DATABASE IF EXISTS ts3116"); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS ts3116 BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); + taos_free_result(pRes); + + char *sql = { + "meters,location=la,groupid=ca current=11.8,voltage=221", + }; + + pRes = taos_query(taos, "use ts3116"); + taos_free_result(pRes); + int32_t totalRows = 0; + char *tmp = (char *)taosMemoryCalloc(1024, 1); + memcpy(tmp, sql, strlen(sql)); + totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + taosMemoryFree(tmp); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + + char *sql1 = { + "meters,location=la,groupid=ca\\=3 current=11.8,voltage=221\nmeters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + }; + + pRes = taos_query(taos, "use ts3116"); + taos_free_result(pRes); + + tmp = (char *)taosMemoryCalloc(1024, 1); + memcpy(tmp, sql1, strlen(sql1)); + totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + taosMemoryFree(tmp); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_td22898_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1195,6 +1299,9 @@ int main(int argc, char *argv[]) { } int ret = 0; + ret = sml_escape_Test(); + ret = sml_ts3116_Test(); + ASSERT(!ret); ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file ASSERT(!ret); // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ From 8e8a3b268b7509f42a3d065899ceb7448a9301c8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 15:54:19 +0800 Subject: [PATCH 3/7] fix:add test cases for escape in schemaless --- source/client/src/clientSmlLine.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index da1bba19dc..6260bcb234 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -452,25 +452,27 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SML_INVALID_DATA; } + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + int32_t ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + return ret; + } + if (keyEscaped){ char *tmp = (char*)taosMemoryMalloc(keyLen); memcpy(tmp, key, keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); - key = tmp; - } - if (valueEscaped){ - char *tmp = (char*)taosMemoryMalloc(valueLen); - memcpy(tmp, value, valueLen); - PROCESS_SLASH_IN_FIELD_VALUE(tmp, valueLen); - value = tmp; + kv.key = tmp; + kv.keyEscaped = keyEscaped; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; - int32_t ret = smlParseValue(&kv, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); - freeSSmlKv(&kv); - return ret; + if (valueEscaped){ + char *tmp = (char*)taosMemoryMalloc(kv.length); + memcpy(tmp, kv.value, kv.length); + PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); + kv.value = tmp; + kv.valueEscaped = valueEscaped; } if (info->dataFormat) { From 2714da26e7873d3209832fb78adae8c29a80b7f7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 17:26:54 +0800 Subject: [PATCH 4/7] fix:[TS-3221] reset max stmt if execute error --- source/client/src/clientSml.c | 8 +++++++- source/client/src/clientSmlLine.c | 12 ++++++------ utils/test/c/sml_test.c | 9 ++++++--- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c972575e59..065f2e1fdb 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1316,9 +1316,15 @@ static int32_t smlInsertData(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, tableData->uid, info->dataFormat); + int measureLen = tableData->sTableNameLen; + char* measure = (char*)taosMemoryMalloc(tableData->sTableNameLen); + memcpy(measure, tableData->sTableName, tableData->sTableNameLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, - (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, + (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf, info->msgBuf.len); + taosMemoryFree(measure); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 6260bcb234..1732473c11 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -155,7 +155,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin char* measure = currElement->measure; int measureLen = currElement->measureLen; if(currElement->measureEscaped){ - measure = taosMemoryMalloc(currElement->measureLen); + measure = (char*)taosMemoryMalloc(currElement->measureLen); memcpy(measure, currElement->measure, currElement->measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); } @@ -230,7 +230,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SML_INVALID_DATA; } - if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(value)) { + if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { valueLenEscaped++; valueEscaped = true; } @@ -349,7 +349,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin char* measure = currElement->measure; int measureLen = currElement->measureLen; if(currElement->measureEscaped){ - measure = taosMemoryMalloc(currElement->measureLen); + measure = (char*)taosMemoryMalloc(currElement->measureLen); memcpy(measure, currElement->measure, currElement->measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); } @@ -460,9 +460,9 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } if (keyEscaped){ - char *tmp = (char*)taosMemoryMalloc(keyLen); - memcpy(tmp, key, keyLen); - PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); + char *tmp = (char*)taosMemoryMalloc(kv.keyLen); + memcpy(tmp, key, kv.keyLen); + PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); kv.key = tmp; kv.keyEscaped = keyEscaped; } diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 515da64081..54fa594e39 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -906,7 +906,7 @@ int sml_escape_Test() { const char *sql[] = { "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_used=176059i,total=1076048383523889174i 1661943960000000000", - "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_free=\"\\\"id,= ei\\\\\\f\" 1661943960000000000", + "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_f\\\\ree=\"\\\"id,= ei\\\\\\f\" 1661943960000000000", }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); @@ -924,6 +924,9 @@ int sml_escape_Test() { int numFields = taos_num_fields(pRes); TAOS_FIELD *fields = taos_fetch_fields(pRes); ASSERT(numFields == 5); + ASSERT(strcmp(fields[1].name, "inode\"i,= s_used") == 0); + ASSERT(strcmp(fields[2].name, "total") == 0); + ASSERT(strcmp(fields[3].name, "inode\"i,= s_f\\\\ree") == 0); ASSERT(strcmp(fields[4].name, "dev\"i,= ce") == 0); TAOS_ROW row = NULL; @@ -932,13 +935,12 @@ int sml_escape_Test() { int64_t ts = *(int64_t *)row[0]; int64_t used = *(int64_t *)row[1]; int64_t total = *(int64_t *)row[2]; - int64_t freed = *(int64_t *)row[3]; if (rowIndex == 0) { ASSERT(ts == 1661943960000); ASSERT(used == 176059); ASSERT(total == 1076048383523889174); - ASSERT(freed == 66932805); + ASSERT(strcmp(row[3], "\"id,= ei\\\\f") == 0); ASSERT(strcmp(row[4], "s\"i,= dc") == 0); } @@ -1300,6 +1302,7 @@ int main(int argc, char *argv[]) { int ret = 0; ret = sml_escape_Test(); + ASSERT(!ret); ret = sml_ts3116_Test(); ASSERT(!ret); ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file From ea939c1efb5a769176361e46d0580c114951c2a1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 17:41:38 +0800 Subject: [PATCH 5/7] fix:add test cases for escape in schemaless --- utils/test/c/sml_test.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 54fa594e39..21964403ea 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -924,10 +924,10 @@ int sml_escape_Test() { int numFields = taos_num_fields(pRes); TAOS_FIELD *fields = taos_fetch_fields(pRes); ASSERT(numFields == 5); - ASSERT(strcmp(fields[1].name, "inode\"i,= s_used") == 0); - ASSERT(strcmp(fields[2].name, "total") == 0); - ASSERT(strcmp(fields[3].name, "inode\"i,= s_f\\\\ree") == 0); - ASSERT(strcmp(fields[4].name, "dev\"i,= ce") == 0); + ASSERT(strncmp(fields[1].name, "inode\"i,= s_used", sizeof("inode\"i,= s_used") - 1) == 0); + ASSERT(strncmp(fields[2].name, "total", sizeof("total") - 1) == 0); + ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\\\ree", sizeof("inode\"i,= s_f\\\\ree") - 1) == 0); + ASSERT(strncmp(fields[4].name, "dev\"i,= ce", sizeof("dev\"i,= ce") - 1) == 0); TAOS_ROW row = NULL; int32_t rowIndex = 0; @@ -940,8 +940,8 @@ int sml_escape_Test() { ASSERT(ts == 1661943960000); ASSERT(used == 176059); ASSERT(total == 1076048383523889174); - ASSERT(strcmp(row[3], "\"id,= ei\\\\f") == 0); - ASSERT(strcmp(row[4], "s\"i,= dc") == 0); + ASSERT(strncmp(row[3], "\"id,= ei\\\\f", sizeof("\"id,= ei\\\\f") - 1) == 0); + ASSERT(strncmp(row[4], "s\"i,= dc", sizeof("s\"i,= dc") - 1) == 0); } rowIndex++; From e61b51b083c67fa47c46c515d3cc86ad3df88f20 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 17:49:40 +0800 Subject: [PATCH 6/7] fix:add test cases for escape in schemaless --- source/client/src/clientSml.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 065f2e1fdb..38c346d280 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1178,6 +1178,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { terrno = 0; taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { + taosHashCleanup(kvHash); return terrno; } } @@ -1251,13 +1252,13 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); terrno = 0; smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); if (terrno == TSDB_CODE_DUP_KEY) { return terrno; } smlInsertMeta(meta->colHash, meta->cols, elements->colArray); - taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); } } uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); From d288e44f921fb5b641391ad41a8f01c898a45234 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Apr 2023 17:18:40 +0800 Subject: [PATCH 7/7] fix:ci error --- source/client/src/clientTmq.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 16a4f55840..6642e716e6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -373,6 +373,9 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; char* topic = taosStrdup(src); + if (topic[0] != '`') { + strtolower(topic, src); + } if (taosArrayPush(container, &topic) == NULL) return -1; return 0; }