From 041b84c13169bc9e0cac81c21f1199ad61a1fb11 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 10 Apr 2024 10:23:48 +0800 Subject: [PATCH 1/4] enh: tag value for nchar --- source/libs/parser/src/parInsertSql.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index afdd6089d2..731480d0e4 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -664,11 +664,13 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, case TSDB_DATA_TYPE_NCHAR: { int32_t output = 0; - void* p = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE); + int64_t realLen = pToken->n << 2; + if (realLen > pSchema->bytes - VARSTR_HEADER_SIZE) realLen = pSchema->bytes - VARSTR_HEADER_SIZE; + void* p = taosMemoryMalloc(realLen); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output)) { if (errno == E2BIG) { taosMemoryFree(p); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); From e495f347c8781b050f969a4a91aa7e09a882e3b4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 18 Apr 2024 17:17:04 +0800 Subject: [PATCH 2/4] enh: acquire conv in batch mode when convert ucs4 to mbs --- include/os/osString.h | 5 +++++ source/client/src/clientImpl.c | 10 ++++++++-- source/os/src/osString.c | 18 ++++++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/include/os/osString.h b/include/os/osString.h index 8e8f1546e7..1aca119293 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -22,6 +22,8 @@ extern "C" { typedef wchar_t TdWchar; typedef int32_t TdUcs4; +typedef void *iconv_t; +typedef enum { M2C = 0, C2M } ConvType; // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. @@ -59,7 +61,10 @@ int64_t taosStr2int64(const char *str); int32_t taosConvInit(void); void taosConvDestroy(); +iconv_t taosAcquireConv(int32_t *idx, ConvType type); +void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type); int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs); +int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len); int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); TdUcs4 *tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 1c4b49df53..51e847d40c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1804,6 +1804,10 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { } static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { + int32_t idx = -1; + iconv_t conv = taosAcquireConv(&idx, C2M); + if (!conv) return TSDB_CODE_TSC_INTERNAL_ERROR; + for (int32_t i = 0; i < numOfCols; ++i) { int32_t type = pResultInfo->fields[i].type; int32_t bytes = pResultInfo->fields[i].bytes; @@ -1811,6 +1815,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) { char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); if (p == NULL) { + taosReleaseConv(idx, conv, C2M); return TSDB_CODE_OUT_OF_MEMORY; } @@ -1821,12 +1826,13 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int if (pCol->offset[j] != -1) { char* pStart = pCol->offset[j] + pCol->pData; - int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p)); + int32_t len = taosUcs4ToMbsEx((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p), conv); if (len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) { tscError( "doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + " "colLength[i]):%p", len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); + taosReleaseConv(idx, conv, C2M); return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -1840,7 +1846,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int pResultInfo->row[i] = pResultInfo->pCol[i].pData; } } - + taosReleaseConv(idx, conv, C2M); return TSDB_CODE_SUCCESS; } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 8c6c0e0d7c..768bbe5cd1 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -129,8 +129,6 @@ typedef struct { int8_t inUse; } SConv; -typedef enum { M2C = 0, C2M } ConvType; - // 0: Mbs --> Ucs4 // 1: Ucs4--> Mbs SConv *gConv[2] = {NULL, NULL}; @@ -269,6 +267,22 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { return (int32_t)(ucs4_max_len - outLen); #endif } + +int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv) { +#ifdef DISALLOW_NCHAR_WITHOUT_ICONV + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); + return -1; +#else + + size_t ucs4_input_len = ucs4_max_len; + size_t outLen = ucs4_max_len; + if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { + return -1; + } + return (int32_t)(ucs4_max_len - outLen); +#endif +} + bool taosValidateEncodec(const char *encodec) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); From 266ac862e32764040fa951d4797de1a8691eec9f Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 18 Apr 2024 17:37:02 +0800 Subject: [PATCH 3/4] enh: error process optimization --- source/client/src/clientImpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 51e847d40c..790bb85fc3 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1827,7 +1827,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int char* pStart = pCol->offset[j] + pCol->pData; int32_t len = taosUcs4ToMbsEx((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p), conv); - if (len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) { + if (len < 0 || len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) { tscError( "doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + " "colLength[i]):%p", @@ -2626,7 +2626,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) { setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; - pRequest->code = code; + code = pRequest->code; tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { From 0d9d0c65f8afd6c0106f06f5145c526f30770f3c Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 18 Apr 2024 18:44:40 +0800 Subject: [PATCH 4/4] enh: error process optimization --- source/client/src/clientImpl.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 790bb85fc3..1d5bc9cfb4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2626,8 +2626,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) { setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; - code = pRequest->code; - tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), + tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code), pRequest->requestId); } else { tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,