diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 995015fe63..63f305ab2d 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -31,7 +31,7 @@ typedef struct SVariant { uint64_t u; double d; char *pz; - wchar_t *wpz; + TdUcs4 *ucs4; SArray *arr; // only for 'in' query to hold value list, not value for a field }; } SVariant; diff --git a/include/os/osString.h b/include/os/osString.h index 80e925e18d..9c6d523ab2 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -20,15 +20,24 @@ extern "C" { #endif +typedef wchar_t TdWchar; +typedef int32_t TdUcs4; + // If the error is in a third-party library, place this header file under the third-party library header file. #ifndef ALLOW_FORBID_FUNC #define iconv_open ICONV_OPEN_FUNC_TAOS_FORBID #define iconv_close ICONV_CLOSE_FUNC_TAOS_FORBID #define iconv ICONV_FUNC_TAOS_FORBID + #define wcwidth WCWIDTH_FUNC_TAOS_FORBID + #define wcswidth WCSWIDTH_FUNC_TAOS_FORBID + #define mbtowc MBTOWC_FUNC_TAOS_FORBID + #define mbstowcs MBSTOWCS_FUNC_TAOS_FORBID + #define wctomb WCTOMB_FUNC_TAOS_FORBID + #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID + #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID + #define wchar_t WCHAR_T_FUNC_TAOS_FORBID #endif -typedef int32_t TdUcs4; - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #define tstrdup(str) _strdup(str) #else @@ -47,8 +56,16 @@ int64_t taosStr2int64(const char *str); int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len); int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); +TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); bool taosValidateEncodec(const char *encodec); +int32_t taosWcharWidth(TdWchar wchar); +int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size); +int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size); +int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size); +int32_t taosWcharToMb(char *pStr, TdWchar wchar); +int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size); + #ifdef __cplusplus } #endif diff --git a/include/util/tcompare.h b/include/util/tcompare.h index 4c80eeb4f6..cc9e8ae464 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -46,7 +46,7 @@ typedef struct SPatternCompareInfo { int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo); -int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo); +int32_t WCSPatternMatch(const TdUcs4 *pattern, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo); int32_t taosArrayCompareString(const void *a, const void *b); diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 703f4402ef..c6aa1cb81d 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -199,8 +199,8 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; - pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); - memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); + pVar->ucs4 = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); + memcpy(pVar->ucs4, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; break; @@ -343,7 +343,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) { case TSDB_DATA_TYPE_NCHAR: { dst[0] = '\''; - taosUcs4ToMbs(pVar->wpz, (taosUcs4len(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); + taosUcs4ToMbs(pVar->ucs4, (taosUcs4len(pVar->ucs4) + 1) * TSDB_NCHAR_SIZE, dst + 1); int32_t len = (int32_t)strlen(dst); dst[len] = '\''; dst[len + 1] = 0; @@ -384,7 +384,7 @@ static FORCE_INLINE int32_t convertToBoolImpl(char *pStr, int32_t len) { } } -static FORCE_INLINE int32_t wcsconvertToBoolImpl(wchar_t *pstr, int32_t len) { +static FORCE_INLINE int32_t wcsconvertToBoolImpl(TdUcs4 *pstr, int32_t len) { if ((wcsncasecmp(pstr, L"true", len) == 0) && (len == 4)) { return TSDB_TRUE; } else if (wcsncasecmp(pstr, L"false", len) == 0 && (len == 5)) { @@ -412,11 +412,11 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { pBuf = realloc(pBuf, newSize + 1); } - taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf); - free(pVariant->wpz); + taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, pBuf); + free(pVariant->ucs4); pBuf[newSize] = 0; } else { - taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest); + taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, *pDest); } } else { @@ -469,17 +469,17 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { // free the binary buffer in the first place if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { - free(pVariant->wpz); + free(pVariant->ucs4); } - pVariant->wpz = pWStr; - *pDestSize = taosUcs4len(pVariant->wpz); + pVariant->ucs4 = pWStr; + *pDestSize = taosUcs4len(pVariant->ucs4); // shrink the allocate memory, no need to check here. - char *tmp = realloc(pVariant->wpz, (*pDestSize + 1) * TSDB_NCHAR_SIZE); + char *tmp = realloc(pVariant->ucs4, (*pDestSize + 1) * TSDB_NCHAR_SIZE); assert(tmp != NULL); - pVariant->wpz = (wchar_t *)tmp; + pVariant->ucs4 = (TdUcs4 *)tmp; } else { int32_t output = 0; @@ -554,7 +554,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result *result = res; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { errno = 0; - wchar_t *endPtr = NULL; + TdUcs4 *endPtr = NULL; SToken token = {0}; token.n = tGetToken(pVariant->pz, &token.type); @@ -564,7 +564,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result } if (token.type == TK_FLOAT) { - double v = wcstod(pVariant->wpz, &endPtr); + double v = wcstod(pVariant->ucs4, &endPtr); if (releaseVariantPtr) { free(pVariant->pz); pVariant->nLen = 0; @@ -583,7 +583,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result setNull((char *)result, type, tDataTypes[type].bytes); return 0; } else { - int64_t val = wcstoll(pVariant->wpz, &endPtr, 10); + int64_t val = wcstoll(pVariant->ucs4, &endPtr, 10); if (releaseVariantPtr) { free(pVariant->pz); pVariant->nLen = 0; @@ -649,7 +649,7 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) { *pDest = ret; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { int32_t ret = 0; - if ((ret = wcsconvertToBoolImpl(pVariant->wpz, pVariant->nLen)) < 0) { + if ((ret = wcsconvertToBoolImpl(pVariant->ucs4, pVariant->nLen)) < 0) { return ret; } *pDest = ret; @@ -899,7 +899,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc return -1; } } else { - wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen); + tasoUcs4Copy((TdUcs4*)payload, pVariant->ucs4, pVariant->nLen); } } } else { @@ -913,7 +913,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc return -1; } } else { - memcpy(p, pVariant->wpz, pVariant->nLen); + memcpy(p, pVariant->ucs4, pVariant->nLen); newlen = pVariant->nLen; } @@ -979,7 +979,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { pVariant->d = v; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { errno = 0; - double v = wcstod(pVariant->wpz, NULL); + double v = wcstod(pVariant->ucs4, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { free(pVariant->pz); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 97c52f44eb..6b2857c411 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3386,7 +3386,7 @@ void filterPrepare(void* expr, void* param) { if (size < (uint32_t)pSchema->bytes) { size = pSchema->bytes; } - // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space. + // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(TdUcs4) space. pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); tVariantDump(pCond, pInfo->q, pSchema->type, true); } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 65d3f51dde..1f333e49e7 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -13,12 +13,11 @@ * along with this program. If not, see . */ -#include "mockCatalogService.h" - #include #include #include #include "tdatablock.h" +#include "mockCatalogService.h" #include "tname.h" #include "ttypes.h" diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 18f6ceb5a0..07108ce34f 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -24,6 +24,9 @@ #include "iconv.h" #endif +extern int wcwidth(wchar_t c); +extern int wcswidth(const wchar_t *s, size_t n); + int64_t taosStr2int64(const char *str) { char *endptr = NULL; return strtoll(str, &endptr, 10); @@ -72,6 +75,11 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { #endif } + +TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) { + memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4)); +} + int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV return -1; @@ -145,6 +153,18 @@ int32_t taosUcs4len(TdUcs4 *ucs4) { return n; } +int32_t taosWcharWidth(TdWchar wchar) { return wcwidth(wchar); } + +int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size) { return wcswidth(pWchar, size); } + +int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size) { return mbtowc(pWchar, pStr, size); } + +int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { return mbstowcs(pWchars, pStrs, size); } + +int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); } + +int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); } + // #ifdef USE_LIBICONV // #include "iconv.h" diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 5e3cf1247c..ff7d2cf733 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -208,7 +208,7 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { if (len1 != len2) { return len1 > len2 ? 1 : -1; } else { - int32_t ret = memcmp((wchar_t *)pLeft, (wchar_t *)pRight, len1); + int32_t ret = memcmp((TdUcs4 *)pLeft, (TdUcs4 *)pRight, len1); if (ret == 0) { return 0; } else { @@ -295,10 +295,10 @@ int32_t patternMatch(const char *patterStr, const char *str, size_t size, const return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { - wchar_t c, c1; - wchar_t matchOne = L'_'; // "_" - wchar_t matchAll = L'%'; // "%" +int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo) { + TdUcs4 c, c1; + TdUcs4 matchOne = L'_'; // "_" + TdUcs4 matchAll = L'%'; // "%" int32_t i = 0; int32_t j = 0; @@ -315,7 +315,7 @@ int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t siz return TSDB_PATTERN_MATCH; } - wchar_t accept[3] = {towupper(c), towlower(c), 0}; + TdUcs4 accept[3] = {towupper(c), towlower(c), 0}; while (1) { size_t n = wcscspn(str, accept); @@ -424,10 +424,10 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); - wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); + char *pattern = calloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - int32_t ret = WCSPatternMatch(pattern, (const wchar_t *)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); + int32_t ret = WCSPatternMatch((TdUcs4*)pattern, (TdUcs4*)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); free(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; @@ -647,7 +647,7 @@ int32_t doCompare(const char *f1, const char *f2, int32_t type, size_t size) { if (t1->len != t2->len) { return t1->len > t2->len ? 1 : -1; } - int32_t ret = memcmp((wchar_t *)t1, (wchar_t *)t2, t2->len); + int32_t ret = memcmp((TdUcs4 *)t1, (TdUcs4 *)t2, t2->len); if (ret == 0) { return ret; } diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c index e4cf09358b..d7a976d52c 100644 --- a/tools/shell/src/backup/shellDarwin.c +++ b/tools/shell/src/backup/shellDarwin.c @@ -28,7 +28,6 @@ int indicator = 1; struct termios oldtio; -extern int wcwidth(wchar_t c); void insertChar(Command *cmd, char *c, int size); @@ -426,7 +425,7 @@ void showOnScreen(Command *cmd) { w.ws_row = 30; } - wchar_t wc; + TdWchar wc; int size = 0; // Print out the command. @@ -441,11 +440,11 @@ void showOnScreen(Command *cmd) { int remain_column = w.ws_col; /* size = cmd->commandSize + prompt_size; */ for (char *str = total_string; size < cmd->commandSize + prompt_size;) { - int ret = mbtowc(&wc, str, MB_CUR_MAX); + int ret = taosMbToWchar(&wc, str, MB_CUR_MAX); if (ret < 0) break; size += ret; /* assert(size >= 0); */ - int width = wcwidth(wc); + int width = taosWcharWidth(wc); if (remain_column > width) { printf("%lc", wc); remain_column -= width; diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index cf0ceded38..fd993998b8 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -21,8 +21,6 @@ #include -extern int wcwidth(wchar_t c); -extern int wcswidth(const wchar_t *s, size_t n); typedef struct { char widthInString; char widthOnScreen; @@ -43,7 +41,7 @@ int countPrefixOnes(unsigned char c) { void getPrevCharSize(const char *str, int pos, int *size, int *width) { assert(pos > 0); - wchar_t wc; + TdWchar wc; *size = 0; *width = 0; @@ -53,25 +51,25 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) { if (str[pos] > 0 || countPrefixOnes((unsigned char )str[pos]) > 1) break; } - int rc = mbtowc(&wc, str + pos, MB_CUR_MAX); + int rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); assert(rc == *size); - *width = wcwidth(wc); + *width = taosWcharWidth(wc); } void getNextCharSize(const char *str, int pos, int *size, int *width) { assert(pos >= 0); - wchar_t wc; - *size = mbtowc(&wc, str + pos, MB_CUR_MAX); - *width = wcwidth(wc); + TdWchar wc; + *size = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); + *width = taosWcharWidth(wc); } void insertChar(Command *cmd, char *c, int size) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - wchar_t wc; - if (mbtowc(&wc, c, size) < 0) return; + TdWchar wc; + if (taosMbToWchar(&wc, c, size) < 0) return; clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); /* update the buffer */ @@ -81,8 +79,8 @@ void insertChar(Command *cmd, char *c, int size) { /* update the values */ cmd->commandSize += size; cmd->cursorOffset += size; - cmd->screenOffset += wcwidth(wc); - cmd->endOffset += wcwidth(wc); + cmd->screenOffset += taosWcharWidth(wc); + cmd->endOffset += taosWcharWidth(wc); showOnScreen(cmd); } @@ -249,10 +247,10 @@ int isReadyGo(Command *cmd) { } void getMbSizeInfo(const char *str, int *size, int *width) { - wchar_t *wc = (wchar_t *)calloc(sizeof(wchar_t), MAX_COMMAND_SIZE); + TdWchar *wc = (TdWchar *)calloc(sizeof(TdWchar), MAX_COMMAND_SIZE); *size = strlen(str); - mbstowcs(wc, str, MAX_COMMAND_SIZE); - *width = wcswidth(wc, MAX_COMMAND_SIZE); + taosMbsToWchars(wc, str, MAX_COMMAND_SIZE); + *width = taosWcharsWidth(wc, MAX_COMMAND_SIZE); free(wc); } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 5a989937d8..1b35afb57d 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -560,12 +560,12 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { } static void shellPrintNChar(const char *str, int length, int width) { - wchar_t tail[3]; + TdWchar tail[3]; int pos = 0, cols = 0, totalCols = 0, tailLen = 0; while (pos < length) { - wchar_t wc; - int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX); + TdWchar wc; + int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); if (bytes == 0) { break; } @@ -577,7 +577,7 @@ static void shellPrintNChar(const char *str, int length, int width) { #ifdef WINDOWS int w = bytes; #else - int w = wcwidth(wc); + int w = taosWcharWidth(wc); #endif if (w <= 0) { continue; diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index cc497688d1..6da05f28df 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -31,7 +31,6 @@ int indicator = 1; struct termios oldtio; -extern int wcwidth(wchar_t c); void insertChar(Command *cmd, char *c, int size); const char *argp_program_version = version; const char *argp_program_bug_address = ""; @@ -456,7 +455,7 @@ void showOnScreen(Command *cmd) { w.ws_row = 30; } - wchar_t wc; + TdWchar wc; int size = 0; // Print out the command. @@ -471,11 +470,11 @@ void showOnScreen(Command *cmd) { int remain_column = w.ws_col; /* size = cmd->commandSize + prompt_size; */ for (char *str = total_string; size < cmd->commandSize + prompt_size;) { - int ret = mbtowc(&wc, str, MB_CUR_MAX); + int ret = taosMbToWchar(&wc, str, MB_CUR_MAX); if (ret < 0) break; size += ret; /* assert(size >= 0); */ - int width = wcwidth(wc); + int width = taosWcharWidth(wc); if (remain_column > width) { printf("%lc", wc); remain_column -= width;