[TD-13767]<fix>: forbid wchar_t.

include/common/tvariant.h

@@ -31,7 +31,7 @@ typedef struct SVariant {
     uint64_t u;
     double   d;
     char    *pz;
-    wchar_t *wpz;
+    TdUcs4  *ucs4;
     SArray  *arr;  // only for 'in' query to hold value list, not value for a field
   };
 } SVariant;

include/os/osString.h

@@ -20,15 +20,24 @@
 extern "C" {
 #endif
 
+typedef wchar_t TdWchar;
+typedef int32_t TdUcs4;
+
 // If the error is in a third-party library, place this header file under the third-party library header file.
 #ifndef ALLOW_FORBID_FUNC
     #define iconv_open ICONV_OPEN_FUNC_TAOS_FORBID
     #define iconv_close ICONV_CLOSE_FUNC_TAOS_FORBID
     #define iconv ICONV_FUNC_TAOS_FORBID
+    #define wcwidth WCWIDTH_FUNC_TAOS_FORBID
+    #define wcswidth WCSWIDTH_FUNC_TAOS_FORBID
+    #define mbtowc MBTOWC_FUNC_TAOS_FORBID
+    #define mbstowcs MBSTOWCS_FUNC_TAOS_FORBID
+    #define wctomb WCTOMB_FUNC_TAOS_FORBID
+    #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID
+    #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID
+    #define wchar_t WCHAR_T_FUNC_TAOS_FORBID
 #endif
 
-typedef int32_t TdUcs4;
-
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   #define tstrdup(str) _strdup(str)
 #else
@@ -47,8 +56,16 @@ int64_t taosStr2int64(const char *str);
 int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs);
 bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len);
 int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
+TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
 bool    taosValidateEncodec(const char *encodec);
 
+int32_t taosWcharWidth(TdWchar wchar);
+int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size);
+int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size);
+int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size);
+int32_t taosWcharToMb(char *pStr, TdWchar wchar);
+int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tcompare.h

@@ -46,7 +46,7 @@ typedef struct SPatternCompareInfo {
 
 int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo);
 
-int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
+int32_t WCSPatternMatch(const TdUcs4 *pattern, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo);
 
 int32_t taosArrayCompareString(const void *a, const void *b);
 

source/common/src/tvariant.c

@@ -199,8 +199,8 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin
     case TSDB_DATA_TYPE_NCHAR: {  // here we get the nchar length from raw binary bits length
       size_t lenInwchar = len / TSDB_NCHAR_SIZE;
 
-      pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE);
+      pVar->ucs4 = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE);
-      memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE);
+      memcpy(pVar->ucs4, pz, lenInwchar * TSDB_NCHAR_SIZE);
       pVar->nLen = (int32_t)len;
 
       break;
@@ -343,7 +343,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) {
 
     case TSDB_DATA_TYPE_NCHAR: {
       dst[0] = '\'';
-      taosUcs4ToMbs(pVar->wpz, (taosUcs4len(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1);
+      taosUcs4ToMbs(pVar->ucs4, (taosUcs4len(pVar->ucs4) + 1) * TSDB_NCHAR_SIZE, dst + 1);
       int32_t len = (int32_t)strlen(dst);
       dst[len] = '\'';
       dst[len + 1] = 0;
@@ -384,7 +384,7 @@ static FORCE_INLINE int32_t convertToBoolImpl(char *pStr, int32_t len) {
   }
 }
 
-static FORCE_INLINE int32_t wcsconvertToBoolImpl(wchar_t *pstr, int32_t len) {
+static FORCE_INLINE int32_t wcsconvertToBoolImpl(TdUcs4 *pstr, int32_t len) {
   if ((wcsncasecmp(pstr, L"true", len) == 0) && (len == 4)) {
     return TSDB_TRUE;
   } else if (wcsncasecmp(pstr, L"false", len) == 0 && (len == 5)) {
@@ -412,11 +412,11 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
         pBuf = realloc(pBuf, newSize + 1);
       }
 
-      taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf);
+      taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, pBuf);
-      free(pVariant->wpz);
+      free(pVariant->ucs4);
       pBuf[newSize] = 0;
     } else {
-      taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest);
+      taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, *pDest);
     }
 
   } else {
@@ -469,17 +469,17 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
 
     // free the binary buffer in the first place
     if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
-      free(pVariant->wpz);
+      free(pVariant->ucs4);
     }
 
-    pVariant->wpz = pWStr;
+    pVariant->ucs4 = pWStr;
-    *pDestSize = taosUcs4len(pVariant->wpz);
+    *pDestSize = taosUcs4len(pVariant->ucs4);
 
     // shrink the allocate memory, no need to check here.
-    char *tmp = realloc(pVariant->wpz, (*pDestSize + 1) * TSDB_NCHAR_SIZE);
+    char *tmp = realloc(pVariant->ucs4, (*pDestSize + 1) * TSDB_NCHAR_SIZE);
     assert(tmp != NULL);
 
-    pVariant->wpz = (wchar_t *)tmp;
+    pVariant->ucs4 = (TdUcs4 *)tmp;
   } else {
     int32_t output = 0;
 
@@ -554,7 +554,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result
     *result = res;
   } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) {
     errno = 0;
-    wchar_t *endPtr = NULL;
+    TdUcs4 *endPtr = NULL;
     
     SToken token = {0};
     token.n = tGetToken(pVariant->pz, &token.type);
@@ -564,7 +564,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result
     }
     
     if (token.type == TK_FLOAT) {
-      double v = wcstod(pVariant->wpz, &endPtr);
+      double v = wcstod(pVariant->ucs4, &endPtr);
       if (releaseVariantPtr) {
         free(pVariant->pz);
         pVariant->nLen = 0;
@@ -583,7 +583,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result
       setNull((char *)result, type, tDataTypes[type].bytes);
       return 0;
     } else {
-      int64_t val = wcstoll(pVariant->wpz, &endPtr, 10);
+      int64_t val = wcstoll(pVariant->ucs4, &endPtr, 10);
       if (releaseVariantPtr) {
         free(pVariant->pz);
         pVariant->nLen = 0;
@@ -649,7 +649,7 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) {
     *pDest = ret;
   } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) {
     int32_t ret = 0;
-    if ((ret = wcsconvertToBoolImpl(pVariant->wpz, pVariant->nLen)) < 0) {
+    if ((ret = wcsconvertToBoolImpl(pVariant->ucs4, pVariant->nLen)) < 0) {
       return ret;
     }
     *pDest = ret;
@@ -899,7 +899,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
               return -1;
             }
           } else {
-            wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen);
+            tasoUcs4Copy((TdUcs4*)payload, pVariant->ucs4, pVariant->nLen);
           }
         }
       } else {
@@ -913,7 +913,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc
               return -1;
             }
           } else {
-            memcpy(p, pVariant->wpz, pVariant->nLen);
+            memcpy(p, pVariant->ucs4, pVariant->nLen);
             newlen = pVariant->nLen;
           }
 
@@ -979,7 +979,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) {
         pVariant->d = v;
       } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) {
         errno = 0;
-        double v = wcstod(pVariant->wpz, NULL);
+        double v = wcstod(pVariant->ucs4, NULL);
         if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) {
           free(pVariant->pz);
           return -1;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -3386,7 +3386,7 @@ void filterPrepare(void* expr, void* param) {
     if (size < (uint32_t)pSchema->bytes) {
       size = pSchema->bytes;
     }
-    // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space.
+    // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(TdUcs4) space.
     pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
     tVariantDump(pCond, pInfo->q, pSchema->type, true);
   }

source/libs/parser/test/mockCatalogService.cpp

@@ -13,12 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "mockCatalogService.h"
-
 #include <iomanip>
 #include <iostream>
 #include <map>
 #include "tdatablock.h"
+#include "mockCatalogService.h"
 
 #include "tname.h"
 #include "ttypes.h"

source/os/src/osString.c

@@ -24,6 +24,9 @@
 #include "iconv.h"
 #endif
 
+extern int wcwidth(wchar_t c);
+extern int wcswidth(const wchar_t *s, size_t n);
+
 int64_t taosStr2int64(const char *str) {
   char *endptr = NULL;
   return strtoll(str, &endptr, 10);
@@ -72,6 +75,11 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
 #endif
 }
 
+
+TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) {
+  memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4));
+}
+
 int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
 #ifdef DISALLOW_NCHAR_WITHOUT_ICONV
   return -1;
@@ -145,6 +153,18 @@ int32_t taosUcs4len(TdUcs4 *ucs4) {
   return n;
 }
 
+int32_t taosWcharWidth(TdWchar wchar) { return wcwidth(wchar); }
+
+int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size) { return wcswidth(pWchar, size); }
+
+int32_t taosMbToWchar(TdWchar *pWchar, const char *pStr, int32_t size) { return mbtowc(pWchar, pStr, size); }
+
+int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { return mbstowcs(pWchars, pStrs, size); }
+
+int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); }
+
+int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); }
+
 // #ifdef USE_LIBICONV
 // #include "iconv.h"
 

source/util/src/tcompare.c

@@ -208,7 +208,7 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
   if (len1 != len2) {
     return len1 > len2 ? 1 : -1;
   } else {
-    int32_t ret = memcmp((wchar_t *)pLeft, (wchar_t *)pRight, len1);
+    int32_t ret = memcmp((TdUcs4 *)pLeft, (TdUcs4 *)pRight, len1);
     if (ret == 0) {
       return 0;
     } else {
@@ -295,10 +295,10 @@ int32_t patternMatch(const char *patterStr, const char *str, size_t size, const
   return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
 }
 
-int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
+int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo) {
-  wchar_t c, c1;
+  TdUcs4 c, c1;
-  wchar_t matchOne = L'_';  // "_"
+  TdUcs4 matchOne = L'_';  // "_"
-  wchar_t matchAll = L'%';  // "%"
+  TdUcs4 matchAll = L'%';  // "%"
 
   int32_t i = 0;
   int32_t j = 0;
@@ -315,7 +315,7 @@ int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t siz
         return TSDB_PATTERN_MATCH;
       }
 
-      wchar_t accept[3] = {towupper(c), towlower(c), 0};
+      TdUcs4 accept[3] = {towupper(c), towlower(c), 0};
       while (1) {
         size_t n = wcscspn(str, accept);
 
@@ -424,10 +424,10 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) {
 
   assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
 
-  wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t));
+  char *pattern = calloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1);
   memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
 
-  int32_t ret = WCSPatternMatch(pattern, (const wchar_t *)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo);
+  int32_t ret = WCSPatternMatch((TdUcs4*)pattern, (TdUcs4*)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo);
   free(pattern);
 
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
@@ -647,7 +647,7 @@ int32_t doCompare(const char *f1, const char *f2, int32_t type, size_t size) {
       if (t1->len != t2->len) {
         return t1->len > t2->len ? 1 : -1;
       }
-      int32_t ret = memcmp((wchar_t *)t1, (wchar_t *)t2, t2->len);
+      int32_t ret = memcmp((TdUcs4 *)t1, (TdUcs4 *)t2, t2->len);
       if (ret == 0) {
         return ret;
       }

tools/shell/src/backup/shellDarwin.c

@@ -28,7 +28,6 @@
 int indicator = 1;
 struct termios oldtio;
 
-extern int wcwidth(wchar_t c);
 void insertChar(Command *cmd, char *c, int size);
 
 
@@ -426,7 +425,7 @@ void showOnScreen(Command *cmd) {
     w.ws_row = 30;
   }
 
-  wchar_t wc;
+  TdWchar wc;
   int size = 0;
 
   // Print out the command.
@@ -441,11 +440,11 @@ void showOnScreen(Command *cmd) {
   int remain_column = w.ws_col;
   /* size = cmd->commandSize + prompt_size; */
   for (char *str = total_string; size < cmd->commandSize + prompt_size;) {
-    int ret = mbtowc(&wc, str, MB_CUR_MAX);
+    int ret = taosMbToWchar(&wc, str, MB_CUR_MAX);
     if (ret < 0) break;
     size += ret;
     /* assert(size >= 0); */
-    int width = wcwidth(wc);
+    int width = taosWcharWidth(wc);
     if (remain_column > width) {
       printf("%lc", wc);
       remain_column -= width;

tools/shell/src/shellCommand.c

@@ -21,8 +21,6 @@
 
 #include <regex.h>
 
-extern int wcwidth(wchar_t c);
-extern int wcswidth(const wchar_t *s, size_t n);
 typedef struct {
   char widthInString;
   char widthOnScreen;
@@ -43,7 +41,7 @@ int countPrefixOnes(unsigned char c) {
 void getPrevCharSize(const char *str, int pos, int *size, int *width) {
   assert(pos > 0);
 
-  wchar_t wc;
+  TdWchar wc;
   *size = 0;
   *width = 0;
 
@@ -53,25 +51,25 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) {
     if (str[pos] > 0 || countPrefixOnes((unsigned char )str[pos]) > 1) break;
   }
 
-  int rc = mbtowc(&wc, str + pos, MB_CUR_MAX);
+  int rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
   assert(rc == *size);
 
-  *width = wcwidth(wc);
+  *width = taosWcharWidth(wc);
 }
 
 void getNextCharSize(const char *str, int pos, int *size, int *width) {
   assert(pos >= 0);
 
-  wchar_t wc;
+  TdWchar wc;
-  *size = mbtowc(&wc, str + pos, MB_CUR_MAX);
+  *size = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
-  *width = wcwidth(wc);
+  *width = taosWcharWidth(wc);
 }
 
 void insertChar(Command *cmd, char *c, int size) {
   assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset);
 
-  wchar_t wc;
+  TdWchar wc;
-  if (mbtowc(&wc, c, size) < 0) return;
+  if (taosMbToWchar(&wc, c, size) < 0) return;
 
   clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
   /* update the buffer */
@@ -81,8 +79,8 @@ void insertChar(Command *cmd, char *c, int size) {
   /* update the values */
   cmd->commandSize += size;
   cmd->cursorOffset += size;
-  cmd->screenOffset += wcwidth(wc);
+  cmd->screenOffset += taosWcharWidth(wc);
-  cmd->endOffset += wcwidth(wc);
+  cmd->endOffset += taosWcharWidth(wc);
   showOnScreen(cmd);
 }
 
@@ -249,10 +247,10 @@ int isReadyGo(Command *cmd) {
 }
 
 void getMbSizeInfo(const char *str, int *size, int *width) {
-  wchar_t *wc = (wchar_t *)calloc(sizeof(wchar_t), MAX_COMMAND_SIZE);
+  TdWchar *wc = (TdWchar *)calloc(sizeof(TdWchar), MAX_COMMAND_SIZE);
   *size = strlen(str);
-  mbstowcs(wc, str, MAX_COMMAND_SIZE);
+  taosMbsToWchars(wc, str, MAX_COMMAND_SIZE);
-  *width = wcswidth(wc, MAX_COMMAND_SIZE);
+  *width = taosWcharsWidth(wc, MAX_COMMAND_SIZE);
   free(wc);
 }
 

tools/shell/src/shellEngine.c

@@ -560,12 +560,12 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) {
 }
 
 static void shellPrintNChar(const char *str, int length, int width) {
-  wchar_t tail[3];
+  TdWchar tail[3];
   int     pos = 0, cols = 0, totalCols = 0, tailLen = 0;
 
   while (pos < length) {
-    wchar_t wc;
+    TdWchar wc;
-    int     bytes = mbtowc(&wc, str + pos, MB_CUR_MAX);
+    int     bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
     if (bytes == 0) {
       break;
     }
@@ -577,7 +577,7 @@ static void shellPrintNChar(const char *str, int length, int width) {
 #ifdef WINDOWS
     int w = bytes;
 #else
-    int w = wcwidth(wc);
+    int w = taosWcharWidth(wc);
 #endif
     if (w <= 0) {
       continue;

tools/shell/src/shellLinux.c

@@ -31,7 +31,6 @@
 int indicator = 1;
 struct termios oldtio;
 
-extern int wcwidth(wchar_t c);
 void insertChar(Command *cmd, char *c, int size);
 const char *argp_program_version = version;
 const char *argp_program_bug_address = "<support@taosdata.com>";
@@ -456,7 +455,7 @@ void showOnScreen(Command *cmd) {
     w.ws_row = 30;
   }
 
-  wchar_t wc;
+  TdWchar wc;
   int size = 0;
 
   // Print out the command.
@@ -471,11 +470,11 @@ void showOnScreen(Command *cmd) {
   int remain_column = w.ws_col;
   /* size = cmd->commandSize + prompt_size; */
   for (char *str = total_string; size < cmd->commandSize + prompt_size;) {
-    int ret = mbtowc(&wc, str, MB_CUR_MAX);
+    int ret = taosMbToWchar(&wc, str, MB_CUR_MAX);
     if (ret < 0) break;
     size += ret;
     /* assert(size >= 0); */
-    int width = wcwidth(wc);
+    int width = taosWcharWidth(wc);
     if (remain_column > width) {
       printf("%lc", wc);
       remain_column -= width;