xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-13767]<fix>: only use iconv.

This commit is contained in:
afwerar 2022-03-18 13:23:48 +08:00
parent 83b62c90ba
commit 394178da68
14 changed files with 502 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmake/cmake.options
View File

@ -18,6 +18,13 @@ IF(${TD_WINDOWS})
ON
)
MESSAGE("build iconv Win32")
option(
BUILD_WITH_ICONV
"If build iconv on Windows"
ON
)
ENDIF ()
IF(${TD_LINUX} MATCHES TRUE)

12
cmake/iconv_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,12 @@
# iconv
ExternalProject_Add(iconv
GIT_REPOSITORY https://github.com/win-iconv/win-iconv.git
GIT_TAG v0.0.8
SOURCE_DIR "${CMAKE_CONTRIB_DIR}/iconv"
BINARY_DIR ""
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

19
contrib/CMakeLists.txt
View File

@ -83,6 +83,11 @@ if(${BUILD_WITH_NURAFT})
cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_NURAFT})
# iconv
if(${BUILD_WITH_ICONV})
cat("${CMAKE_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_ICONV})
# download dependencies
configure_file(${CONTRIB_TMP_FILE} "${CMAKE_CONTRIB_DIR}/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@ -208,11 +213,7 @@ endif(${BUILD_WITH_TRAFT})
# LIBUV
if(${BUILD_WITH_UV})
if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
MESSAGE("Windows need set no-sign-compare")
add_compile_options(-Wno-sign-compare)
endif ()
if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
if (${TD_WINDOWS})
file(READ "libuv/include/uv.h" CONTENTS)
string(REGEX REPLACE "/([\r]*)\nstruct uv_tcp_s {" "/\\1\ntypedef BOOL (PASCAL *LPFN_CONNECTEX) (SOCKET s, const struct sockaddr* name, int namelen, PVOID lpSendBuffer, DWORD dwSendDataLength,LPDWORD lpdwBytesSent, LPOVERLAPPED lpOverlapped);\\1\nstruct uv_tcp_s {" CONTENTS_NEW "${CONTENTS}")
file(WRITE "libuv/include/uv.h" "${CONTENTS_NEW}")
@ -249,10 +250,14 @@ endif(${BUILD_WITH_SQLITE})
# pthread
if(${BUILD_PTHREAD})
ADD_DEFINITIONS("-DPTW32_STATIC_LIB")
add_subdirectory(pthread-win32)
add_definitions(-DPTW32_STATIC_LIB)
add_subdirectory(pthread)
endif(${BUILD_PTHREAD})
# iconv
if(${BUILD_WITH_ICONV})
add_subdirectory(iconv)
endif(${BUILD_WITH_ICONV})
# ================================================================================================
# Build test

31
include/os/osFile.h
View File

@ -22,15 +22,6 @@ extern "C" {
#include "osSocket.h"
#if defined(WINDOWS)
typedef int32_t FileFd;
typedef int32_t SocketFd;
#else
typedef int32_t FileFd;
typedef int32_t SocketFd;
#endif
int64_t taosRead(FileFd fd, void *buf, int64_t count);
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC
#define open OPEN_FUNC_TAOS_FORBID
@ -42,6 +33,7 @@ int64_t taosRead(FileFd fd, void *buf, int64_t count);
#define close CLOSE_FUNC_TAOS_FORBID
#define fclose FCLOSE_FUNC_TAOS_FORBID
#define fsync FSYNC_FUNC_TAOS_FORBID
#define getline GETLINE_FUNC_TAOS_FORBID
// #define fflush FFLUSH_FUNC_TAOS_FORBID
#endif
@ -49,15 +41,6 @@ int64_t taosRead(FileFd fd, void *buf, int64_t count);
#define PATH_MAX 256
#endif
typedef int32_t FileFd;
typedef struct TdFile {
pthread_rwlock_t rwlock;
int refId;
FileFd fd;
FILE *fp;
} * TdFilePtr, TdFile;
typedef struct TdFile *TdFilePtr;
#define TD_FILE_CTEATE 0x0001
@ -95,10 +78,6 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
#if defined(WINDOWS)
#define __restrict__
#endif // WINDOWS
int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
int32_t taosEOFFile(TdFilePtr pFile);
@ -111,15 +90,7 @@ int32_t taosRemoveFile(const char *path);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
#if defined(_TD_DARWIN_64)
typedef int32_t SocketFd;
int64_t taosSendFile(SocketFd fdDst, FileFd pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(FILE *pFileOut, FILE *pFileIn, int64_t *offset, int64_t size);
#else
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
#endif
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
bool taosValidFile(TdFilePtr pFile);

23
include/os/osString.h
View File

@ -20,16 +20,19 @@
extern "C" {
#endif
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC
#define iconv_open ICONV_OPEN_FUNC_TAOS_FORBID
#define iconv_close ICONV_CLOSE_FUNC_TAOS_FORBID
#define iconv ICONV_FUNC_TAOS_FORBID
#endif
typedef int32_t TdUcs4;
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#define tstrdup(str) _strdup(str)
#define tstrndup(str, size) _strndup(str, size)
int32_t tgetline(char **lineptr, size_t *n, FILE *stream);
int32_t twcslen(const wchar_t *wcs);
#else
#define tstrdup(str) strdup(str)
#define tstrndup(str, size) strndup(str, size)
#define tgetline(lineptr, n, stream) getline(lineptr, n, stream)
#define twcslen wcslen
#endif
#define tstrncpy(dst, src, size) \
@ -38,12 +41,12 @@ extern "C" {
(dst)[(size)-1] = 0; \
} while (0)
int32_t taosUcs4len(TdUcs4 *ucs4);
int64_t taosStr2int64(const char *str);
// USE_LIBICONV
int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len);
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
bool taosValidateEncodec(const char *encodec);
#ifdef __cplusplus

2
include/util/tdef.h
View File

@ -41,7 +41,7 @@ extern const int32_t TYPE_BYTES[15];
#define DOUBLE_BYTES sizeof(double)
#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
#define TSDB_KEYSIZE sizeof(TSKEY)
#define TSDB_NCHAR_SIZE sizeof(int32_t)
#define TSDB_NCHAR_SIZE sizeof(TdUcs4)
// NULL definition
#define TSDB_DATA_BOOL_NULL 0x02

10
source/common/src/tvariant.c
View File

@ -343,7 +343,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) {
case TSDB_DATA_TYPE_NCHAR: {
dst[0] = '\'';
taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1);
taosUcs4ToMbs(pVar->wpz, (taosUcs4len(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1);
int32_t len = (int32_t)strlen(dst);
dst[len] = '\'';
dst[len + 1] = 0;
@ -460,8 +460,8 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
}
if (*pDest == pVariant->pz) {
wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE);
bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
TdUcs4 *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE);
bool ret = taosMbsToUcs4(pDst, nLen, pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
if (!ret) {
tfree(pWStr);
return -1;
@ -473,7 +473,7 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
}
pVariant->wpz = pWStr;
*pDestSize = twcslen(pVariant->wpz);
*pDestSize = taosUcs4len(pVariant->wpz);
// shrink the allocate memory, no need to check here.
char *tmp = realloc(pVariant->wpz, (*pDestSize + 1) * TSDB_NCHAR_SIZE);
@ -483,7 +483,7 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) {
} else {
int32_t output = 0;
bool ret = taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output);
bool ret = taosMbsToUcs4(pDst, nLen, (TdUcs4*)*pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output);
if (!ret) {
return -1;
}

4
source/libs/parser/src/parInsert.c
View File

@ -622,7 +622,7 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0;
const char* rowEnd = tdRowEnd(rb->pBuf);
if (!taosMbsToUcs4(value, len, (char*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
}
varDataSetLen(rowEnd, output);
@ -725,7 +725,7 @@ static int32_t KvRowAppend(const void *value, int32_t len, void *param) {
} else if (TSDB_DATA_TYPE_NCHAR == type) {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0;
if (!taosMbsToUcs4(value, len, varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
}

8
source/libs/scalar/src/filter.c
View File

@ -1813,7 +1813,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
if(type == TSDB_DATA_TYPE_NCHAR &&
(unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)){
char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
if (len < 0){
qError("filterInitValFieldData taosUcs4ToMbs error 1");
return TSDB_CODE_QRY_APP_ERROR;
@ -2992,7 +2992,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){
char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData));
if (len < 0){
qError("castConvert1 taosUcs4ToMbs error");
}else{
@ -3052,7 +3052,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
} else {
if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){
char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData));
if (len < 0){
qError("castConvert1 taosUcs4ToMbs error");
}else{
@ -3433,7 +3433,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
varDataCopy(dst, src);
continue;
}
bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4*)varDataVal(dst), bufSize, &len);
if(!ret) {
qError("filterConverNcharColumns taosMbsToUcs4 error");
return TSDB_CODE_FAILED;

2
source/libs/scalar/src/sclvector.c
View File

@ -316,7 +316,7 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t
tmp = realloc(tmp, bufSize);
}
int len = taosUcs4ToMbs(varDataVal(pIn->data), varDataLen(pIn->data), tmp);
int len = taosUcs4ToMbs((TdUcs4*)varDataVal(pIn->data), varDataLen(pIn->data), tmp);
if (len < 0){
sclError("castConvert taosUcs4ToMbs error 1");
tfree(tmp);

7
source/os/CMakeLists.txt
View File

@ -5,9 +5,14 @@ target_include_directories(
PUBLIC "${CMAKE_SOURCE_DIR}/include/os"
PUBLIC "${CMAKE_SOURCE_DIR}/include"
PUBLIC "${CMAKE_SOURCE_DIR}/include/util"
PUBLIC "${CMAKE_SOURCE_DIR}/contrib/pthread-win32"
PUBLIC "${CMAKE_SOURCE_DIR}/contrib/pthread"
PUBLIC "${CMAKE_SOURCE_DIR}/contrib/gnuregex"
)
# iconv
find_path(IconvApiIncludes iconv.h PATHS)
if(NOT IconvApiIncludes)
add_definitions(-DDISALLOW_NCHAR_WITHOUT_ICONV)
endif ()
target_link_libraries(
os pthread dl rt m
)

17
source/os/src/osFile.c
View File

@ -46,6 +46,23 @@ extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */
#define O_TEXT LINUX_FILE_NO_TEXT_OPTION
#endif
#if defined(WINDOWS)
typedef int32_t FileFd;
typedef int32_t SocketFd;
#else
typedef int32_t FileFd;
typedef int32_t SocketFd;
#endif
typedef int32_t FileFd;
typedef struct TdFile {
pthread_rwlock_t rwlock;
int refId;
FileFd fd;
FILE *fp;
} * TdFilePtr, TdFile;
#define FILE_WITH_LOCK 1
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) {

2
source/os/src/osLocale.c
View File

@ -81,7 +81,7 @@ void taosSetSystemLocale(const char *inLocale, const char *inCharSet) {
}
if (!taosValidateEncodec(inCharSet)) {
printf("Invalid charset:%s, please set the valid charset in config file", inCharSet);
printf("Invalid charset:%s, please set the valid charset in config file\n", inCharSet);
exit(-1);
}
}

714
source/os/src/osString.c
View File

@ -13,287 +13,32 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#define _DEFAULT_SOURCE
#include "os.h"
#include "tdef.h"
#include <wchar.h>
#include <wctype.h>
// #include "tdef.h"
// #include <wchar.h>
// #include <wctype.h>
#ifndef DISALLOW_NCHAR_WITHOUT_ICONV
#include "iconv.h"
#endif
int64_t taosStr2int64(const char *str) {
char *endptr = NULL;
return strtoll(str, &endptr, 10);
}
#ifdef USE_LIBICONV
#include "iconv.h"
int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
size_t ucs4_input_len = ucs4_max_len;
size_t outLen = ucs4_max_len;
if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
iconv_close(cd);
return -1;
}
iconv_close(cd);
return (int32_t)(ucs4_max_len - outLen);
}
bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
memset(ucs4, 0, ucs4_max_len);
iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
size_t ucs4_input_len = mbsLength;
size_t outLeft = ucs4_max_len;
if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
iconv_close(cd);
bool taosCheckNcharValid(void) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
return false;
}
iconv_close(cd);
if (len != NULL) {
*len = (int32_t)(ucs4_max_len - outLeft);
if (*len < 0) {
return false;
}
}
return true;
}
bool taosValidateEncodec(const char *encodec) {
iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
if (cd == (iconv_t)(-1)) {
return false;
}
iconv_close(cd);
return true;
}
#else
int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
mbstate_t state = {0};
int32_t len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state);
if (len < 0) {
return -1;
}
memset(&state, 0, sizeof(state));
len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state);
if (len < 0) {
return -1;
}
return len;
}
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
memset(ucs4, 0, ucs4_max_len);
mbstate_t state = {0};
int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);
*len = retlen;
return retlen >= 0;
}
bool taosValidateEncodec(const char *encodec) {
return true;
}
#endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
/*
* windows implementation
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <sys/types.h>
#include <stdio.h>
#include <assert.h>
#include <errno.h>
#include <stdint.h>
#if STDC_HEADERS
#include <stdlib.h>
#else
char *malloc(), *realloc();
#endif
/* Always add at least this many bytes when extending the buffer. */
#define MIN_CHUNK 64
/* Read up to (and including) a TERMINATOR from STREAM into *LINEPTR
+ OFFSET (and null-terminate it). *LINEPTR is a pointer returned from
malloc (or NULL), pointing to *N characters of space. It is realloc'd
as necessary. Return the number of characters read (not including the
null terminator), or -1 on error or EOF. On a -1 return, the caller
should check feof(), if not then errno has been set to indicate
the error. */
int32_t getstr(char **lineptr, size_t *n, FILE *stream, char terminator, int32_t offset) {
int32_t nchars_avail; /* Allocated but unused chars in *LINEPTR. */
char * read_pos; /* Where we're reading into *LINEPTR. */
int32_t ret;
if (!lineptr || !n || !stream) {
errno = EINVAL;
return -1;
}
if (!*lineptr) {
*n = MIN_CHUNK;
*lineptr = malloc(*n);
if (!*lineptr) {
errno = ENOMEM;
return -1;
}
}
nchars_avail = (int32_t)(*n - offset);
read_pos = *lineptr + offset;
for (;;) {
int32_t save_errno;
register int32_t c = getc(stream);
save_errno = errno;
/* We always want at least one char left in the buffer, since we
always (unless we get an error while reading the first char)
NUL-terminate the line buffer. */
assert((*lineptr + *n) == (read_pos + nchars_avail));
if (nchars_avail < 2) {
if (*n > MIN_CHUNK)
*n *= 2;
else
*n += MIN_CHUNK;
nchars_avail = (int32_t)(*n + *lineptr - read_pos);
char* lineptr1 = realloc(*lineptr, *n);
if (!lineptr1) {
errno = ENOMEM;
return -1;
}
*lineptr = lineptr1;
read_pos = *n - nchars_avail + *lineptr;
assert((*lineptr + *n) == (read_pos + nchars_avail));
}
if (ferror(stream)) {
/* Might like to return partial line, but there is no
place for us to store errno. And we don't want to just
lose errno. */
errno = save_errno;
return -1;
}
if (c == EOF) {
/* Return partial line, if any. */
if (read_pos == *lineptr)
return -1;
else
break;
}
*read_pos++ = c;
nchars_avail--;
if (c == terminator) /* Return the line. */
break;
}
/* Done - NUL terminate and return the number of chars read. */
*read_pos = '\0';
ret = (int32_t)(read_pos - (*lineptr + offset));
return ret;
}
int32_t tgetline(char **lineptr, size_t *n, FILE *stream) { return getstr(lineptr, n, stream, '\n', 0); }
/*
* Get next token from string *stringp, where tokens are possibly-empty
* strings separated by characters from delim.
*
* Writes NULs into the string at *stringp to end tokens.
* delim need not remain constant from call to call.
* On return, *stringp points past the last NUL written (if there might
* be further tokens), or is NULL (if there are definitely no moretokens).
*
* If *stringp is NULL, strsep returns NULL.
*/
char *strsep(char **stringp, const char *delim) {
char * s;
const char *spanp;
int32_t c, sc;
char *tok;
if ((s = *stringp) == NULL)
return (NULL);
for (tok = s;;) {
c = *s++;
spanp = delim;
do {
if ((sc = *spanp++) == c) {
if (c == 0)
s = NULL;
else
s[-1] = 0;
*stringp = s;
return (tok);
}
} while (sc != 0);
}
/* NOTREACHED */
}
char *getpass(const char *prefix) {
static char passwd[TSDB_PASSWORD_LEN] = {0};
memset(passwd, 0, TSDB_PASSWORD_LEN);
//printf("%s", prefix);
int32_t index = 0;
char ch;
while (index < TSDB_PASSWORD_LEN) {
ch = getch();
if (ch == '\n' || ch == '\r') {
break;
} else {
passwd[index++] = ch;
}
}
return passwd;
}
int32_t twcslen(const wchar_t *wcs) {
int32_t *wstr = (int32_t *)wcs;
if (NULL == wstr) {
return 0;
}
int32_t n = 0;
while (1) {
if (0 == *wstr++) {
break;
}
n++;
}
return n;
}
int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) {
for (int32_t i = 0; i < bytes; i += TSDB_NCHAR_SIZE) {
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
for (int32_t i = 0; i < bytes; i += sizeof(TdUcs4)) {
int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i);
int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i);
@ -327,38 +72,413 @@ int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) {
#endif
}
/* Copy memory to memory until the specified number of bytes
has been copied, return pointer to following byte.
Overlap is NOT handled correctly. */
void *mempcpy(void *dest, const void *src, size_t len) {
return (char*)memcpy(dest, src, len) + len;
}
/* Copy SRC to DEST, returning the address of the terminating '\0' in DEST. */
char *stpcpy (char *dest, const char *src) {
size_t len = strlen (src);
return (char*)memcpy(dest, src, len + 1) + len;
}
/* Copy no more than N characters of SRC to DEST, returning the address of
the terminating '\0' in DEST, if any, or else DEST + N. */
char *stpncpy (char *dest, const char *src, size_t n) {
size_t size = strnlen (src, n);
memcpy (dest, src, size);
dest += size;
if (size == n)
return dest;
return memset (dest, '\0', n - size);
}
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
return -1;
#else
/*
* linux and darwin implementation
*/
int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize) {
return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / ncharSize);
iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
size_t ucs4_input_len = ucs4_max_len;
size_t outLen = ucs4_max_len;
if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
iconv_close(cd);
return -1;
}
iconv_close(cd);
return (int32_t)(ucs4_max_len - outLen);
#endif
}
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
return -1;
#else
memset(ucs4, 0, ucs4_max_len);
iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
size_t ucs4_input_len = mbsLength;
size_t outLeft = ucs4_max_len;
if (iconv(cd, (char**)&mbs, &ucs4_input_len, (char**)&ucs4, &outLeft) == -1) {
iconv_close(cd);
return false;
}
iconv_close(cd);
if (len != NULL) {
*len = (int32_t)(ucs4_max_len - outLeft);
if (*len < 0) {
return false;
}
}
return true;
#endif
}
bool taosValidateEncodec(const char *encodec) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
return false;
#else
iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
if (cd == (iconv_t)(-1)) {
return false;
}
iconv_close(cd);
return true;
#endif
}
int32_t taosUcs4len(TdUcs4 *ucs4) {
TdUcs4 *wstr = (TdUcs4 *)ucs4;
if (NULL == wstr) {
return 0;
}
int32_t n = 0;
while (1) {
if (0 == *wstr++) {
break;
}
n++;
}
return n;
}
// #ifdef USE_LIBICONV
// #include "iconv.h"
// int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
// iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
// size_t ucs4_input_len = ucs4_max_len;
// size_t outLen = ucs4_max_len;
// if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
// iconv_close(cd);
// return -1;
// }
// iconv_close(cd);
// return (int32_t)(ucs4_max_len - outLen);
// }
// bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
// memset(ucs4, 0, ucs4_max_len);
// iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
// size_t ucs4_input_len = mbsLength;
// size_t outLeft = ucs4_max_len;
// if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
// iconv_close(cd);
// return false;
// }
// iconv_close(cd);
// if (len != NULL) {
// *len = (int32_t)(ucs4_max_len - outLeft);
// if (*len < 0) {
// return false;
// }
// }
// return true;
// }
// bool taosValidateEncodec(const char *encodec) {
// iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
// if (cd == (iconv_t)(-1)) {
// return false;
// }
// iconv_close(cd);
// return true;
// }
// #else
// int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
// mbstate_t state = {0};
// int32_t len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state);
// if (len < 0) {
// return -1;
// }
// memset(&state, 0, sizeof(state));
// len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state);
// if (len < 0) {
// return -1;
// }
// return len;
// }
// bool taosMbsToUcs4(const char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
// memset(ucs4, 0, ucs4_max_len);
// mbstate_t state = {0};
// int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);
// *len = retlen;
// return retlen >= 0;
// }
// bool taosValidateEncodec(const char *encodec) {
// return true;
// }
// #endif
// #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
// /*
// * windows implementation
// */
// #ifdef HAVE_CONFIG_H
// #include <config.h>
// #endif
// #include <sys/types.h>
// #include <stdio.h>
// #include <assert.h>
// #include <errno.h>
// #include <stdint.h>
// #if STDC_HEADERS
// #include <stdlib.h>
// #else
// char *malloc(), *realloc();
// #endif
// /* Always add at least this many bytes when extending the buffer. */
// #define MIN_CHUNK 64
// /* Read up to (and including) a TERMINATOR from STREAM into *LINEPTR
// + OFFSET (and null-terminate it). *LINEPTR is a pointer returned from
// malloc (or NULL), pointing to *N characters of space. It is realloc'd
// as necessary. Return the number of characters read (not including the
// null terminator), or -1 on error or EOF. On a -1 return, the caller
// should check feof(), if not then errno has been set to indicate
// the error. */
// int32_t getstr(char **lineptr, size_t *n, FILE *stream, char terminator, int32_t offset) {
// int32_t nchars_avail; /* Allocated but unused chars in *LINEPTR. */
// char * read_pos; /* Where we're reading into *LINEPTR. */
// int32_t ret;
// if (!lineptr || !n || !stream) {
// errno = EINVAL;
// return -1;
// }
// if (!*lineptr) {
// *n = MIN_CHUNK;
// *lineptr = malloc(*n);
// if (!*lineptr) {
// errno = ENOMEM;
// return -1;
// }
// }
// nchars_avail = (int32_t)(*n - offset);
// read_pos = *lineptr + offset;
// for (;;) {
// int32_t save_errno;
// register int32_t c = getc(stream);
// save_errno = errno;
// /* We always want at least one char left in the buffer, since we
// always (unless we get an error while reading the first char)
// NUL-terminate the line buffer. */
// assert((*lineptr + *n) == (read_pos + nchars_avail));
// if (nchars_avail < 2) {
// if (*n > MIN_CHUNK)
// *n *= 2;
// else
// *n += MIN_CHUNK;
// nchars_avail = (int32_t)(*n + *lineptr - read_pos);
// char* lineptr1 = realloc(*lineptr, *n);
// if (!lineptr1) {
// errno = ENOMEM;
// return -1;
// }
// *lineptr = lineptr1;
// read_pos = *n - nchars_avail + *lineptr;
// assert((*lineptr + *n) == (read_pos + nchars_avail));
// }
// if (ferror(stream)) {
// /* Might like to return partial line, but there is no
// place for us to store errno. And we don't want to just
// lose errno. */
// errno = save_errno;
// return -1;
// }
// if (c == EOF) {
// /* Return partial line, if any. */
// if (read_pos == *lineptr)
// return -1;
// else
// break;
// }
// *read_pos++ = c;
// nchars_avail--;
// if (c == terminator) /* Return the line. */
// break;
// }
// /* Done - NUL terminate and return the number of chars read. */
// *read_pos = '\0';
// ret = (int32_t)(read_pos - (*lineptr + offset));
// return ret;
// }
// int32_t tgetline(char **lineptr, size_t *n, FILE *stream) { return getstr(lineptr, n, stream, '\n', 0); }
// /*
// * Get next token from string *stringp, where tokens are possibly-empty
// * strings separated by characters from delim.
// *
// * Writes NULs into the string at *stringp to end tokens.
// * delim need not remain constant from call to call.
// * On return, *stringp points past the last NUL written (if there might
// * be further tokens), or is NULL (if there are definitely no moretokens).
// *
// * If *stringp is NULL, strsep returns NULL.
// */
// char *strsep(char **stringp, const char *delim) {
// char * s;
// const char *spanp;
// int32_t c, sc;
// char *tok;
// if ((s = *stringp) == NULL)
// return (NULL);
// for (tok = s;;) {
// c = *s++;
// spanp = delim;
// do {
// if ((sc = *spanp++) == c) {
// if (c == 0)
// s = NULL;
// else
// s[-1] = 0;
// *stringp = s;
// return (tok);
// }
// } while (sc != 0);
// }
// /* NOTREACHED */
// }
// char *getpass(const char *prefix) {
// static char passwd[TSDB_PASSWORD_LEN] = {0};
// memset(passwd, 0, TSDB_PASSWORD_LEN);
// //printf("%s", prefix);
// int32_t index = 0;
// char ch;
// while (index < TSDB_PASSWORD_LEN) {
// ch = getch();
// if (ch == '\n' || ch == '\r') {
// break;
// } else {
// passwd[index++] = ch;
// }
// }
// return passwd;
// }
// int32_t twcslen(const wchar_t *wcs) {
// int32_t *wstr = (int32_t *)wcs;
// if (NULL == wstr) {
// return 0;
// }
// int32_t n = 0;
// while (1) {
// if (0 == *wstr++) {
// break;
// }
// n++;
// }
// return n;
// }
// int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) {
// for (int32_t i = 0; i < bytes; i += TSDB_NCHAR_SIZE) {
// int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i);
// int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i);
// if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
// return f1 - f2;
// } else if (f1 == 0 && f2 == 0) {
// return 0;
// }
// if (f1 != f2) {
// return f1 - f2;
// }
// }
// return 0;
// #if 0
// int32_t ucs4_max_len = bytes + 4;
// char *f1_mbs = calloc(bytes, 1);
// char *f2_mbs = calloc(bytes, 1);
// if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) {
// return -1;
// }
// if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) {
// return -1;
// }
// int32_t ret = strcmp(f1_mbs, f2_mbs);
// free(f1_mbs);
// free(f2_mbs);
// return ret;
// #endif
// }
// /* Copy memory to memory until the specified number of bytes
// has been copied, return pointer to following byte.
// Overlap is NOT handled correctly. */
// void *mempcpy(void *dest, const void *src, size_t len) {
// return (char*)memcpy(dest, src, len) + len;
// }
// /* Copy SRC to DEST, returning the address of the terminating '\0' in DEST. */
// char *stpcpy (char *dest, const char *src) {
// size_t len = strlen (src);
// return (char*)memcpy(dest, src, len + 1) + len;
// }
// /* Copy no more than N characters of SRC to DEST, returning the address of
// the terminating '\0' in DEST, if any, or else DEST + N. */
// char *stpncpy (char *dest, const char *src, size_t n) {
// size_t size = strnlen (src, n);
// memcpy (dest, src, size);
// dest += size;
// if (size == n)
// return dest;
// return memset (dest, '\0', n - size);
// }
// #else
// /*
// * linux and darwin implementation
// */
// int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize) {
// return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / ncharSize);
// }
// #endif