xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-225]handle the error in converting mbs to nchar

This commit is contained in:
Haojun Liao 2021-01-08 17:13:22 +08:00
parent 612d908b2d
commit ba1ec97b00
6 changed files with 29 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/client/src/tscParseInsert.c
View File

@ -332,7 +332,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload,
return tscInvalidSQLErrMsg(msg, "illegal double data", pToken->z); return tscInvalidSQLErrMsg(msg, "illegal double data", pToken->z);
} }
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv)) { if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
return tscInvalidSQLErrMsg(msg, "illegal double data", pToken->z); return tscInvalidSQLErrMsg(msg, "illegal double data", pToken->z);
} }
@ -359,7 +359,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload,
setVardataNull(payload, TSDB_DATA_TYPE_NCHAR); setVardataNull(payload, TSDB_DATA_TYPE_NCHAR);
} else { } else {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
size_t output = 0; int32_t output = 0;
if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
char buf[512] = {0}; char buf[512] = {0};
snprintf(buf, tListLen(buf), "%s", strerror(errno)); snprintf(buf, tListLen(buf), "%s", strerror(errno));

2
src/client/src/tscPrepare.c
View File

@ -616,7 +616,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
switch (bind->buffer_type) { switch (bind->buffer_type) {
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
size_t output = 0; int32_t output = 0;
if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) { if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) {
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }

39
src/common/src/tvariant.c
View File

@ -341,7 +341,6 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
return 0; return 0;
} }
// todo handle the error
static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
char tmpBuf[40] = {0}; char tmpBuf[40] = {0};
@ -364,8 +363,11 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
if (*pDest == pVariant->pz) { if (*pDest == pVariant->pz) {
wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE);
taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
if (!ret) {
return -1;
}
// free the binary buffer in the first place // free the binary buffer in the first place
if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
free(pVariant->wpz); free(pVariant->wpz);
@ -380,11 +382,14 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
pVariant->wpz = (wchar_t *)tmp; pVariant->wpz = (wchar_t *)tmp;
} else { } else {
size_t output = -1; int32_t output = 0;
taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output); bool ret = taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output);
if (!ret) {
return -1;
}
if (pDestSize != NULL) { if (pDestSize != NULL) {
*pDestSize = (int32_t)output; *pDestSize = output;
} }
} }
@ -416,11 +421,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result
*result = (int64_t) pVariant->dKey; *result = (int64_t) pVariant->dKey;
} else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
SStrToken token = {.z = pVariant->pz, .n = pVariant->nLen}; SStrToken token = {.z = pVariant->pz, .n = pVariant->nLen};
int32_t n = tSQLGetToken(pVariant->pz, &token.type); /*int32_t n = */tSQLGetToken(pVariant->pz, &token.type);
if (token.type == TK_MINUS || token.type == TK_PLUS) {
// decide if pVariant->pz is NULL or not
tSQLGetToken(pVariant->pz + n, &token.type);
}
if (token.type == TK_NULL) { if (token.type == TK_NULL) {
if (releaseVariantPtr) { if (releaseVariantPtr) {
@ -549,8 +550,6 @@ static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) {
/* /*
* transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType * transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType
* to column type defined in schema * to column type defined in schema
*
* todo handle the return value
*/ */
int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix) { int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix) {
if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) { if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) {
@ -709,7 +708,9 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
*(uint32_t *)payload = TSDB_DATA_NCHAR_NULL; *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL;
} else { } else {
if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) { if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) {
toNchar(pVariant, &payload, &newlen); if (toNchar(pVariant, &payload, &newlen) != 0) {
return -1;
}
} else { } else {
wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen); wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen);
} }
@ -721,7 +722,9 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
char *p = varDataVal(payload); char *p = varDataVal(payload);
if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) { if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) {
toNchar(pVariant, &p, &newlen); if (toNchar(pVariant, &p, &newlen) != 0) {
return -1;
}
} else { } else {
wcsncpy((wchar_t *)p, pVariant->wpz, pVariant->nLen); wcsncpy((wchar_t *)p, pVariant->wpz, pVariant->nLen);
newlen = pVariant->nLen; newlen = pVariant->nLen;
@ -805,7 +808,9 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) {
} }
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) { if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) {
toNchar(pVariant, &pVariant->pz, &pVariant->nLen); if (toNchar(pVariant, &pVariant->pz, &pVariant->nLen) != 0) {
return -1;
}
} }
pVariant->nType = type; pVariant->nType = type;
break; break;

2
src/cq/src/cqMain.c
View File

@ -341,7 +341,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
val = ((char*)val) - sizeof(VarDataLenT); val = ((char*)val) - sizeof(VarDataLenT);
} else if (c->type == TSDB_DATA_TYPE_NCHAR) { } else if (c->type == TSDB_DATA_TYPE_NCHAR) {
char buf[TSDB_MAX_NCHAR_LEN]; char buf[TSDB_MAX_NCHAR_LEN];
size_t len = taos_fetch_lengths(tres)[i]; int32_t len = taos_fetch_lengths(tres)[i];
taosMbsToUcs4(val, len, buf, sizeof(buf), &len); taosMbsToUcs4(val, len, buf, sizeof(buf), &len);
memcpy(val + sizeof(VarDataLenT), buf, len); memcpy(val + sizeof(VarDataLenT), buf, len);
varDataLen(val) = len; varDataLen(val) = len;

2
src/os/inc/osString.h
View File

@ -47,7 +47,7 @@ extern "C" {
// USE_LIBICONV // USE_LIBICONV
int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t *len); bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int *len);
int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes); int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes);
bool taosValidateEncodec(const char *encodec); bool taosValidateEncodec(const char *encodec);
char * taosCharsetReplace(char *charsetstr); char * taosCharsetReplace(char *charsetstr);

4
src/os/src/detail/osString.c
View File

@ -46,7 +46,7 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
return (int32_t)(ucs4_max_len - outLen); return (int32_t)(ucs4_max_len - outLen);
} }
bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) { bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
memset(ucs4, 0, ucs4_max_len); memset(ucs4, 0, ucs4_max_len);
iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
size_t ucs4_input_len = mbsLength; size_t ucs4_input_len = mbsLength;
@ -92,7 +92,7 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
return len; return len;
} }
bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) { bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
memset(ucs4, 0, ucs4_max_len); memset(ucs4, 0, ucs4_max_len);
mbstate_t state = {0}; mbstate_t state = {0};
int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state); int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);