xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[td-168] fix bug in handling variable nchar string

This commit is contained in:
hjxilinx 2020-05-04 23:57:28 +08:00
parent d87cefab53
commit 37c05411a1
8 changed files with 27 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/client/src/tscParseInsert.c
View File

@ -305,32 +305,37 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload,
case TSDB_DATA_TYPE_BINARY:
// binary data cannot be null-terminated char string, otherwise the last char of the string is lost
if (pToken->type == TK_NULL) {
*(int16_t*) payload = sizeof(int8_t);
payload += VARSTR_HEADER_SIZE;
*payload = TSDB_DATA_BINARY_NULL;
} else { // too long values will return invalid sql, not be truncated automatically
if (pToken->n > pSchema->bytes) {
return tscInvalidSQLErrMsg(msg, "string data overflow", pToken->z);
}
strncpy(payload, pToken->z, pToken->n);
if (pToken->n < pSchema->bytes) {
payload[pToken->n] = 0; // add the null-terminated char if the length of the string is shorter than the available space
}
STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n);
}
break;
case TSDB_DATA_TYPE_NCHAR:
if (pToken->type == TK_NULL) {
*(uint32_t *)payload = TSDB_DATA_NCHAR_NULL;
*(int16_t*) payload = sizeof(int32_t);
payload += VARSTR_HEADER_SIZE;
*(uint32_t*) payload = TSDB_DATA_NCHAR_NULL;
} else {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
if (!taosMbsToUcs4(pToken->z, pToken->n, payload, pSchema->bytes)) {
int32_t resLen = -1;
if (!taosMbsToUcs4(pToken->z, pToken->n, payload + VARSTR_HEADER_SIZE, pSchema->bytes, &resLen)) {
char buf[512] = {0};
snprintf(buf, 512, "%s", strerror(errno));
return tscInvalidSQLErrMsg(msg, buf, pToken->z);
}
*(uint16_t*)payload = (uint16_t) (resLen * TSDB_NCHAR_SIZE);
}
break;

2
src/client/src/tscPrepare.c
View File

@ -300,7 +300,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
break;
case TSDB_DATA_TYPE_NCHAR:
if (!taosMbsToUcs4(bind->buffer, *bind->length, data + param->offset, param->bytes)) {
if (!taosMbsToUcs4(bind->buffer, *bind->length, data + param->offset, param->bytes, NULL)) {
return TSDB_CODE_INVALID_VALUE;
}
return TSDB_CODE_SUCCESS;

2
src/client/src/tscSubquery.c
View File

@ -1934,7 +1934,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF
/* string terminated char for binary data*/
memset(pRes->buffer[columnIndex], 0, pField->bytes + TSDB_NCHAR_SIZE);
if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes, pRes->buffer[columnIndex])) {
if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes - VARSTR_HEADER_SIZE, pRes->buffer[columnIndex])) {
pRes->tsrow[columnIndex] = pRes->buffer[columnIndex];
} else {
tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow);

6
src/client/src/tscUtil.c
View File

@ -2132,12 +2132,12 @@ char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t colum
char* pData = ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row;
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
int32_t realLen = *(int16_t*) pData;
if (realLen < pInfo->pSqlExpr->resBytes - sizeof(int16_t)) { // todo refactor
int32_t realLen = varDataLen(pData);
if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor
*(char*) (pData + realLen + sizeof(int16_t)) = 0;
}
return pData + sizeof(int16_t); // head is the length of binary/nchar data
return pData + VARSTR_HEADER_SIZE; // head is the length of binary/nchar data
} else {
return pData;
}

23
src/common/src/tdataformat.c
View File

@ -172,28 +172,9 @@ int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
// set offset
*(VarDataOffsetT *)POINTER_DRIFT(row, toffset) = dataRowLen(row);
// set length
VarDataLenT slen = 0;
if (isNull(value, type)) {
slen = (type == TSDB_DATA_TYPE_BINARY) ? sizeof(int8_t) : sizeof(int32_t);
} else {
if (type == TSDB_DATA_TYPE_BINARY) {
slen = strnlen((char *)value, bytes);
} else {
slen = wcsnlen((wchar_t *)value, (bytes) / TSDB_NCHAR_SIZE) * TSDB_NCHAR_SIZE;
}
}
ASSERT(slen <= bytes);
*(VarDataLenT *)ptr = slen;
ptr = POINTER_DRIFT(ptr, sizeof(VarDataLenT));
memcpy((void *)ptr, value, slen);
dataRowLen(row) += (sizeof(int16_t) + slen);
memcpy(ptr, value, varDataTLen(value));
dataRowLen(row) += varDataTLen(value);
break;
default:
memcpy(POINTER_DRIFT(row, toffset), value, TYPE_BYTES[type]);

4
src/query/src/tvariant.c
View File

@ -408,7 +408,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
if (*pDest == pVariant->pz) {
wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE);
taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE);
taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
// free the binary buffer in the first place
if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
@ -424,7 +424,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
pVariant->wpz = (wchar_t *)tmp;
} else {
taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE);
taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
}
return 0;

2
src/util/inc/tutil.h
View File

@ -144,7 +144,7 @@ void getTmpfilePath(const char *fileNamePattern, char *dstPath);
int32_t taosInitTimer(void (*callback)(int), int32_t ms);
void taosUninitTimer();
bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len);
bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t* len);
int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes);

6
src/util/src/tutil.c
View File

@ -490,7 +490,7 @@ bool taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
#endif
}
bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len) {
bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t* len) {
memset(ucs4, 0, ucs4_max_len);
#ifdef USE_LIBICONV
iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
@ -501,6 +501,10 @@ bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len)
return false;
}
iconv_close(cd);
if (len != NULL) {
*len = outLen;
}
return true;
#else
mbstate_t state = {0};