Merge pull request #13987 from taosdata/fix/TD-16591
fix:raw data error in json type
This commit is contained in:
commit
e50b10dbef
|
@ -162,6 +162,7 @@ typedef struct SReqResultInfo {
|
|||
int32_t precision;
|
||||
bool convertUcs4;
|
||||
int32_t payloadLen;
|
||||
char* convertJson;
|
||||
} SReqResultInfo;
|
||||
|
||||
typedef struct SRequestSendRecvBody {
|
||||
|
@ -242,6 +243,7 @@ static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool conver
|
|||
taosMemoryFreeClear(msg->resInfo.pCol);
|
||||
taosMemoryFreeClear(msg->resInfo.length);
|
||||
taosMemoryFreeClear(msg->resInfo.convertBuf);
|
||||
taosMemoryFreeClear(msg->resInfo.convertJson);
|
||||
}
|
||||
setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4, false);
|
||||
return &msg->resInfo;
|
||||
|
|
|
@ -212,6 +212,7 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
|
|||
taosMemoryFreeClear(pResInfo->pCol);
|
||||
taosMemoryFreeClear(pResInfo->fields);
|
||||
taosMemoryFreeClear(pResInfo->userFields);
|
||||
taosMemoryFreeClear(pResInfo->convertJson);
|
||||
|
||||
if (pResInfo->convertBuf != NULL) {
|
||||
for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
|
||||
|
|
|
@ -1339,67 +1339,6 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
|
|||
}
|
||||
}
|
||||
|
||||
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
|
||||
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
|
||||
} else if (type == TSDB_DATA_TYPE_JSON && colLength[i] > 0) {
|
||||
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
|
||||
if (p == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
pResultInfo->convertBuf[i] = p;
|
||||
int32_t len = 0;
|
||||
SResultColumn* pCol = &pResultInfo->pCol[i];
|
||||
for (int32_t j = 0; j < numOfRows; ++j) {
|
||||
if (pCol->offset[j] != -1) {
|
||||
char* pStart = pCol->offset[j] + pCol->pData;
|
||||
|
||||
int32_t jsonInnerType = *pStart;
|
||||
char* jsonInnerData = pStart + CHAR_BYTES;
|
||||
char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
|
||||
if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
|
||||
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
|
||||
varDataSetLen(dst, strlen(varDataVal(dst)));
|
||||
} else if (jsonInnerType == TD_TAG_JSON) {
|
||||
char* jsonString = parseTagDatatoJson(pStart);
|
||||
STR_TO_VARSTR(dst, jsonString);
|
||||
taosMemoryFree(jsonString);
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
|
||||
*(char*)varDataVal(dst) = '\"';
|
||||
int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
|
||||
varDataVal(dst) + CHAR_BYTES);
|
||||
if (length <= 0) {
|
||||
tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset);
|
||||
length = 0;
|
||||
}
|
||||
varDataSetLen(dst, length + CHAR_BYTES * 2);
|
||||
*(char*)POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES) = '\"';
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
|
||||
double jsonVd = *(double*)(jsonInnerData);
|
||||
sprintf(varDataVal(dst), "%.9lf", jsonVd);
|
||||
varDataSetLen(dst, strlen(varDataVal(dst)));
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
|
||||
sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
|
||||
varDataSetLen(dst, strlen(varDataVal(dst)));
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
|
||||
if (len + varDataTLen(dst) > colLength[i]) {
|
||||
p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst));
|
||||
if (p == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
pResultInfo->convertBuf[i] = p;
|
||||
}
|
||||
p = pResultInfo->convertBuf[i] + len;
|
||||
memcpy(p, dst, varDataTLen(dst));
|
||||
pCol->offset[j] = len;
|
||||
len += varDataTLen(dst);
|
||||
}
|
||||
}
|
||||
|
||||
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
|
||||
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
|
||||
}
|
||||
|
@ -1408,6 +1347,171 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows){
|
||||
char* p = (char*)pResultInfo->pData;
|
||||
|
||||
int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
|
||||
int32_t* colLength = (int32_t*)(p + len);
|
||||
len += sizeof(int32_t) * numOfCols;
|
||||
|
||||
char* pStart = p + len;
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
int32_t colLen = htonl(colLength[i]);
|
||||
|
||||
if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
|
||||
int32_t* offset = (int32_t*)pStart;
|
||||
int32_t lenTmp = numOfRows * sizeof(int32_t);
|
||||
len += lenTmp;
|
||||
pStart += lenTmp;
|
||||
|
||||
for (int32_t j = 0; j < numOfRows; ++j) {
|
||||
if (offset[j] == -1) {
|
||||
continue;
|
||||
}
|
||||
char* data = offset[j] + pStart;
|
||||
|
||||
int32_t jsonInnerType = *data;
|
||||
char* jsonInnerData = data + CHAR_BYTES;
|
||||
if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
|
||||
len += (VARSTR_HEADER_SIZE + strlen(TSDB_DATA_NULL_STR_L));
|
||||
} else if (jsonInnerType & TD_TAG_JSON) {
|
||||
len += (VARSTR_HEADER_SIZE + ((const STag*)(data))->len);
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
|
||||
len += varDataTLen(jsonInnerData) + CHAR_BYTES * 2;
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
|
||||
len += (VARSTR_HEADER_SIZE + 32);
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
|
||||
len += (VARSTR_HEADER_SIZE + 5);
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
|
||||
}
|
||||
} else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
|
||||
int32_t lenTmp = numOfRows * sizeof(int32_t);
|
||||
len += (lenTmp + colLen);
|
||||
pStart += lenTmp;
|
||||
} else {
|
||||
int32_t lenTmp = BitmapLen(pResultInfo->numOfRows);
|
||||
len += (lenTmp + colLen);
|
||||
pStart += lenTmp;
|
||||
}
|
||||
pStart += colLen;
|
||||
}
|
||||
return len;
|
||||
}
|
||||
|
||||
static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) {
|
||||
bool needConvert = false;
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
|
||||
needConvert = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(!needConvert) return TSDB_CODE_SUCCESS;
|
||||
|
||||
char* p = (char*)pResultInfo->pData;
|
||||
int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows);
|
||||
|
||||
pResultInfo->convertJson = taosMemoryCalloc(1, dataLen);
|
||||
if(pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
|
||||
char* p1 = pResultInfo->convertJson;
|
||||
|
||||
int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
|
||||
memcpy(p1, p, len);
|
||||
|
||||
p += len;
|
||||
p1 += len;
|
||||
|
||||
len = sizeof(int32_t) * numOfCols;
|
||||
int32_t* colLength = (int32_t*)p;
|
||||
int32_t* colLength1 = (int32_t*)p1;
|
||||
memcpy(p1, p, len);
|
||||
p += len;
|
||||
p1 += len;
|
||||
|
||||
char* pStart = p;
|
||||
char* pStart1 = p1;
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
int32_t colLen = htonl(colLength[i]);
|
||||
int32_t colLen1 = htonl(colLength1[i]);
|
||||
ASSERT(colLen < dataLen);
|
||||
|
||||
if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
|
||||
int32_t* offset = (int32_t*)pStart;
|
||||
int32_t* offset1 = (int32_t*)pStart1;
|
||||
len = numOfRows * sizeof(int32_t);
|
||||
memcpy(pStart1, pStart, len);
|
||||
pStart += len;
|
||||
pStart1 += len;
|
||||
|
||||
len = 0;
|
||||
for (int32_t j = 0; j < numOfRows; ++j) {
|
||||
if (offset[j] == -1) {
|
||||
continue;
|
||||
}
|
||||
char* data = offset[j] + pStart;
|
||||
|
||||
int32_t jsonInnerType = *data;
|
||||
char* jsonInnerData = data + CHAR_BYTES;
|
||||
char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
|
||||
if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
|
||||
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
|
||||
varDataSetLen(dst, strlen(varDataVal(dst)));
|
||||
} else if (jsonInnerType & TD_TAG_JSON) {
|
||||
char* jsonString = parseTagDatatoJson(data);
|
||||
STR_TO_VARSTR(dst, jsonString);
|
||||
taosMemoryFree(jsonString);
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
|
||||
*(char*)varDataVal(dst) = '\"';
|
||||
int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
|
||||
varDataVal(dst) + CHAR_BYTES);
|
||||
if (length <= 0) {
|
||||
tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset);
|
||||
length = 0;
|
||||
}
|
||||
varDataSetLen(dst, length + CHAR_BYTES * 2);
|
||||
*(char*)POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES) = '\"';
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
|
||||
double jsonVd = *(double*)(jsonInnerData);
|
||||
sprintf(varDataVal(dst), "%.9lf", jsonVd);
|
||||
varDataSetLen(dst, strlen(varDataVal(dst)));
|
||||
} else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
|
||||
sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
|
||||
varDataSetLen(dst, strlen(varDataVal(dst)));
|
||||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
|
||||
offset1[j]= len;
|
||||
memcpy(pStart1 + len, dst, varDataTLen(dst));
|
||||
len += varDataTLen(dst);
|
||||
}
|
||||
colLen1 = len;
|
||||
colLength1[i] = htonl(len);
|
||||
} else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
|
||||
len = numOfRows * sizeof(int32_t);
|
||||
memcpy(pStart1, pStart, len);
|
||||
pStart += len;
|
||||
pStart1 += len;
|
||||
memcpy(pStart1, pStart, colLen);
|
||||
} else {
|
||||
len = BitmapLen(pResultInfo->numOfRows);
|
||||
memcpy(pStart1, pStart, len);
|
||||
pStart += len;
|
||||
pStart1 += len;
|
||||
memcpy(pStart1, pStart, colLen);
|
||||
|
||||
}
|
||||
pStart += colLen;
|
||||
pStart1 += colLen1;
|
||||
}
|
||||
|
||||
pResultInfo->pData = pResultInfo->convertJson;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows,
|
||||
bool convertUcs4) {
|
||||
assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL);
|
||||
|
@ -1419,6 +1523,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
|
|||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
code = doConvertJson(pResultInfo, numOfCols, numOfRows);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
|
||||
char* p = (char*)pResultInfo->pData;
|
||||
|
||||
|
@ -1462,8 +1570,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
|
|||
pStart += colLength[i];
|
||||
}
|
||||
|
||||
// convert UCS4-LE encoded character to native multi-bytes character in current data block.
|
||||
if (convertUcs4) {
|
||||
if(convertUcs4){
|
||||
code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength);
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue