xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fea:add select json logic

This commit is contained in:
wangmm0220 2022-05-18 22:13:38 +08:00
parent 2393e53602
commit e818e27902
5 changed files with 184 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/util/tdef.h
View File

@ -232,7 +232,7 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAGS 128
#define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_MAX_TAG_CONDITIONS 1024
#define TSDB_MAX_JSON_TAG_LEN 16384 #define TSDB_MAX_JSON_TAG_LEN (16384 + VARSTR_HEADER_SIZE)
#define TSDB_AUTH_LEN 16 #define TSDB_AUTH_LEN 16
#define TSDB_PASSWORD_LEN 32 #define TSDB_PASSWORD_LEN 32

192
source/client/src/clientImpl.c
View File

@ -268,7 +268,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR) { if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR) {
pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE; pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE;
} else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
} }
@ -745,6 +745,105 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
#include "cJSON.h"
static char* parseTagDatatoJson(void *p){
char* string = NULL;
cJSON *json = cJSON_CreateObject();
if (json == NULL)
{
goto end;
}
int16_t nCols = kvRowNCols(p);
char tagJsonKey[256] = {0};
for (int j = 0; j < nCols; ++j) {
SColIdx * pColIdx = kvRowColIdxAt(p, j);
void* val = (kvRowColVal(p, pColIdx));
if (j == 0){
if(*(char*)val == TSDB_DATA_TYPE_NULL){
string = taosMemoryCalloc(1, 8);
sprintf(varDataVal(string), "%s", TSDB_DATA_NULL_STR_L);
varDataSetLen(string, strlen(varDataVal(string)));
goto end;
}
continue;
}
// json key encode by binary
memset(tagJsonKey, 0, sizeof(tagJsonKey));
memcpy(tagJsonKey, varDataVal(val), varDataLen(val));
// json value
val += varDataTLen(val);
char* realData = POINTER_SHIFT(val, CHAR_BYTES);
char type = *(char*)val;
if(type == TSDB_DATA_TYPE_NULL) {
cJSON* value = cJSON_CreateNull();
if (value == NULL)
{
goto end;
}
cJSON_AddItemToObject(json, tagJsonKey, value);
}else if(type == TSDB_DATA_TYPE_NCHAR) {
cJSON* value = NULL;
if (varDataLen(realData) > 0){
char *tagJsonValue = taosMemoryCalloc(varDataLen(realData), 1);
int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(realData), varDataLen(realData), tagJsonValue);
if (length < 0) {
tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
(char*)val);
taosMemoryFree(tagJsonValue);
goto end;
}
value = cJSON_CreateString(tagJsonValue);
taosMemoryFree(tagJsonValue);
if (value == NULL)
{
goto end;
}
}else if(varDataLen(realData) == 0){
value = cJSON_CreateString("");
}else{
ASSERT(0);
}
cJSON_AddItemToObject(json, tagJsonKey, value);
}else if(type == TSDB_DATA_TYPE_DOUBLE){
double jsonVd = *(double*)(realData);
cJSON* value = cJSON_CreateNumber(jsonVd);
if (value == NULL)
{
goto end;
}
cJSON_AddItemToObject(json, tagJsonKey, value);
}else if(type == TSDB_DATA_TYPE_BIGINT){
int64_t jsonVd = *(int64_t*)(realData);
cJSON* value = cJSON_CreateNumber((double)jsonVd);
if (value == NULL)
{
goto end;
}
cJSON_AddItemToObject(json, tagJsonKey, value);
}else if (type == TSDB_DATA_TYPE_BOOL) {
char jsonVd = *(char*)(realData);
cJSON* value = cJSON_CreateBool(jsonVd);
if (value == NULL)
{
goto end;
}
cJSON_AddItemToObject(json, tagJsonKey, value);
}
else{
tscError("unsupportted json value");
}
}
string = cJSON_PrintUnformatted(json);
end:
cJSON_Delete(json);
return string;
}
#include "tdataformat.h"
static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) {
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
int32_t type = pResultInfo->fields[i].type; int32_t type = pResultInfo->fields[i].type;
@ -775,9 +874,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
pResultInfo->row[i] = pResultInfo->pCol[i].pData; pResultInfo->row[i] = pResultInfo->pCol[i].pData;
} }else if (type == TSDB_DATA_TYPE_JSON && colLength[i] > 0) {
if (type == TSDB_DATA_TYPE_JSON) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
if (p == NULL) { if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -789,48 +886,51 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t j = 0; j < numOfRows; ++j) {
if (pCol->offset[j] != -1) { if (pCol->offset[j] != -1) {
char* pStart = pCol->offset[j] + pCol->pData; char* pStart = pCol->offset[j] + pCol->pData;
char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
int32_t jsonInnerType = *pStart; char *jsonString = parseTagDatatoJson(pStart);
char* jsonInnerData = pStart + CHAR_BYTES; STR_TO_VARSTR(dst, jsonString);
char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; taosMemoryFree(jsonString);
if (jsonInnerType == TSDB_DATA_TYPE_NULL) { // int32_t jsonInnerType = *pStart;
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); // char* jsonInnerData = pStart + CHAR_BYTES;
varDataSetLen(dst, strlen(varDataVal(dst))); // char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
} else if (jsonInnerType == TSDB_DATA_TYPE_JSON) { // if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
int32_t length = // sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst)); // varDataSetLen(dst, strlen(varDataVal(dst)));
// } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) {
if (length <= 0) { // int32_t length =
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, // taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst));
varDataVal(jsonInnerData)); //
length = 0; // if (length <= 0) {
} // tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
varDataSetLen(dst, length); // varDataVal(jsonInnerData));
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" // length = 0;
*(char*)varDataVal(dst) = '\"'; // }
int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), // varDataSetLen(dst, length);
varDataVal(dst) + CHAR_BYTES); // } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
if (length <= 0) { // *(char*)varDataVal(dst) = '\"';
tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, // int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
varDataVal(jsonInnerData)); // varDataVal(dst) + CHAR_BYTES);
length = 0; // if (length <= 0) {
} // tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
varDataSetLen(dst, length + CHAR_BYTES * 2); // varDataVal(jsonInnerData));
*(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"'; // length = 0;
} else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { // }
double jsonVd = *(double*)(jsonInnerData); // varDataSetLen(dst, length + CHAR_BYTES * 2);
sprintf(varDataVal(dst), "%.9lf", jsonVd); // *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
varDataSetLen(dst, strlen(varDataVal(dst))); // } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
} else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) { // double jsonVd = *(double*)(jsonInnerData);
int64_t jsonVd = *(int64_t*)(jsonInnerData); // sprintf(varDataVal(dst), "%.9lf", jsonVd);
sprintf(varDataVal(dst), "%" PRId64, jsonVd); // varDataSetLen(dst, strlen(varDataVal(dst)));
varDataSetLen(dst, strlen(varDataVal(dst))); // } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) {
} else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { // int64_t jsonVd = *(int64_t*)(jsonInnerData);
sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); // sprintf(varDataVal(dst), "%" PRId64, jsonVd);
varDataSetLen(dst, strlen(varDataVal(dst))); // varDataSetLen(dst, strlen(varDataVal(dst)));
} else { // } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
ASSERT(0); // sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
} // varDataSetLen(dst, strlen(varDataVal(dst)));
// } else {
// ASSERT(0);
// }
if (len + varDataTLen(dst) > colLength[i]) { if (len + varDataTLen(dst) > colLength[i]) {
p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst)); p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst));

25
source/common/src/tdatablock.c
View File

@ -116,18 +116,21 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
int32_t type = pColumnInfoData->info.type; int32_t type = pColumnInfoData->info.type;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
int32_t dataLen = varDataTLen(pData); int32_t dataLen = 0;
if (type == TSDB_DATA_TYPE_JSON) { if (type == TSDB_DATA_TYPE_JSON) {
if (*pData == TSDB_DATA_TYPE_NULL) { // if (*pData == TSDB_DATA_TYPE_NULL) {
dataLen = 0; // dataLen = 0;
} else if (*pData == TSDB_DATA_TYPE_NCHAR) { // } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
dataLen = varDataTLen(pData + CHAR_BYTES); // dataLen = varDataTLen(pData + CHAR_BYTES);
} else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { // } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
dataLen = LONG_BYTES; // dataLen = LONG_BYTES;
} else if (*pData == TSDB_DATA_TYPE_BOOL) { // } else if (*pData == TSDB_DATA_TYPE_BOOL) {
dataLen = CHAR_BYTES; // dataLen = CHAR_BYTES;
} // }
dataLen += CHAR_BYTES; // dataLen += CHAR_BYTES;
dataLen = kvRowLen(pData);
}else {
dataLen = varDataTLen(pData);
} }
SVarColAttr* pAttr = &pColumnInfoData->varmeta; SVarColAttr* pAttr = &pColumnInfoData->varmeta;

7
source/libs/executor/src/scanoperator.c
View File

@ -1467,7 +1467,12 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
STR_TO_VARSTR(str, mr.me.name); STR_TO_VARSTR(str, mr.me.name);
colDataAppend(pDst, count, str, false); colDataAppend(pDst, count, str, false);
} else { // it is a tag value } else { // it is a tag value
const char* p = metaGetTableTagVal(&mr.me, pExprInfo[j].base.pParam[0].pCol->colId); const char* p = NULL;
if(pDst->info.type == TSDB_DATA_TYPE_JSON){
p = mr.me.ctbEntry.pTags;
}else{
p = metaGetTableTagVal(&mr.me, pExprInfo[j].base.pParam[0].pCol->colId);
}
colDataAppend(pDst, count, p, (p == NULL)); colDataAppend(pDst, count, p, (p == NULL));
} }
} }

26
tools/shell/src/shellEngine.c
View File

@ -315,6 +315,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
break; break;
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:
memcpy(buf, val, length); memcpy(buf, val, length);
buf[length] = 0; buf[length] = 0;
taosFprintfFile(pFile, "\'%s\'", buf); taosFprintfFile(pFile, "\'%s\'", buf);
@ -384,19 +385,25 @@ void shellPrintNChar(const char *str, int32_t length, int32_t width) {
while (pos < length) { while (pos < length) {
TdWchar wc; TdWchar wc;
int32_t bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); int32_t bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
if (bytes == 0) { if (bytes <= 0) {
break;
}
pos += bytes;
if (pos > length) {
break; break;
} }
if (pos + bytes > length) {
break;
}
int w = 0;
#ifdef WINDOWS #ifdef WINDOWS
int32_t w = bytes; w = bytes;
#else #else
int32_t w = taosWcharWidth(wc); if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
w = bytes;
}else{
w = taosWcharWidth(wc);
}
#endif #endif
pos += bytes;
if (w <= 0) { if (w <= 0) {
continue; continue;
} }
@ -496,6 +503,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
break; break;
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:
shellPrintNChar(val, length, width); shellPrintNChar(val, length, width);
break; break;
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
@ -604,7 +612,6 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
return TMAX(25, width); return TMAX(25, width);
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
if (field->bytes > shell.args.displayWidth) { if (field->bytes > shell.args.displayWidth) {
return TMAX(shell.args.displayWidth, width); return TMAX(shell.args.displayWidth, width);
@ -612,7 +619,8 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
return TMAX(field->bytes, width); return TMAX(field->bytes, width);
} }
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON: {
int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
if (bytes > shell.args.displayWidth) { if (bytes > shell.args.displayWidth) {
return TMAX(shell.args.displayWidth, width); return TMAX(shell.args.displayWidth, width);