xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat:add new logic for new tag format

This commit is contained in:
wangmm0220 2022-05-31 20:18:02 +08:00
parent 816cea07f9
commit f89ce48e66
4 changed files with 60 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
source/libs/parser/src/parInsert.c
View File

@ -958,6 +958,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
SArray *pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal));
SToken sToken;
bool isParseBindParam = false;
bool isJson = false;
for (int i = 0; i < pCxt->tags.numOfBound; ++i) {
NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken);
@ -994,6 +995,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
if(code != TSDB_CODE_SUCCESS){
goto end;
}
isJson = true;
}else{
STagVal val = {0};
code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, &val, &pCxt->msg);
@ -1014,7 +1016,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
}
STag* pTag = NULL;
code = tTagNew(pTagVals, 1, false, &pTag);
code = tTagNew(pTagVals, 1, isJson, &pTag);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
@ -1404,7 +1406,6 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
.pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK),
.pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK),
.totalNum = 0,
.pTagVals = NULL,
.pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT),
.pStmtCb = pContext->pStmtCb};
@ -1526,6 +1527,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN
int32_t code = TSDB_CODE_SUCCESS;
SSchema* pSchema = pDataBlock->pTableMeta->schema;
bool isJson = false;
for (int c = 0; c < tags->numOfBound; ++c) {
if (bind[c].is_null && bind[c].is_null[0]) {
continue;
@ -1537,42 +1539,56 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN
if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
colLen = bind[c].length[0];
}
STagVal val = {0};
if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){
val.pData = (uint8_t*)bind[c].buffer;
val.nData = colLen;
}else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){
int32_t output = 0;
void *p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE);
if(p == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
if (colLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
code = buildSyntaxErrMsg(&pBuf, "json string too long than 4095", bind[c].buffer);
goto end;
}
if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
if (errno == E2BIG) {
taosMemoryFree(p);
code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
isJson = true;
char *tmp = taosMemoryCalloc(1, colLen + 1);
memcpy(tmp, bind[c].buffer, colLen);
code = parseJsontoTagData(tmp, pTagArray, &pBuf);
taosMemoryFree(tmp);
if(code != TSDB_CODE_SUCCESS){
goto end;
}
}else{
STagVal val = {0};
if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){
val.pData = (uint8_t*)bind[c].buffer;
val.nData = colLen;
}else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){
int32_t output = 0;
void *p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE);
if(p == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
char buf[512] = {0};
snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
taosMemoryFree(p);
code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer);
goto end;
if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
if (errno == E2BIG) {
taosMemoryFree(p);
code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
goto end;
}
char buf[512] = {0};
snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
taosMemoryFree(p);
code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer);
goto end;
}
val.pData = p;
val.nData = output;
}else{
memcpy(&val.i64, bind[c].buffer, colLen);
}
val.pData = p;
val.nData = output;
}else{
memcpy(&val.i64, bind[c].buffer, colLen);
taosArrayPush(pTagArray, &val);
}
taosArrayPush(pTagArray, &val);
}
STag* pTag = NULL;
// TODO: stmt support json
if (0 != tTagNew(pTagArray, 1, false, &pTag)) {
if (0 != tTagNew(pTagArray, 1, isJson, &pTag)) {
code = buildInvalidOperationMsg(&pBuf, "out of memory");
goto end;
}

6
source/libs/parser/src/parTranslater.c
View File

@ -4235,7 +4235,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla
}
}
code = tTagNew(pTagArray, 1, false, ppTag);
code = tTagNew(pTagArray, 1, isJson, ppTag);
end:
if(isJson){
@ -4302,7 +4302,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
}
++index;
}
code = tTagNew(pTagArray, 1, false, ppTag);
code = tTagNew(pTagArray, 1, isJson, ppTag);
end:
if(isJson){
@ -4584,7 +4584,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
if (TSDB_CODE_SUCCESS != code) {
break;
}
code = tTagNew(pTagVals, 1, false, &pTag);
code = tTagNew(pTagVals, 1, true, &pTag);
}while(0);
for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
STagVal *p = (STagVal *)taosArrayGet(pTagVals, i);

2
source/libs/scalar/src/sclvector.c
View File

@ -951,7 +951,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
char *data = tTagValToData(value, true);
colDataAppend(pOutputCol, i, data, data == NULL);
if(value && IS_VAR_DATA_TYPE(value->type) && data){
taosMemoryFree(data)
taosMemoryFree(data);
}
}
}

40
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -1105,24 +1105,16 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
nodesDestroyNode(opNode);
}
#if 0
TEST(columnTest, json_column_arith_op) {
scltInitLogFile();
char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
char rightv[256] = {0};
memcpy(rightv, rightvTmp, strlen(rightvTmp));
SKVRowBuilder kvRowBuilder;
tdInitKVRowBuilder(&kvRowBuilder);
parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
char *tmp = (char *)taosMemoryRealloc(row, kvRowLen(row)+1);
if(tmp == NULL){
ASSERT_TRUE(0);
}
memmove(tmp+1, tmp, kvRowLen(tmp));
*tmp = TSDB_DATA_TYPE_JSON;
row = tmp;
SArray *tags = taosArrayInit(1, sizeof(STagVal));
parseJsontoTagData(rightv, tags, NULL);
STag* row = NULL;
tTagNew(tags, 1, true, &row);
const int32_t len = 8;
EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV,
@ -1176,10 +1168,10 @@ TEST(columnTest, json_column_arith_op) {
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]);
}
tdDestroyKVRowBuilder(&kvRowBuilder);
taosArrayDestroy(tags);
taosMemoryFree(row);
}
#endif
void *prepareNchar(char* rightData){
int32_t len = 0;
int32_t inputLen = strlen(rightData);
@ -1189,24 +1181,17 @@ void *prepareNchar(char* rightData){
varDataSetLen(t, len);
return t;
}
#if 0
TEST(columnTest, json_column_logic_op) {
scltInitLogFile();
char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}";
char rightv[256] = {0};
memcpy(rightv, rightvTmp, strlen(rightvTmp));
SKVRowBuilder kvRowBuilder;
tdInitKVRowBuilder(&kvRowBuilder);
parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
char *tmp = (char *)taosMemoryRealloc(row, kvRowLen(row)+1);
if(tmp == NULL){
ASSERT_TRUE(0);
}
memmove(tmp+1, tmp, kvRowLen(tmp));
*tmp = TSDB_DATA_TYPE_JSON;
row = tmp;
SArray *tags = taosArrayInit(1, sizeof(STagVal));
parseJsontoTagData(rightv, tags, NULL);
STag* row = NULL;
tTagNew(tags, 1, true, &row);
const int32_t len = 9;
const int32_t len1 = 4;
@ -1306,10 +1291,9 @@ TEST(columnTest, json_column_logic_op) {
taosMemoryFree(rightData);
}
tdDestroyKVRowBuilder(&kvRowBuilder);
taosArrayDestroy(tags);
taosMemoryFree(row);
}
#endif
TEST(columnTest, smallint_value_add_int_column) {
scltInitLogFile();