From fc905406d52d79266f27fbf0a9d0e96f6c260994 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 14:08:24 +0800 Subject: [PATCH 1/7] chore: validate tag val --- source/common/src/tdataformat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f97f9c0c11..5eebf05596 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1247,6 +1247,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (isJson) { n += tPutCStr(p ? p + n : p, pTagVal->pKey); } else { + ASSERTS(pTagVal->cid > 0, "Invalid tag cid %" PRIi16, pTagVal->cid); n += tPutI16v(p ? p + n : p, pTagVal->cid); } From 7d8864ec89ff4f18ff31aba677935500f8300349 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 14:22:49 +0800 Subject: [PATCH 2/7] chore: validate tag val --- source/common/src/tdataformat.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 5eebf05596..fe5374d5da 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1247,8 +1247,8 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (isJson) { n += tPutCStr(p ? p + n : p, pTagVal->pKey); } else { - ASSERTS(pTagVal->cid > 0, "Invalid tag cid %" PRIi16, pTagVal->cid); n += tPutI16v(p ? p + n : p, pTagVal->cid); + ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type @@ -1273,6 +1273,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tGetCStr(p + n, &pTagVal->pKey); } else { n += tGetI16v(p + n, &pTagVal->cid); + ASSERTS(pTagVal->cid < 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type From 4cc22a1b90b8bc2860d8694361f05c7444d76189 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 14:26:34 +0800 Subject: [PATCH 3/7] chore: validate tag val --- source/common/src/tdataformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index fe5374d5da..cf0dd9ecdb 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1273,7 +1273,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tGetCStr(p + n, &pTagVal->pKey); } else { n += tGetI16v(p + n, &pTagVal->cid); - ASSERTS(pTagVal->cid < 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); + ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type From 210a304cc123df9ce01266f9871f3028fba078f0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 4 Dec 2023 14:41:28 +0800 Subject: [PATCH 4/7] Update tdataformat.c --- source/common/src/tdataformat.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index cf0dd9ecdb..02dfbfebfe 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1273,7 +1273,6 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tGetCStr(p + n, &pTagVal->pKey); } else { n += tGetI16v(p + n, &pTagVal->cid); - ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type From 08be3d790cc906727f2c4a3bfa250b37664d4664 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 20:08:46 +0800 Subject: [PATCH 5/7] fix: ignore NULL tag --- source/libs/parser/src/parInsertSql.c | 7 +++++-- tests/script/tsim/parser/columnValue_bigint.sim | 2 ++ tests/script/tsim/parser/columnValue_bool.sim | 2 ++ tests/script/tsim/parser/columnValue_float.sim | 2 ++ tests/script/tsim/parser/columnValue_int.sim | 14 ++++++++++++++ tests/script/tsim/parser/columnValue_smallint.sim | 2 ++ tests/script/tsim/parser/columnValue_tinyint.sim | 2 ++ tests/script/tsim/parser/null_char.sim | 2 ++ 8 files changed, 31 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 9347f94e8d..a3b6f01e3a 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -700,7 +700,8 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SSchema* pTagSchema, SToken* pToken, SArray* pTagName, SArray* pTagVals, STag** pTag) { - if (!isNullValue(pTagSchema->type, pToken)) { + bool isNull = isNullValue(pTagSchema->type, pToken); + if (!isNull) { taosArrayPush(pTagName, pTagSchema->name); } @@ -709,13 +710,15 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - if (isNullValue(pTagSchema->type, pToken)) { + if (isNull) { return tTagNew(pTagVals, 1, true, pTag); } else { return parseJsontoTagData(pToken->z, pTagVals, pTag, &pCxt->msg); } } + if (isNull) return 0; + STagVal val = {0}; int32_t code = parseTagToken(ppSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg); diff --git a/tests/script/tsim/parser/columnValue_bigint.sim b/tests/script/tsim/parser/columnValue_bigint.sim index 0a024029a5..056855eea2 100644 --- a/tests/script/tsim/parser/columnValue_bigint.sim +++ b/tests/script/tsim/parser/columnValue_bigint.sim @@ -17,6 +17,7 @@ sql create table mt_bigint (ts timestamp, c bigint) tags (tagname bigint) ## case 00: static create table for test tag values sql create table st_bigint_0 using mt_bigint tags (NULL) +sql show create table st_bigint_0 sql show tags from st_bigint_0 if $data05 != NULL then return -1 @@ -173,6 +174,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_bigint_16 using mt_bigint tags (NULL) values (now, NULL) +sql show create table st_bigint_16 sql show tags from st_bigint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_bool.sim b/tests/script/tsim/parser/columnValue_bool.sim index ad4232d884..2553e6805a 100644 --- a/tests/script/tsim/parser/columnValue_bool.sim +++ b/tests/script/tsim/parser/columnValue_bool.sim @@ -17,6 +17,7 @@ sql create table mt_bool (ts timestamp, c bool) tags (tagname bool) ## case 00: static create table for test tag values sql create table st_bool_0 using mt_bool tags (NULL) +sql show create table st_bool_0 sql show tags from st_bool_0 if $data05 != NULL then print ==1== expect: NULL, actually: $data05 @@ -291,6 +292,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_bool_16 using mt_bool tags (NULL) values (now, NULL) +sql show create table st_bool_16 sql show tags from st_bool_16 if $data05 != NULL then print ==33== expect: NULL, actually: $data00 diff --git a/tests/script/tsim/parser/columnValue_float.sim b/tests/script/tsim/parser/columnValue_float.sim index b2db7dff2b..4dcda33224 100644 --- a/tests/script/tsim/parser/columnValue_float.sim +++ b/tests/script/tsim/parser/columnValue_float.sim @@ -17,6 +17,7 @@ sql create table mt_float (ts timestamp, c float) tags (tagname float) ## case 00: static create table for test tag values sql create table st_float_0 using mt_float tags (NULL) +sql show create table st_float_0 sql show tags from st_float_0 if $data05 != NULL then return -1 @@ -299,6 +300,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_float_16 using mt_float tags (NULL) values (now, NULL) +sql show create table st_float_16 sql show tags from st_float_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 4a3b8ebd0b..9f0fb6e294 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -17,6 +17,7 @@ sql create table mt_int (ts timestamp, c int) tags (tagname int) ## case 00: static create table for test tag values sql create table st_int_0 using mt_int tags (NULL) +sql show create table st_int_0 sql show tags from st_int_0 if $data05 != NULL then return -1 @@ -27,6 +28,18 @@ if $data05 != NULL then return -1 endi +sql insert into mt_int(tbname,tagname,ts,c) values("st_int_00",NULL,now,1) values("st_int_01", NULL, now,2) +sql show create table st_int_00 +sql show create table st_int_01 +sql show tags from st_int_00 +if $data05 != NULL then + return -1 +endi +sql show tags from st_int_01 +if $data05 != NULL then + return -1 +endi + sql_error create table st_int_2 using mt_int tags ('NULL') sql_error create table st_int_3 using mt_int tags ('NULL') sql_error create table st_int_4 using mt_int tags ("NULL") @@ -172,6 +185,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_int_16 using mt_int tags (NULL) values (now, NULL) +sql show create table st_int_16 sql show tags from st_int_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_smallint.sim b/tests/script/tsim/parser/columnValue_smallint.sim index eb364f3630..f9be6ebd52 100644 --- a/tests/script/tsim/parser/columnValue_smallint.sim +++ b/tests/script/tsim/parser/columnValue_smallint.sim @@ -20,6 +20,7 @@ sql create table mt_smallint (ts timestamp, c smallint) tags (tagname smallint) ## case 00: static create table for test tag values sql create table st_smallint_0 using mt_smallint tags (NULL) +sql show create table st_smallint_0 sql show tags from st_smallint_0 if $data05 != NULL then return -1 @@ -175,6 +176,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_smallint_16 using mt_smallint tags (NULL) values (now, NULL) +sql show create table st_smallint_16 sql show tags from st_smallint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_tinyint.sim b/tests/script/tsim/parser/columnValue_tinyint.sim index d7938aa739..7d0f10a30d 100644 --- a/tests/script/tsim/parser/columnValue_tinyint.sim +++ b/tests/script/tsim/parser/columnValue_tinyint.sim @@ -17,6 +17,7 @@ sql create table mt_tinyint (ts timestamp, c tinyint) tags (tagname tinyint) ## case 00: static create table for test tag values sql create table st_tinyint_0 using mt_tinyint tags (NULL) +sql show create table st_tinyint_0 sql show tags from st_tinyint_0 if $data05 != NULL then print expect NULL, actually: $data05 @@ -173,6 +174,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_tinyint_16 using mt_tinyint tags (NULL) values (now, NULL) +sql show create table st_tinyint_16 sql show tags from st_tinyint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/null_char.sim b/tests/script/tsim/parser/null_char.sim index 098f4061c1..9d476b5227 100644 --- a/tests/script/tsim/parser/null_char.sim +++ b/tests/script/tsim/parser/null_char.sim @@ -13,6 +13,7 @@ sql use $db #### case 0: field NULL, or 'NULL' sql create table mt1 (ts timestamp, col1 int, col2 bigint, col3 float, col4 double, col5 binary(8), col6 bool, col7 smallint, col8 tinyint, col9 nchar(8)) tags (tag1 binary(8), tag2 nchar(8), tag3 int, tag4 bigint, tag5 bool, tag6 float) sql create table st1 using mt1 tags (NULL, 'NULL', 100, 1000, 'false', 9.123) +sql show create table st1 sql insert into st1 values ('2019-01-01 09:00:00.000', 123, -123, 3.0, 4.0, 'binary', true, 1000, 121, 'nchar') sql insert into st1 values ('2019-01-01 09:00:01.000', '456', '456', '3.33', '4.444', 'binary', 'true', '1001', '122', 'nchar') sql insert into st1 values ('2019-01-01 09:00:02.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) @@ -117,6 +118,7 @@ if $rows != 0 then endi sql create table st3 using mt2 tags (NULL, 'ABC', 103, 'FALSE') +sql show create table st3 sql insert into st3 (ts, col1) values(now, 1) sql select tag1, tag2, tag3, tag5 from st3 if $rows != 1 then From f3a075ee9e5b897c949fba69e3c2d2f46ff65264 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 21:00:13 +0800 Subject: [PATCH 6/7] test: check null json tag --- tests/system-test/2-query/json_tag.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index ebd580efd4..8db92f38a2 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -394,9 +394,12 @@ class TDTestCase: tdSql.execute(f"create table if not exists {dbname}.jsons3(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") tdSql.execute(f"insert into {dbname}.jsons3_1 using {dbname}.jsons3 tags('{{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}}') values(1591060618000, 3, false, 'json3', '你是3')") tdSql.execute(f"insert into {dbname}.jsons3_2 using {dbname}.jsons3 tags('{{\"tag1\":5,\"tag2\":\"beijing\"}}') values (1591060638000, 2, true, 'json3', 'sss')") + tdSql.execute(f"insert into {dbname}.jsons3_3 using {dbname}.jsons3 tags(NULL) values (1591060638000, 2, true, 'json3', 'sss')") tdSql.query(f"select 'sss',33,a.jtag->'tag3' from {dbname}.jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") tdSql.checkData(0, 0, "sss") tdSql.checkData(0, 2, "true") + tdSql.query(f"show create table jsons3_3") + tdSql.checkNotEqual(tdSql.queryResult[0][1].find("TAGS (null)"), 0) res = tdSql.getColNameList(f"select 'sss',33,a.jtag->'tag3' from {dbname}.jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") cname_list = [] From 646d807a28acd2549c0b6fd731d05e3be7f30e93 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 22:43:04 +0800 Subject: [PATCH 7/7] test: check null tag when insert by stb --- tests/script/tsim/insert/insert_stb.sim | 23 ++++++++++++++++++++ tests/script/tsim/parser/columnValue_int.sim | 12 ---------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim index 4c0797e2a7..14ff2261c2 100644 --- a/tests/script/tsim/insert/insert_stb.sim +++ b/tests/script/tsim/insert/insert_stb.sim @@ -66,6 +66,29 @@ if $rows != 1 then return -1 endi +sql insert into st(tbname, ts, f, t) values('ct2',now,20,NULL)('ct3',now,30,NULL) +sql insert into st(tbname, t, ts, f) values('ct4',NULL, now,20)('ct5',NULL, now,30) +sql show create table ct2 +sql show create table ct3 +sql show create table ct4 +sql show create table ct5 +sql show tags from ct2 +if $data05 != NULL then + return -1 +endi +sql show tags from ct3 +if $data05 != NULL then + return -1 +endi +sql show tags from ct4 +if $data05 != NULL then + return -1 +endi +sql show tags from ct5 +if $data05 != NULL then + return -1 +endi + sql_error insert into d2.st values(now, 1, 1) sql_error insert into d2.st(ts, f) values(now, 1); sql_error insert into d2.st(ts, f, tbname) values(now, 1); diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 9f0fb6e294..e68ae6f13f 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -28,18 +28,6 @@ if $data05 != NULL then return -1 endi -sql insert into mt_int(tbname,tagname,ts,c) values("st_int_00",NULL,now,1) values("st_int_01", NULL, now,2) -sql show create table st_int_00 -sql show create table st_int_01 -sql show tags from st_int_00 -if $data05 != NULL then - return -1 -endi -sql show tags from st_int_01 -if $data05 != NULL then - return -1 -endi - sql_error create table st_int_2 using mt_int tags ('NULL') sql_error create table st_int_3 using mt_int tags ('NULL') sql_error create table st_int_4 using mt_int tags ("NULL")