xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #19295 from taosdata/refact/submit_req_marks 

opti:json parse logic for schemaless
This commit is contained in:
WANG MINGMING 2022-12-30 18:45:08 +08:00 committed by GitHub
parent b1d4993a0e 3220ee9f1a
commit 28b0cf949c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 952 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
source/client/inc/clientSml.h
View File

@ -69,6 +69,7 @@ extern "C" {
#define VALUE "_value"
#define VALUE_LEN 6
#define OTD_JSON_FIELDS_NUM 4
#define MAX_RETRY_TIMES 5
typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
@ -177,9 +178,10 @@ typedef struct {
int32_t lineNum;
SSmlMsgBuf msgBuf;
// cJSON *root; // for parse json
int8_t offset[4];
cJSON *root; // for parse json
int8_t offset[OTD_JSON_FIELDS_NUM];
SSmlLineInfo *lines; // element is SSmlLineInfo
bool parseJsonByLib;
//
SArray *preLineTagKV;
@ -206,9 +208,9 @@ typedef int32_t (*_equal_fn_sml)(const void *, const void *);
SSmlHandle *smlBuildSmlInfo(TAOS *taos);
void smlDestroyInfo(SSmlHandle *info);
void smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
void smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
SArray *smlJsonParseTags(char *start, char *end);
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
//SArray *smlJsonParseTags(char *start, char *end);
bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
@ -226,6 +228,7 @@ int32_t is_same_child_table_telnet(const void *a, const void *b);
int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len);
int32_t smlClearForRerun(SSmlHandle *info);
int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg);
uint8_t smlGetTimestampLen(int64_t num);
int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);

15
source/client/src/clientSml.c
View File

@ -1008,12 +1008,16 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols
return TSDB_CODE_SUCCESS;
}
static void smlDestroyTableInfo(SSmlTableInfo *tag) {
static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) {
for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) {
SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i);
taosHashCleanup(kvHash);
}
if(info->parseJsonByLib){
SSmlLineInfo *key = (SSmlLineInfo *)(tag->key);
if(key != NULL) taosMemoryFree(key->tags);
}
taosMemoryFree(tag->key);
taosArrayDestroy(tag->cols);
taosArrayDestroy(tag->tags);
@ -1028,7 +1032,7 @@ void smlDestroyInfo(SSmlHandle *info) {
NodeList *tmp = info->childTables;
while (tmp) {
if (tmp->data.used) {
smlDestroyTableInfo((SSmlTableInfo *)tmp->data.value);
smlDestroyTableInfo(info, (SSmlTableInfo *)tmp->data.value);
}
NodeList *t = tmp->next;
taosMemoryFree(tmp);
@ -1055,6 +1059,9 @@ void smlDestroyInfo(SSmlHandle *info) {
if (!info->dataFormat) {
for (int i = 0; i < info->lineNum; i++) {
taosArrayDestroy(info->lines[i].colArray);
if(info->parseJsonByLib){
taosMemoryFree(info->lines[i].tags);
}
}
taosMemoryFree(info->lines);
}
@ -1251,7 +1258,7 @@ int32_t smlClearForRerun(SSmlHandle *info) {
NodeList *pList = info->childTables;
while (pList) {
if (pList->data.used) {
smlDestroyTableInfo((SSmlTableInfo *)pList->data.value);
smlDestroyTableInfo(info, (SSmlTableInfo *)pList->data.value);
pList->data.used = false;
}
pList = pList->next;
@ -1267,11 +1274,13 @@ int32_t smlClearForRerun(SSmlHandle *info) {
pList = pList->next;
}
if (!info->dataFormat){
if (unlikely(info->lines != NULL)) {
uError("SML:0x%" PRIx64 " info->lines != NULL", info->id);
return TSDB_CODE_SML_INVALID_DATA;
}
info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo));
}
memset(&info->preLine, 0, sizeof(SSmlLineInfo));
info->currSTableMeta = NULL;

1093
source/client/src/clientSmlJson.c
View File

File diff suppressed because it is too large Load Diff

44
source/client/test/smlTest.cpp
View File

@ -411,28 +411,28 @@ TEST(testCase, smlParseCols_Test) {
smlDestroyInfo(info);
}
//TEST(testCase, smlGetTimestampLen_Test) {
// uint8_t len = smlGetTimestampLen(0);
// ASSERT_EQ(len, 1);
//
// len = smlGetTimestampLen(1);
// ASSERT_EQ(len, 1);
//
// len = smlGetTimestampLen(10);
// ASSERT_EQ(len, 2);
//
// len = smlGetTimestampLen(390);
// ASSERT_EQ(len, 3);
//
// len = smlGetTimestampLen(-1);
// ASSERT_EQ(len, 1);
//
// len = smlGetTimestampLen(-10);
// ASSERT_EQ(len, 2);
//
// len = smlGetTimestampLen(-390);
// ASSERT_EQ(len, 3);
//}
TEST(testCase, smlGetTimestampLen_Test) {
uint8_t len = smlGetTimestampLen(0);
ASSERT_EQ(len, 1);
len = smlGetTimestampLen(1);
ASSERT_EQ(len, 1);
len = smlGetTimestampLen(10);
ASSERT_EQ(len, 2);
len = smlGetTimestampLen(390);
ASSERT_EQ(len, 3);
len = smlGetTimestampLen(-1);
ASSERT_EQ(len, 1);
len = smlGetTimestampLen(-10);
ASSERT_EQ(len, 2);
len = smlGetTimestampLen(-390);
ASSERT_EQ(len, 3);
}
TEST(testCase, smlParseNumber_Test) {
SSmlKv kv = {0};

5
tests/system-test/1-insert/opentsdb_json_taosc_insert.py
View File

@ -1719,7 +1719,6 @@ class TDTestCase:
print(err.errno)
def runAll(self):
"""
for value_type in ["obj", "default"]:
self.initCheckCase(value_type)
self.symbolsCheckCase(value_type)
@ -1759,7 +1758,7 @@ class TDTestCase:
self.batchErrorInsertCheckCase()
self.chineseCheckCase()
# self.spellCheckCase()
self.tbnameTagsColsNameCheckCase()
# self.tbnameTagsColsNameCheckCase()
# # MultiThreads
# self.sStbStbDdataInsertMultiThreadCheckCase()
# self.sStbStbDdataAtInsertMultiThreadCheckCase()
@ -1772,7 +1771,7 @@ class TDTestCase:
# self.sStbStbDdataDtsMtInsertMultiThreadCheckCase()
# self.sStbDtbDdataDtsMtInsertMultiThreadCheckCase()
# self.lengthIcreaseCrashCheckCase()
"""
def run(self):
print("running {}".format(__file__))
self.createDb()

13
tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py
View File

@ -243,7 +243,7 @@ class TDTestCase:
if t_add_tag is not None:
sql_seq = f'{stb_name} {ts} {value} t0={t0} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8} t9={t8}'
if id_change_tag is not None:
sql_seq = f'{stb_name} {ts} {value} t0={t0} {id}={tb_name} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}'
sql_seq = f'{stb_name} {ts} {value} {id}={tb_name} t0={t0} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}'
if id_double_tag is not None:
sql_seq = f'{stb_name} {ts} {value} {id}=\"{tb_name}_1\" t0={t0} t1={t1} {id}=\"{tb_name}_2\" t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}'
if t_add_tag is not None:
@ -1126,9 +1126,9 @@ class TDTestCase:
self._conn.schemaless_insert([input_sql], TDSmlProtocolType.TELNET.value, None)
query_sql = 'select * from `rFa$sta`'
query_res = tdSql.query(query_sql, True)
tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), 9.223372036854776e+18, '2147483647i32', 'L"ncharTagValue"', '32767i16', '9223372036854775807i64', '22.123456789f64', '"ddzhiksj"', '11.12345f32', 'true', '127Ii8')])
tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), 9.223372036854776e+18, 'true', '127Ii8', '32767i16', '2147483647i32', '9223372036854775807i64', '11.12345f32', '22.123456789f64', '"ddzhiksj"', 'L"ncharTagValue"')])
col_tag_res = tdSql.getColNameList(query_sql)
tdSql.checkEqual(col_tag_res, ['_ts', '_value', '"t$3"', 't!@#$%^&*()_+[];:<>?,9', 't#2', 't%4', 't&6', 't*7', 't^5', 'Tt!0', 'tT@1'])
tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'Tt!0', 'tT@1', 't#2', '"t$3"', 't%4', 't^5', 't&6', 't*7', 't!@#$%^&*()_+[];:<>?,9'])
tdSql.execute('drop table `rFa$sta`')
def tcpKeywordsCheckCase(self, protocol="telnet-tcp"):
@ -1207,7 +1207,6 @@ class TDTestCase:
tdLog.info(f'{sys._getframe().f_code.co_name}() function is running')
tdCom.cleanTb(dbname="test")
input_sql = self.genSqlList()[0]
print(input_sql)
self.multiThreadRun(self.genMultiThreadSeq(input_sql))
tdSql.query(f"show tables;")
tdSql.checkRows(5)
@ -1416,8 +1415,8 @@ class TDTestCase:
self.symbolsCheckCase()
self.tsCheckCase()
self.openTstbTelnetTsCheckCase()
#self.idSeqCheckCase()
#self.idLetterCheckCase()
# self.idSeqCheckCase()
self.idLetterCheckCase()
self.noIdCheckCase()
self.maxColTagCheckCase()
self.stbTbNameCheckCase()
@ -1450,7 +1449,7 @@ class TDTestCase:
self.spellCheckCase()
self.pointTransCheckCase()
self.defaultTypeCheckCase()
#self.tbnameTagsColsNameCheckCase()
self.tbnameTagsColsNameCheckCase()
# # # MultiThreads
# self.stbInsertMultiThreadCheckCase()
# self.sStbStbDdataInsertMultiThreadCheckCase()