xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

opti:logic in parse raw line & make log more specific

This commit is contained in:
wangmm0220 2023-12-18 15:16:33 +08:00
parent 93f3a3135f
commit 7f8479f953
3 changed files with 56 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
source/client/inc/clientSml.h
View File

@ -171,7 +171,6 @@ typedef struct {
int8_t precision;
bool reRun;
bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol)
bool isRawLine;
int32_t ttl;
int32_t uid; // used for automatic create child table

69
source/client/src/clientSml.c
View File

@ -1640,6 +1640,37 @@ int32_t smlClearForRerun(SSmlHandle *info) {
return TSDB_CODE_SUCCESS;
}
static bool getLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd,
int numLines, int i, char** tmp, int *len){
if (lines) {
*tmp = lines[i];
*len = strlen(*tmp);
} else if (rawLine) {
*tmp = rawLine;
while (rawLine < rawLineEnd) {
if (*(rawLine++) == '\n') {
break;
}
(*len)++;
}
if (info->protocol == TSDB_SML_LINE_PROTOCOL && (*tmp)[0] == '#') { // this line is comment
false;
}
}
if(rawLine != NULL){
char* print = taosMemoryCalloc(*len + 1, 1);
memcpy(print, *tmp, *len);
uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id,
numLines, info->protocol, *len, print);
taosMemoryFree(print);
}else{
uDebug("SML:0x%" PRIx64 " smlParseLine is not numLines:%d, protocol:%d, len:%d, data:%s", info->id,
numLines, info->protocol, *len, *tmp);
}
return true;
}
static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) {
uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id);
int32_t code = TSDB_CODE_SUCCESS;
@ -1661,25 +1692,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
while (i < numLines) {
char *tmp = NULL;
int len = 0;
if (lines) {
tmp = lines[i];
len = strlen(tmp);
} else if (rawLine) {
tmp = rawLine;
while (rawLine < rawLineEnd) {
if (*(rawLine++) == '\n') {
break;
}
len++;
}
if (info->protocol == TSDB_SML_LINE_PROTOCOL && tmp[0] == '#') { // this line is comment
continue;
}
if(!getLine(info, lines, rawLine, rawLineEnd, numLines, i, &tmp, &len)){
continue;
}
uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id,
info->isRawLine, numLines, info->protocol, len, info->isRawLine ? "rawdata" : tmp);
if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
if (info->dataFormat) {
SSmlLineInfo element = {0};
@ -1699,7 +1714,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE;
}
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, info->isRawLine ? "rawdata" : tmp);
if(rawLine != NULL){
char* print = taosMemoryCalloc(len + 1, 1);
memcpy(print, tmp, len);
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print);
taosMemoryFree(print);
}else{
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp);
}
return code;
}
if (info->reRun) {
@ -1828,7 +1850,6 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
return (TAOS_RES *)request;
}
info->pRequest = request;
info->isRawLine = rawLine != NULL;
info->ttl = ttl;
info->precision = precision;
info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol;
@ -1934,8 +1955,7 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi
reqid);
}
TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
int precision, int32_t ttl, int64_t reqid) {
static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol){
int numLines = 0;
*totalRows = 0;
char *tmp = lines;
@ -1948,6 +1968,11 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int
tmp = lines + i + 1;
}
}
}
TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
int precision, int32_t ttl, int64_t reqid) {
getRawLineLen(lines, len, totalRows, protocol);
return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid);
}

10
source/libs/parser/src/parInsertSml.c
View File

@ -210,7 +210,15 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
SSmlKv* kv = (SSmlKv*)data;
if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){
ret = TSDB_CODE_SML_INVALID_DATA;
uInfo("SML smlBuildCol error col not same %s", pColSchema->name);
char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1);
if(tmp){
memcpy(tmp, kv->key, kv->keyLen);
uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)",
tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name);
taosMemoryFree(tmp);
}else{
uError("SML smlBuildCol out of memory");
}
goto end;
}
if (kv->type == TSDB_DATA_TYPE_NCHAR) {