xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #24113 from taosdata/mark/3.0 

opti:logic in parse raw line & make log more specific
This commit is contained in:
Haojun Liao 2023-12-20 15:25:26 +08:00 committed by GitHub
parent 667a8013b7 1f70f51ac8
commit a0d1933134
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 57 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
source/client/inc/clientSml.h
View File

@ -171,7 +171,6 @@ typedef struct {
int8_t precision;
bool reRun;
bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol)
bool isRawLine;
int32_t ttl;
int32_t uid; // used for automatic create child table

69
source/client/src/clientSml.c
View File

@ -1640,6 +1640,37 @@ int32_t smlClearForRerun(SSmlHandle *info) {
return TSDB_CODE_SUCCESS;
}
static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLineEnd,
int numLines, int i, char** tmp, int *len){
if (lines) {
*tmp = lines[i];
*len = strlen(*tmp);
} else if (*rawLine) {
*tmp = *rawLine;
while (*rawLine < rawLineEnd) {
if (*((*rawLine)++) == '\n') {
break;
}
(*len)++;
}
if (info->protocol == TSDB_SML_LINE_PROTOCOL && (*tmp)[0] == '#') { // this line is comment
return false;
}
}
if(*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)){
char* print = taosMemoryCalloc(*len + 1, 1);
memcpy(print, *tmp, *len);
uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id,
numLines, info->protocol, *len, print);
taosMemoryFree(print);
}else{
uDebug("SML:0x%" PRIx64 " smlParseLine is not numLines:%d, protocol:%d, len:%d, data:%s", info->id,
numLines, info->protocol, *len, *tmp);
}
return true;
}
static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) {
uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id);
int32_t code = TSDB_CODE_SUCCESS;
@ -1661,25 +1692,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
while (i < numLines) {
char *tmp = NULL;
int len = 0;
if (lines) {
tmp = lines[i];
len = strlen(tmp);
} else if (rawLine) {
tmp = rawLine;
while (rawLine < rawLineEnd) {
if (*(rawLine++) == '\n') {
break;
}
len++;
}
if (info->protocol == TSDB_SML_LINE_PROTOCOL && tmp[0] == '#') { // this line is comment
continue;
}
if(!getLine(info, lines, &rawLine, rawLineEnd, numLines, i, &tmp, &len)){
continue;
}
uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id,
info->isRawLine, numLines, info->protocol, len, info->isRawLine ? "rawdata" : tmp);
if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
if (info->dataFormat) {
SSmlLineInfo element = {0};
@ -1699,7 +1714,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE;
}
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, info->isRawLine ? "rawdata" : tmp);
if(rawLine != NULL){
char* print = taosMemoryCalloc(len + 1, 1);
memcpy(print, tmp, len);
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print);
taosMemoryFree(print);
}else{
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp);
}
return code;
}
if (info->reRun) {
@ -1828,7 +1850,6 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
return (TAOS_RES *)request;
}
info->pRequest = request;
info->isRawLine = rawLine != NULL;
info->ttl = ttl;
info->precision = precision;
info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol;
@ -1934,8 +1955,7 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi
reqid);
}
TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
int precision, int32_t ttl, int64_t reqid) {
static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol){
int numLines = 0;
*totalRows = 0;
char *tmp = lines;
@ -1948,6 +1968,11 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int
tmp = lines + i + 1;
}
}
}
TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
int precision, int32_t ttl, int64_t reqid) {
getRawLineLen(lines, len, totalRows, protocol);
return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid);
}

2
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -184,7 +184,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj));
}
if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT64) {
pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj));
pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int64_t *)pRow->pObj));
}
pSdb->tableVer[pRow->type]++;

10
source/libs/parser/src/parInsertSml.c
View File

@ -210,7 +210,15 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
SSmlKv* kv = (SSmlKv*)data;
if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){
ret = TSDB_CODE_SML_INVALID_DATA;
uInfo("SML smlBuildCol error col not same %s", pColSchema->name);
char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1);
if(tmp){
memcpy(tmp, kv->key, kv->keyLen);
uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)",
tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name);
taosMemoryFree(tmp);
}else{
uError("SML smlBuildCol out of memory");
}
goto end;
}
if (kv->type == TSDB_DATA_TYPE_NCHAR) {