xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: support batch loading of csv files

This commit is contained in:
Xiaoyu Wang 2022-10-11 17:28:58 +08:00
parent 18fa48ae57
commit 0563eb2475
3 changed files with 134 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
source/client/src/clientImpl.c
View File

@ -922,7 +922,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
pRequest->code = code1; pRequest->code = code1;
} }
if (pRequest->code == TSDB_CODE_SUCCESS && pWrapper->pParseCtx->needMultiParse) { if (pRequest->code == TSDB_CODE_SUCCESS && NULL != pWrapper->pParseCtx && pWrapper->pParseCtx->needMultiParse) {
code = continueInsertFromCsv(pWrapper, pRequest); code = continueInsertFromCsv(pWrapper, pRequest);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
return; return;

245
source/client/src/clientSml.c
View File

@ -209,7 +209,7 @@ static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashOb
} }
static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
if(pBuf->buf){ if (pBuf->buf) {
memset(pBuf->buf, 0, pBuf->len); memset(pBuf->buf, 0, pBuf->len);
if (msg1) strncat(pBuf->buf, msg1, pBuf->len); if (msg1) strncat(pBuf->buf, msg1, pBuf->len);
int32_t left = pBuf->len - strlen(pBuf->buf); int32_t left = pBuf->len - strlen(pBuf->buf);
@ -256,15 +256,15 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
while (result <= length) { while (result <= length) {
result *= 2; result *= 2;
} }
if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){ if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE;
} else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
} }
if (type == TSDB_DATA_TYPE_NCHAR){ if (type == TSDB_DATA_TYPE_NCHAR) {
result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
}else if (type == TSDB_DATA_TYPE_BINARY){ } else if (type == TSDB_DATA_TYPE_BINARY) {
result = result + VARSTR_HEADER_SIZE; result = result + VARSTR_HEADER_SIZE;
} }
return result; return result;
@ -274,7 +274,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
ESchemaAction *action, bool isTag) { ESchemaAction *action, bool isTag) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
for (int j = 0; j < taosArrayGetSize(cols); ++j) { for (int j = 0; j < taosArrayGetSize(cols); ++j) {
if(j == 0 && !isTag) continue; if (j == 0 && !isTag) continue;
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j);
code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -286,12 +286,12 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) {
SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
int32_t i = 0; int32_t i = 0;
for ( ;i < length; i++) { for (; i < length; i++) {
taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES);
} }
if (isTag){ if (isTag) {
i = 0; i = 0;
} else { } else {
i = 1; i = 1;
@ -306,7 +306,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool
return 0; return 0;
} }
static int32_t getBytes(uint8_t type, int32_t length){ static int32_t getBytes(uint8_t type, int32_t length) {
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
return smlFindNearestPowerOf2(length, type); return smlFindNearestPowerOf2(length, type);
} else { } else {
@ -314,21 +314,22 @@ static int32_t getBytes(uint8_t type, int32_t length){
} }
} }
static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray* results, int32_t numOfCols, bool isTag) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols,
SArray *results, int32_t numOfCols, bool isTag) {
for (int j = 0; j < taosArrayGetSize(cols); ++j) { for (int j = 0; j < taosArrayGetSize(cols); ++j) {
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j);
ESchemaAction action = SCHEMA_ACTION_NULL; ESchemaAction action = SCHEMA_ACTION_NULL;
smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info);
if(action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG){ if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) {
SField field = {0}; SField field = {0};
field.type = kv->type; field.type = kv->type;
field.bytes = getBytes(kv->type, kv->length); field.bytes = getBytes(kv->type, kv->length);
memcpy(field.name, kv->key, kv->keyLen); memcpy(field.name, kv->key, kv->keyLen);
taosArrayPush(results, &field); taosArrayPush(results, &field);
}else if(action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE){ } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen);
uint16_t newIndex = *index; uint16_t newIndex = *index;
if(isTag) newIndex -= numOfCols; if (isTag) newIndex -= numOfCols;
SField *field = (SField *)taosArrayGet(results, newIndex); SField *field = (SField *)taosArrayGet(results, newIndex);
field->bytes = getBytes(kv->type, kv->length); field->bytes = getBytes(kv->type, kv->length);
} }
@ -336,12 +337,11 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
//static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, // static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ // int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns, SArray* pTags, static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta,
STableMeta *pTableMeta, ESchemaAction action){ ESchemaAction action) {
SRequestObj *pRequest = NULL;
SRequestObj* pRequest = NULL;
SMCreateStbReq pReq = {0}; SMCreateStbReq pReq = {0};
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SCmdMsgInfo pCmdMsg = {0}; SCmdMsgInfo pCmdMsg = {0};
@ -363,24 +363,24 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns,
goto end; goto end;
} }
if (action == SCHEMA_ACTION_CREATE_STABLE){ if (action == SCHEMA_ACTION_CREATE_STABLE) {
pReq.colVer = 1; pReq.colVer = 1;
pReq.tagVer = 1; pReq.tagVer = 1;
pReq.suid = 0; pReq.suid = 0;
pReq.source = TD_REQ_FROM_APP; pReq.source = TD_REQ_FROM_APP;
} else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE){ } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
pReq.colVer = pTableMeta->sversion; pReq.colVer = pTableMeta->sversion;
pReq.tagVer = pTableMeta->tversion + 1; pReq.tagVer = pTableMeta->tversion + 1;
pReq.suid = pTableMeta->uid; pReq.suid = pTableMeta->uid;
pReq.source = TD_REQ_FROM_TAOX; pReq.source = TD_REQ_FROM_TAOX;
} else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE){ } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) {
pReq.colVer = pTableMeta->sversion + 1; pReq.colVer = pTableMeta->sversion + 1;
pReq.tagVer = pTableMeta->tversion; pReq.tagVer = pTableMeta->tversion;
pReq.suid = pTableMeta->uid; pReq.suid = pTableMeta->uid;
pReq.source = TD_REQ_FROM_TAOX; pReq.source = TD_REQ_FROM_TAOX;
} }
if (pReq.numOfTags == 0){ if (pReq.numOfTags == 0) {
pReq.numOfTags = 1; pReq.numOfTags = 1;
SField field = {0}; SField field = {0};
field.type = TSDB_DATA_TYPE_NCHAR; field.type = TSDB_DATA_TYPE_NCHAR;
@ -412,7 +412,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns,
launchQueryImpl(pRequest, &pQuery, true, NULL); launchQueryImpl(pRequest, &pQuery, true, NULL);
if(pRequest->code == TSDB_CODE_SUCCESS){ if (pRequest->code == TSDB_CODE_SUCCESS) {
catalogRemoveTableMeta(info->pCatalog, pName); catalogRemoveTableMeta(info->pCatalog, pName);
} }
code = pRequest->code; code = pRequest->code;
@ -425,11 +425,11 @@ end:
} }
static int32_t smlModifyDBSchemas(SSmlHandle *info) { static int32_t smlModifyDBSchemas(SSmlHandle *info) {
int32_t code = 0; int32_t code = 0;
SHashObj *hashTmp = NULL; SHashObj *hashTmp = NULL;
STableMeta *pTableMeta = NULL; STableMeta *pTableMeta = NULL;
SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
strcpy(pName.dbname, info->pRequest->pDb); strcpy(pName.dbname, info->pRequest->pDb);
SRequestConnInfo conn = {0}; SRequestConnInfo conn = {0};
@ -451,8 +451,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) {
SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField));
SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField));
smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true);
smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false);
@ -463,8 +463,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
} }
info->cost.numOfCreateSTables++; info->cost.numOfCreateSTables++;
} else if (code == TSDB_CODE_SUCCESS) { } else if (code == TSDB_CODE_SUCCESS) {
hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true,
taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); HASH_NO_LOCK);
for (uint16_t i = pTableMeta->tableInfo.numOfColumns; for (uint16_t i = pTableMeta->tableInfo.numOfColumns;
i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES);
@ -475,22 +475,25 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
if (action != SCHEMA_ACTION_NULL){ if (action != SCHEMA_ACTION_NULL) {
SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); SArray *pColumns =
SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
SArray *pTags =
taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
SField field = {0}; SField field = {0};
field.type = pTableMeta->schema[i].type; field.type = pTableMeta->schema[i].type;
field.bytes = pTableMeta->schema[i].bytes; field.bytes = pTableMeta->schema[i].bytes;
strcpy(field.name, pTableMeta->schema[i].name); strcpy(field.name, pTableMeta->schema[i].name);
if(i < pTableMeta->tableInfo.numOfColumns){ if (i < pTableMeta->tableInfo.numOfColumns) {
taosArrayPush(pColumns, &field); taosArrayPush(pColumns, &field);
}else{ } else {
taosArrayPush(pTags, &field); taosArrayPush(pTags, &field);
} }
} }
smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, pTableMeta->tableInfo.numOfColumns, true); smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags,
pTableMeta->tableInfo.numOfColumns, true);
code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -518,23 +521,26 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
if (action != SCHEMA_ACTION_NULL){ if (action != SCHEMA_ACTION_NULL) {
SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); SArray *pColumns =
SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
SArray *pTags =
taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
SField field = {0}; SField field = {0};
field.type = pTableMeta->schema[i].type; field.type = pTableMeta->schema[i].type;
field.bytes = pTableMeta->schema[i].bytes; field.bytes = pTableMeta->schema[i].bytes;
strcpy(field.name, pTableMeta->schema[i].name); strcpy(field.name, pTableMeta->schema[i].name);
if(i < pTableMeta->tableInfo.numOfColumns){ if (i < pTableMeta->tableInfo.numOfColumns) {
taosArrayPush(pColumns, &field); taosArrayPush(pColumns, &field);
}else{ } else {
taosArrayPush(pTags, &field); taosArrayPush(pTags, &field);
} }
} }
smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, pTableMeta->tableInfo.numOfColumns, false); smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns,
pTableMeta->tableInfo.numOfColumns, false);
code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -847,7 +853,7 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t
static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) {
int64_t ts = 0; int64_t ts = 0;
if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
// uError("SML:data:%s,len:%d", data, len); // uError("SML:data:%s,len:%d", data, len);
ts = smlParseInfluxTime(info, data, len); ts = smlParseInfluxTime(info, data, len);
} else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
ts = smlParseOpenTsdbTime(info, data, len); ts = smlParseOpenTsdbTime(info, data, len);
@ -877,7 +883,7 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
if (smlIsBinary(pVal->value, pVal->length)) { if (smlIsBinary(pVal->value, pVal->length)) {
pVal->type = TSDB_DATA_TYPE_BINARY; pVal->type = TSDB_DATA_TYPE_BINARY;
pVal->length -= BINARY_ADD_LEN; pVal->length -= BINARY_ADD_LEN;
if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){ if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
} }
pVal->value += (BINARY_ADD_LEN - 1); pVal->value += (BINARY_ADD_LEN - 1);
@ -887,7 +893,7 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
if (smlIsNchar(pVal->value, pVal->length)) { if (smlIsNchar(pVal->value, pVal->length)) {
pVal->type = TSDB_DATA_TYPE_NCHAR; pVal->type = TSDB_DATA_TYPE_NCHAR;
pVal->length -= NCHAR_ADD_LEN; pVal->length -= NCHAR_ADD_LEN;
if(pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
} }
pVal->value += (NCHAR_ADD_LEN - 1); pVal->value += (NCHAR_ADD_LEN - 1);
@ -1063,7 +1069,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab
continue; continue;
} }
if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
} }
@ -1224,7 +1230,7 @@ static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *c
kv->value = value; kv->value = value;
kv->length = valueLen; kv->length = valueLen;
if (isTag) { if (isTag) {
if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
} }
kv->type = TSDB_DATA_TYPE_NCHAR; kv->type = TSDB_DATA_TYPE_NCHAR;
@ -1362,8 +1368,8 @@ static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) {
static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) {
SHashObj *s1 = *(SHashObj **)key1; SHashObj *s1 = *(SHashObj **)key1;
SHashObj *s2 = *(SHashObj **)key2; SHashObj *s2 = *(SHashObj **)key2;
SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN);
SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN);
ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP);
ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP);
if (kv1->i < kv2->i) { if (kv1->i < kv2->i) {
@ -1375,12 +1381,12 @@ static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) {
} }
} }
static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){ static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *cols) {
if(dataFormat){ if (dataFormat) {
void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT); void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT);
if(p == NULL){ if (p == NULL) {
taosArrayPush(oneTable->cols, &cols); taosArrayPush(oneTable->cols, &cols);
}else{ } else {
taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1397,9 +1403,9 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col
} }
void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT); void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT);
if(p == NULL){ if (p == NULL) {
taosArrayPush(oneTable->cols, &kvHash); taosArrayPush(oneTable->cols, &kvHash);
}else{ } else {
taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1488,15 +1494,15 @@ static void smlDestroyInfo(SSmlHandle *info) {
taosMemoryFreeClear(info); taosMemoryFreeClear(info);
} }
static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLProtocolType protocol, int8_t precision){ static SSmlHandle *smlBuildSmlInfo(STscObj *pTscObj, SRequestObj *request, SMLProtocolType protocol, int8_t precision) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SSmlHandle* info = (SSmlHandle*)taosMemoryCalloc(1, sizeof(SSmlHandle)); SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle));
if (NULL == info) { if (NULL == info) {
return NULL; return NULL;
} }
info->id = smlGenId(); info->id = smlGenId();
info->pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); info->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == info->pQuery) { if (NULL == info->pQuery) {
uError("SML:0x%" PRIx64 " create info->pQuery error", info->id); uError("SML:0x%" PRIx64 " create info->pQuery error", info->id);
goto cleanup; goto cleanup;
@ -1511,8 +1517,8 @@ static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLPr
} }
((SVnodeModifOpStmt *)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; ((SVnodeModifOpStmt *)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV;
if (pTscObj){ if (pTscObj) {
info->taos = pTscObj; info->taos = pTscObj;
code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code);
@ -1528,7 +1534,7 @@ static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLPr
info->dataFormat = true; info->dataFormat = true;
} }
if(request){ if (request) {
info->pRequest = request; info->pRequest = request;
info->msgBuf.buf = info->pRequest->msgBuf; info->msgBuf.buf = info->pRequest->msgBuf;
info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE;
@ -1827,10 +1833,11 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) {
} }
pVal->length = (int16_t)strlen(value->valuestring); pVal->length = (int16_t)strlen(value->valuestring);
if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){ if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
} }
if (pVal->type == TSDB_DATA_TYPE_NCHAR && pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ if (pVal->type == TSDB_DATA_TYPE_NCHAR &&
pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
} }
@ -2058,7 +2065,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) {
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql);
int ret = smlParseInfluxString(sql, &elements, &info->msgBuf); int ret = smlParseInfluxString(sql, &elements, &info->msgBuf);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id);
return ret; return ret;
@ -2314,7 +2321,8 @@ static int32_t smlInsertData(SSmlHandle *info) {
(*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid
code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat,
(*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->msgBuf.buf, info->msgBuf.len); (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
info->msgBuf.buf, info->msgBuf.len);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBindData failed", info->id); uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
return code; return code;
@ -2336,7 +2344,12 @@ static int32_t smlInsertData(SSmlHandle *info) {
SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary;
atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);
launchAsyncQuery(info->pRequest, info->pQuery, NULL, NULL); SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper));
if (pWrapper == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pWrapper->pRequest = info->pRequest;
launchAsyncQuery(info->pRequest, info->pQuery, NULL, pWrapper);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2417,41 +2430,41 @@ static int smlProcess(SSmlHandle *info, char *lines[], int numLines) {
} }
static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) { static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) {
// SCatalog *catalog = NULL; // SCatalog *catalog = NULL;
// int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog); // int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog);
// if (code != TSDB_CODE_SUCCESS) { // if (code != TSDB_CODE_SUCCESS) {
// uError("SML get catalog error %d", code); // uError("SML get catalog error %d", code);
// return code; // return code;
// } // }
// //
// SName name; // SName name;
// tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db)); // tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db));
// char dbFname[TSDB_DB_FNAME_LEN] = {0}; // char dbFname[TSDB_DB_FNAME_LEN] = {0};
// tNameGetFullDbName(&name, dbFname); // tNameGetFullDbName(&name, dbFname);
// SDbCfgInfo pInfo = {0}; // SDbCfgInfo pInfo = {0};
// //
// SRequestConnInfo conn = {0}; // SRequestConnInfo conn = {0};
// conn.pTrans = taos->pAppInfo->pTransporter; // conn.pTrans = taos->pAppInfo->pTransporter;
// conn.requestId = request->requestId; // conn.requestId = request->requestId;
// conn.requestObjRefId = request->self; // conn.requestObjRefId = request->self;
// conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp); // conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp);
// //
// code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo); // code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo);
// if (code != TSDB_CODE_SUCCESS) { // if (code != TSDB_CODE_SUCCESS) {
// return code; // return code;
// } // }
// taosArrayDestroy(pInfo.pRetensions); // taosArrayDestroy(pInfo.pRetensions);
// //
// if (!pInfo.schemaless) { // if (!pInfo.schemaless) {
// return TSDB_CODE_SML_INVALID_DB_CONF; // return TSDB_CODE_SML_INVALID_DB_CONF;
// } // }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void smlInsertCallback(void *param, void *res, int32_t code) { static void smlInsertCallback(void *param, void *res, int32_t code) {
SRequestObj *pRequest = (SRequestObj *)res; SRequestObj *pRequest = (SRequestObj *)res;
SSmlHandle *info = (SSmlHandle *)param; SSmlHandle *info = (SSmlHandle *)param;
int32_t rows = taos_affected_rows(pRequest); int32_t rows = taos_affected_rows(pRequest);
uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf);
Params *pParam = info->params; Params *pParam = info->params;
@ -2461,7 +2474,7 @@ static void smlInsertCallback(void *param, void *res, int32_t code) {
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pParam->request->code = code; pParam->request->code = code;
pParam->request->body.resInfo.numOfRows += rows; pParam->request->body.resInfo.numOfRows += rows;
}else{ } else {
pParam->request->body.resInfo.numOfRows += info->affectedRows; pParam->request->body.resInfo.numOfRows += info->affectedRows;
} }
if (pParam->cnt == pParam->total) { if (pParam->cnt == pParam->total) {
@ -2497,20 +2510,20 @@ static void smlInsertCallback(void *param, void *res, int32_t code) {
* *
*/ */
TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
if (NULL == taos) { if (NULL == taos) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL; return NULL;
} }
SRequestObj* request = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT);
if(!request){ if (!request) {
uError("SML:taos_schemaless_insert error request is null"); uError("SML:taos_schemaless_insert error request is null");
return NULL; return NULL;
} }
int batchs = 0; int batchs = 0;
STscObj* pTscObj = request->pTscObj; STscObj *pTscObj = request->pTscObj;
pTscObj->schemalessType = 1; pTscObj->schemalessType = 1;
SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf};
@ -2526,7 +2539,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
goto end; goto end;
} }
if(isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS){ if (isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS) {
request->code = TSDB_CODE_SML_INVALID_DB_CONF; request->code = TSDB_CODE_SML_INVALID_DB_CONF;
smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL); smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL);
goto end; goto end;
@ -2551,9 +2564,9 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
goto end; goto end;
} }
if(protocol == TSDB_SML_JSON_PROTOCOL){ if (protocol == TSDB_SML_JSON_PROTOCOL) {
numLines = 1; numLines = 1;
}else if(numLines <= 0){ } else if (numLines <= 0) {
request->code = TSDB_CODE_SML_INVALID_DATA; request->code = TSDB_CODE_SML_INVALID_DATA;
smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL); smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL);
goto end; goto end;
@ -2562,14 +2575,14 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
batchs = ceil(((double)numLines) / LINE_BATCH); batchs = ceil(((double)numLines) / LINE_BATCH);
params.total = batchs; params.total = batchs;
for (int i = 0; i < batchs; ++i) { for (int i = 0; i < batchs; ++i) {
SRequestObj* req = (SRequestObj*)createRequest(pTscObj->id, TSDB_SQL_INSERT); SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT);
if(!req){ if (!req) {
request->code = TSDB_CODE_OUT_OF_MEMORY; request->code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML:taos_schemaless_insert error request is null"); uError("SML:taos_schemaless_insert error request is null");
goto end; goto end;
} }
SSmlHandle* info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision); SSmlHandle *info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision);
if(!info){ if (!info) {
request->code = TSDB_CODE_OUT_OF_MEMORY; request->code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML:taos_schemaless_insert error SSmlHandle is null"); uError("SML:taos_schemaless_insert error SSmlHandle is null");
goto end; goto end;
@ -2599,8 +2612,8 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
end: end:
taosThreadSpinDestroy(&params.lock); taosThreadSpinDestroy(&params.lock);
tsem_destroy(&params.sem); tsem_destroy(&params.sem);
// ((STscObj *)taos)->schemalessType = 0; // ((STscObj *)taos)->schemalessType = 0;
pTscObj->schemalessType = 1; pTscObj->schemalessType = 1;
uDebug("resultend:%s", request->msgBuf); uDebug("resultend:%s", request->msgBuf);
return (TAOS_RES*)request; return (TAOS_RES *)request;
} }

4
source/libs/parser/src/parser.c
View File

@ -215,6 +215,10 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata
} }
void qDestroyParseContext(SParseContext* pCxt) { void qDestroyParseContext(SParseContext* pCxt) {
if (NULL == pCxt) {
return;
}
taosArrayDestroy(pCxt->pTableMetaPos); taosArrayDestroy(pCxt->pTableMetaPos);
taosArrayDestroy(pCxt->pTableVgroupPos); taosArrayDestroy(pCxt->pTableVgroupPos);
taosMemoryFree(pCxt); taosMemoryFree(pCxt);