diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index a6aca2fddf..d94a671c45 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -92,6 +92,26 @@ extern "C" { } \ } +#define SML_CHECK_CODE(CMD) \ + code = (CMD); \ + if (TSDB_CODE_SUCCESS != code) { \ + lino = __LINE__; \ + goto END; \ + } + +#define SML_CHECK_NULL(CMD) \ + if (NULL == (CMD)) { \ + code = terrno; \ + lino = __LINE__; \ + goto END; \ + } + +#define RETURN \ + if (code != 0){ \ + uError("%s failed code:%d line:%d", __FUNCTION__ , code, lino); \ + } \ + return code; + typedef enum { SCHEMA_ACTION_NULL, SCHEMA_ACTION_CREATE_STABLE, @@ -191,7 +211,6 @@ typedef struct { cJSON *root; // for parse json int8_t offset[OTD_JSON_FIELDS_NUM]; SSmlLineInfo *lines; // element is SSmlLineInfo - bool parseJsonByLib; SArray *tagJsonArray; SArray *valueJsonArray; @@ -211,13 +230,8 @@ typedef struct { extern int64_t smlFactorNS[]; extern int64_t smlFactorS[]; -typedef int32_t (*_equal_fn_sml)(const void *, const void *); - int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle); void smlDestroyInfo(SSmlHandle *info); -int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); -int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); -bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); @@ -237,7 +251,7 @@ void smlDestroyTableInfo(void *para); void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); -int32_t smlParseJSON(SSmlHandle *info, char *payload); +int32_t smlParseJSONExt(SSmlHandle *info, char *payload); int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); @@ -246,7 +260,8 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements); int32_t smlJoinMeasureTag(SSmlLineInfo *elements); void smlBuildTsKv(SSmlKv *kv, int64_t ts); -int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); +int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); +int32_t smlParseEndTelnetJsonUnFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); static inline bool smlDoubleToInt64OverFlow(double num) { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 80f583bbee..e8221c8b8d 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -104,6 +104,9 @@ kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ kvVal->u = result; +#define IS_COMMENT(protocol,data) \ + (protocol == TSDB_SML_LINE_PROTOCOL && data == '#') + int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL}; int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; @@ -135,7 +138,7 @@ void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2 if (pBuf->buf == NULL) { return; } - (void)memset(pBuf->buf, 0, pBuf->len); + pBuf->buf[0] = 0; if (msg1) { (void)strncat(pBuf->buf, msg1, pBuf->len - 1); } @@ -166,23 +169,23 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u } int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, SSmlTableInfo **tInfo) { + int32_t code = 0; + int32_t lino = 0; SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); - if (!tag) { - return terrno; - } + SML_CHECK_NULL(tag) tag->sTableName = measure; tag->sTableNameLen = measureLen; tag->cols = taosArrayInit(numRows, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - taosMemoryFree(tag); - return terrno; - } - + SML_CHECK_NULL(tag->cols) *tInfo = tag; - return TSDB_CODE_SUCCESS; + return code; + +END: + taosMemoryFree(tag); + uError("%s failed code:%d line:%d", __FUNCTION__ , code, lino); + return code; } void smlBuildTsKv(SSmlKv *kv, int64_t ts) { @@ -194,7 +197,13 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts) { } static void smlDestroySTableMeta(void *para) { + if (para == NULL) { + return; + } SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; + if (meta == NULL) { + return; + } taosHashCleanup(meta->tagHash); taosHashCleanup(meta->colHash); taosArrayDestroy(meta->tags); @@ -204,39 +213,26 @@ static void smlDestroySTableMeta(void *para) { } int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta **sMeta) { - int32_t code = TSDB_CODE_SUCCESS; - char *measure = currElement->measure; - int measureLen = currElement->measureLen; - if (currElement->measureEscaped) { - measure = (char *)taosMemoryMalloc(measureLen); - if (measure == NULL) { - return terrno; - } - (void)memcpy(measure, currElement->measure, measureLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); - smlStrReplace(measure, measureLen); - } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; STableMeta *pTableMeta = NULL; - code = smlGetMeta(info, measure, measureLen, &pTableMeta); + + int measureLen = currElement->measureLen; + char *measure = (char *)taosMemoryMalloc(measureLen); + SML_CHECK_NULL(measure); + (void)memcpy(measure, currElement->measure, measureLen); if (currElement->measureEscaped) { - taosMemoryFree(measure); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); } + smlStrReplace(measure, measureLen); + code = smlGetMeta(info, measure, measureLen, &pTableMeta); + taosMemoryFree(measure); if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; - return code; - } - code = smlBuildSTableMeta(info->dataFormat, sMeta); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pTableMeta); - return code; - } - (*sMeta)->tableMeta = pTableMeta; - code = taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES); - if (code != TSDB_CODE_SUCCESS) { - smlDestroySTableMeta(*sMeta); - return code; + goto END; } + SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, sMeta)); for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { SSchema *col = pTableMeta->schema + i; SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; @@ -250,16 +246,19 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml } if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush((*sMeta)->cols, &kv) == NULL) { - return terrno; - } + SML_CHECK_NULL(taosArrayPush((*sMeta)->cols, &kv)); } else { - if (taosArrayPush((*sMeta)->tags, &kv) == NULL) { - return terrno; - } + SML_CHECK_NULL(taosArrayPush((*sMeta)->tags, &kv)); } } - return TSDB_CODE_SUCCESS; + SML_CHECK_CODE(taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES)); + (*sMeta)->tableMeta = pTableMeta; + return code; + +END: + smlDestroySTableMeta(sMeta); + taosMemoryFreeClear(pTableMeta); + RETURN } bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { @@ -305,7 +304,6 @@ bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { goto END; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - if (maxKV == NULL) { goto END; } @@ -376,103 +374,70 @@ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { int32_t code = TSDB_CODE_SUCCESS; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); + int32_t lino = 0; + SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); SSmlTableInfo *tinfo = NULL; if (unlikely(oneTable == NULL)) { - code = smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - code = taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); - if (code != 0) { - smlDestroyTableInfo(&tinfo); - return code; - } + SML_CHECK_CODE(smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo)); + SML_CHECK_CODE(taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES)); tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); - if (tinfo->tags == NULL) { - smlDestroyTableInfo(&tinfo); - return terrno; - } + SML_CHECK_NULL(tinfo->tags); for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); - if (kv == NULL) { - smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); if (kv->keyEscaped) kv->key = NULL; if (kv->valueEscaped) kv->value = NULL; } - code = smlSetCTableName(tinfo, info->tbnameKey); - if (code != TSDB_CODE_SUCCESS) { - smlDestroyTableInfo(&tinfo); - return code; - } - code = getTableUid(info, elements, tinfo); - if (code != TSDB_CODE_SUCCESS) { - smlDestroyTableInfo(&tinfo); - return code; - } + SML_CHECK_CODE(smlSetCTableName(tinfo, info->tbnameKey)); + SML_CHECK_CODE(getTableUid(info, elements, tinfo)); if (info->dataFormat) { info->currSTableMeta->uid = tinfo->uid; - code = smlInitTableDataCtx(info->pQuery, info->currSTableMeta, &tinfo->tableDataCtx); - if (code != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(&tinfo); - return code; - } + SML_CHECK_CODE(smlInitTableDataCtx(info->pQuery, info->currSTableMeta, &tinfo->tableDataCtx)); } } else { tinfo = *oneTable; } - if (tinfo == NULL) { - uError("smlProcessChildTable failed to get child table info"); - return TSDB_CODE_SML_INTERNAL_ERROR; - } if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; return TSDB_CODE_SUCCESS; + +END: + smlDestroyTableInfo(&tinfo); + RETURN } -int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { - if (info->dataFormat) { - uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); - int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - if (elements->colArray == NULL) { - return terrno; - } - } - if (taosArrayPush(elements->colArray, kvTs) == NULL) { - return terrno; - } - if (taosArrayPush(elements->colArray, kv) == NULL) { - return terrno; - } - } - info->preLine = *elements; +int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { + int32_t code = 0; + int32_t lino = 0; + uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__ , kvTs->i); + SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0)); + SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1)); + SML_CHECK_CODE(smlBuildRow(info->currTableDataCtx)); - return TSDB_CODE_SUCCESS; +END: + clearColValArraySml(info->currTableDataCtx->pValues); + RETURN +} + +int32_t smlParseEndTelnetJsonUnFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { + int32_t code = 0; + int32_t lino = 0; + uDebug("SML:0x%" PRIx64 " %s format false, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); + if (elements->colArray == NULL) { + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + SML_CHECK_NULL(elements->colArray); + } + SML_CHECK_NULL(taosArrayPush(elements->colArray, kvTs)); + SML_CHECK_NULL (taosArrayPush(elements->colArray, kv)); + +END: + RETURN } int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) { if (info->dataFormat) { - uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i); + uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); if (ret == TSDB_CODE_SUCCESS) { ret = smlBuildRow(info->currTableDataCtx); @@ -481,11 +446,11 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) clearColValArraySml(info->currTableDataCtx->pValues); taosArrayClearP(info->escapedStringList, taosMemoryFree); if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + uError("SML:0x%" PRIx64 " %s smlBuildCol error:%d", info->id, __FUNCTION__, ret); return ret; } } else { - uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i); + uDebug("SML:0x%" PRIx64 " %s format false, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); taosArraySet(elements->colArray, 0, kvTs); } info->preLine = *elements; @@ -494,15 +459,15 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) } static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnameKey) { - bool autoChildName = false; - size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); + int32_t code = 0; + int32_t lino = 0; + bool autoChildName = false; + size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); if (delimiter > 0 && tbnameKey == NULL) { size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if (tag == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(tag); totalNameLen += tag->length; } if (totalNameLen < TSDB_TABLE_NAME_LEN) { @@ -510,12 +475,10 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam } } if (autoChildName) { - (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + childTableName[0] = '\0'; for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if (tag == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(tag); (void)strncat(childTableName, tag->value, TMIN(tag->length, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName))); if (i != taosArrayGetSize(tags) - 1) { (void)strncat(childTableName, tsSmlAutoChildTableNameDelimiter, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName)); @@ -533,12 +496,9 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if (tag == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(tag); // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { - (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); tstrncpy(childTableName, tag->value, TMIN(TSDB_TABLE_NAME_LEN, tag->length + 1)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); @@ -549,24 +509,21 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam } } - return TSDB_CODE_SUCCESS; +END: + RETURN } int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { - int32_t code = smlParseTableName(oneTable->tags, oneTable->childTableName, tbnameKey); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + int32_t code = 0; + int32_t lino = 0; + SArray *dst = NULL; + SML_CHECK_CODE(smlParseTableName(oneTable->tags, oneTable->childTableName, tbnameKey)); if (strlen(oneTable->childTableName) == 0) { - SArray *dst = taosArrayDup(oneTable->tags, NULL); - if (dst == NULL) { - return terrno; - } + dst = taosArrayDup(oneTable->tags, NULL); + SML_CHECK_NULL(dst); if (oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN) { - uError("SML:smlSetCTableName super table name is too long"); - taosArrayDestroy(dst); - return TSDB_CODE_SML_INTERNAL_ERROR; + SML_CHECK_CODE(TSDB_CODE_SML_INTERNAL_ERROR); } char superName[TSDB_TABLE_NAME_LEN] = {0}; RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; @@ -578,13 +535,12 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { rName.stbFullName = oneTable->sTableName; } - code = buildChildTableName(&rName); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - taosArrayDestroy(dst); + SML_CHECK_CODE(buildChildTableName(&rName)); } - return TSDB_CODE_SUCCESS; + +END: + taosArrayDestroy(dst); + RETURN } int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { @@ -608,42 +564,29 @@ int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo * } int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { + int32_t code = 0; + int32_t lino = 0; SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - if (!meta) { - return terrno; - } - + SML_CHECK_NULL(meta); if (unlikely(!isDataFormat)) { meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->tagHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } - + SML_CHECK_NULL(meta->tagHash); meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->colHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + SML_CHECK_NULL(meta->colHash); } meta->tags = taosArrayInit(32, sizeof(SSmlKv)); - if (meta->tags == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + SML_CHECK_NULL(meta->tags); meta->cols = taosArrayInit(32, sizeof(SSmlKv)); - if (meta->cols == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + SML_CHECK_NULL(meta->cols); *sMeta = meta; return TSDB_CODE_SUCCESS; -cleanup: - smlDestroySTableMeta(meta); - return TSDB_CODE_OUT_OF_MEMORY; +END: + smlDestroySTableMeta(&meta); + uError("%s failed code:%d line:%d", __FUNCTION__ , code, lino); + return code; } int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { @@ -720,104 +663,6 @@ int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { - const char *pVal = kvVal->value; - int32_t len = kvVal->length; - char *endptr = NULL; - double result = taosStr2Double(pVal, &endptr); - if (pVal == endptr) { - smlBuildInvalidDataMsg(msg, "invalid data", pVal); - return false; - } - - int32_t left = len - (endptr - pVal); - if (left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)) { - kvVal->type = TSDB_DATA_TYPE_DOUBLE; - kvVal->d = result; - } else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)) { - if (!IS_VALID_FLOAT(result)) { - smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_FLOAT; - kvVal->f = (float)result; - } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { - if (smlDoubleToInt64OverFlow(result)) { - errno = 0; - int64_t tmp = taosStr2Int64(pVal, &endptr, 10); - if (errno == ERANGE) { - smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_BIGINT; - kvVal->i = tmp; - return true; - } - kvVal->type = TSDB_DATA_TYPE_BIGINT; - kvVal->i = (int64_t)result; - } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) { - if (result >= (double)UINT64_MAX || result < 0) { - errno = 0; - uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); - if (errno == ERANGE || result < 0) { - smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UBIGINT; - kvVal->u = tmp; - return true; - } - kvVal->type = TSDB_DATA_TYPE_UBIGINT; - kvVal->u = result; - } else if (left == 3 && strncasecmp(endptr, "i32", left) == 0) { - if (!IS_VALID_INT(result)) { - smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_INT; - kvVal->i = result; - } else if (left == 3 && strncasecmp(endptr, "u32", left) == 0) { - if (!IS_VALID_UINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UINT; - kvVal->u = result; - } else if (left == 3 && strncasecmp(endptr, "i16", left) == 0) { - if (!IS_VALID_SMALLINT(result)) { - smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_SMALLINT; - kvVal->i = result; - } else if (left == 3 && strncasecmp(endptr, "u16", left) == 0) { - if (!IS_VALID_USMALLINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_USMALLINT; - kvVal->u = result; - } else if (left == 2 && strncasecmp(endptr, "i8", left) == 0) { - if (!IS_VALID_TINYINT(result)) { - smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_TINYINT; - kvVal->i = result; - } else if (left == 2 && strncasecmp(endptr, "u8", left) == 0) { - if (!IS_VALID_UTINYINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UTINYINT; - kvVal->u = result; - } else { - smlBuildInvalidDataMsg(msg, "invalid data", pVal); - return false; - } - return true; -} - int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, STableMeta **pTableMeta) { *pTableMeta = NULL; @@ -829,14 +674,11 @@ int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, ST conn.requestId = info->pRequest->requestId; conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(measureLen, TSDB_TABLE_NAME_LEN - 1); (void)memcpy(pName.tname, measure, measureLen); + pName.tname[len] = 0; - int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - return TSDB_CODE_SUCCESS; + return catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); } static int64_t smlGenId() { @@ -855,8 +697,9 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; if (index) { if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. db type: %d, point type: %d, key: %s", info->id, - colField[*index].type, kv->type, kv->key); + snprintf(info->msgBuf.buf, info->msgBuf.len, "SML:0x%" PRIx64 " %s point type and db type mismatch. db type: %s, point type: %s, key: %s", + info->id, __FUNCTION__, tDataTypes[colField[*index].type].name, tDataTypes[kv->type].name, kv->key); + uError("%s", info->msgBuf.buf); return TSDB_CODE_SML_INVALID_DATA; } @@ -910,65 +753,46 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *checkDumplicateCols, ESchemaAction *action, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } - code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + SML_CHECK_NULL(kv); + SML_CHECK_CODE(smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info)); } for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); if (taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL) { - return TSDB_CODE_PAR_DUPLICATED_COLUMN; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } - return TSDB_CODE_SUCCESS; +END: + RETURN } static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (hashTmp == NULL) { - code = terrno; - goto END; - } + SML_CHECK_NULL(hashTmp); int32_t i = 0; for (; i < length; i++) { - code = taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); - if (code != 0) { - goto END; - } - } - - if (isTag) { - i = 0; - } else { - i = 1; + SML_CHECK_CODE(taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES)); } + i = isTag ? 0 : 1; for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - code = terrno; - goto END; - } + SML_CHECK_NULL(kv); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - code = TSDB_CODE_SML_INVALID_DATA; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } } END: taosHashCleanup(hashTmp); - return code; + RETURN } static int32_t getBytes(uint8_t type, int32_t length) { @@ -982,36 +806,28 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); ESchemaAction action = SCHEMA_ACTION_NULL; - int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); - if (code != 0) { - return code; - } + SML_CHECK_CODE(smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info)); if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); - (void)memcpy(field.name, kv->key, kv->keyLen); - if (taosArrayPush(results, &field) == NULL) { - return terrno; - } + (void)memcpy(field.name, kv->key, TMIN(kv->keyLen, sizeof(field.name) - 1)); + SML_CHECK_NULL(taosArrayPush(results, &field)); } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); if (index == NULL) { - uError("smlBuildFieldsList get error, key:%s", kv->key); - return TSDB_CODE_SML_INVALID_DATA; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); - if (field == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(field); field->bytes = getBytes(kv->type, kv->length); } } @@ -1020,84 +836,67 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO int32_t len = 0; for (int j = 0; j < taosArrayGetSize(results); ++j) { SField *field = taosArrayGet(results, j); - if (field == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(field); len += field->bytes; } if (len > maxLen) { return isTag ? TSDB_CODE_PAR_INVALID_TAGS_LENGTH : TSDB_CODE_PAR_INVALID_ROW_LENGTH; } - return TSDB_CODE_SUCCESS; +END: + RETURN } -static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, +static FORCE_INLINE void smlBuildCreateStbReq(SMCreateStbReq *pReq, int32_t colVer, int32_t tagVer, tb_uid_t suid, int8_t source){ + pReq->colVer = colVer; + pReq->tagVer = tagVer; + pReq->suid = suid; + pReq->source = source; +} +static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray **pTags, STableMeta *pTableMeta, ESchemaAction action) { SRequestObj *pRequest = NULL; SMCreateStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SCmdMsgInfo pCmdMsg = {0}; char *pSql = NULL; // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); - pReq.pTags = pTags; - pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pTags = *pTags; + pReq.numOfTags = taosArrayGetSize(*pTags); + *pTags = NULL; pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); - if (pReq.pColumns == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(pReq.pColumns); for (int32_t i = 0; i < pReq.numOfColumns; ++i) { SField *pField = taosArrayGet(pColumns, i); - if (pField == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(pField); SFieldWithOptions fieldWithOption = {0}; setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); - if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(taosArrayPush(pReq.pColumns, &fieldWithOption)); } if (action == SCHEMA_ACTION_CREATE_STABLE) { - pReq.colVer = 1; - pReq.tagVer = 1; - pReq.suid = 0; - pReq.source = TD_REQ_FROM_APP; pSql = "sml_create_stable"; + smlBuildCreateStbReq(&pReq, 1, 1, 0, TD_REQ_FROM_APP); } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - pReq.colVer = pTableMeta->sversion; - pReq.tagVer = pTableMeta->tversion + 1; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; pSql = (action == SCHEMA_ACTION_ADD_TAG) ? "sml_add_tag" : "sml_modify_tag_size"; + smlBuildCreateStbReq(&pReq, pTableMeta->sversion, pTableMeta->tversion + 1, pTableMeta->uid, TD_REQ_FROM_TAOX); } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { - pReq.colVer = pTableMeta->sversion + 1; - pReq.tagVer = pTableMeta->tversion; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; + smlBuildCreateStbReq(&pReq, pTableMeta->sversion + 1, pTableMeta->tversion, pTableMeta->uid, TD_REQ_FROM_TAOX); } else { - uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); - code = TSDB_CODE_SML_INVALID_DATA; - goto end; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } - code = buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } + SML_CHECK_CODE(buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0)); pRequest->syncQuery = true; if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; + SML_CHECK_CODE(TSDB_CODE_PAR_DB_NOT_SPECIFIED); } if (pReq.numOfTags == 0) { @@ -1106,40 +905,30 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, field.type = TSDB_DATA_TYPE_NCHAR; field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; tstrncpy(field.name, tsSmlTagName, sizeof(field.name)); - if (taosArrayPush(pReq.pTags, &field) == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(taosArrayPush(pReq.pTags, &field)); } pReq.commentLen = -1; pReq.igExists = true; - code = tNameExtractFullName(pName, pReq.name); - if (TSDB_CODE_SUCCESS != code) { - goto end; - } + SML_CHECK_CODE(tNameExtractFullName(pName, pReq.name)); pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); if (pCmdMsg.msgLen < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + uError("failed to serialize create stable request1, code:%d, terrno:%d", pCmdMsg.msgLen, terrno); + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { - code = terrno; - goto end; - } - - if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; + SML_CHECK_NULL(pCmdMsg.pMsg); + code = tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); + if (code < 0) { taosMemoryFree(pCmdMsg.pMsg); - goto end; + uError("failed to serialize create stable request2, code:%d, terrno:%d", code, terrno); + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } SQuery pQuery = {0}; - (void)memset(&pQuery, 0, sizeof(pQuery)); pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; @@ -1148,26 +937,144 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, launchQueryImpl(pRequest, &pQuery, true, NULL); // no need to check return value if (pRequest->code == TSDB_CODE_SUCCESS) { - code = catalogRemoveTableMeta(info->pCatalog, pName); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } + SML_CHECK_CODE(catalogRemoveTableMeta(info->pCatalog, pName)); } code = pRequest->code; -end: +END: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); + RETURN +} + +static int32_t smlCreateTable(SSmlHandle *info, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, + SName *pName, STableMeta **pTableMeta){ + int32_t code = 0; + int32_t lino = 0; + SArray *pColumns = NULL; + SArray *pTags = NULL; + SML_CHECK_CODE(smlCheckAuth(info, conn, NULL, AUTH_TYPE_WRITE)); + uDebug("SML:0x%" PRIx64 " %s create table:%s", info->id, __FUNCTION__, pName->tname); + pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + SML_CHECK_NULL(pColumns); + pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + SML_CHECK_NULL(pTags); + SML_CHECK_CODE(smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true)); + SML_CHECK_CODE(smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false)); + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, &pTags, NULL, SCHEMA_ACTION_CREATE_STABLE)); + info->cost.numOfCreateSTables++; + taosMemoryFreeClear(*pTableMeta); + + SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); + +END: + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + RETURN +} + +static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTableMeta, SSmlSTableMeta *sTableData){ + int32_t code = 0; + int32_t lino = 0; + *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + (pTableMeta)->tableInfo.numOfColumns, sizeof(SField)); + SML_CHECK_NULL(pColumns); + *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + (pTableMeta)->tableInfo.numOfTags, sizeof(SField)); + SML_CHECK_NULL(pTags); + for (uint16_t i = 0; i < (pTableMeta)->tableInfo.numOfColumns + (pTableMeta)->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = (pTableMeta)->schema[i].type; + field.bytes = (pTableMeta)->schema[i].bytes; + tstrncpy(field.name, (pTableMeta)->schema[i].name, sizeof(field.name)); + if (i < (pTableMeta)->tableInfo.numOfColumns) { + SML_CHECK_NULL(taosArrayPush(*pColumns, &field)); + } else { + SML_CHECK_NULL(taosArrayPush(*pTags, &field)); + } + } +END: + RETURN +} +static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, + SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ + ESchemaAction action = SCHEMA_ACTION_NULL; + SArray *pColumns = NULL; + SArray *pTags = NULL; + int32_t code = 0; + int32_t lino = 0; + SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true)); + + if (action != SCHEMA_ACTION_NULL) { + SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); + uDebug("SML:0x%" PRIx64 " %s change table tag, table:%s, action:%d", info->id, __FUNCTION__, pName->tname, + action); + SML_CHECK_CODE(smlBuildFields(&pColumns, &pTags, *pTableMeta, sTableData)); + SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, pTags, + (*pTableMeta)->tableInfo.numOfColumns, true)); + + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, &pTags, (*pTableMeta), action)); + + info->cost.numOfAlterTagSTables++; + taosMemoryFreeClear(*pTableMeta); + SML_CHECK_CODE(catalogRefreshTableMeta(info->pCatalog, conn, pName, -1)); + SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); + } + +END: + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + RETURN +} + +static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, + SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ + ESchemaAction action = SCHEMA_ACTION_NULL; + SArray *pColumns = NULL; + SArray *pTags = NULL; + int32_t code = 0; + int32_t lino = 0; + SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false)); + + if (action != SCHEMA_ACTION_NULL) { + SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); + uDebug("SML:0x%" PRIx64 " %s change table col, table:%s, action:%d", info->id, __FUNCTION__, pName->tname, + action); + SML_CHECK_CODE(smlBuildFields(&pColumns, &pTags, *pTableMeta, sTableData)); + SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, pColumns, + (*pTableMeta)->tableInfo.numOfColumns, false)); + + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, &pTags, (*pTableMeta), action)); + + info->cost.numOfAlterColSTables++; + taosMemoryFreeClear(*pTableMeta); + SML_CHECK_CODE(catalogRefreshTableMeta(info->pCatalog, conn, pName, -1)); + SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); + } + +END: + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + RETURN +} + +static int32_t smlBuildTempHash(SHashObj *hashTmp, STableMeta *pTableMeta, uint16_t start, uint16_t end){ + int32_t code = 0; + int32_t lino = 0; + for (uint16_t i = start; i < end; i++) { + SML_CHECK_CODE(taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES)); + } + +END: return code; } static int32_t smlModifyDBSchemas(SSmlHandle *info) { - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s start, format:%d, needModifySchema:%d", info->id, __FUNCTION__, info->dataFormat, info->needModifySchema); if (info->dataFormat && !info->needModifySchema) { return TSDB_CODE_SUCCESS; } int32_t code = 0; + int32_t lino = 0; SHashObj *hashTmp = NULL; STableMeta *pTableMeta = NULL; @@ -1188,345 +1095,102 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); char *measure = taosMemoryMalloc(superTableLen); - if (measure == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(measure); (void)memcpy(measure, superTable, superTableLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); + if (info->protocol == TSDB_SML_LINE_PROTOCOL){ + PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); + } smlStrReplace(measure, superTableLen); - (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - (void)memcpy(pName.tname, measure, superTableLen); + size_t nameLen = TMIN(superTableLen, TSDB_TABLE_NAME_LEN - 1); + (void)memcpy(pName.tname, measure, nameLen); + pName.tname[nameLen] = '\0'; taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { - code = smlCheckAuth(info, &conn, NULL, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); - SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); - if (pColumns == NULL) { - code = terrno; - goto end; - } - SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); - if (pTags == NULL) { - code = terrno; - taosArrayDestroy(pColumns); - goto end; - } - code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - taosArrayDestroy(pColumns); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - info->cost.numOfCreateSTables++; - taosMemoryFreeClear(pTableMeta); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } + SML_CHECK_CODE(smlCreateTable(info, &conn, sTableData, &pName, &pTableMeta)); } else if (code == TSDB_CODE_SUCCESS) { if (smlIsPKTable(pTableMeta)) { - code = TSDB_CODE_SML_NOT_SUPPORT_PK; - goto end; - } - - hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, - HASH_NO_LOCK); - if (hashTmp == NULL) { - code = terrno; - goto end; - } - for (uint16_t i = pTableMeta->tableInfo.numOfColumns; - i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - if (code != 0) { - goto end; - } - } - - ESchemaAction action = SCHEMA_ACTION_NULL; - code = - smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - if (action != SCHEMA_ACTION_NULL) { - code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname, - action); - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - if (pColumns == NULL) { - code = terrno; - goto end; - } - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - if (pTags == NULL) { - taosArrayDestroy(pColumns); - code = terrno; - goto end; - } - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); - if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush(pColumns, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } else { - if (taosArrayPush(pTags, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } - } - code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, - pTableMeta->tableInfo.numOfColumns, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - if (taosArrayGetSize(pTags) + pTableMeta->tableInfo.numOfColumns > TSDB_MAX_COLUMNS) { - uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); - code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - if (taosArrayGetSize(pTags) > TSDB_MAX_TAGS) { - uError("SML:0x%" PRIx64 " too many tags than 128", info->id); - code = TSDB_CODE_PAR_INVALID_TAGS_NUM; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterTagSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } + SML_CHECK_CODE(TSDB_CODE_SML_NOT_SUPPORT_PK); } + hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(hashTmp); + SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags)); + SML_CHECK_CODE(smlModifyTag(info, hashTmp, &conn, sTableData, &pName, &pTableMeta)); taosHashClear(hashTmp); - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - } - action = SCHEMA_ACTION_NULL; - code = - smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - if (action != SCHEMA_ACTION_NULL) { - code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname, - action); - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - if (pColumns == NULL) { - code = terrno; - goto end; - } - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - if (pTags == NULL) { - taosArrayDestroy(pColumns); - code = terrno; - goto end; - } - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); - if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush(pColumns, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } else { - if (taosArrayPush(pTags, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } - } - - code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, - pTableMeta->tableInfo.numOfColumns, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - if (taosArrayGetSize(pColumns) + pTableMeta->tableInfo.numOfTags > TSDB_MAX_COLUMNS) { - uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); - code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterColSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } - } + SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns)); + SML_CHECK_CODE(smlModifyCols(info, hashTmp, &conn, sTableData, &pName, &pTableMeta)); needCheckMeta = true; taosHashCleanup(hashTmp); hashTmp = NULL; } else { - uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); - goto end; + uError("SML:0x%" PRIx64 " %s load table meta error: %s", info->id, __FUNCTION__, tstrerror(code)); + goto END; } if (needCheckMeta) { - code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, - sTableData->tags, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); - goto end; - } - code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); - goto end; - } + SML_CHECK_CODE(smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, sTableData->tags, true)); + SML_CHECK_CODE(smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false)); } taosMemoryFreeClear(sTableData->tableMeta); sTableData->tableMeta = pTableMeta; - uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, + uDebug("SML:0x%" PRIx64 " %s modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, __FUNCTION__, pTableMeta->uid, pTableMeta->sversion, pTableMeta->tversion); tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s end success, format:%d, needModifySchema:%d", info->id, __FUNCTION__, info->dataFormat, info->needModifySchema); return TSDB_CODE_SUCCESS; -end: +END: taosHashCancelIterate(info->superTables, tmp); taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error - uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, + uError("SML:0x%" PRIx64 " %s end failed:%d:%s, format:%d, needModifySchema:%d", info->id, __FUNCTION__, code, tstrerror(code), info->dataFormat, info->needModifySchema); return code; } static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SHashObj *checkDuplicate) { + int32_t code = 0; + int32_t lino = 0; terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { - if (taosArrayPush(metaArray, kv) == NULL) { - return terrno; - } + SML_CHECK_NULL(taosArrayPush(metaArray, kv)); if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { - return TSDB_CODE_PAR_DUPLICATED_COLUMN; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } else if (terrno == TSDB_CODE_DUP_KEY) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } } - return TSDB_CODE_SUCCESS; + +END: + RETURN } static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj *checkDuplicate) { + int32_t code = 0; + int32_t lino = 0; for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); if (index) { SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); - if (value == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(value); if (isTag) { if (kv->length > value->length) { @@ -1536,7 +1200,7 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } if (kv->type != value->type) { smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - return TSDB_CODE_SML_NOT_SAME_TYPE; + SML_CHECK_CODE(TSDB_CODE_SML_NOT_SAME_TYPE); } if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger @@ -1546,25 +1210,19 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols size_t tmp = taosArrayGetSize(metaArray); if (tmp > INT16_MAX) { smlBuildInvalidDataMsg(msg, "too many cols or tags", kv->key); - uError("too many cols or tags"); - return TSDB_CODE_SML_INVALID_DATA; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } int16_t size = tmp; - int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); - if (ret == 0) { - if (taosArrayPush(metaArray, kv) == NULL) { - return terrno; - } - if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { - return TSDB_CODE_PAR_DUPLICATED_COLUMN; - } - } else { - return ret; + SML_CHECK_CODE(taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES)); + SML_CHECK_NULL(taosArrayPush(metaArray, kv)); + if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } } - return TSDB_CODE_SUCCESS; +END: + RETURN } void smlDestroyTableInfo(void *para) { @@ -1588,8 +1246,7 @@ void freeSSmlKv(void *data) { } void smlDestroyInfo(SSmlHandle *info) { - if (!info) return; - // qDestroyQuery(info->pQuery); + if (info == NULL) return; taosHashCleanup(info->pVgHash); taosHashCleanup(info->childTables); @@ -1614,111 +1271,89 @@ void smlDestroyInfo(SSmlHandle *info) { if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); - if (info->parseJsonByLib) { - taosMemoryFree(info->lines[i].tags); - } if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) { taosMemoryFree(info->lines[i].measureTag); } } taosMemoryFree(info->lines); } - + if(info->protocol == TSDB_SML_JSON_PROTOCOL) { + taosMemoryFreeClear(info->preLine.tags); + } cJSON_Delete(info->root); taosMemoryFreeClear(info); } int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); - if (NULL == info) { - return terrno; - } - if (taos != NULL) { + SML_CHECK_NULL(info); + if (taos != NULL){ info->taos = acquireTscObj(*(int64_t *)taos); - if (info->taos == NULL) { - code = TSDB_CODE_TSC_DISCONNECTED; - goto FAILED; - } - code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto FAILED; - } + SML_CHECK_NULL(info->taos); + SML_CHECK_CODE(catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog)); } info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + SML_CHECK_NULL(info->pVgHash); info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(info->childTables); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(info->tableUids); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { - uError("create SSmlHandle hash obj failed"); - code = terrno; - goto FAILED; - } + SML_CHECK_NULL(info->superTables); taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); - code = smlInitHandle(&info->pQuery); - if (code != TSDB_CODE_SUCCESS) { - goto FAILED; - } + SML_CHECK_CODE(smlInitHandle(&info->pQuery)); info->dataFormat = true; - info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); + SML_CHECK_NULL(info->tagJsonArray); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); + SML_CHECK_NULL(info->valueJsonArray); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + SML_CHECK_NULL(info->preLineTagKV); info->escapedStringList = taosArrayInit(8, POINTER_BYTES); - if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || - info->preLineTagKV == NULL || info->escapedStringList == NULL) { - uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); - code = terrno; - goto FAILED; - } + SML_CHECK_NULL(info->escapedStringList); *handle = info; - return code; + info = NULL; -FAILED: +END: smlDestroyInfo(info); - return code; + RETURN } static int32_t smlPushCols(SArray *colsArray, SArray *cols) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!kvHash) { - uError("SML:smlDealCols failed to allocate memory"); - return terrno; - } + SML_CHECK_NULL(kvHash); for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - taosHashCleanup(kvHash); - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); terrno = 0; - int32_t code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { - taosHashCleanup(kvHash); - return TSDB_CODE_PAR_DUPLICATED_COLUMN; - } - if (code != TSDB_CODE_SUCCESS) { - taosHashCleanup(kvHash); - return code; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } + SML_CHECK_CODE(code); } - if (taosArrayPush(colsArray, &kvHash) == NULL) { - taosHashCleanup(kvHash); - return terrno; - } - return TSDB_CODE_SUCCESS; + SML_CHECK_NULL(taosArrayPush(colsArray, &kvHash)); + return code; +END: + taosHashCleanup(kvHash); + RETURN } -static int32_t smlParseLineBottom(SSmlHandle *info) { - uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, +static int32_t smlParseEnd(SSmlHandle *info) { + uDebug("SML:0x%" PRIx64 " %s start, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); + int32_t code = 0; + int32_t lino = 0; if (info->dataFormat) return TSDB_CODE_SUCCESS; for (int32_t i = 0; i < info->lineNum; i++) { @@ -1728,10 +1363,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen); if (tmp) tinfo = *tmp; - } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, - elements->measureLen + elements->tagsLen); - if (tmp) tinfo = *tmp; } else { SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); @@ -1754,97 +1385,74 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { return TSDB_CODE_PAR_TOO_MANY_COLUMNS; } - int ret = smlPushCols(tinfo->cols, elements->colArray); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } + SML_CHECK_CODE(smlPushCols(tinfo->cols, elements->colArray)); SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); if (tableMeta) { // update meta - uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s update meta, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, - (*tableMeta)->tagHash); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, - (*tableMeta)->colHash); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed, ret:%d", info->id, ret); - return ret; - } + SML_CHECK_CODE(smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, + (*tableMeta)->tagHash)); + SML_CHECK_CODE(smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, + (*tableMeta)->colHash)); } else { - uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s add meta, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = NULL; - ret = smlBuildSTableMeta(info->dataFormat, &meta); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - ret = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " put measuer to hash failed", info->id); - return ret; - } - ret = smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " insert meta failed:%s", info->id, tstrerror(ret)); - return ret; + SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, &meta)); + code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + smlDestroySTableMeta(&meta); + SML_CHECK_CODE(code); } + SML_CHECK_CODE(smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL)); + SML_CHECK_CODE(smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash)); } } - uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); + uDebug("SML:0x%" PRIx64 " %s end, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); - return TSDB_CODE_SUCCESS; +END: + RETURN } static int32_t smlInsertData(SSmlHandle *info) { - int32_t code = TSDB_CODE_SUCCESS; - uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + char *measure = NULL; + SSmlTableInfo **oneTable = NULL; + uDebug("SML:0x%" PRIx64 " %s start, format:%d", info->id, __FUNCTION__, info->dataFormat); if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); - if (info->pRequest->dbList == NULL) { - return terrno; - } + SML_CHECK_NULL(info->pRequest->dbList); } char *data = (char *)taosArrayReserve(info->pRequest->dbList, 1); - if (data == NULL) { - return terrno; - } + SML_CHECK_NULL(data); SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); (void)tNameGetFullDbName(&pName, data); // ignore - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); + oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); while (oneTable) { SSmlTableInfo *tableData = *oneTable; int measureLen = tableData->sTableNameLen; - char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); - if (measure == NULL) { - return terrno; - } + measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); + SML_CHECK_NULL(measure); (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + if (info->protocol == TSDB_SML_LINE_PROTOCOL){ + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + } smlStrReplace(measure, measureLen); - (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); (void)memcpy(pName.tname, measure, measureLen); + pName.tname[measureLen] = '\0'; if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); - if (info->pRequest->tableList == NULL) { - return terrno; - } + SML_CHECK_NULL(info->pRequest->tableList); } - if (taosArrayPush(info->pRequest->tableList, &pName) == NULL) { - return terrno; - } - + SML_CHECK_NULL(taosArrayPush(info->pRequest->tableList, &pName)); tstrncpy(pName.tname, tableData->childTableName, sizeof(pName.tname)); SRequestConnInfo conn = {0}; @@ -1853,61 +1461,33 @@ static int32_t smlInsertData(SSmlHandle *info) { conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } + SML_CHECK_CODE(smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE)); - SVgroupInfo vg; - code = catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName); - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } - code = taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " taosHashPut failed. table name: %s", info->id, tableData->childTableName); - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } + SVgroupInfo vg = {0}; + SML_CHECK_CODE(catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg)); + SML_CHECK_CODE(taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg))); SSmlSTableMeta **pMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); - if (unlikely(NULL == pMeta || NULL == (*pMeta)->tableMeta)) { - uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName); - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return TSDB_CODE_SML_INTERNAL_ERROR; + if (unlikely(NULL == pMeta || NULL == *pMeta || NULL == (*pMeta)->tableMeta)) { + uError("SML:0x%" PRIx64 " %s NULL == pMeta. table name: %s", info->id, __FUNCTION__, tableData->childTableName); + SML_CHECK_CODE(TSDB_CODE_SML_INTERNAL_ERROR); } // use tablemeta of stable to save vgid and uid of child table (*pMeta)->tableMeta->vgId = vg.vgId; (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid - uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, + uDebug("SML:0x%" PRIx64 " %s table:%s, uid:%" PRIu64 ", format:%d", info->id, __FUNCTION__, pName.tname, tableData->uid, info->dataFormat); - code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, + SML_CHECK_CODE(smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf, - info->msgBuf.len); - taosMemoryFree(measure); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBindData failed", info->id); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } + info->msgBuf.len)); + taosMemoryFreeClear(measure); oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); } - code = smlBuildOutput(info->pQuery, info->pVgHash); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildOutput failed", info->id); - return code; - } + SML_CHECK_CODE(smlBuildOutput(info->pQuery, info->pVgHash)); info->cost.insertRpcTime = taosGetTimestampUs(); SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; @@ -1915,10 +1495,15 @@ static int32_t smlInsertData(SSmlHandle *info) { launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code - uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, + uDebug("SML:0x%" PRIx64 " %s end, format:%d, code:%d,%s", info->id, __FUNCTION__, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code)); return info->pRequest->code; + +END: + taosMemoryFree(measure); + taosHashCancelIterate(info->childTables, oneTable); + RETURN } static void smlPrintStatisticInfo(SSmlHandle *info) { @@ -1934,6 +1519,8 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { } int32_t smlClearForRerun(SSmlHandle *info) { + int32_t code = 0; + int32_t lino = 0; info->reRun = false; taosHashClear(info->childTables); @@ -1941,18 +1528,14 @@ int32_t smlClearForRerun(SSmlHandle *info) { taosHashClear(info->tableUids); if (!info->dataFormat) { - if (unlikely(info->lines != NULL)) { - uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); - return TSDB_CODE_SML_INVALID_DATA; - } info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); - if (unlikely(info->lines == NULL)) { - uError("SML:0x%" PRIx64 " info->lines == NULL", info->id); - return terrno; - } + SML_CHECK_NULL(info->lines); } taosArrayClearP(info->escapedStringList, taosMemoryFree); + if(info->protocol == TSDB_SML_JSON_PROTOCOL) { + taosMemoryFreeClear(info->preLine.tags); + } (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; @@ -1960,7 +1543,26 @@ int32_t smlClearForRerun(SSmlHandle *info) { SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot); stmt->freeHashFunc(stmt->pTableBlockHashObj); stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - return TSDB_CODE_SUCCESS; + SML_CHECK_NULL(stmt->pTableBlockHashObj); + +END: + RETURN +} + +static void printRaw(int64_t id, int lineNum, int numLines, ELogLevel level, char* data, int32_t len){ + char *print = taosMemoryMalloc(len + 1); + if (print == NULL) { + uError("SML:0x%" PRIx64 " smlParseLine failed. code : %d", id, terrno); + return; + } + (void)memcpy(print, data, len); + print[len] = '\0'; + if (level == DEBUG_DEBUG){ + uDebug("SML:0x%" PRIx64 " smlParseLine is raw, line %d/%d : %s", id, lineNum, numLines, print); + }else if (level == DEBUG_ERROR){ + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d/%d : %s", id, lineNum, numLines, print); + } + taosMemoryFree(print); } static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLineEnd, int numLines, int i, char **tmp, @@ -1976,42 +1578,38 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi } (*len)++; } - if (info->protocol == TSDB_SML_LINE_PROTOCOL && (*tmp)[0] == '#') { // this line is comment + if (IS_COMMENT(info->protocol,(*tmp)[0])) { // this line is comment return false; } } if (*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)) { - char *print = taosMemoryCalloc(*len + 1, 1); - if (print != NULL) { - (void)memcpy(print, *tmp, *len); - uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, - info->protocol, *len, print); - taosMemoryFree(print); - } else { - uError("SML:0x%" PRIx64 " smlParseLine taosMemoryCalloc failed", info->id); - } + printRaw(info->id, i, numLines, DEBUG_DEBUG, *tmp, *len); } else { - uDebug("SML:0x%" PRIx64 " smlParseLine is not numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, - info->protocol, *len, *tmp); + uDebug("SML:0x%" PRIx64 " smlParseLine is not raw, line %d/%d : %s", info->id, i, numLines, *tmp); } return true; } -static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { - uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); + +static int32_t smlParseJson(SSmlHandle *info, char *lines[], char *rawLine) { + int32_t code = TSDB_CODE_SUCCESS; + if (lines) { + code = smlParseJSONExt(info, *lines); + } else if (rawLine) { + code = smlParseJSONExt(info, rawLine); + } + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed code:%d", __FUNCTION__ , code); + } + return code; +} + +static int32_t smlParseStart(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { + uDebug("SML:0x%" PRIx64 " %s start", info->id, __FUNCTION__); int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - if (lines) { - code = smlParseJSON(info, *lines); - } else if (rawLine) { - code = smlParseJSON(info, rawLine); - } - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseJSON failed:%s", info->id, lines ? *lines : rawLine); - return code; - } - return code; + return smlParseJson(info, lines, rawLine); } char *oldRaw = rawLine; @@ -2037,26 +1635,17 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } else { code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i); } - } else { - code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { if (rawLine != NULL) { - char *print = taosMemoryCalloc(len + 1, 1); - if (print == NULL) { - uError("SML:0x%" PRIx64 " smlParseLine failed. out of memory", info->id); - return code; - } - (void)memcpy(print, tmp, len); - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); - taosMemoryFree(print); + printRaw(info->id, i, numLines, DEBUG_ERROR, tmp, len); } else { - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); + uError("SML:0x%" PRIx64 " %s failed. line %d : %s", info->id, __FUNCTION__, i, tmp); } return code; } if (info->reRun) { - uDebug("SML:0x%" PRIx64 " smlParseLine re run", info->id); + uDebug("SML:0x%" PRIx64 " %s re run", info->id, __FUNCTION__); i = 0; rawLine = oldRaw; code = smlClearForRerun(info); @@ -2067,32 +1656,24 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } i++; } - uDebug("SML:0x%" PRIx64 " smlParseLine end", info->id); + uDebug("SML:0x%" PRIx64 " %s end", info->id, __FUNCTION__); return code; } static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t retryNum = 0; info->cost.parseTime = taosGetTimestampUs(); - code = smlParseLine(info, lines, rawLine, rawLineEnd, numLines); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code)); - return code; - } - code = smlParseLineBottom(info); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlParseLineBottom error : %s", info->id, tstrerror(code)); - return code; - } + SML_CHECK_CODE(smlParseStart(info, lines, rawLine, rawLineEnd, numLines)); + SML_CHECK_CODE(smlParseEnd(info)); info->cost.lineNum = info->lineNum; info->cost.numOfSTables = taosHashGetSize(info->superTables); info->cost.numOfCTables = taosHashGetSize(info->childTables); - info->cost.schemaTime = taosGetTimestampUs(); do { @@ -2105,19 +1686,12 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum); } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlModifyDBSchemas error : %s", info->id, tstrerror(code)); - return code; - } - + SML_CHECK_CODE(code); info->cost.insertBindTime = taosGetTimestampUs(); - code = smlInsertData(info); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlInsertData error : %s", info->id, tstrerror(code)); - return code; - } + SML_CHECK_CODE(smlInsertData(info)); - return code; +END: + RETURN } void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd) { @@ -2160,29 +1734,17 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { - int32_t code = TSDB_CODE_SUCCESS; - if (NULL == taos) { - uError("SML:taos_schemaless_insert error taos is null"); - return NULL; - } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRequestObj *request = NULL; - SSmlHandle *info = NULL; - int cnt = 0; + SSmlHandle *info = NULL; + int cnt = 0; while (1) { - code = createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid, &request); - if (TSDB_CODE_SUCCESS != code) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } + SML_CHECK_CODE(createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid, &request)); + SSmlMsgBuf msg = {request->msgBufLen, request->msgBuf}; + request->code = smlBuildSmlInfo(taos, &info); + SML_CHECK_CODE(request->code); - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - code = smlBuildSmlInfo(taos, &info); - if (code != TSDB_CODE_SUCCESS) { - request->code = code; - smlBuildInvalidDataMsg(&msg, "init SSmlHandle failed", NULL); - uError("SML:taos_schemaless_insert error SSmlHandle is null, err msg:%s", tstrerror(code)); - goto end; - } info->pRequest = request; info->pRequest->pQuery = info->pQuery; info->ttl = ttl; @@ -2198,20 +1760,20 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); - goto end; + goto END; } if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); - goto end; + goto END; } if (protocol == TSDB_SML_LINE_PROTOCOL && (precision < TSDB_SML_TIMESTAMP_NOT_CONFIGURED || precision > TSDB_SML_TIMESTAMP_NANO_SECONDS)) { request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; smlBuildInvalidDataMsg(&msg, "precision invalidate for line protocol", NULL); - goto end; + goto END; } if (protocol == TSDB_SML_JSON_PROTOCOL) { @@ -2219,7 +1781,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, } else if (numLines <= 0) { request->code = TSDB_CODE_SML_INVALID_DATA; smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL); - goto end; + goto END; } code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); @@ -2249,7 +1811,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } -end: +END: smlDestroyInfo(info); return (TAOS_RES *)request; } @@ -2275,6 +1837,16 @@ end: TAOS_RES *taos_schemaless_insert_ttl_with_reqid_tbname_key(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { + if (taos == NULL || lines == NULL || numLines < 0) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + for (int i = 0; i < numLines; i++){ + if (lines[i] == NULL){ + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + } return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid, tbnameKey); } @@ -2298,26 +1870,32 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi reqid); } -static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol) { +static int32_t getRawLineLen(char *lines, int len, int protocol) { int numLines = 0; - *totalRows = 0; char *tmp = lines; for (int i = 0; i < len; i++) { if (lines[i] == '\n' || i == len - 1) { - numLines++; - if (tmp[0] != '#' || protocol != TSDB_SML_LINE_PROTOCOL) { // ignore comment - (*totalRows)++; + if (!IS_COMMENT(protocol, tmp[0])) { // ignore comment + numLines++; } tmp = lines + i + 1; } } + return numLines; } TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { - getRawLineLen(lines, len, totalRows, protocol); - return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid, + if (taos == NULL || lines == NULL || len < 0) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + int numLines = getRawLineLen(lines, len, protocol); + if (totalRows != NULL){ + *totalRows = numLines; + } + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, numLines, protocol, precision, ttl, reqid, tbnameKey); } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index ece1ddf61f..9a27a30d84 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -21,259 +21,10 @@ #define OTD_JSON_SUB_FIELDS_NUM 2 -#define JUMP_JSON_SPACE(start) \ - while (*(start)) { \ - if (unlikely(*(start) > 32)) \ - break; \ - else \ - (start)++; \ - } - -static int32_t smlJsonGetObj(char **payload) { - int leftBracketCnt = 0; - bool isInQuote = false; - while (**payload) { - if (**payload == '"' && *((*payload) - 1) != '\\') { - isInQuote = !isInQuote; - } else if (!isInQuote && unlikely(**payload == '{')) { - leftBracketCnt++; - (*payload)++; - continue; - } else if (!isInQuote && unlikely(**payload == '}')) { - leftBracketCnt--; - (*payload)++; - if (leftBracketCnt == 0) { - return 0; - } else if (leftBracketCnt < 0) { - return -1; - } - continue; - } - (*payload)++; - } - return -1; -} - -int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { - int index = 0; - while (*(*start)) { - if ((*start)[0] != '"') { - (*start)++; - continue; - } - - if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { - uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start); - return TSDB_CODE_TSC_INVALID_JSON; - } - - char *sTmp = *start; - if ((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' && (*start)[4] == 'r' && (*start)[5] == 'i' && - (*start)[6] == 'c' && (*start)[7] == '"') { - (*start) += 8; - bool isInQuote = false; - while (*(*start)) { - if (unlikely(!isInQuote && *(*start) == '"')) { - (*start)++; - offset[index++] = *start - sTmp; - element->measure = (*start); - isInQuote = true; - continue; - } - if (unlikely(isInQuote && *(*start) == '"')) { - element->measureLen = (*start) - element->measure; - (*start)++; - break; - } - (*start)++; - } - } else if ((*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' && (*start)[4] == 'e' && - (*start)[5] == 's' && (*start)[6] == 't' && (*start)[7] == 'a' && (*start)[8] == 'm' && - (*start)[9] == 'p' && (*start)[10] == '"') { - (*start) += 11; - bool hasColon = false; - while (*(*start)) { - if (unlikely(!hasColon && *(*start) == ':')) { - (*start)++; - JUMP_JSON_SPACE((*start)) - offset[index++] = *start - sTmp; - element->timestamp = (*start); - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->timestampLen = tmp - (*start); - *start = tmp; - } - break; - } - hasColon = true; - continue; - } - if (unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))) { - element->timestampLen = (*start) - element->timestamp; - break; - } - (*start)++; - } - } else if ((*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' && (*start)[4] == 'u' && - (*start)[5] == 'e' && (*start)[6] == '"') { - (*start) += 7; - - bool hasColon = false; - while (*(*start)) { - if (unlikely(!hasColon && *(*start) == ':')) { - (*start)++; - JUMP_JSON_SPACE((*start)) - offset[index++] = *start - sTmp; - element->cols = (*start); - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->colsLen = tmp - (*start); - *start = tmp; - } - break; - } - hasColon = true; - continue; - } - if (unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))) { - element->colsLen = (*start) - element->cols; - break; - } - (*start)++; - } - } else if ((*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' && (*start)[4] == 's' && - (*start)[5] == '"') { - (*start) += 6; - - while (*(*start)) { - if (unlikely(*(*start) == ':')) { - (*start)++; - JUMP_JSON_SPACE((*start)) - offset[index++] = *start - sTmp; - element->tags = (*start); - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->tagsLen = tmp - (*start); - *start = tmp; - } - break; - } - (*start)++; - } - } - if (*(*start) == '\0') { - break; - } - if (*(*start) == '}') { - (*start)++; - break; - } - (*start)++; - } - - if (unlikely(index != OTD_JSON_FIELDS_NUM) || element->tags == NULL || element->cols == NULL || - element->measure == NULL || element->timestamp == NULL) { - uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM); - return TSDB_CODE_TSC_INVALID_JSON; - } - return 0; -} - -int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { - int index = 0; - while (*(*start)) { - if ((*start)[0] != '"') { - (*start)++; - continue; - } - - if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { - uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start); - return TSDB_CODE_TSC_INVALID_JSON; - } - - if ((*start)[1] == 'm') { - (*start) += offset[index++]; - element->measure = *start; - while (*(*start)) { - if (unlikely(*(*start) == '"')) { - element->measureLen = (*start) - element->measure; - (*start)++; - break; - } - (*start)++; - } - } else if ((*start)[1] == 't' && (*start)[2] == 'i') { - (*start) += offset[index++]; - element->timestamp = *start; - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->timestampLen = tmp - (*start); - *start = tmp; - } - } else { - while (*(*start)) { - if (unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)) { - element->timestampLen = (*start) - element->timestamp; - break; - } - (*start)++; - } - } - } else if ((*start)[1] == 'v') { - (*start) += offset[index++]; - element->cols = *start; - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->colsLen = tmp - (*start); - *start = tmp; - } - } else { - while (*(*start)) { - if (unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)) { - element->colsLen = (*start) - element->cols; - break; - } - (*start)++; - } - } - } else if ((*start)[1] == 't' && (*start)[2] == 'a') { - (*start) += offset[index++]; - element->tags = (*start); - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->tagsLen = tmp - (*start); - *start = tmp; - } - } - if (*(*start) == '}') { - (*start)++; - break; - } - (*start)++; - } - - if (unlikely(index != 0 && index != OTD_JSON_FIELDS_NUM)) { - uError("elements != %d", OTD_JSON_FIELDS_NUM); - return TSDB_CODE_TSC_INVALID_JSON; - } - return TSDB_CODE_SUCCESS; -} - static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { elements->measureLen = strlen(metric->valuestring); if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - uError("OTD:0x%" PRIx64 " Metric length is 0 or large than 192", info->id); + uError("SML:0x%" PRIx64 " Metric length is 0 or large than 192", info->id); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -293,7 +44,7 @@ static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) { child = child->next; } if (*marks[i] == NULL) { - uError("smlGetJsonElements error, not find mark:%d:%s", i, jsonName[i]); + uError("SML %s error, not find mark:%d:%s", __FUNCTION__, i, jsonName[i]); return TSDB_CODE_TSC_INVALID_JSON; } } @@ -302,7 +53,7 @@ static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) { static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { if (strcasecmp(typeStr, "bool") != 0) { - uError("OTD:invalid type(%s) for JSON Bool", typeStr); + uError("SML:invalid type(%s) for JSON Bool", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } pVal->type = TSDB_DATA_TYPE_BOOL; @@ -316,7 +67,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // tinyint if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { if (!IS_VALID_TINYINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_TINYINT; @@ -327,7 +78,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // smallint if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { if (!IS_VALID_SMALLINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_SMALLINT; @@ -338,7 +89,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // int if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { if (!IS_VALID_INT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(int)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_INT; @@ -362,7 +113,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // float if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { if (!IS_VALID_FLOAT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(float)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_FLOAT; @@ -379,7 +130,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { } // if reach here means type is unsupported - uError("OTD:invalid type(%s) for JSON Number", typeStr); + uError("SML:invalid type(%s) for JSON Number", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } @@ -391,7 +142,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { } else if (strcasecmp(typeStr, "nchar") == 0) { pVal->type = TSDB_DATA_TYPE_NCHAR; } else { - uError("OTD:invalid type(%s) for JSON String", typeStr); + uError("SML:invalid type(%s) for JSON String", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } pVal->length = strlen(value->valuestring); @@ -474,7 +225,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { case cJSON_String: { int32_t ret = smlConvertJSONString(kv, "binary", root); if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse binary value from JSON Obj"); + uError("SML:Failed to parse binary value from JSON Obj"); return ret; } break; @@ -482,7 +233,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { case cJSON_Object: { int32_t ret = smlParseValueFromJSONObj(root, kv); if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse value from JSON Obj"); + uError("SML:Failed to parse value from JSON Obj"); return ret; } break; @@ -511,7 +262,7 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ } size_t keyLen = strlen(tag->string); if (unlikely(IS_INVALID_COL_LEN(keyLen))) { - uError("OTD:Tag key length is 0 or too large than 64"); + uError("SML:Tag key length is 0 or too large than 64"); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -539,28 +290,24 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ } static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { - int32_t ret = 0; + if (is_same_child_table_telnet(elements, &info->preLine) == 0) { + elements->measureTag = info->preLine.measureTag; + return TSDB_CODE_SUCCESS; + } + int32_t code = 0; + int32_t lino = 0; if(info->dataFormat){ - ret = smlProcessSuperTable(info, elements); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - } - ret = smlProcessTagJson(info, tags); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - ret = smlJoinMeasureTag(elements); - if(ret != 0){ - return ret; + SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } + SML_CHECK_CODE(smlProcessTagJson(info, tags)); + SML_CHECK_CODE(smlJoinMeasureTag(elements)); return smlProcessChildTable(info, elements); + +END: + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + RETURN } static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { @@ -678,7 +425,8 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { } static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { - int32_t ret = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; cJSON *metricJson = NULL; cJSON *tsJson = NULL; @@ -688,57 +436,27 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo int32_t size = cJSON_GetArraySize(root); // outmost json fields has to be exactly 4 if (size != OTD_JSON_FIELDS_NUM) { - uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); + uError("SML:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); return TSDB_CODE_TSC_INVALID_JSON; } cJSON **marks[OTD_JSON_FIELDS_NUM] = {&metricJson, &tsJson, &valueJson, &tagsJson}; - ret = smlGetJsonElements(root, marks); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - return ret; - } - + SML_CHECK_CODE(smlGetJsonElements(root, marks)); // Parse metric - ret = smlParseMetricFromJSON(info, metricJson, elements); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); - return ret; - } - + SML_CHECK_CODE(smlParseMetricFromJSON(info, metricJson, elements)); // Parse metric value SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN}; - ret = smlParseValueFromJSON(valueJson, &kv); - if (unlikely(ret)) { - uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); - return ret; - } + SML_CHECK_CODE(smlParseValueFromJSON(valueJson, &kv)); // Parse tags - bool needFree = info->dataFormat; elements->tags = cJSON_PrintUnformatted(tagsJson); - if (elements->tags == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; - } - elements->tagsLen = strlen(elements->tags); - if (is_same_child_table_telnet(elements, &info->preLine) != 0) { - ret = smlParseTagsFromJSON(info, tagsJson, elements); - if (unlikely(ret)) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - taosMemoryFree(elements->tags); - elements->tags = NULL; - return ret; - } - } else { - elements->measureTag = info->preLine.measureTag; - } + SML_CHECK_NULL(elements->tags); - if (needFree) { - taosMemoryFree(elements->tags); - elements->tags = NULL; - } + elements->tagsLen = strlen(elements->tags); + SML_CHECK_CODE(smlParseTagsFromJSON(info, tagsJson, elements)); if (unlikely(info->reRun)) { - return TSDB_CODE_SUCCESS; + goto END; } // Parse timestamp @@ -747,29 +465,34 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo if (unlikely(ts < 0)) { char* tmp = cJSON_PrintUnformatted(tsJson); if (tmp == NULL) { - uError("cJSON_PrintUnformatted failed since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %" PRId64, info->id, info->msgBuf.buf, ts); + uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %" PRId64, info->id, info->msgBuf.buf, ts); } else { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); + uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); taosMemoryFree(tmp); } - return TSDB_CODE_INVALID_TIMESTAMP; + SML_CHECK_CODE(TSDB_CODE_INVALID_TIMESTAMP); } SSmlKv kvTs = {0}; smlBuildTsKv(&kvTs, ts); + if (info->dataFormat){ + code = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); + } else { + code = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); + } + SML_CHECK_CODE(code); + taosMemoryFreeClear(info->preLine.tags); + info->preLine = *elements; + elements->tags = NULL; - return smlParseEndTelnetJson(info, elements, &kvTs, &kv); +END: + taosMemoryFree(elements->tags); + RETURN } -static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { +int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { int32_t payloadNum = 0; int32_t ret = TSDB_CODE_SUCCESS; - if (unlikely(payload == NULL)) { - uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - info->root = cJSON_Parse(payload); if (unlikely(info->root == NULL)) { uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); @@ -782,27 +505,11 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { } else if (cJSON_IsObject(info->root)) { payloadNum = 1; } else { - uError("SML:0x%" PRIx64 " Invalid JSON Payload 3:%s", info->id, payload); + uError("SML:0x%" PRIx64 " Invalid JSON type:%s", info->id, payload); return TSDB_CODE_TSC_INVALID_JSON; } - if (unlikely(info->lines != NULL)) { - for (int i = 0; i < info->lineNum; i++) { - taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); - if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); - } - taosMemoryFree(info->lines); - info->lines = NULL; - } info->lineNum = payloadNum; - info->dataFormat = true; - - ret = smlClearForRerun(info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - info->parseJsonByLib = true; cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child; int cnt = 0; @@ -811,6 +518,7 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { if (info->dataFormat) { SSmlLineInfo element = {0}; ret = smlParseJSONStringExt(info, dataPoint, &element); + if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); } else { ret = smlParseJSONStringExt(info, dataPoint, info->lines + cnt); } @@ -836,164 +544,3 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (info->offset[0] == 0) { - ret = smlJsonParseObjFirst(start, elements, info->offset); - } else { - ret = smlJsonParseObj(start, elements, info->offset); - } - - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; - - if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { - smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; - - if (unlikely(elements->colsLen == 0)) { - uError("SML:colsLen == 0"); - return TSDB_CODE_TSC_INVALID_VALUE; - } else if (unlikely(elements->cols[0] == '{')) { - char tmp = elements->cols[elements->colsLen]; - elements->cols[elements->colsLen] = '\0'; - cJSON *valueJson = cJSON_Parse(elements->cols); - if (unlikely(valueJson == NULL)) { - uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols); - elements->cols[elements->colsLen] = tmp; - return TSDB_CODE_TSC_INVALID_JSON; - } - if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){ - cJSON_Delete(valueJson); - elements->cols[elements->colsLen] = tmp; - return terrno; - } - ret = smlParseValueFromJSONObj(valueJson, &kv); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols); - elements->cols[elements->colsLen] = tmp; - return TSDB_CODE_TSC_INVALID_VALUE; - } - elements->cols[elements->colsLen] = tmp; - } else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - // Parse tags - if (is_same_child_table_telnet(elements, &info->preLine) != 0) { - char tmp = *(elements->tags + elements->tagsLen); - *(elements->tags + elements->tagsLen) = 0; - cJSON *tagsJson = cJSON_Parse(elements->tags); - *(elements->tags + elements->tagsLen) = tmp; - if (unlikely(tagsJson == NULL)) { - uError("SML:0x%" PRIx64 " parse json tag failed:%s", info->id, elements->tags); - return TSDB_CODE_TSC_INVALID_JSON; - } - - if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){ - cJSON_Delete(tagsJson); - uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id); - return terrno; - } - ret = smlParseTagsFromJSON(info, tagsJson, elements); - if (unlikely(ret)) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - return ret; - } - } else { - elements->measureTag = info->preLine.measureTag; - } - - if (unlikely(info->reRun)) { - return TSDB_CODE_SUCCESS; - } - - // Parse timestamp - // notice!!! put ts back to tag to ensure get meta->precision - int64_t ts = 0; - if (unlikely(elements->timestampLen == 0)) { - uError("OTD:0x%" PRIx64 " elements->timestampLen == 0", info->id); - return TSDB_CODE_INVALID_TIMESTAMP; - } else if (elements->timestamp[0] == '{') { - char tmp = elements->timestamp[elements->timestampLen]; - elements->timestamp[elements->timestampLen] = '\0'; - cJSON *tsJson = cJSON_Parse(elements->timestamp); - ts = smlParseTSFromJSON(info, tsJson); - if (unlikely(ts < 0)) { - uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload:%s", info->id, elements->timestamp); - elements->timestamp[elements->timestampLen] = tmp; - cJSON_Delete(tsJson); - return TSDB_CODE_INVALID_TIMESTAMP; - } - elements->timestamp[elements->timestampLen] = tmp; - cJSON_Delete(tsJson); - } else { - ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); - if (unlikely(ts < 0)) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } - SSmlKv kvTs = {0}; - smlBuildTsKv(&kvTs, ts); - - return smlParseEndTelnetJson(info, elements, &kvTs, &kv); -} - -int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 1 << 15; - int32_t ret = TSDB_CODE_SUCCESS; - - uDebug("SML:0x%" PRIx64 "json:%s", info->id, payload); - int cnt = 0; - char *dataPointStart = payload; - while (1) { - if (info->dataFormat) { - SSmlLineInfo element = {0}; - ret = smlParseJSONString(info, &dataPointStart, &element); - if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); - } else { - if (cnt >= payloadNum) { - payloadNum = payloadNum << 1; - void *tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); - if (tmp == NULL) { - ret = terrno; - return ret; - } - info->lines = (SSmlLineInfo *)tmp; - (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); - } - ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); - if ((info->lines + cnt)->measure == NULL) break; - } - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uError("SML:0x%" PRIx64 " Invalid JSON Payload 1:%s", info->id, payload); - return smlParseJSONExt(info, payload); - } - - if (unlikely(info->reRun)) { - cnt = 0; - dataPointStart = payload; - info->lineNum = payloadNum; - ret = smlClearForRerun(info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - continue; - } - - cnt++; - if (*dataPointStart == '\0') break; - } - info->lineNum = cnt; - - return TSDB_CODE_SUCCESS; -} diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index c1f3431698..b54f4e0beb 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -63,7 +63,7 @@ static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t le int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); if (unlikely(ts == -1)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid timestamp", data); return TSDB_CODE_SML_INVALID_DATA; } return ts; @@ -84,7 +84,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 'l' || pVal->value[0] == 'L') { // nchar - if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) { + if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') { pVal->type = TSDB_DATA_TYPE_NCHAR; pVal->length -= NCHAR_ADD_LEN; if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { @@ -97,7 +97,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 'g' || pVal->value[0] == 'G') { // geometry - if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= sizeof("POINT")+3) { + if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') { int32_t code = initCtxGeomFromText(); if (code != TSDB_CODE_SUCCESS) { return code; @@ -124,7 +124,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 'b' || pVal->value[0] == 'B') { // varbinary - if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) { + if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') { pVal->type = TSDB_DATA_TYPE_VARBINARY; if(isHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){ if(!isValidateHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){ @@ -298,7 +298,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - return TSDB_CODE_TSC_INVALID_JSON; + return TSDB_CODE_SML_INVALID_DATA; } cnt++; @@ -311,31 +311,24 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { + int32_t code = 0; + int32_t lino = 0; bool isSameCTable = IS_SAME_CHILD_TABLE; if(isSameCTable){ return TSDB_CODE_SUCCESS; } - int32_t ret = 0; if(info->dataFormat){ - ret = smlProcessSuperTable(info, elements); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } + SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } - - ret = smlProcessTagLine(info, sql, sqlEnd); - if(ret != 0){ - if (info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - + SML_CHECK_CODE(smlProcessTagLine(info, sql, sqlEnd)); return smlProcessChildTable(info, elements); + +END: + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + RETURN } static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) { @@ -353,7 +346,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL const char *escapeChar = NULL; while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid data", *sql); return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(IS_EQUAL(*sql,escapeChar))) { @@ -370,7 +363,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid key or key is too long than 64", key); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -404,18 +397,18 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL valueLen = *sql - value; if (unlikely(quoteNum != 0 && quoteNum != 2)) { - smlBuildInvalidDataMsg(&info->msgBuf, "unbalanced quotes", value); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line unbalanced quotes", value); return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(valueLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid value", value); return TSDB_CODE_SML_INVALID_DATA; } SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; int32_t ret = smlParseValue(&kv, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + uError("SML:0x%" PRIx64 " %s parse value error:%d.", info->id, __FUNCTION__, ret); return ret; } @@ -437,11 +430,6 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } (void)memcpy(tmp, kv.value, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); - if(kv.type == TSDB_DATA_TYPE_GEOMETRY) { - uError("SML:0x%" PRIx64 " smlParseColLine error, invalid GEOMETRY type.", info->id); - taosMemoryFree((void*)kv.value); - return TSDB_CODE_TSC_INVALID_VALUE; - } if(kv.type == TSDB_DATA_TYPE_VARBINARY){ taosMemoryFree((void*)kv.value); } @@ -510,7 +498,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } elements->measureLen = sql - elements->measure; if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) { - smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line measure is empty or too large than 192", NULL); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -557,7 +545,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->colsLen = sql - elements->cols; if (unlikely(elements->colsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "cols is empty", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line cols is empty", NULL); return TSDB_CODE_SML_INVALID_DATA; } @@ -574,7 +562,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine int64_t ts = smlParseInfluxTime(info, elements->timestamp, elements->timestampLen); if (unlikely(ts <= 0)) { - uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); + uError("SML:0x%" PRIx64 " %s error:%" PRId64, info->id, __FUNCTION__, ts); return TSDB_CODE_INVALID_TIMESTAMP; } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index e8601e33bc..dd264da11e 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -148,31 +148,21 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS return TSDB_CODE_SUCCESS; } - int32_t ret = 0; + int32_t code = 0; + int32_t lino = 0; if(info->dataFormat){ - ret = smlProcessSuperTable(info, elements); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } + SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } + SML_CHECK_CODE(smlProcessTagTelnet(info, data, sqlEnd)); + SML_CHECK_CODE(smlJoinMeasureTag(elements)); - ret = smlProcessTagTelnet(info, data, sqlEnd); - if(ret != 0){ - if (info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; + code = smlProcessChildTable(info, elements); + +END: + if(info->reRun){ + return TSDB_CODE_SUCCESS; } - - ret = smlJoinMeasureTag(elements); - if(ret != 0){ - return ret; - } - - return smlProcessChildTable(info, elements); + RETURN } // format: =[ =] @@ -182,14 +172,14 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine // parse metric smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen); if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid measure", sql); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } // parse timestamp smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen); if (unlikely(!elements->timestamp || elements->timestampLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid timestamp", sql); return TSDB_CODE_SML_INVALID_DATA; } @@ -199,19 +189,21 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); if (unlikely(ts < 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet parse timestamp failed", sql); return TSDB_CODE_INVALID_TIMESTAMP; } // parse value smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); if (unlikely(!elements->cols || elements->colsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid value", sql); return TSDB_CODE_TSC_INVALID_VALUE; } SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; - if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + int ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " %s parse value error:%d.", info->id, __FUNCTION__, ret); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -220,11 +212,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->tags = sql; elements->tagsLen = sqlEnd - sql; if (unlikely(!elements->tags || elements->tagsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid tag value", sql); return TSDB_CODE_TSC_INVALID_VALUE; } - int ret = smlParseTelnetTags(info, sql, sqlEnd, elements); + ret = smlParseTelnetTags(info, sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } @@ -239,5 +231,12 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); } - return smlParseEndTelnetJson(info, elements, &kvTs, &kv); + if (info->dataFormat){ + ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); + } else { + ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); + } + info->preLine = *elements; + + return ret; } \ No newline at end of file diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index dc6a302924..338457bec4 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -68,6 +68,15 @@ TEST(testCase, smlParseInfluxString_Test) { taosArrayDestroy(elements.colArray); elements.colArray = nullptr; + // case 0 false + tmp = "st,t1=3 c3=\""; + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); + ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; + // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; (void)memcpy(sql, tmp, strlen(tmp) + 1); @@ -591,6 +600,104 @@ TEST(testCase, smlParseTelnetLine_Test) { // smlDestroyInfo(info); //} +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + char *endptr = NULL; + double result = taosStr2Double(pVal, &endptr); + if (pVal == endptr) { + smlBuildInvalidDataMsg(msg, "invalid data", pVal); + return false; + } + + int32_t left = len - (endptr - pVal); + if (left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)) { + kvVal->type = TSDB_DATA_TYPE_DOUBLE; + kvVal->d = result; + } else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)) { + if (!IS_VALID_FLOAT(result)) { + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_FLOAT; + kvVal->f = (float)result; + } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { + if (smlDoubleToInt64OverFlow(result)) { + errno = 0; + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); + if (errno == ERANGE) { + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_BIGINT; + kvVal->i = tmp; + return true; + } + kvVal->type = TSDB_DATA_TYPE_BIGINT; + kvVal->i = (int64_t)result; + } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) { + if (result >= (double)UINT64_MAX || result < 0) { + errno = 0; + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); + if (errno == ERANGE || result < 0) { + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UBIGINT; + kvVal->u = tmp; + return true; + } + kvVal->type = TSDB_DATA_TYPE_UBIGINT; + kvVal->u = result; + } else if (left == 3 && strncasecmp(endptr, "i32", left) == 0) { + if (!IS_VALID_INT(result)) { + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_INT; + kvVal->i = result; + } else if (left == 3 && strncasecmp(endptr, "u32", left) == 0) { + if (!IS_VALID_UINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UINT; + kvVal->u = result; + } else if (left == 3 && strncasecmp(endptr, "i16", left) == 0) { + if (!IS_VALID_SMALLINT(result)) { + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_SMALLINT; + kvVal->i = result; + } else if (left == 3 && strncasecmp(endptr, "u16", left) == 0) { + if (!IS_VALID_USMALLINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_USMALLINT; + kvVal->u = result; + } else if (left == 2 && strncasecmp(endptr, "i8", left) == 0) { + if (!IS_VALID_TINYINT(result)) { + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_TINYINT; + kvVal->i = result; + } else if (left == 2 && strncasecmp(endptr, "u8", left) == 0) { + if (!IS_VALID_UTINYINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UTINYINT; + kvVal->u = result; + } else { + smlBuildInvalidDataMsg(msg, "invalid data", pVal); + return false; + } + return true; +} + TEST(testCase, smlParseNumber_performance_Test) { char msg[256] = {0}; SSmlMsgBuf msgBuf; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index d07fe9c5ac..7566b69c02 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -38,7 +38,7 @@ int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnod SVnodeObj *pVnode = *ppVnode; if (pVnode && num < size) { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - // dTrace("vgId:%d, acquire vnode list, ref:%d", pVnode->vgId, refCount); + dTrace("vgId:%d,acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); pVnodes[num++] = (*ppVnode); pIter = taosHashIterate(pMgmt->hash, pIter); } else { @@ -52,7 +52,7 @@ int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnod SVnodeObj *pVnode = *ppVnode; if (pVnode && num < size) { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - // dTrace("vgId:%d, acquire vnode list, ref:%d", pVnode->vgId, refCount); + dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); pVnodes[num++] = (*ppVnode); pIter = taosHashIterate(pMgmt->closedHash, pIter); } else { @@ -84,7 +84,7 @@ int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeOb SVnodeObj *pVnode = *ppVnode; if (pVnode && num < size) { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - // dTrace("vgId:%d, acquire vnode list, ref:%d", pVnode->vgId, refCount); + dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); pVnodes[num++] = (*ppVnode); pIter = taosHashIterate(pMgmt->hash, pIter); } else { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index a285043382..682c179270 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -103,7 +103,7 @@ SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict) { pVnode = NULL; } else { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - // dTrace("vgId:%d, acquire vnode, ref:%d", pVnode->vgId, refCount); + dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); } (void)taosThreadRwlockUnlock(&pMgmt->lock); @@ -115,16 +115,24 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { return vmAcquireVno void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { if (pVnode == NULL) return; - (void)taosThreadRwlockRdlock(&pMgmt->lock); + //(void)taosThreadRwlockRdlock(&pMgmt->lock); int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - // dTrace("vgId:%d, release vnode, ref:%d", pVnode->vgId, refCount); - (void)taosThreadRwlockUnlock(&pMgmt->lock); + dTrace("vgId:%d, release vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); + //(void)taosThreadRwlockUnlock(&pMgmt->lock); } static void vmFreeVnodeObj(SVnodeObj **ppVnode) { if (!ppVnode || !(*ppVnode)) return; SVnodeObj *pVnode = *ppVnode; + + int32_t refCount = atomic_load_32(&pVnode->refCount); + while (refCount > 0) { + dWarn("vgId:%d, vnode is refenced, retry to free in 200ms, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); + taosMsleep(200); + refCount = atomic_load_32(&pVnode->refCount); + } + taosMemoryFree(pVnode->path); taosMemoryFree(pVnode); ppVnode[0] = NULL; diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index fd593e0638..c151529aea 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -30,9 +30,18 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &epSet); if (epSet.numOfEps <= 1) { - pMsg->pCont = NULL; - pMsg->code = TSDB_CODE_MNODE_NOT_FOUND; - return; + if (epSet.numOfEps == 0) { + pMsg->pCont = NULL; + pMsg->code = TSDB_CODE_MNODE_NOT_FOUND; + return; + } + // dnode is not the mnode or mnode leader and This ensures that the function correctly handles cases where the + // dnode cannot obtain a valid epSet and avoids returning an incorrect or misleading epSet. + if (strcmp(epSet.eps[0].fqdn, tsLocalFqdn) == 0 && epSet.eps[0].port == tsServerPort) { + pMsg->pCont = NULL; + pMsg->code = TSDB_CODE_MNODE_NOT_FOUND; + return; + } } int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 34ecda6ce7..8c5f628034 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2018,7 +2018,12 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { cleanupAfterGroupResultGen(pMiaInfo, pRes); code = doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); QUERY_CHECK_CODE(code, lino, _end); - break; + if (pRes->info.rows == 0) { + // After filtering for last group, the result is empty, so we need to continue to process next group + continue; + } else { + break; + } } else { // continue pRes->info.id.groupId = pMiaInfo->groupId; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index cca35d9c9a..b5cdf1e4ee 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -468,35 +468,38 @@ end: int32_t smlInitHandle(SQuery** query) { *query = NULL; SQuery* pQuery = NULL; + SVnodeModifyOpStmt* stmt = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pQuery); - if (NULL == pQuery) { - uError("create pQuery error"); - return code; + if (code != 0) { + uError("SML create pQuery error"); + goto END; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = false; pQuery->msgType = TDMT_VND_SUBMIT; - SVnodeModifyOpStmt* stmt = NULL; code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&stmt); - if (NULL == stmt) { - uError("create SVnodeModifyOpStmt error"); - qDestroyQuery(pQuery); - return code; + if (code != 0) { + uError("SML create SVnodeModifyOpStmt error"); + goto END; } stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (stmt->pTableBlockHashObj == NULL){ - uError("create pTableBlockHashObj error"); - qDestroyQuery(pQuery); - nodesDestroyNode((SNode*)stmt); - return terrno; + uError("SML create pTableBlockHashObj error"); + code = terrno; + goto END; } stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeArrayFunc = insDestroyVgroupDataCxtList; pQuery->pRoot = (SNode*)stmt; *query = pQuery; + return code; - return TSDB_CODE_SUCCESS; +END: + nodesDestroyNode((SNode*)stmt); + qDestroyQuery(pQuery); + return code; } int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1faa010663..52c40ef0a4 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3757,7 +3757,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { bool partionByTbname = hasTbnameFunction(pSelect->pPartitionByList); FOREACH(pPartKey, pSelect->pPartitionByList) { if (nodesEqualNode(pPartKey, *pNode)) { - return pCxt->currClause == SQL_CLAUSE_HAVING ? DEAL_RES_IGNORE_CHILD : rewriteExprToGroupKeyFunc(pCxt, pNode); + return pSelect->hasAggFuncs ? rewriteExprToGroupKeyFunc(pCxt, pNode) : DEAL_RES_IGNORE_CHILD; } if ((partionByTbname) && QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index c8c37b11bc..1c53baf360 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -664,7 +664,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy // set status if (pWal->vers.firstVer == -1) { - pWal->vers.firstVer = 0; + pWal->vers.firstVer = index; } pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalCkHead) + cyptedBodyLen; diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index 3b98ec30ce..3847f1dec4 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -344,9 +344,72 @@ class TDTestCase: tdSql.query(sql, queryTimes=1) tdSql.checkRows(55) - # TODO Fix Me! - sql = "explain SELECT count(*), timediff(_wend, last(ts)), timediff('2018-09-20 01:00:00', _wstart) FROM meters WHERE ts >= '2018-09-20 00:00:00.000' AND ts < '2018-09-20 01:00:00.000' PARTITION BY concat(tbname, 'asd') INTERVAL(5m) having(concat(tbname, 'asd') like '%asd');" - tdSql.error(sql, -2147473664) # Error: Planner internal error + sql = "SELECT count(*), timediff(_wend, last(ts)), timediff('2018-09-20 01:00:00', _wstart) FROM meters WHERE ts >= '2018-09-20 00:00:00.000' AND ts < '2018-09-20 01:00:00.000' PARTITION BY concat(tbname, 'asd') INTERVAL(5m) having(concat(tbname, 'asd') like '%asd');" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(60) + + sql = "SELECT count(*), timediff(_wend, last(ts)), timediff('2018-09-20 01:00:00', _wstart) FROM meters WHERE ts >= '2018-09-20 00:00:00.000' AND ts < '2018-09-20 01:00:00.000' PARTITION BY concat(tbname, 'asd') INTERVAL(5m) having(concat(tbname, 'asd') like 'asd%');" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(0) + + sql = "SELECT c1 FROM meters PARTITION BY c1 HAVING c1 > 0 slimit 2 limit 10" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(20) + + sql = "SELECT t1 FROM meters PARTITION BY t1 HAVING(t1 = 1)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(20000) + + sql = "SELECT concat(t2, 'asd') FROM meters PARTITION BY t2 HAVING(t2 like '%5')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10000) + tdSql.checkData(0, 0, 'tb5asd') + + sql = "SELECT concat(t2, 'asd') FROM meters PARTITION BY concat(t2, 'asd') HAVING(concat(t2, 'asd')like '%5%')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10000) + tdSql.checkData(0, 0, 'tb5asd') + + sql = "SELECT avg(c1) FROM meters PARTITION BY tbname, t1 HAVING(t1 = 1)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(2) + + sql = "SELECT count(*) FROM meters PARTITION BY concat(tbname, 'asd') HAVING(concat(tbname, 'asd') like '%asd')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10) + + sql = "SELECT count(*), concat(tbname, 'asd') FROM meters PARTITION BY concat(tbname, 'asd') HAVING(concat(tbname, 'asd') like '%asd')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10) + + sql = "SELECT count(*) FROM meters PARTITION BY t1 HAVING(t1 < 4) order by t1 +1" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(4) + + sql = "SELECT count(*), t1 + 100 FROM meters PARTITION BY t1 HAVING(t1 < 4) order by t1 +1" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(4) + + sql = "SELECT count(*), t1 + 100 FROM meters PARTITION BY t1 INTERVAL(1d) HAVING(t1 < 4) order by t1 +1 desc" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(280) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd' and count(*) = 118)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(1) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd' and count(*) != 118)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(69) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd') order by count(*) asc limit 10" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd' or concat(t3, 'asd') like '%3asd') order by count(*) asc limit 10000" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(140) + def run(self): self.prepareTestEnv() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py index e0632686d4..40879d5c66 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py @@ -17,8 +17,6 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - - updatecfgDict = {'sDebugFlag':143, 'wDebugFlag':143} def __init__(self): self.vgroups = 1 self.ctbNum = 10 diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 55fa336221..a8d4fafb03 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -105,6 +105,113 @@ int smlProcess_telnet_Test() { return code; } +int smlProcess_telnet0_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql1[] = {"sysif.bytes.out 1479496100 1.3E0 host=web01 interface=eth0"}; + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + + const char *sql2[] = {"sysif.bytes.out 1479496700 1.6E0 host=web01 interface=eth0"}; + pRes = taos_schemaless_insert(taos, (char **)sql2, sizeof(sql2) / sizeof(sql2[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + + const char *sql3[] = {"sysif.bytes.out 1479496300 1.1E0 interface=eth0 host=web01"}; + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + + taos_close(taos); + + return code; +} + +int smlProcess_json0_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[{\"metric\":\"syscpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}}]"}; + + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } + ASSERT(code == 0); + + + const char *sql2[] = { + "[{\"metric\":\"syscpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":1}" + "},{\"metric\":\"syscpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}" + "}]", + }; + + char *sql3[1] = {0}; + for (int i = 0; i < 1; i++) { + sql3[i] = taosMemoryCalloc(1, 1024); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql3[i]); + } + + ASSERT(code == 0); + + taos_close(taos); + + return code; +} + int smlProcess_json1_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1775,6 +1882,21 @@ int sml_td24559_Test() { pRes = taos_query(taos, "create database if not exists td24559"); taos_free_result(pRes); + const char *sql1[] = { + "sttb,t1=1 f1=283i32,f2=g\"\" 1632299372000", + "sttb,t1=1 f2=G\"Point(4.343 89.342)\",f1=106i32 1632299373000", + }; + + pRes = taos_query(taos, "use td24559"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + int code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code); + taos_free_result(pRes); + const char *sql[] = { "stb,t1=1 f1=283i32,f2=g\"Point(4.343 89.342)\" 1632299372000", "stb,t1=1 f2=G\"Point(4.343 89.342)\",f1=106i32 1632299373000", @@ -1788,7 +1910,7 @@ int sml_td24559_Test() { pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); - int code = taos_errno(pRes); + code = taos_errno(pRes); printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); @@ -2136,7 +2258,8 @@ int main(int argc, char *argv[]) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } - int ret = 0; + int ret = smlProcess_json0_Test(); + ASSERT(!ret); ret = sml_ts5528_test(); ASSERT(!ret); ret = sml_td29691_Test(); @@ -2173,6 +2296,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = smlProcess_telnet_Test(); ASSERT(!ret); + ret = smlProcess_telnet0_Test(); + ASSERT(!ret); ret = smlProcess_json1_Test(); ASSERT(!ret); ret = smlProcess_json2_Test();