From a2fa41ccc77797da594464221768c43cabc51448 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Wed, 9 Oct 2024 09:51:19 +0800 Subject: [PATCH 01/43] Update 01-arch.md --- docs/zh/26-tdinternal/01-arch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/26-tdinternal/01-arch.md b/docs/zh/26-tdinternal/01-arch.md index 04e47797a8..8aa69e45d5 100644 --- a/docs/zh/26-tdinternal/01-arch.md +++ b/docs/zh/26-tdinternal/01-arch.md @@ -178,7 +178,7 @@ TDengine 集群可以容纳单个、多个甚至几千个数据节点。应用 TDengine 存储的数据包括采集的时序数据以及库、表相关的元数据、标签数据等，这些数据具体分为三部分： -- 时序数据：TDengine 的核心存储对象，存放于 vnode 里，由 data、head 和 last 三个文件组成，数据量大，查询量取决于应用场景。允许乱序写入，但暂时不支持删除操作，并且仅在 update 参数设置为 1 时允许更新操作。通过采用一个采集点一张表的模型，一个时间段的数据是连续存储，对单张表的写入是简单的追加操作，一次读，可以读到多条记录，这样保证对单个采集点的插入和查询操作，性能达到最优。 +- 时序数据：时序数据是 TDengine 的核心存储对象，它们被存储在 vnode 中。时序数据由 data、head、sma 和 stt 4 类文件组成，这些文件共同构成了时序数据的完整存储结构。由于时序数据的特点是数据量大且查询需求取决于具体应用场景，因此 TDengine 采用了“一个数据采集点一张表”的模型来优化存储和查询性能。在这种模型下，一个时间段内的数据是连续存储的，对单张表的写入是简单的追加操作，一次读取可以获取多条记录。这种设计确保了单个数据采集点的写入和查询操作都能达到最优性能。 - 数据表元数据：包含标签信息和 Table Schema 信息，存放于 vnode 里的 meta 文件，支持增删改查四个标准操作。数据量很大，有 N 张表，就有 N 条记录，因此采用 LRU 存储，支持标签数据的索引。TDengine 支持多核多线程并发查询。只要计算内存足够，元数据全内存存储，千万级别规模的标签数据过滤结果能毫秒级返回。在内存资源不足的情况下，仍然可以支持数千万张表的快速查询。 - 数据库元数据：存放于 mnode 里，包含系统节点、用户、DB、STable Schema 等信息，支持增删改查四个标准操作。这部分数据的量不大，可以全内存保存，而且由于客户端有缓存，查询量也不大。因此目前的设计虽是集中式存储管理，但不会构成性能瓶颈。 @@ -321,4 +321,4 @@ TDengine 采用了一种数据驱动的策略来实现缓存数据的持久化 此外，TDengine 还提供了数据分级存储的功能，允许用户将不同时间段的数据存储在不同存储设备的目录中，以此实现将“热”数据和“冷”数据分开存储。这样做可以充分利用各种存储资源，同时节约成本。例如，对于最新采集且需要频繁访问的数据，由于其读取性能要求较高，用户可以配置将这些数据存储在高性能的固态硬盘上。而对于超过一定期限、查询需求较低的数据，则可以将其存储在成本相对较低的机械硬盘上。 -为了进一步降低存储成本，TDengine 还支持将时序数据存储在对象存储系统中。通过其创新性的设计，在大多数情况下，从对象存储系统中查询时序数据的性能接近本地硬盘的一半，而在某些场景下，性能甚至可以与本地硬盘相媲美。同时，TDengine 还允许用户对存储在对象存储中的时序数据执行删除和更新操作。 \ No newline at end of file +为了进一步降低存储成本，TDengine 还支持将时序数据存储在对象存储系统中。通过其创新性的设计，在大多数情况下，从对象存储系统中查询时序数据的性能接近本地硬盘的一半，而在某些场景下，性能甚至可以与本地硬盘相媲美。同时，TDengine 还允许用户对存储在对象存储中的时序数据执行删除和更新操作。 From eaae6d4bfda93383a1b7c30d64e987eda79ee93f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 21 Oct 2024 17:30:12 +0800 Subject: [PATCH 02/43] enh:[TD-32166]refactor code in sml --- source/client/inc/clientSml.h | 24 +- source/client/src/clientSml.c | 1218 ++++++++----------------- source/client/src/clientSmlJson.c | 68 +- source/client/src/clientSmlLine.c | 28 +- source/client/src/clientSmlTelnet.c | 38 +- source/client/test/smlTest.cpp | 98 ++ source/libs/parser/src/parInsertSml.c | 23 +- 7 files changed, 592 insertions(+), 905 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index a6aca2fddf..3098b124b7 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -92,6 +92,22 @@ extern "C" { } \ } +#define SML_CHECK_CODE(CMD) \ + do { \ + code = (CMD); \ + if (TSDB_CODE_SUCCESS != code) { \ + goto END; \ + } \ + } while (0) + +#define SML_CHECK_NULL(CMD) \ + do { \ + if (NULL == (CMD)) { \ + code = terrno; \ + goto END; \ + } \ + } while (0) + typedef enum { SCHEMA_ACTION_NULL, SCHEMA_ACTION_CREATE_STABLE, @@ -211,13 +227,10 @@ typedef struct { extern int64_t smlFactorNS[]; extern int64_t smlFactorS[]; -typedef int32_t (*_equal_fn_sml)(const void *, const void *); - int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle); void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); -int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); -bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); +int smlJsonParseObj(char **start, char *end, SSmlLineInfo *element, int8_t *offset); void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); @@ -246,7 +259,8 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements); int32_t smlJoinMeasureTag(SSmlLineInfo *elements); void smlBuildTsKv(SSmlKv *kv, int64_t ts); -int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); +int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); +int32_t smlParseEndTelnetJsonUnFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); static inline bool smlDoubleToInt64OverFlow(double num) { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 80f583bbee..25abeb52fb 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -104,6 +104,9 @@ kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ kvVal->u = result; +#define IS_COMMENT(protocol,data) \ + (protocol == TSDB_SML_LINE_PROTOCOL && data == '#') + int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL}; int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; @@ -224,6 +227,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; + taosMemoryFreeClear(pTableMeta); return code; } code = smlBuildSTableMeta(info->dataFormat, sMeta); @@ -305,7 +309,6 @@ bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { goto END; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - if (maxKV == NULL) { goto END; } @@ -380,49 +383,23 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); SSmlTableInfo *tinfo = NULL; if (unlikely(oneTable == NULL)) { - code = smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - code = taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); - if (code != 0) { - smlDestroyTableInfo(&tinfo); - return code; - } + SML_CHECK_CODE(smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo)); + SML_CHECK_CODE(taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES)); tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); - if (tinfo->tags == NULL) { - smlDestroyTableInfo(&tinfo); - return terrno; - } + SML_CHECK_NULL(tinfo->tags); for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); - if (kv == NULL) { - smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); if (kv->keyEscaped) kv->key = NULL; if (kv->valueEscaped) kv->value = NULL; } - code = smlSetCTableName(tinfo, info->tbnameKey); - if (code != TSDB_CODE_SUCCESS) { - smlDestroyTableInfo(&tinfo); - return code; - } - code = getTableUid(info, elements, tinfo); - if (code != TSDB_CODE_SUCCESS) { - smlDestroyTableInfo(&tinfo); - return code; - } + SML_CHECK_CODE(smlSetCTableName(tinfo, info->tbnameKey)); + SML_CHECK_CODE(getTableUid(info, elements, tinfo)); if (info->dataFormat) { info->currSTableMeta->uid = tinfo->uid; - code = smlInitTableDataCtx(info->pQuery, info->currSTableMeta, &tinfo->tableDataCtx); - if (code != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(&tinfo); - return code; - } + SML_CHECK_CODE(smlInitTableDataCtx(info->pQuery, info->currSTableMeta, &tinfo->tableDataCtx)); } } else { tinfo = *oneTable; @@ -433,41 +410,41 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { } if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; return TSDB_CODE_SUCCESS; + +END: + smlDestroyTableInfo(&tinfo); + return code; } -int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { - if (info->dataFormat) { - uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); - int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - if (elements->colArray == NULL) { - return terrno; - } - } - if (taosArrayPush(elements->colArray, kvTs) == NULL) { - return terrno; - } - if (taosArrayPush(elements->colArray, kv) == NULL) { - return terrno; - } +int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { + int32_t code = 0; + uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); + SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0)); + SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1)); + SML_CHECK_CODE(smlBuildRow(info->currTableDataCtx)); + + info->preLine = *elements; +END: + clearColValArraySml(info->currTableDataCtx->pValues); + if (unlikely(code != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); } + return code; +} + +int32_t smlParseEndTelnetJsonUnFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { + int32_t code = 0; + uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); + if (elements->colArray == NULL) { + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + SML_CHECK_NULL(elements->colArray); + } + SML_CHECK_NULL(taosArrayPush(elements->colArray, kvTs)); + SML_CHECK_NULL (taosArrayPush(elements->colArray, kv)); info->preLine = *elements; - return TSDB_CODE_SUCCESS; +END: + return code; } int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) { @@ -494,15 +471,14 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) } static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnameKey) { + int32_t code = 0; bool autoChildName = false; size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); if (delimiter > 0 && tbnameKey == NULL) { size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if (tag == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(tag); totalNameLen += tag->length; } if (totalNameLen < TSDB_TABLE_NAME_LEN) { @@ -513,9 +489,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if (tag == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(tag); (void)strncat(childTableName, tag->value, TMIN(tag->length, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName))); if (i != taosArrayGetSize(tags) - 1) { (void)strncat(childTableName, tsSmlAutoChildTableNameDelimiter, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName)); @@ -533,9 +507,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if (tag == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(tag); // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); @@ -549,24 +521,22 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam } } - return TSDB_CODE_SUCCESS; +END: + return code; } int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { - int32_t code = smlParseTableName(oneTable->tags, oneTable->childTableName, tbnameKey); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + int32_t code = 0; + SArray *dst = NULL; + SML_CHECK_CODE(smlParseTableName(oneTable->tags, oneTable->childTableName, tbnameKey)); if (strlen(oneTable->childTableName) == 0) { - SArray *dst = taosArrayDup(oneTable->tags, NULL); - if (dst == NULL) { - return terrno; - } + dst = taosArrayDup(oneTable->tags, NULL); + SML_CHECK_NULL(dst); if (oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN) { uError("SML:smlSetCTableName super table name is too long"); - taosArrayDestroy(dst); - return TSDB_CODE_SML_INTERNAL_ERROR; + code = TSDB_CODE_SML_INTERNAL_ERROR; + goto END; } char superName[TSDB_TABLE_NAME_LEN] = {0}; RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; @@ -578,13 +548,12 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { rName.stbFullName = oneTable->sTableName; } - code = buildChildTableName(&rName); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - taosArrayDestroy(dst); + SML_CHECK_CODE(buildChildTableName(&rName)); } - return TSDB_CODE_SUCCESS; + +END: + taosArrayDestroy(dst); + return code; } int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { @@ -608,40 +577,25 @@ int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo * } int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { + int32_t code = 0; SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - if (!meta) { - return terrno; - } - + SML_CHECK_NULL(meta); if (unlikely(!isDataFormat)) { meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->tagHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } - + SML_CHECK_NULL(meta->tagHash); meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->colHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + SML_CHECK_NULL(meta->colHash); } meta->tags = taosArrayInit(32, sizeof(SSmlKv)); - if (meta->tags == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + SML_CHECK_NULL(meta->tags); meta->cols = taosArrayInit(32, sizeof(SSmlKv)); - if (meta->cols == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + SML_CHECK_NULL(meta->cols); *sMeta = meta; return TSDB_CODE_SUCCESS; -cleanup: +END: smlDestroySTableMeta(meta); return TSDB_CODE_OUT_OF_MEMORY; } @@ -720,104 +674,6 @@ int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { - const char *pVal = kvVal->value; - int32_t len = kvVal->length; - char *endptr = NULL; - double result = taosStr2Double(pVal, &endptr); - if (pVal == endptr) { - smlBuildInvalidDataMsg(msg, "invalid data", pVal); - return false; - } - - int32_t left = len - (endptr - pVal); - if (left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)) { - kvVal->type = TSDB_DATA_TYPE_DOUBLE; - kvVal->d = result; - } else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)) { - if (!IS_VALID_FLOAT(result)) { - smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_FLOAT; - kvVal->f = (float)result; - } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { - if (smlDoubleToInt64OverFlow(result)) { - errno = 0; - int64_t tmp = taosStr2Int64(pVal, &endptr, 10); - if (errno == ERANGE) { - smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_BIGINT; - kvVal->i = tmp; - return true; - } - kvVal->type = TSDB_DATA_TYPE_BIGINT; - kvVal->i = (int64_t)result; - } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) { - if (result >= (double)UINT64_MAX || result < 0) { - errno = 0; - uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); - if (errno == ERANGE || result < 0) { - smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UBIGINT; - kvVal->u = tmp; - return true; - } - kvVal->type = TSDB_DATA_TYPE_UBIGINT; - kvVal->u = result; - } else if (left == 3 && strncasecmp(endptr, "i32", left) == 0) { - if (!IS_VALID_INT(result)) { - smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_INT; - kvVal->i = result; - } else if (left == 3 && strncasecmp(endptr, "u32", left) == 0) { - if (!IS_VALID_UINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UINT; - kvVal->u = result; - } else if (left == 3 && strncasecmp(endptr, "i16", left) == 0) { - if (!IS_VALID_SMALLINT(result)) { - smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_SMALLINT; - kvVal->i = result; - } else if (left == 3 && strncasecmp(endptr, "u16", left) == 0) { - if (!IS_VALID_USMALLINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_USMALLINT; - kvVal->u = result; - } else if (left == 2 && strncasecmp(endptr, "i8", left) == 0) { - if (!IS_VALID_TINYINT(result)) { - smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_TINYINT; - kvVal->i = result; - } else if (left == 2 && strncasecmp(endptr, "u8", left) == 0) { - if (!IS_VALID_UTINYINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UTINYINT; - kvVal->u = result; - } else { - smlBuildInvalidDataMsg(msg, "invalid data", pVal); - return false; - } - return true; -} - int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, STableMeta **pTableMeta) { *pTableMeta = NULL; @@ -832,11 +688,7 @@ int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, ST (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); (void)memcpy(pName.tname, measure, measureLen); - int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - return TSDB_CODE_SUCCESS; + return catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); } static int64_t smlGenId() { @@ -913,53 +765,34 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } - code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + SML_CHECK_NULL(kv); + SML_CHECK_CODE(smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info)); } for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); if (taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL) { - return TSDB_CODE_PAR_DUPLICATED_COLUMN; + code = TSDB_CODE_PAR_DUPLICATED_COLUMN; + goto END; } } - return TSDB_CODE_SUCCESS; +END: + return code; } static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (hashTmp == NULL) { - code = terrno; - goto END; - } + SML_CHECK_NULL(hashTmp); int32_t i = 0; for (; i < length; i++) { - code = taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); - if (code != 0) { - goto END; - } - } - - if (isTag) { - i = 0; - } else { - i = 1; + SML_CHECK_CODE(taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES)); } + i = isTag ? 0 : 1; for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - code = terrno; - goto END; - } + SML_CHECK_NULL(kv); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { code = TSDB_CODE_SML_INVALID_DATA; goto END; @@ -982,36 +815,29 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); ESchemaAction action = SCHEMA_ACTION_NULL; - int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); - if (code != 0) { - return code; - } + SML_CHECK_CODE(smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info)); if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); - (void)memcpy(field.name, kv->key, kv->keyLen); - if (taosArrayPush(results, &field) == NULL) { - return terrno; - } + (void)memcpy(field.name, kv->key, MIN(kv->keyLen, sizeof(field.name) - 1)); + SML_CHECK_NULL(taosArrayPush(results, &field)); } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); if (index == NULL) { uError("smlBuildFieldsList get error, key:%s", kv->key); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto END; } uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); - if (field == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(field); field->bytes = getBytes(kv->type, kv->length); } } @@ -1020,18 +846,23 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO int32_t len = 0; for (int j = 0; j < taosArrayGetSize(results); ++j) { SField *field = taosArrayGet(results, j); - if (field == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(field); len += field->bytes; } if (len > maxLen) { return isTag ? TSDB_CODE_PAR_INVALID_TAGS_LENGTH : TSDB_CODE_PAR_INVALID_ROW_LENGTH; } - return TSDB_CODE_SUCCESS; +END: + return code; } +static FORCE_INLINE void smlBuildCreateStbReq(SMCreateStbReq *pReq, int32_t colVer, int32_t tagVer, tb_uid_t suid, int8_t source){ + pReq->colVer = colVer; + pReq->tagVer = tagVer; + pReq->suid = suid; + pReq->source = source; +} static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, ESchemaAction action) { SRequestObj *pRequest = NULL; @@ -1046,58 +877,37 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.numOfTags = taosArrayGetSize(pTags); pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); - if (pReq.pColumns == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(pReq.pColumns); for (int32_t i = 0; i < pReq.numOfColumns; ++i) { SField *pField = taosArrayGet(pColumns, i); - if (pField == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(pField); SFieldWithOptions fieldWithOption = {0}; setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); - if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(taosArrayPush(pReq.pColumns, &fieldWithOption)); } if (action == SCHEMA_ACTION_CREATE_STABLE) { - pReq.colVer = 1; - pReq.tagVer = 1; - pReq.suid = 0; - pReq.source = TD_REQ_FROM_APP; pSql = "sml_create_stable"; + smlBuildCreateStbReq(&pReq, 1, 1, 0, TD_REQ_FROM_APP); } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - pReq.colVer = pTableMeta->sversion; - pReq.tagVer = pTableMeta->tversion + 1; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; pSql = (action == SCHEMA_ACTION_ADD_TAG) ? "sml_add_tag" : "sml_modify_tag_size"; + smlBuildCreateStbReq(&pReq, pTableMeta->sversion, pTableMeta->tversion + 1, pTableMeta->uid, TD_REQ_FROM_TAOX); } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { - pReq.colVer = pTableMeta->sversion + 1; - pReq.tagVer = pTableMeta->tversion; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; + smlBuildCreateStbReq(&pReq, pTableMeta->sversion + 1, pTableMeta->tversion, pTableMeta->uid, TD_REQ_FROM_TAOX); } else { uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); code = TSDB_CODE_SML_INVALID_DATA; - goto end; + goto END; } - code = buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } + SML_CHECK_CODE(buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0)); pRequest->syncQuery = true; if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; + goto END; } if (pReq.numOfTags == 0) { @@ -1106,40 +916,29 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, field.type = TSDB_DATA_TYPE_NCHAR; field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; tstrncpy(field.name, tsSmlTagName, sizeof(field.name)); - if (taosArrayPush(pReq.pTags, &field) == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(taosArrayPush(pReq.pTags, &field)); } pReq.commentLen = -1; pReq.igExists = true; - code = tNameExtractFullName(pName, pReq.name); - if (TSDB_CODE_SUCCESS != code) { - goto end; - } + SML_CHECK_CODE(tNameExtractFullName(pName, pReq.name)); pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); if (pCmdMsg.msgLen < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + code = pCmdMsg.msgLen; + goto END; } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { - code = terrno; - goto end; - } - - if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; + SML_CHECK_NULL(pCmdMsg.pMsg); + code = tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); + if (code < 0) { taosMemoryFree(pCmdMsg.pMsg); - goto end; + goto END; } SQuery pQuery = {0}; - (void)memset(&pQuery, 0, sizeof(pQuery)); pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; @@ -1148,19 +947,134 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, launchQueryImpl(pRequest, &pQuery, true, NULL); // no need to check return value if (pRequest->code == TSDB_CODE_SUCCESS) { - code = catalogRemoveTableMeta(info->pCatalog, pName); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } + SML_CHECK_CODE(catalogRemoveTableMeta(info->pCatalog, pName)); } code = pRequest->code; -end: +END: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); return code; } +static int32_t smlCreateTable(SSmlHandle *info, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, + SName *pName, STableMeta **pTableMeta){ + int32_t code = 0; + SArray *pColumns = NULL; + SArray *pTags = NULL; + SML_CHECK_CODE(smlCheckAuth(info, conn, NULL, AUTH_TYPE_WRITE)); + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName->tname); + pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + SML_CHECK_NULL(pColumns); + pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + SML_CHECK_NULL(pTags); + SML_CHECK_CODE(smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true)); + SML_CHECK_CODE(smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false)); + pTags = NULL; + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE)); + info->cost.numOfCreateSTables++; + taosMemoryFreeClear(*pTableMeta); + + SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); + +END: + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + return code; +} + +static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTableMeta, SSmlSTableMeta *sTableData){ + int32_t code = 0; + *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + (pTableMeta)->tableInfo.numOfColumns, sizeof(SField)); + SML_CHECK_NULL(pColumns); + *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + (pTableMeta)->tableInfo.numOfTags, sizeof(SField)); + SML_CHECK_NULL(pTags); + for (uint16_t i = 0; i < (pTableMeta)->tableInfo.numOfColumns + (pTableMeta)->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = (pTableMeta)->schema[i].type; + field.bytes = (pTableMeta)->schema[i].bytes; + tstrncpy(field.name, (pTableMeta)->schema[i].name, sizeof(field.name)); + if (i < (pTableMeta)->tableInfo.numOfColumns) { + SML_CHECK_NULL(taosArrayPush(*pColumns, &field)); + } else { + SML_CHECK_NULL(taosArrayPush(*pTags, &field)); + } + } +END: + return code; +} +static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, + SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ + ESchemaAction action = SCHEMA_ACTION_NULL; + SArray *pColumns = NULL; + SArray *pTags = NULL; + int32_t code = 0; + SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true)); + + if (action != SCHEMA_ACTION_NULL) { + SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName->tname, + action); + SML_CHECK_CODE(smlBuildFields(&pColumns, &pTags, *pTableMeta, sTableData)); + SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, pTags, + (*pTableMeta)->tableInfo.numOfColumns, true)); + + pTags = NULL; + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, (*pTableMeta), action)); + + info->cost.numOfAlterTagSTables++; + taosMemoryFreeClear(*pTableMeta); + SML_CHECK_CODE(catalogRefreshTableMeta(info->pCatalog, conn, pName, -1)); + SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); + } + +END: + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + return code; +} + +static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, + SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ + ESchemaAction action = SCHEMA_ACTION_NULL; + SArray *pColumns = NULL; + SArray *pTags = NULL; + int32_t code = 0; + SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false)); + + if (action != SCHEMA_ACTION_NULL) { + SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName->tname, + action); + SML_CHECK_CODE(smlBuildFields(&pColumns, &pTags, *pTableMeta, sTableData)); + SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, pColumns, + (*pTableMeta)->tableInfo.numOfColumns, false)); + + pTags = NULL; + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, (*pTableMeta), action)); + + info->cost.numOfAlterColSTables++; + taosMemoryFreeClear(*pTableMeta); + SML_CHECK_CODE(catalogRefreshTableMeta(info->pCatalog, conn, pName, -1)); + SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); + } + + END: + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + return code; +} + +static int32_t smlBuildTempHash(SHashObj *hashTmp, STableMeta *pTableMeta, uint16_t start, uint16_t end){ + int32_t code = 0; + for (uint16_t i = start; i < end; i++) { + SML_CHECK_CODE(taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES)); + } + +END: + return code; +} + static int32_t smlModifyDBSchemas(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema); @@ -1188,286 +1102,43 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); char *measure = taosMemoryMalloc(superTableLen); - if (measure == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(measure); (void)memcpy(measure, superTable, superTableLen); PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); smlStrReplace(measure, superTableLen); (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - (void)memcpy(pName.tname, measure, superTableLen); + (void)memcpy(pName.tname, measure, MIN(superTableLen, TSDB_TABLE_NAME_LEN - 1)); taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { - code = smlCheckAuth(info, &conn, NULL, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); - SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); - if (pColumns == NULL) { - code = terrno; - goto end; - } - SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); - if (pTags == NULL) { - code = terrno; - taosArrayDestroy(pColumns); - goto end; - } - code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - taosArrayDestroy(pColumns); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - info->cost.numOfCreateSTables++; - taosMemoryFreeClear(pTableMeta); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } + SML_CHECK_CODE(smlCreateTable(info, &conn, sTableData, &pName, &pTableMeta)); } else if (code == TSDB_CODE_SUCCESS) { if (smlIsPKTable(pTableMeta)) { code = TSDB_CODE_SML_NOT_SUPPORT_PK; - goto end; - } - - hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, - HASH_NO_LOCK); - if (hashTmp == NULL) { - code = terrno; - goto end; - } - for (uint16_t i = pTableMeta->tableInfo.numOfColumns; - i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - if (code != 0) { - goto end; - } - } - - ESchemaAction action = SCHEMA_ACTION_NULL; - code = - smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - if (action != SCHEMA_ACTION_NULL) { - code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname, - action); - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - if (pColumns == NULL) { - code = terrno; - goto end; - } - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - if (pTags == NULL) { - taosArrayDestroy(pColumns); - code = terrno; - goto end; - } - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); - if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush(pColumns, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } else { - if (taosArrayPush(pTags, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } - } - code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, - pTableMeta->tableInfo.numOfColumns, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - if (taosArrayGetSize(pTags) + pTableMeta->tableInfo.numOfColumns > TSDB_MAX_COLUMNS) { - uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); - code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - if (taosArrayGetSize(pTags) > TSDB_MAX_TAGS) { - uError("SML:0x%" PRIx64 " too many tags than 128", info->id); - code = TSDB_CODE_PAR_INVALID_TAGS_NUM; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterTagSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } + goto END; } + hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(hashTmp); + SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags)); + SML_CHECK_CODE(smlModifyTag(info, hashTmp, &conn, sTableData, &pName, &pTableMeta)); taosHashClear(hashTmp); - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - } - action = SCHEMA_ACTION_NULL; - code = - smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - if (action != SCHEMA_ACTION_NULL) { - code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname, - action); - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - if (pColumns == NULL) { - code = terrno; - goto end; - } - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - if (pTags == NULL) { - taosArrayDestroy(pColumns); - code = terrno; - goto end; - } - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); - if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush(pColumns, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } else { - if (taosArrayPush(pTags, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } - } - - code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, - pTableMeta->tableInfo.numOfColumns, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - if (taosArrayGetSize(pColumns) + pTableMeta->tableInfo.numOfTags > TSDB_MAX_COLUMNS) { - uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); - code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterColSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } - } + SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns)); + SML_CHECK_CODE(smlModifyCols(info, hashTmp, &conn, sTableData, &pName, &pTableMeta)); needCheckMeta = true; taosHashCleanup(hashTmp); hashTmp = NULL; } else { uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); - goto end; + goto END; } if (needCheckMeta) { - code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, - sTableData->tags, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); - goto end; - } - code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); - goto end; - } + SML_CHECK_CODE(smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, sTableData->tags, true)); + SML_CHECK_CODE(smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false)); } taosMemoryFreeClear(sTableData->tableMeta); @@ -1481,7 +1152,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { return TSDB_CODE_SUCCESS; -end: +END: taosHashCancelIterate(info->superTables, tmp); taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); @@ -1493,40 +1164,37 @@ end: } static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SHashObj *checkDuplicate) { + int32_t code = 0; terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { - if (taosArrayPush(metaArray, kv) == NULL) { - return terrno; - } + SML_CHECK_NULL(taosArrayPush(metaArray, kv)); if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { - return TSDB_CODE_PAR_DUPLICATED_COLUMN; + code = TSDB_CODE_PAR_DUPLICATED_COLUMN; + goto END; } } else if (terrno == TSDB_CODE_DUP_KEY) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } } - return TSDB_CODE_SUCCESS; + +END: + return code; } static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj *checkDuplicate) { + int32_t code = 0; for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); if (index) { SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); - if (value == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(value); if (isTag) { if (kv->length > value->length) { @@ -1536,7 +1204,8 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } if (kv->type != value->type) { smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - return TSDB_CODE_SML_NOT_SAME_TYPE; + code = TSDB_CODE_SML_NOT_SAME_TYPE; + goto END; } if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger @@ -1547,24 +1216,21 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols if (tmp > INT16_MAX) { smlBuildInvalidDataMsg(msg, "too many cols or tags", kv->key); uError("too many cols or tags"); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto END; } int16_t size = tmp; - int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); - if (ret == 0) { - if (taosArrayPush(metaArray, kv) == NULL) { - return terrno; - } - if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { - return TSDB_CODE_PAR_DUPLICATED_COLUMN; - } - } else { - return ret; + SML_CHECK_CODE(taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES)); + SML_CHECK_NULL(taosArrayPush(metaArray, kv)); + if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { + code = TSDB_CODE_PAR_DUPLICATED_COLUMN; + goto END; } } } - return TSDB_CODE_SUCCESS; +END: + return code; } void smlDestroyTableInfo(void *para) { @@ -1631,94 +1297,71 @@ void smlDestroyInfo(SSmlHandle *info) { int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { int32_t code = TSDB_CODE_SUCCESS; SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); - if (NULL == info) { - return terrno; - } - if (taos != NULL) { + SML_CHECK_NULL(info); + if (taos != NULL){ info->taos = acquireTscObj(*(int64_t *)taos); - if (info->taos == NULL) { - code = TSDB_CODE_TSC_DISCONNECTED; - goto FAILED; - } - code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto FAILED; - } + SML_CHECK_NULL(info->taos); + SML_CHECK_CODE(catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog)); } info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + SML_CHECK_NULL(info->pVgHash); info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(info->childTables); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(info->tableUids); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { - uError("create SSmlHandle hash obj failed"); - code = terrno; - goto FAILED; - } + SML_CHECK_NULL(info->superTables); taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); - code = smlInitHandle(&info->pQuery); - if (code != TSDB_CODE_SUCCESS) { - goto FAILED; - } + SML_CHECK_CODE(smlInitHandle(&info->pQuery)); info->dataFormat = true; - info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); + SML_CHECK_NULL(info->tagJsonArray); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); + SML_CHECK_NULL(info->valueJsonArray); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + SML_CHECK_NULL(info->preLineTagKV); info->escapedStringList = taosArrayInit(8, POINTER_BYTES); - if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || - info->preLineTagKV == NULL || info->escapedStringList == NULL) { - uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); - code = terrno; - goto FAILED; - } + SML_CHECK_NULL(info->escapedStringList); *handle = info; return code; -FAILED: +END: smlDestroyInfo(info); return code; } static int32_t smlPushCols(SArray *colsArray, SArray *cols) { + int32_t code = TSDB_CODE_SUCCESS; SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!kvHash) { - uError("SML:smlDealCols failed to allocate memory"); - return terrno; - } + SML_CHECK_NULL(kvHash); for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - taosHashCleanup(kvHash); - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); terrno = 0; - int32_t code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { - taosHashCleanup(kvHash); - return TSDB_CODE_PAR_DUPLICATED_COLUMN; - } - if (code != TSDB_CODE_SUCCESS) { - taosHashCleanup(kvHash); - return code; + code = TSDB_CODE_PAR_DUPLICATED_COLUMN; + goto END; } + SML_CHECK_CODE(code); } - if (taosArrayPush(colsArray, &kvHash) == NULL) { - taosHashCleanup(kvHash); - return terrno; - } - return TSDB_CODE_SUCCESS; + SML_CHECK_NULL(taosArrayPush(colsArray, &kvHash)); + return code; +END: + taosHashCleanup(kvHash); + return code; } static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); + int32_t code = 0; if (info->dataFormat) return TSDB_CODE_SUCCESS; for (int32_t i = 0; i < info->lineNum; i++) { @@ -1728,10 +1371,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen); if (tmp) tinfo = *tmp; - } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, - elements->measureLen + elements->tagsLen); - if (tmp) tinfo = *tmp; } else { SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); @@ -1754,81 +1393,62 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { return TSDB_CODE_PAR_TOO_MANY_COLUMNS; } - int ret = smlPushCols(tinfo->cols, elements->colArray); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } + SML_CHECK_CODE(smlPushCols(tinfo->cols, elements->colArray)); SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); if (tableMeta) { // update meta uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, - (*tableMeta)->tagHash); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, - (*tableMeta)->colHash); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed, ret:%d", info->id, ret); - return ret; - } + SML_CHECK_CODE(smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, + (*tableMeta)->tagHash)); + SML_CHECK_CODE(smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, + (*tableMeta)->colHash)); } else { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = NULL; - ret = smlBuildSTableMeta(info->dataFormat, &meta); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - ret = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); - if (ret != TSDB_CODE_SUCCESS) { + SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, &meta)); + SML_CHECK_CODE(taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES)); + code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + smlDestroySTableMeta(meta); uError("SML:0x%" PRIx64 " put measuer to hash failed", info->id); - return ret; - } - ret = smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " insert meta failed:%s", info->id, tstrerror(ret)); - return ret; + goto END; } + SML_CHECK_CODE(smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL)); + SML_CHECK_CODE(smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash)); } } uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); - return TSDB_CODE_SUCCESS; +END: + return code; } static int32_t smlInsertData(SSmlHandle *info) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + char *measure = NULL; + SSmlTableInfo **oneTable = NULL; uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat); if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); - if (info->pRequest->dbList == NULL) { - return terrno; - } + SML_CHECK_NULL(info->pRequest->dbList); } char *data = (char *)taosArrayReserve(info->pRequest->dbList, 1); - if (data == NULL) { - return terrno; - } + SML_CHECK_NULL(data); SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); (void)tNameGetFullDbName(&pName, data); // ignore - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); + oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); while (oneTable) { SSmlTableInfo *tableData = *oneTable; int measureLen = tableData->sTableNameLen; - char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); - if (measure == NULL) { - return terrno; - } + measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); + SML_CHECK_NULL(measure); (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); @@ -1837,14 +1457,9 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); - if (info->pRequest->tableList == NULL) { - return terrno; - } + SML_CHECK_NULL(info->pRequest->tableList); } - if (taosArrayPush(info->pRequest->tableList, &pName) == NULL) { - return terrno; - } - + SML_CHECK_NULL(taosArrayPush(info->pRequest->tableList, &pName)); tstrncpy(pName.tname, tableData->childTableName, sizeof(pName.tname)); SRequestConnInfo conn = {0}; @@ -1853,36 +1468,18 @@ static int32_t smlInsertData(SSmlHandle *info) { conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } + SML_CHECK_CODE(smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE)); - SVgroupInfo vg; - code = catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName); - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } - code = taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " taosHashPut failed. table name: %s", info->id, tableData->childTableName); - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } + SVgroupInfo vg = {0}; + SML_CHECK_CODE(catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg)); + SML_CHECK_CODE(taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg))); SSmlSTableMeta **pMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); - if (unlikely(NULL == pMeta || NULL == (*pMeta)->tableMeta)) { + if (unlikely(NULL == pMeta || NULL == *pMeta || NULL == (*pMeta)->tableMeta)) { uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName); - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return TSDB_CODE_SML_INTERNAL_ERROR; + code = TSDB_CODE_SML_INTERNAL_ERROR; + goto END; } // use tablemeta of stable to save vgid and uid of child table @@ -1891,23 +1488,14 @@ static int32_t smlInsertData(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, tableData->uid, info->dataFormat); - code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, + SML_CHECK_CODE(smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf, - info->msgBuf.len); - taosMemoryFree(measure); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBindData failed", info->id); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } + info->msgBuf.len)); + taosMemoryFreeClear(measure); oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); } - code = smlBuildOutput(info->pQuery, info->pVgHash); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildOutput failed", info->id); - return code; - } + SML_CHECK_CODE(smlBuildOutput(info->pQuery, info->pVgHash)); info->cost.insertRpcTime = taosGetTimestampUs(); SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; @@ -1919,6 +1507,11 @@ static int32_t smlInsertData(SSmlHandle *info) { tstrerror(info->pRequest->code)); return info->pRequest->code; + +END: + taosMemoryFree(measure); + taosHashCancelIterate(info->childTables, oneTable); + return code; } static void smlPrintStatisticInfo(SSmlHandle *info) { @@ -1958,11 +1551,33 @@ int32_t smlClearForRerun(SSmlHandle *info) { info->currTableDataCtx = NULL; SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot); + if (stmt == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } stmt->freeHashFunc(stmt->pTableBlockHashObj); stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (stmt->pTableBlockHashObj == NULL) { + uError("SML:0x%" PRIx64 " stmt->pTableBlockHashObj == NULL", info->id); + return terrno; + } return TSDB_CODE_SUCCESS; } +static void printRaw(int64_t id, int lineNum, int numLines, ELogLevel level, char* data, int32_t len){ + char *print = taosMemoryCalloc(len + 1, 1); + if (print == NULL) { + uError("SML:0x%" PRIx64 " smlParseLine failed. code : %d", id, terrno); + return; + } + (void)memcpy(print, data, len); + if (level == DEBUG_DEBUG){ + uDebug("SML:0x%" PRIx64 " smlParseLine is raw, line %d/%d : %s", id, lineNum, numLines, print); + }else if (level == DEBUG_ERROR){ + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d/%d : %s", id, lineNum, numLines, print); + } + taosMemoryFree(print); +} + static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLineEnd, int numLines, int i, char **tmp, int *len) { if (lines) { @@ -1976,24 +1591,15 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi } (*len)++; } - if (info->protocol == TSDB_SML_LINE_PROTOCOL && (*tmp)[0] == '#') { // this line is comment + if (IS_COMMENT(info->protocol,(*tmp)[0])) { // this line is comment return false; } } if (*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)) { - char *print = taosMemoryCalloc(*len + 1, 1); - if (print != NULL) { - (void)memcpy(print, *tmp, *len); - uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, - info->protocol, *len, print); - taosMemoryFree(print); - } else { - uError("SML:0x%" PRIx64 " smlParseLine taosMemoryCalloc failed", info->id); - } + printRaw(info->id, i, numLines, DEBUG_DEBUG, *tmp, *len); } else { - uDebug("SML:0x%" PRIx64 " smlParseLine is not numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, - info->protocol, *len, *tmp); + uDebug("SML:0x%" PRIx64 " smlParseLine is not raw, line %d/%d : %s", info->id, i, numLines, *tmp); } return true; } @@ -2042,14 +1648,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } if (code != TSDB_CODE_SUCCESS) { if (rawLine != NULL) { - char *print = taosMemoryCalloc(len + 1, 1); - if (print == NULL) { - uError("SML:0x%" PRIx64 " smlParseLine failed. out of memory", info->id); - return code; - } - (void)memcpy(print, tmp, len); - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); - taosMemoryFree(print); + printRaw(info->id, i, numLines, DEBUG_ERROR, tmp, len); } else { uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); } @@ -2078,21 +1677,12 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL info->cost.parseTime = taosGetTimestampUs(); - code = smlParseLine(info, lines, rawLine, rawLineEnd, numLines); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code)); - return code; - } - code = smlParseLineBottom(info); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlParseLineBottom error : %s", info->id, tstrerror(code)); - return code; - } + SML_CHECK_CODE(smlParseLine(info, lines, rawLine, rawLineEnd, numLines)); + SML_CHECK_CODE(smlParseLineBottom(info)); info->cost.lineNum = info->lineNum; info->cost.numOfSTables = taosHashGetSize(info->superTables); info->cost.numOfCTables = taosHashGetSize(info->childTables); - info->cost.schemaTime = taosGetTimestampUs(); do { @@ -2105,18 +1695,11 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum); } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlModifyDBSchemas error : %s", info->id, tstrerror(code)); - return code; - } - + SML_CHECK_CODE(code); info->cost.insertBindTime = taosGetTimestampUs(); - code = smlInsertData(info); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlInsertData error : %s", info->id, tstrerror(code)); - return code; - } + SML_CHECK_CODE(smlInsertData(info)); +END: return code; } @@ -2160,29 +1743,16 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { - int32_t code = TSDB_CODE_SUCCESS; - if (NULL == taos) { - uError("SML:taos_schemaless_insert error taos is null"); - return NULL; - } + int32_t code = TSDB_CODE_SUCCESS; SRequestObj *request = NULL; - SSmlHandle *info = NULL; - int cnt = 0; + SSmlHandle *info = NULL; + int cnt = 0; while (1) { - code = createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid, &request); - if (TSDB_CODE_SUCCESS != code) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - + SML_CHECK_CODE(createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid, &request)); SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - code = smlBuildSmlInfo(taos, &info); - if (code != TSDB_CODE_SUCCESS) { - request->code = code; - smlBuildInvalidDataMsg(&msg, "init SSmlHandle failed", NULL); - uError("SML:taos_schemaless_insert error SSmlHandle is null, err msg:%s", tstrerror(code)); - goto end; - } + request->code = smlBuildSmlInfo(taos, &info); + SML_CHECK_CODE(request->code); + info->pRequest = request; info->pRequest->pQuery = info->pQuery; info->ttl = ttl; @@ -2198,20 +1768,20 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); - goto end; + goto END; } if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); - goto end; + goto END; } if (protocol == TSDB_SML_LINE_PROTOCOL && (precision < TSDB_SML_TIMESTAMP_NOT_CONFIGURED || precision > TSDB_SML_TIMESTAMP_NANO_SECONDS)) { request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; smlBuildInvalidDataMsg(&msg, "precision invalidate for line protocol", NULL); - goto end; + goto END; } if (protocol == TSDB_SML_JSON_PROTOCOL) { @@ -2219,7 +1789,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, } else if (numLines <= 0) { request->code = TSDB_CODE_SML_INVALID_DATA; smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL); - goto end; + goto END; } code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); @@ -2249,7 +1819,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } -end: + END: smlDestroyInfo(info); return (TAOS_RES *)request; } @@ -2275,6 +1845,16 @@ end: TAOS_RES *taos_schemaless_insert_ttl_with_reqid_tbname_key(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { + if (taos == NULL || lines == NULL || numLines <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + for (int i = 0; i < numLines; i++){ + if (lines[i] == NULL){ + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + } return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid, tbnameKey); } @@ -2298,26 +1878,32 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi reqid); } -static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol) { +static int32_t getRawLineLen(char *lines, int len, int protocol) { int numLines = 0; - *totalRows = 0; char *tmp = lines; for (int i = 0; i < len; i++) { if (lines[i] == '\n' || i == len - 1) { - numLines++; - if (tmp[0] != '#' || protocol != TSDB_SML_LINE_PROTOCOL) { // ignore comment - (*totalRows)++; + if (!IS_COMMENT(protocol, tmp[0])) { // ignore comment + numLines++; } tmp = lines + i + 1; } } + return numLines; } TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { - getRawLineLen(lines, len, totalRows, protocol); - return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid, + if (taos == NULL || lines == NULL || len <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + int numLines = getRawLineLen(lines, len, protocol); + if (totalRows != NULL){ + *totalRows = numLines; + } + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, numLines, protocol, precision, ttl, reqid, tbnameKey); } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index ece1ddf61f..9568074018 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -184,7 +184,7 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { return 0; } -int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { +int smlJsonParseObj(char **start, char *end, SSmlLineInfo *element, int8_t *offset) { int index = 0; while (*(*start)) { if ((*start)[0] != '"') { @@ -197,6 +197,10 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { return TSDB_CODE_TSC_INVALID_JSON; } + if ((*start) + offset[index] >= end){ + return TSDB_CODE_TSC_INVALID_JSON; + } + if ((*start)[1] == 'm') { (*start) += offset[index++]; element->measure = *start; @@ -539,28 +543,19 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ } static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { - int32_t ret = 0; + int32_t code = 0; if(info->dataFormat){ - ret = smlProcessSuperTable(info, elements); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - } - ret = smlProcessTagJson(info, tags); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - ret = smlJoinMeasureTag(elements); - if(ret != 0){ - return ret; + SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } + SML_CHECK_CODE(smlProcessTagJson(info, tags)); + SML_CHECK_CODE(smlJoinMeasureTag(elements)); return smlProcessChildTable(info, elements); + +END: + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return code; } static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { @@ -757,19 +752,18 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo } SSmlKv kvTs = {0}; smlBuildTsKv(&kvTs, ts); - - return smlParseEndTelnetJson(info, elements, &kvTs, &kv); + if (info->dataFormat){ + ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); + } else { + ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); + } + return ret; } static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { int32_t payloadNum = 0; int32_t ret = TSDB_CODE_SUCCESS; - if (unlikely(payload == NULL)) { - uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - info->root = cJSON_Parse(payload); if (unlikely(info->root == NULL)) { uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); @@ -836,13 +830,13 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) { +static int32_t smlParseJSONString(SSmlHandle *info, char **start, char *end, SSmlLineInfo *elements) { int32_t ret = TSDB_CODE_SUCCESS; if (info->offset[0] == 0) { ret = smlJsonParseObjFirst(start, elements, info->offset); } else { - ret = smlJsonParseObj(start, elements, info->offset); + ret = smlJsonParseObj(start, end, elements, info->offset); } if (ret != TSDB_CODE_SUCCESS) { @@ -870,7 +864,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * elements->cols[elements->colsLen] = tmp; return TSDB_CODE_TSC_INVALID_JSON; } - if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){ + if (taosArrayPush(info->valueJsonArray, &valueJson) == NULL){ cJSON_Delete(valueJson); elements->cols[elements->colsLen] = tmp; return terrno; @@ -945,20 +939,26 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * SSmlKv kvTs = {0}; smlBuildTsKv(&kvTs, ts); - return smlParseEndTelnetJson(info, elements, &kvTs, &kv); + if (info->dataFormat){ + ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); + } else { + ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); + } + return ret; } int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 1 << 15; + int32_t payloadNum = 1 << 10; int32_t ret = TSDB_CODE_SUCCESS; uDebug("SML:0x%" PRIx64 "json:%s", info->id, payload); int cnt = 0; char *dataPointStart = payload; + char *dataPointEnd = payload + strlen(payload); while (1) { if (info->dataFormat) { SSmlLineInfo element = {0}; - ret = smlParseJSONString(info, &dataPointStart, &element); + ret = smlParseJSONString(info, &dataPointStart, dataPointEnd, &element); if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); } else { if (cnt >= payloadNum) { @@ -971,7 +971,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { info->lines = (SSmlLineInfo *)tmp; (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); } - ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); + ret = smlParseJSONString(info, &dataPointStart, dataPointEnd, info->lines + cnt); if ((info->lines + cnt)->measure == NULL) break; } if (unlikely(ret != TSDB_CODE_SUCCESS)) { diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index c1f3431698..5ecf4e2206 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -298,7 +298,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - return TSDB_CODE_TSC_INVALID_JSON; + return TSDB_CODE_SML_INVALID_DATA; } cnt++; @@ -311,31 +311,23 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { + int32_t code = 0; bool isSameCTable = IS_SAME_CHILD_TABLE; if(isSameCTable){ return TSDB_CODE_SUCCESS; } - int32_t ret = 0; if(info->dataFormat){ - ret = smlProcessSuperTable(info, elements); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } + SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } - - ret = smlProcessTagLine(info, sql, sqlEnd); - if(ret != 0){ - if (info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - + SML_CHECK_CODE(smlProcessTagLine(info, sql, sqlEnd)); return smlProcessChildTable(info, elements); + +END: + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return code; } static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) { diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index e8601e33bc..bf422675ef 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -148,31 +148,20 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS return TSDB_CODE_SUCCESS; } - int32_t ret = 0; + int32_t code = 0; if(info->dataFormat){ - ret = smlProcessSuperTable(info, elements); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - } - - ret = smlProcessTagTelnet(info, data, sqlEnd); - if(ret != 0){ - if (info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - - ret = smlJoinMeasureTag(elements); - if(ret != 0){ - return ret; + SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } + SML_CHECK_CODE(smlProcessTagTelnet(info, data, sqlEnd)); + SML_CHECK_CODE(smlJoinMeasureTag(elements)); return smlProcessChildTable(info, elements); + +END: + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return code; } // format: =[ =] @@ -239,5 +228,10 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); } - return smlParseEndTelnetJson(info, elements, &kvTs, &kv); + if (info->dataFormat){ + ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); + } else { + ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); + } + return ret; } \ No newline at end of file diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index dc6a302924..bcd12a393a 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -591,6 +591,104 @@ TEST(testCase, smlParseTelnetLine_Test) { // smlDestroyInfo(info); //} +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + char *endptr = NULL; + double result = taosStr2Double(pVal, &endptr); + if (pVal == endptr) { + smlBuildInvalidDataMsg(msg, "invalid data", pVal); + return false; + } + + int32_t left = len - (endptr - pVal); + if (left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)) { + kvVal->type = TSDB_DATA_TYPE_DOUBLE; + kvVal->d = result; + } else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)) { + if (!IS_VALID_FLOAT(result)) { + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_FLOAT; + kvVal->f = (float)result; + } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { + if (smlDoubleToInt64OverFlow(result)) { + errno = 0; + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); + if (errno == ERANGE) { + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_BIGINT; + kvVal->i = tmp; + return true; + } + kvVal->type = TSDB_DATA_TYPE_BIGINT; + kvVal->i = (int64_t)result; + } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) { + if (result >= (double)UINT64_MAX || result < 0) { + errno = 0; + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); + if (errno == ERANGE || result < 0) { + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UBIGINT; + kvVal->u = tmp; + return true; + } + kvVal->type = TSDB_DATA_TYPE_UBIGINT; + kvVal->u = result; + } else if (left == 3 && strncasecmp(endptr, "i32", left) == 0) { + if (!IS_VALID_INT(result)) { + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_INT; + kvVal->i = result; + } else if (left == 3 && strncasecmp(endptr, "u32", left) == 0) { + if (!IS_VALID_UINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UINT; + kvVal->u = result; + } else if (left == 3 && strncasecmp(endptr, "i16", left) == 0) { + if (!IS_VALID_SMALLINT(result)) { + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_SMALLINT; + kvVal->i = result; + } else if (left == 3 && strncasecmp(endptr, "u16", left) == 0) { + if (!IS_VALID_USMALLINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_USMALLINT; + kvVal->u = result; + } else if (left == 2 && strncasecmp(endptr, "i8", left) == 0) { + if (!IS_VALID_TINYINT(result)) { + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_TINYINT; + kvVal->i = result; + } else if (left == 2 && strncasecmp(endptr, "u8", left) == 0) { + if (!IS_VALID_UTINYINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UTINYINT; + kvVal->u = result; + } else { + smlBuildInvalidDataMsg(msg, "invalid data", pVal); + return false; + } + return true; +} + TEST(testCase, smlParseNumber_performance_Test) { char msg[256] = {0}; SSmlMsgBuf msgBuf; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index cca35d9c9a..b1ff8cb733 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -468,35 +468,38 @@ end: int32_t smlInitHandle(SQuery** query) { *query = NULL; SQuery* pQuery = NULL; + SVnodeModifyOpStmt* stmt = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pQuery); - if (NULL == pQuery) { + if (code != 0) { uError("create pQuery error"); - return code; + goto END; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = false; pQuery->msgType = TDMT_VND_SUBMIT; - SVnodeModifyOpStmt* stmt = NULL; code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&stmt); - if (NULL == stmt) { + if (code != 0) { uError("create SVnodeModifyOpStmt error"); - qDestroyQuery(pQuery); - return code; + goto END; } stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (stmt->pTableBlockHashObj == NULL){ uError("create pTableBlockHashObj error"); - qDestroyQuery(pQuery); - nodesDestroyNode((SNode*)stmt); - return terrno; + code = terrno; + goto END; } stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeArrayFunc = insDestroyVgroupDataCxtList; pQuery->pRoot = (SNode*)stmt; *query = pQuery; + return code; - return TSDB_CODE_SUCCESS; +END: + nodesDestroyNode((SNode*)stmt); + qDestroyQuery(pQuery); + return code; } int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { From 5ee1c9b70a9cbf99ec71bf2517753d6d8305ef53 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 21 Oct 2024 21:01:33 +0800 Subject: [PATCH 03/43] fix retry error --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 3090903805..e63e17e584 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -29,9 +29,16 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &epSet); if (epSet.numOfEps <= 1) { - pMsg->pCont = NULL; - pMsg->code = TSDB_CODE_MNODE_NOT_FOUND; - return; + if (epSet.numOfEps == 0) { + pMsg->pCont = NULL; + pMsg->code = TSDB_CODE_MNODE_NOT_FOUND; + return; + } + if (strcmp(epSet.eps[0].fqdn, tsLocalFqdn) == 0 && epSet.eps[0].port == tsServerPort) { + pMsg->pCont = NULL; + pMsg->code = TSDB_CODE_MNODE_NOT_FOUND; + return; + } } int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); From d91b771d9c3b22619a0cf52bcb0fff6f395627ad Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 22 Oct 2024 09:13:45 +0800 Subject: [PATCH 04/43] fix retry error --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index e63e17e584..1340ee9277 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -34,6 +34,8 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { pMsg->code = TSDB_CODE_MNODE_NOT_FOUND; return; } + // dnode is not the mnode or mnode leader and This ensures that the function correctly handles cases where the + // dnode cannot obtain a valid epSet and avoids returning an incorrect or misleading epSet. if (strcmp(epSet.eps[0].fqdn, tsLocalFqdn) == 0 && epSet.eps[0].port == tsServerPort) { pMsg->pCont = NULL; pMsg->code = TSDB_CODE_MNODE_NOT_FOUND; From 2e8a5348c9c9438f4341e5838104f420b8ceef5e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 22 Oct 2024 17:32:18 +0800 Subject: [PATCH 05/43] enh:[TD-32166]refactor code in sml --- source/client/inc/clientSml.h | 24 ++++++--------- source/client/src/clientSml.c | 58 ++++++++++++++--------------------- 2 files changed, 33 insertions(+), 49 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 3098b124b7..88831ce815 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -92,21 +92,17 @@ extern "C" { } \ } -#define SML_CHECK_CODE(CMD) \ - do { \ - code = (CMD); \ - if (TSDB_CODE_SUCCESS != code) { \ - goto END; \ - } \ - } while (0) +#define SML_CHECK_CODE(CMD) \ + code = (CMD); \ + if (TSDB_CODE_SUCCESS != code) { \ + goto END; \ + } -#define SML_CHECK_NULL(CMD) \ - do { \ - if (NULL == (CMD)) { \ - code = terrno; \ - goto END; \ - } \ - } while (0) +#define SML_CHECK_NULL(CMD) \ + if (NULL == (CMD)) { \ + code = terrno; \ + goto END; \ + } typedef enum { SCHEMA_ACTION_NULL, diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 25abeb52fb..c5842bb0bb 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -207,19 +207,18 @@ static void smlDestroySTableMeta(void *para) { } int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta **sMeta) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + STableMeta *pTableMeta = NULL; + char *measure = currElement->measure; int measureLen = currElement->measureLen; if (currElement->measureEscaped) { measure = (char *)taosMemoryMalloc(measureLen); - if (measure == NULL) { - return terrno; - } + SML_CHECK_NULL(measure); (void)memcpy(measure, currElement->measure, measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); } - STableMeta *pTableMeta = NULL; code = smlGetMeta(info, measure, measureLen, &pTableMeta); if (currElement->measureEscaped) { taosMemoryFree(measure); @@ -227,20 +226,9 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; - taosMemoryFreeClear(pTableMeta); - return code; - } - code = smlBuildSTableMeta(info->dataFormat, sMeta); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pTableMeta); - return code; - } - (*sMeta)->tableMeta = pTableMeta; - code = taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES); - if (code != TSDB_CODE_SUCCESS) { - smlDestroySTableMeta(*sMeta); - return code; + goto END; } + SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, sMeta)); for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { SSchema *col = pTableMeta->schema + i; SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; @@ -254,16 +242,19 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml } if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush((*sMeta)->cols, &kv) == NULL) { - return terrno; - } + SML_CHECK_NULL(taosArrayPush((*sMeta)->cols, &kv)); } else { - if (taosArrayPush((*sMeta)->tags, &kv) == NULL) { - return terrno; - } + SML_CHECK_NULL(taosArrayPush((*sMeta)->tags, &kv)); } } - return TSDB_CODE_SUCCESS; + SML_CHECK_CODE(taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES)); + (*sMeta)->tableMeta = pTableMeta; + return code; + +END: + smlDestroySTableMeta(*sMeta); + taosMemoryFreeClear(pTableMeta); + return code; } bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { @@ -863,7 +854,7 @@ static FORCE_INLINE void smlBuildCreateStbReq(SMCreateStbReq *pReq, int32_t colV pReq->suid = suid; pReq->source = source; } -static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, +static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray **pTags, STableMeta *pTableMeta, ESchemaAction action) { SRequestObj *pRequest = NULL; SMCreateStbReq pReq = {0}; @@ -873,8 +864,9 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); - pReq.pTags = pTags; - pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pTags = *pTags; + pReq.numOfTags = taosArrayGetSize(*pTags); + *pTags = NULL; pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); SML_CHECK_NULL(pReq.pColumns); @@ -970,8 +962,7 @@ static int32_t smlCreateTable(SSmlHandle *info, SRequestConnInfo *conn, SSmlSTab SML_CHECK_NULL(pTags); SML_CHECK_CODE(smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true)); SML_CHECK_CODE(smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false)); - pTags = NULL; - SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE)); + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, &pTags, NULL, SCHEMA_ACTION_CREATE_STABLE)); info->cost.numOfCreateSTables++; taosMemoryFreeClear(*pTableMeta); @@ -1019,8 +1010,7 @@ static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInf SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, pTags, (*pTableMeta)->tableInfo.numOfColumns, true)); - pTags = NULL; - SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, (*pTableMeta), action)); + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, &pTags, (*pTableMeta), action)); info->cost.numOfAlterTagSTables++; taosMemoryFreeClear(*pTableMeta); @@ -1050,8 +1040,7 @@ static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnIn SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, pColumns, (*pTableMeta)->tableInfo.numOfColumns, false)); - pTags = NULL; - SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, (*pTableMeta), action)); + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, &pTags, (*pTableMeta), action)); info->cost.numOfAlterColSTables++; taosMemoryFreeClear(*pTableMeta); @@ -1409,7 +1398,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { info->lineNum); SSmlSTableMeta *meta = NULL; SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, &meta)); - SML_CHECK_CODE(taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES)); code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { smlDestroySTableMeta(meta); From 21a7b6dc16d0d4cdf60d348e86330723c723d3ed Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 22 Oct 2024 17:43:30 +0800 Subject: [PATCH 06/43] enh:[TD-32166]remove useless code in sml --- source/client/inc/clientSml.h | 4 +- source/client/src/clientSml.c | 4 +- source/client/src/clientSmlJson.c | 422 +----------------------------- 3 files changed, 4 insertions(+), 426 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 88831ce815..a562ca3226 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -225,8 +225,6 @@ extern int64_t smlFactorS[]; int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle); void smlDestroyInfo(SSmlHandle *info); -int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); -int smlJsonParseObj(char **start, char *end, SSmlLineInfo *element, int8_t *offset); void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); @@ -246,7 +244,7 @@ void smlDestroyTableInfo(void *para); void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); -int32_t smlParseJSON(SSmlHandle *info, char *payload); +int32_t smlParseJSONExt(SSmlHandle *info, char *payload); int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c5842bb0bb..9f9671f340 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1597,9 +1597,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { if (lines) { - code = smlParseJSON(info, *lines); + code = smlParseJSONExt(info, *lines); } else if (rawLine) { - code = smlParseJSON(info, rawLine); + code = smlParseJSONExt(info, rawLine); } if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseJSON failed:%s", info->id, lines ? *lines : rawLine); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 9568074018..d44f9cac9c 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -21,259 +21,6 @@ #define OTD_JSON_SUB_FIELDS_NUM 2 -#define JUMP_JSON_SPACE(start) \ - while (*(start)) { \ - if (unlikely(*(start) > 32)) \ - break; \ - else \ - (start)++; \ - } - -static int32_t smlJsonGetObj(char **payload) { - int leftBracketCnt = 0; - bool isInQuote = false; - while (**payload) { - if (**payload == '"' && *((*payload) - 1) != '\\') { - isInQuote = !isInQuote; - } else if (!isInQuote && unlikely(**payload == '{')) { - leftBracketCnt++; - (*payload)++; - continue; - } else if (!isInQuote && unlikely(**payload == '}')) { - leftBracketCnt--; - (*payload)++; - if (leftBracketCnt == 0) { - return 0; - } else if (leftBracketCnt < 0) { - return -1; - } - continue; - } - (*payload)++; - } - return -1; -} - -int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { - int index = 0; - while (*(*start)) { - if ((*start)[0] != '"') { - (*start)++; - continue; - } - - if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { - uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start); - return TSDB_CODE_TSC_INVALID_JSON; - } - - char *sTmp = *start; - if ((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' && (*start)[4] == 'r' && (*start)[5] == 'i' && - (*start)[6] == 'c' && (*start)[7] == '"') { - (*start) += 8; - bool isInQuote = false; - while (*(*start)) { - if (unlikely(!isInQuote && *(*start) == '"')) { - (*start)++; - offset[index++] = *start - sTmp; - element->measure = (*start); - isInQuote = true; - continue; - } - if (unlikely(isInQuote && *(*start) == '"')) { - element->measureLen = (*start) - element->measure; - (*start)++; - break; - } - (*start)++; - } - } else if ((*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' && (*start)[4] == 'e' && - (*start)[5] == 's' && (*start)[6] == 't' && (*start)[7] == 'a' && (*start)[8] == 'm' && - (*start)[9] == 'p' && (*start)[10] == '"') { - (*start) += 11; - bool hasColon = false; - while (*(*start)) { - if (unlikely(!hasColon && *(*start) == ':')) { - (*start)++; - JUMP_JSON_SPACE((*start)) - offset[index++] = *start - sTmp; - element->timestamp = (*start); - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->timestampLen = tmp - (*start); - *start = tmp; - } - break; - } - hasColon = true; - continue; - } - if (unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))) { - element->timestampLen = (*start) - element->timestamp; - break; - } - (*start)++; - } - } else if ((*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' && (*start)[4] == 'u' && - (*start)[5] == 'e' && (*start)[6] == '"') { - (*start) += 7; - - bool hasColon = false; - while (*(*start)) { - if (unlikely(!hasColon && *(*start) == ':')) { - (*start)++; - JUMP_JSON_SPACE((*start)) - offset[index++] = *start - sTmp; - element->cols = (*start); - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->colsLen = tmp - (*start); - *start = tmp; - } - break; - } - hasColon = true; - continue; - } - if (unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))) { - element->colsLen = (*start) - element->cols; - break; - } - (*start)++; - } - } else if ((*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' && (*start)[4] == 's' && - (*start)[5] == '"') { - (*start) += 6; - - while (*(*start)) { - if (unlikely(*(*start) == ':')) { - (*start)++; - JUMP_JSON_SPACE((*start)) - offset[index++] = *start - sTmp; - element->tags = (*start); - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->tagsLen = tmp - (*start); - *start = tmp; - } - break; - } - (*start)++; - } - } - if (*(*start) == '\0') { - break; - } - if (*(*start) == '}') { - (*start)++; - break; - } - (*start)++; - } - - if (unlikely(index != OTD_JSON_FIELDS_NUM) || element->tags == NULL || element->cols == NULL || - element->measure == NULL || element->timestamp == NULL) { - uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM); - return TSDB_CODE_TSC_INVALID_JSON; - } - return 0; -} - -int smlJsonParseObj(char **start, char *end, SSmlLineInfo *element, int8_t *offset) { - int index = 0; - while (*(*start)) { - if ((*start)[0] != '"') { - (*start)++; - continue; - } - - if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { - uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start); - return TSDB_CODE_TSC_INVALID_JSON; - } - - if ((*start) + offset[index] >= end){ - return TSDB_CODE_TSC_INVALID_JSON; - } - - if ((*start)[1] == 'm') { - (*start) += offset[index++]; - element->measure = *start; - while (*(*start)) { - if (unlikely(*(*start) == '"')) { - element->measureLen = (*start) - element->measure; - (*start)++; - break; - } - (*start)++; - } - } else if ((*start)[1] == 't' && (*start)[2] == 'i') { - (*start) += offset[index++]; - element->timestamp = *start; - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->timestampLen = tmp - (*start); - *start = tmp; - } - } else { - while (*(*start)) { - if (unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)) { - element->timestampLen = (*start) - element->timestamp; - break; - } - (*start)++; - } - } - } else if ((*start)[1] == 'v') { - (*start) += offset[index++]; - element->cols = *start; - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->colsLen = tmp - (*start); - *start = tmp; - } - } else { - while (*(*start)) { - if (unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)) { - element->colsLen = (*start) - element->cols; - break; - } - (*start)++; - } - } - } else if ((*start)[1] == 't' && (*start)[2] == 'a') { - (*start) += offset[index++]; - element->tags = (*start); - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->tagsLen = tmp - (*start); - *start = tmp; - } - } - if (*(*start) == '}') { - (*start)++; - break; - } - (*start)++; - } - - if (unlikely(index != 0 && index != OTD_JSON_FIELDS_NUM)) { - uError("elements != %d", OTD_JSON_FIELDS_NUM); - return TSDB_CODE_TSC_INVALID_JSON; - } - return TSDB_CODE_SUCCESS; -} - static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { elements->measureLen = strlen(metric->valuestring); if (IS_INVALID_TABLE_LEN(elements->measureLen)) { @@ -760,7 +507,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo return ret; } -static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { +int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { int32_t payloadNum = 0; int32_t ret = TSDB_CODE_SUCCESS; @@ -830,170 +577,3 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseJSONString(SSmlHandle *info, char **start, char *end, SSmlLineInfo *elements) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (info->offset[0] == 0) { - ret = smlJsonParseObjFirst(start, elements, info->offset); - } else { - ret = smlJsonParseObj(start, end, elements, info->offset); - } - - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; - - if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { - smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; - - if (unlikely(elements->colsLen == 0)) { - uError("SML:colsLen == 0"); - return TSDB_CODE_TSC_INVALID_VALUE; - } else if (unlikely(elements->cols[0] == '{')) { - char tmp = elements->cols[elements->colsLen]; - elements->cols[elements->colsLen] = '\0'; - cJSON *valueJson = cJSON_Parse(elements->cols); - if (unlikely(valueJson == NULL)) { - uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols); - elements->cols[elements->colsLen] = tmp; - return TSDB_CODE_TSC_INVALID_JSON; - } - if (taosArrayPush(info->valueJsonArray, &valueJson) == NULL){ - cJSON_Delete(valueJson); - elements->cols[elements->colsLen] = tmp; - return terrno; - } - ret = smlParseValueFromJSONObj(valueJson, &kv); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols); - elements->cols[elements->colsLen] = tmp; - return TSDB_CODE_TSC_INVALID_VALUE; - } - elements->cols[elements->colsLen] = tmp; - } else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - // Parse tags - if (is_same_child_table_telnet(elements, &info->preLine) != 0) { - char tmp = *(elements->tags + elements->tagsLen); - *(elements->tags + elements->tagsLen) = 0; - cJSON *tagsJson = cJSON_Parse(elements->tags); - *(elements->tags + elements->tagsLen) = tmp; - if (unlikely(tagsJson == NULL)) { - uError("SML:0x%" PRIx64 " parse json tag failed:%s", info->id, elements->tags); - return TSDB_CODE_TSC_INVALID_JSON; - } - - if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){ - cJSON_Delete(tagsJson); - uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id); - return terrno; - } - ret = smlParseTagsFromJSON(info, tagsJson, elements); - if (unlikely(ret)) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - return ret; - } - } else { - elements->measureTag = info->preLine.measureTag; - } - - if (unlikely(info->reRun)) { - return TSDB_CODE_SUCCESS; - } - - // Parse timestamp - // notice!!! put ts back to tag to ensure get meta->precision - int64_t ts = 0; - if (unlikely(elements->timestampLen == 0)) { - uError("OTD:0x%" PRIx64 " elements->timestampLen == 0", info->id); - return TSDB_CODE_INVALID_TIMESTAMP; - } else if (elements->timestamp[0] == '{') { - char tmp = elements->timestamp[elements->timestampLen]; - elements->timestamp[elements->timestampLen] = '\0'; - cJSON *tsJson = cJSON_Parse(elements->timestamp); - ts = smlParseTSFromJSON(info, tsJson); - if (unlikely(ts < 0)) { - uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload:%s", info->id, elements->timestamp); - elements->timestamp[elements->timestampLen] = tmp; - cJSON_Delete(tsJson); - return TSDB_CODE_INVALID_TIMESTAMP; - } - elements->timestamp[elements->timestampLen] = tmp; - cJSON_Delete(tsJson); - } else { - ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); - if (unlikely(ts < 0)) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } - SSmlKv kvTs = {0}; - smlBuildTsKv(&kvTs, ts); - - if (info->dataFormat){ - ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); - } else { - ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); - } - return ret; -} - -int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 1 << 10; - int32_t ret = TSDB_CODE_SUCCESS; - - uDebug("SML:0x%" PRIx64 "json:%s", info->id, payload); - int cnt = 0; - char *dataPointStart = payload; - char *dataPointEnd = payload + strlen(payload); - while (1) { - if (info->dataFormat) { - SSmlLineInfo element = {0}; - ret = smlParseJSONString(info, &dataPointStart, dataPointEnd, &element); - if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); - } else { - if (cnt >= payloadNum) { - payloadNum = payloadNum << 1; - void *tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); - if (tmp == NULL) { - ret = terrno; - return ret; - } - info->lines = (SSmlLineInfo *)tmp; - (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); - } - ret = smlParseJSONString(info, &dataPointStart, dataPointEnd, info->lines + cnt); - if ((info->lines + cnt)->measure == NULL) break; - } - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uError("SML:0x%" PRIx64 " Invalid JSON Payload 1:%s", info->id, payload); - return smlParseJSONExt(info, payload); - } - - if (unlikely(info->reRun)) { - cnt = 0; - dataPointStart = payload; - info->lineNum = payloadNum; - ret = smlClearForRerun(info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - continue; - } - - cnt++; - if (*dataPointStart == '\0') break; - } - info->lineNum = cnt; - - return TSDB_CODE_SUCCESS; -} From 574998f02835372490b6d5b9cfeec555a34ebb78 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 22 Oct 2024 17:50:26 +0800 Subject: [PATCH 07/43] enh:[TD-32166]remove useless code in sml --- source/client/src/clientSml.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 9f9671f340..c17ff1413e 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -197,7 +197,13 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts) { } static void smlDestroySTableMeta(void *para) { + if (para == NULL) { + return; + } SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; + if (meta == NULL) { + return; + } taosHashCleanup(meta->tagHash); taosHashCleanup(meta->colHash); taosArrayDestroy(meta->tags); @@ -252,7 +258,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml return code; END: - smlDestroySTableMeta(*sMeta); + smlDestroySTableMeta(sMeta); taosMemoryFreeClear(pTableMeta); return code; } @@ -587,7 +593,7 @@ int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { return TSDB_CODE_SUCCESS; END: - smlDestroySTableMeta(meta); + smlDestroySTableMeta(&meta); return TSDB_CODE_OUT_OF_MEMORY; } @@ -1400,7 +1406,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, &meta)); code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { - smlDestroySTableMeta(meta); + smlDestroySTableMeta(&meta); uError("SML:0x%" PRIx64 " put measuer to hash failed", info->id); goto END; } From 9a3184578dc0c352aa96bbb22179881a8aeae2dc Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Wed, 23 Oct 2024 09:26:28 +0800 Subject: [PATCH 08/43] docs:Update 07-supported.md --- docs/zh/14-reference/07-supported.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/07-supported.md b/docs/zh/14-reference/07-supported.md index 10ca237653..fcb20b8ac1 100644 --- a/docs/zh/14-reference/07-supported.md +++ b/docs/zh/14-reference/07-supported.md @@ -14,7 +14,7 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" | M1 | | | | | | | | ● | 注：1) ● 表示经过官方测试验证， ○ 表示非官方测试验证，E 表示仅企业版支持。 - 2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/RedHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。 + 2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/CentOS Stream/RedHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。 ## TDengine 客户端和连接器支持的平台列表 From ea1da6db9e8fef7edf4ff0c8f87246590a4edf6e Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:35:01 +0800 Subject: [PATCH 09/43] docs:Update 01-taosd.md --- docs/zh/14-reference/01-components/01-taosd.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index fbf086bf6b..b724d46bde 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -33,7 +33,7 @@ taosd 命令行参数如下 | secondEp | taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，缺省值：无 | | fqdn | 启动 taosd 后所监听的服务地址，缺省值：所在服务器上配置的第一个 hostname | | serverPort | 启动 taosd 后所监听的端口，缺省值：6030 | -| numOfRpcSessions | 允许一个客户端能创建的最大连接数，取值范围 100-100000，缺省值：30000 | +| numOfRpcSessions | 允许一个 dnode 能发起的最大连接数，取值范围 100-100000，缺省值：30000 | | timeToGetAvailableConn | 获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值：500000 | ### 监控相关 @@ -458,4 +458,4 @@ TDengine 的日志文件主要包括普通日志和慢日志两种类型。 3. 多个客户端的日志存储在相应日志路径下的同一个 taosSlowLog.yyyy.mm.dd 文件里。 4. 慢日志文件不自动删除，不压缩。 5. 使用和普通日志文件相同的三个参数 logDir, minimalLogDirGB, asyncLog。另外两个参数 numOfLogLines，logKeepDays 不适用于慢日志。 - \ No newline at end of file + From 75c8727c52909818b0ed5ac3a25ecced9b9a508b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 24 Oct 2024 15:48:57 +0800 Subject: [PATCH 10/43] enh:[TD-32166]refactor code in sml --- source/client/inc/clientSml.h | 8 ++ source/client/src/clientSml.c | 167 ++++++++++++++-------------- source/client/src/clientSmlJson.c | 89 +++++---------- source/client/src/clientSmlLine.c | 3 +- source/client/src/clientSmlTelnet.c | 7 +- utils/test/c/sml_test.c | 104 ++++++++++++++++- 6 files changed, 230 insertions(+), 148 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index a562ca3226..8558ec46dc 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -95,15 +95,23 @@ extern "C" { #define SML_CHECK_CODE(CMD) \ code = (CMD); \ if (TSDB_CODE_SUCCESS != code) { \ + lino = __LINE__; \ goto END; \ } #define SML_CHECK_NULL(CMD) \ if (NULL == (CMD)) { \ code = terrno; \ + lino = __LINE__; \ goto END; \ } +#define RETURN \ + if (code != 0){ \ + uError("%s failed code:%d line:%d", __FUNCTION__ , code, lino); \ + } \ + return code; + typedef enum { SCHEMA_ACTION_NULL, SCHEMA_ACTION_CREATE_STABLE, diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c17ff1413e..5f4327b1cd 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -213,22 +213,20 @@ static void smlDestroySTableMeta(void *para) { } int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta **sMeta) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; STableMeta *pTableMeta = NULL; - char *measure = currElement->measure; - int measureLen = currElement->measureLen; + int measureLen = currElement->measureLen; + char *measure = (char *)taosMemoryMalloc(measureLen); + SML_CHECK_NULL(measure); + (void)memcpy(measure, currElement->measure, measureLen); if (currElement->measureEscaped) { - measure = (char *)taosMemoryMalloc(measureLen); - SML_CHECK_NULL(measure); - (void)memcpy(measure, currElement->measure, measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); - smlStrReplace(measure, measureLen); } + smlStrReplace(measure, measureLen); code = smlGetMeta(info, measure, measureLen, &pTableMeta); - if (currElement->measureEscaped) { - taosMemoryFree(measure); - } + taosMemoryFree(measure); if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; @@ -260,7 +258,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml END: smlDestroySTableMeta(sMeta); taosMemoryFreeClear(pTableMeta); - return code; + RETURN } bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { @@ -376,8 +374,8 @@ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { int32_t code = TSDB_CODE_SUCCESS; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); + int32_t lino = 0; + SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); SSmlTableInfo *tinfo = NULL; if (unlikely(oneTable == NULL)) { SML_CHECK_CODE(smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo)); @@ -401,47 +399,40 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { } else { tinfo = *oneTable; } - if (tinfo == NULL) { - uError("smlProcessChildTable failed to get child table info"); - return TSDB_CODE_SML_INTERNAL_ERROR; - } if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; return TSDB_CODE_SUCCESS; END: smlDestroyTableInfo(&tinfo); - return code; + RETURN } int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { int32_t code = 0; - uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); + int32_t lino = 0; + uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__ , kvTs->i); SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0)); SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1)); SML_CHECK_CODE(smlBuildRow(info->currTableDataCtx)); - info->preLine = *elements; END: clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(code != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - } - return code; + RETURN } int32_t smlParseEndTelnetJsonUnFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { int32_t code = 0; - uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); + int32_t lino = 0; + uDebug("SML:0x%" PRIx64 " %s format false, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); if (elements->colArray == NULL) { elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); SML_CHECK_NULL(elements->colArray); } SML_CHECK_NULL(taosArrayPush(elements->colArray, kvTs)); SML_CHECK_NULL (taosArrayPush(elements->colArray, kv)); - info->preLine = *elements; END: - return code; + RETURN } int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) { @@ -469,8 +460,9 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnameKey) { int32_t code = 0; - bool autoChildName = false; - size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); + int32_t lino = 0; + bool autoChildName = false; + size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); if (delimiter > 0 && tbnameKey == NULL) { size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { @@ -519,11 +511,12 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam } END: - return code; + RETURN } int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { int32_t code = 0; + int32_t lino = 0; SArray *dst = NULL; SML_CHECK_CODE(smlParseTableName(oneTable->tags, oneTable->childTableName, tbnameKey)); @@ -531,7 +524,6 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { dst = taosArrayDup(oneTable->tags, NULL); SML_CHECK_NULL(dst); if (oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN) { - uError("SML:smlSetCTableName super table name is too long"); code = TSDB_CODE_SML_INTERNAL_ERROR; goto END; } @@ -550,7 +542,7 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { END: taosArrayDestroy(dst); - return code; + RETURN } int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { @@ -575,6 +567,7 @@ int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo * int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { int32_t code = 0; + int32_t lino = 0; SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); SML_CHECK_NULL(meta); if (unlikely(!isDataFormat)) { @@ -594,7 +587,8 @@ int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { END: smlDestroySTableMeta(&meta); - return TSDB_CODE_OUT_OF_MEMORY; + uError("%s failed code:%d line:%d", __FUNCTION__ , code, lino); + return code; } int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { @@ -704,8 +698,9 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; if (index) { if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. db type: %d, point type: %d, key: %s", info->id, - colField[*index].type, kv->type, kv->key); + snprintf(info->msgBuf.buf, info->msgBuf.len, "SML:0x%" PRIx64 " %s point type and db type mismatch. db type: %d, point type: %d, key: %s", + info->id, __FUNCTION__, colField[*index].type, kv->type, kv->key); + uError("%s", info->msgBuf.buf); return TSDB_CODE_SML_INVALID_DATA; } @@ -759,6 +754,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *checkDumplicateCols, ESchemaAction *action, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); @@ -775,11 +771,12 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH } } END: - return code; + RETURN } static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SML_CHECK_NULL(hashTmp); int32_t i = 0; @@ -798,7 +795,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool END: taosHashCleanup(hashTmp); - return code; + RETURN } static int32_t getBytes(uint8_t type, int32_t length) { @@ -813,6 +810,7 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = TSDB_CODE_SUCCESS; for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); SML_CHECK_NULL(kv); @@ -822,7 +820,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); - (void)memcpy(field.name, kv->key, MIN(kv->keyLen, sizeof(field.name) - 1)); + (void)memcpy(field.name, kv->key, TMIN(kv->keyLen, sizeof(field.name) - 1)); SML_CHECK_NULL(taosArrayPush(results, &field)); } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); @@ -851,7 +849,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO } END: - return code; + RETURN } static FORCE_INLINE void smlBuildCreateStbReq(SMCreateStbReq *pReq, int32_t colVer, int32_t tagVer, tb_uid_t suid, int8_t source){ @@ -865,6 +863,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SRequestObj *pRequest = NULL; SMCreateStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SCmdMsgInfo pCmdMsg = {0}; char *pSql = NULL; @@ -952,16 +951,17 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, END: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); - return code; + RETURN } static int32_t smlCreateTable(SSmlHandle *info, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ int32_t code = 0; + int32_t lino = 0; SArray *pColumns = NULL; SArray *pTags = NULL; SML_CHECK_CODE(smlCheckAuth(info, conn, NULL, AUTH_TYPE_WRITE)); - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName->tname); + uDebug("SML:0x%" PRIx64 " %s create table:%s", info->id, __FUNCTION__, pName->tname); pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); SML_CHECK_NULL(pColumns); pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); @@ -977,11 +977,12 @@ static int32_t smlCreateTable(SSmlHandle *info, SRequestConnInfo *conn, SSmlSTab END: taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - return code; + RETURN } static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTableMeta, SSmlSTableMeta *sTableData){ int32_t code = 0; + int32_t lino = 0; *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + (pTableMeta)->tableInfo.numOfColumns, sizeof(SField)); SML_CHECK_NULL(pColumns); *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + (pTableMeta)->tableInfo.numOfTags, sizeof(SField)); @@ -998,7 +999,7 @@ static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTa } } END: - return code; + RETURN } static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ @@ -1006,11 +1007,12 @@ static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInf SArray *pColumns = NULL; SArray *pTags = NULL; int32_t code = 0; + int32_t lino = 0; SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true)); if (action != SCHEMA_ACTION_NULL) { SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName->tname, + uDebug("SML:0x%" PRIx64 " %s change table tag, table:%s, action:%d", info->id, __FUNCTION__, pName->tname, action); SML_CHECK_CODE(smlBuildFields(&pColumns, &pTags, *pTableMeta, sTableData)); SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, pTags, @@ -1027,7 +1029,7 @@ static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInf END: taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - return code; + RETURN } static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, @@ -1036,11 +1038,12 @@ static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnIn SArray *pColumns = NULL; SArray *pTags = NULL; int32_t code = 0; + int32_t lino = 0; SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false)); if (action != SCHEMA_ACTION_NULL) { SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName->tname, + uDebug("SML:0x%" PRIx64 " %s change table col, table:%s, action:%d", info->id, __FUNCTION__, pName->tname, action); SML_CHECK_CODE(smlBuildFields(&pColumns, &pTags, *pTableMeta, sTableData)); SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, pColumns, @@ -1054,14 +1057,15 @@ static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnIn SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); } - END: +END: taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - return code; + RETURN } static int32_t smlBuildTempHash(SHashObj *hashTmp, STableMeta *pTableMeta, uint16_t start, uint16_t end){ int32_t code = 0; + int32_t lino = 0; for (uint16_t i = start; i < end; i++) { SML_CHECK_CODE(taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES)); } @@ -1071,12 +1075,13 @@ END: } static int32_t smlModifyDBSchemas(SSmlHandle *info) { - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s start, format:%d, needModifySchema:%d", info->id, __FUNCTION__, info->dataFormat, info->needModifySchema); if (info->dataFormat && !info->needModifySchema) { return TSDB_CODE_SUCCESS; } int32_t code = 0; + int32_t lino = 0; SHashObj *hashTmp = NULL; STableMeta *pTableMeta = NULL; @@ -1099,10 +1104,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { char *measure = taosMemoryMalloc(superTableLen); SML_CHECK_NULL(measure); (void)memcpy(measure, superTable, superTableLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); + if (info->protocol == TSDB_SML_LINE_PROTOCOL){ + PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); + } smlStrReplace(measure, superTableLen); (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - (void)memcpy(pName.tname, measure, MIN(superTableLen, TSDB_TABLE_NAME_LEN - 1)); + (void)memcpy(pName.tname, measure, TMIN(superTableLen, TSDB_TABLE_NAME_LEN - 1)); taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1160,6 +1167,7 @@ END: static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SHashObj *checkDuplicate) { int32_t code = 0; + int32_t lino = 0; terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1177,12 +1185,13 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } END: - return code; + RETURN } static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj *checkDuplicate) { int32_t code = 0; + int32_t lino = 0; for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); SML_CHECK_NULL(kv); @@ -1225,7 +1234,7 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } END: - return code; + RETURN } void smlDestroyTableInfo(void *para) { @@ -1249,8 +1258,7 @@ void freeSSmlKv(void *data) { } void smlDestroyInfo(SSmlHandle *info) { - if (!info) return; - // qDestroyQuery(info->pQuery); + if (info == NULL) return; taosHashCleanup(info->pVgHash); taosHashCleanup(info->childTables); @@ -1275,22 +1283,20 @@ void smlDestroyInfo(SSmlHandle *info) { if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); - if (info->parseJsonByLib) { - taosMemoryFree(info->lines[i].tags); - } if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) { taosMemoryFree(info->lines[i].measureTag); } } taosMemoryFree(info->lines); } - + taosMemoryFreeClear(info->preLine.tags); cJSON_Delete(info->root); taosMemoryFreeClear(info); } int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); SML_CHECK_NULL(info); if (taos != NULL){ @@ -1327,11 +1333,12 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { END: smlDestroyInfo(info); - return code; + RETURN } static int32_t smlPushCols(SArray *colsArray, SArray *cols) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SML_CHECK_NULL(kvHash); for (size_t i = 0; i < taosArrayGetSize(cols); i++) { @@ -1350,13 +1357,14 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { return code; END: taosHashCleanup(kvHash); - return code; + RETURN } static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); int32_t code = 0; + int32_t lino = 0; if (info->dataFormat) return TSDB_CODE_SUCCESS; for (int32_t i = 0; i < info->lineNum; i++) { @@ -1417,11 +1425,12 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); END: - return code; + RETURN } static int32_t smlInsertData(SSmlHandle *info) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; char *measure = NULL; SSmlTableInfo **oneTable = NULL; uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat); @@ -1444,7 +1453,9 @@ static int32_t smlInsertData(SSmlHandle *info) { measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); SML_CHECK_NULL(measure); (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + if (info->protocol == TSDB_SML_LINE_PROTOCOL){ + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + } smlStrReplace(measure, measureLen); (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); (void)memcpy(pName.tname, measure, measureLen); @@ -1505,7 +1516,7 @@ static int32_t smlInsertData(SSmlHandle *info) { END: taosMemoryFree(measure); taosHashCancelIterate(info->childTables, oneTable); - return code; + RETURN } static void smlPrintStatisticInfo(SSmlHandle *info) { @@ -1521,6 +1532,8 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { } int32_t smlClearForRerun(SSmlHandle *info) { + int32_t code = 0; + int32_t lino = 0; info->reRun = false; taosHashClear(info->childTables); @@ -1528,33 +1541,23 @@ int32_t smlClearForRerun(SSmlHandle *info) { taosHashClear(info->tableUids); if (!info->dataFormat) { - if (unlikely(info->lines != NULL)) { - uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); - return TSDB_CODE_SML_INVALID_DATA; - } info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); - if (unlikely(info->lines == NULL)) { - uError("SML:0x%" PRIx64 " info->lines == NULL", info->id); - return terrno; - } + SML_CHECK_NULL(info->lines); } taosArrayClearP(info->escapedStringList, taosMemoryFree); + taosMemoryFreeClear(info->preLine.tags); (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot); - if (stmt == NULL){ - return TSDB_CODE_SML_INVALID_DATA; - } stmt->freeHashFunc(stmt->pTableBlockHashObj); stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (stmt->pTableBlockHashObj == NULL) { - uError("SML:0x%" PRIx64 " stmt->pTableBlockHashObj == NULL", info->id); - return terrno; - } - return TSDB_CODE_SUCCESS; + SML_CHECK_NULL(stmt->pTableBlockHashObj); + +END: + RETURN } static void printRaw(int64_t id, int lineNum, int numLines, ELogLevel level, char* data, int32_t len){ @@ -1667,6 +1670,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t retryNum = 0; info->cost.parseTime = taosGetTimestampUs(); @@ -1694,7 +1698,7 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL SML_CHECK_CODE(smlInsertData(info)); END: - return code; + RETURN } void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd) { @@ -1738,6 +1742,7 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRequestObj *request = NULL; SSmlHandle *info = NULL; int cnt = 0; diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index d44f9cac9c..feb0b4645a 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -290,7 +290,12 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ } static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { + if (is_same_child_table_telnet(elements, &info->preLine) == 0) { + elements->measureTag = info->preLine.measureTag; + return TSDB_CODE_SUCCESS; + } int32_t code = 0; + int32_t lino = 0; if(info->dataFormat){ SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } @@ -302,7 +307,7 @@ END: if(info->reRun){ return TSDB_CODE_SUCCESS; } - return code; + RETURN } static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { @@ -420,7 +425,8 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { } static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { - int32_t ret = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; cJSON *metricJson = NULL; cJSON *tsJson = NULL; @@ -435,52 +441,22 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo } cJSON **marks[OTD_JSON_FIELDS_NUM] = {&metricJson, &tsJson, &valueJson, &tagsJson}; - ret = smlGetJsonElements(root, marks); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - return ret; - } - + SML_CHECK_CODE(smlGetJsonElements(root, marks)); // Parse metric - ret = smlParseMetricFromJSON(info, metricJson, elements); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); - return ret; - } - + SML_CHECK_CODE(smlParseMetricFromJSON(info, metricJson, elements)); // Parse metric value SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN}; - ret = smlParseValueFromJSON(valueJson, &kv); - if (unlikely(ret)) { - uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); - return ret; - } + SML_CHECK_CODE(smlParseValueFromJSON(valueJson, &kv)); // Parse tags - bool needFree = info->dataFormat; elements->tags = cJSON_PrintUnformatted(tagsJson); - if (elements->tags == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; - } - elements->tagsLen = strlen(elements->tags); - if (is_same_child_table_telnet(elements, &info->preLine) != 0) { - ret = smlParseTagsFromJSON(info, tagsJson, elements); - if (unlikely(ret)) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - taosMemoryFree(elements->tags); - elements->tags = NULL; - return ret; - } - } else { - elements->measureTag = info->preLine.measureTag; - } + SML_CHECK_NULL(elements->tags); - if (needFree) { - taosMemoryFree(elements->tags); - elements->tags = NULL; - } + elements->tagsLen = strlen(elements->tags); + SML_CHECK_CODE(smlParseTagsFromJSON(info, tagsJson, elements)); if (unlikely(info->reRun)) { - return TSDB_CODE_SUCCESS; + goto END; } // Parse timestamp @@ -489,22 +465,29 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo if (unlikely(ts < 0)) { char* tmp = cJSON_PrintUnformatted(tsJson); if (tmp == NULL) { - uError("cJSON_PrintUnformatted failed since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %" PRId64, info->id, info->msgBuf.buf, ts); } else { uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); taosMemoryFree(tmp); } - return TSDB_CODE_INVALID_TIMESTAMP; + code = TSDB_CODE_INVALID_TIMESTAMP; + goto END; } SSmlKv kvTs = {0}; smlBuildTsKv(&kvTs, ts); if (info->dataFormat){ - ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); + code = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); } else { - ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); + code = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); } - return ret; + SML_CHECK_CODE(code); + taosMemoryFreeClear(info->preLine.tags); + info->preLine = *elements; + elements->tags = NULL; + +END: + taosMemoryFree(elements->tags); + RETURN } int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { @@ -527,23 +510,7 @@ int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_TSC_INVALID_JSON; } - if (unlikely(info->lines != NULL)) { - for (int i = 0; i < info->lineNum; i++) { - taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); - if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); - } - taosMemoryFree(info->lines); - info->lines = NULL; - } info->lineNum = payloadNum; - info->dataFormat = true; - - ret = smlClearForRerun(info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - info->parseJsonByLib = true; cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child; int cnt = 0; @@ -552,6 +519,8 @@ int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { if (info->dataFormat) { SSmlLineInfo element = {0}; ret = smlParseJSONStringExt(info, dataPoint, &element); + if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); + } else { ret = smlParseJSONStringExt(info, dataPoint, info->lines + cnt); } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 5ecf4e2206..e2e60ee7c4 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -312,6 +312,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { int32_t code = 0; + int32_t lino = 0; bool isSameCTable = IS_SAME_CHILD_TABLE; if(isSameCTable){ return TSDB_CODE_SUCCESS; @@ -327,7 +328,7 @@ END: if(info->reRun){ return TSDB_CODE_SUCCESS; } - return code; + RETURN } static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) { diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index bf422675ef..4deb6a34cd 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -149,19 +149,20 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS } int32_t code = 0; + int32_t lino = 0; if(info->dataFormat){ SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } SML_CHECK_CODE(smlProcessTagTelnet(info, data, sqlEnd)); SML_CHECK_CODE(smlJoinMeasureTag(elements)); - return smlProcessChildTable(info, elements); + code = smlProcessChildTable(info, elements); END: if(info->reRun){ return TSDB_CODE_SUCCESS; } - return code; + RETURN } // format: =[ =] @@ -233,5 +234,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } else { ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); } + info->preLine = *elements; + return ret; } \ No newline at end of file diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 1d8d82ccb9..20f2d7dc25 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -105,6 +105,102 @@ int smlProcess_telnet_Test() { return code; } +int smlProcess_json0_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}}]"}; + + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } + ASSERT(code == 0); + + + const char *sql2[] = { + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":1}" + "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}" + "}]", + }; + + char *sql3[1] = {0}; + for (int i = 0; i < 1; i++) { + sql3[i] = taosMemoryCalloc(1, 1024); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql3[i]); + } + + ASSERT(code == 0); + + + // TD-22903 + const char *sql4[] = { + "[{\"metric\": \"test_us\", \"timestamp\": {\"value\": 1626006833639, \"type\": \"ms\"}, \"value\": true, \"tags\": {\"t0\": true}}, {\"metric\": \"test_us\", \"timestamp\": {\"value\": 1626006833638, \"type\": \"ms\"}, \"value\": false, \"tags\": {\"t0\": true}}]" + }; + char *sql5[1] = {0}; + for (int i = 0; i < 1; i++) { + sql5[i] = taosMemoryCalloc(1, 1024); + ASSERT(sql5[i] != NULL); + (void)strncpy(sql5[i], sql4[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql5[i]); + } + ASSERT(code == 0); + + taos_close(taos); + + return code; +} + int smlProcess_json1_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -2135,8 +2231,8 @@ int main(int argc, char *argv[]) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } - int ret = 0; - ret = sml_ts5528_test(); +// int ret = smlProcess_json0_Test(); + int ret = sml_ts5528_test(); ASSERT(!ret); ret = sml_td29691_Test(); ASSERT(ret); @@ -2146,8 +2242,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_td18789_Test(); ASSERT(!ret); - ret = sml_td24070_Test(); - ASSERT(!ret); +// ret = sml_td24070_Test(); +// ASSERT(!ret); ret = sml_td23881_Test(); ASSERT(ret); ret = sml_escape_Test(); From 4090aad6fd2c53a940e9472a8d3614357d557704 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 24 Oct 2024 16:18:39 +0800 Subject: [PATCH 11/43] enh:[TD-32166]refactor code in sml --- source/client/src/clientSml.c | 8 +++- utils/test/c/sml_test.c | 82 ++++++++++++++++++++--------------- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5f4327b1cd..1b4643a958 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1289,7 +1289,9 @@ void smlDestroyInfo(SSmlHandle *info) { } taosMemoryFree(info->lines); } - taosMemoryFreeClear(info->preLine.tags); + if(info->protocol == TSDB_SML_JSON_PROTOCOL) { + taosMemoryFreeClear(info->preLine.tags); + } cJSON_Delete(info->root); taosMemoryFreeClear(info); } @@ -1546,7 +1548,9 @@ int32_t smlClearForRerun(SSmlHandle *info) { } taosArrayClearP(info->escapedStringList, taosMemoryFree); - taosMemoryFreeClear(info->preLine.tags); + if(info->protocol == TSDB_SML_JSON_PROTOCOL) { + taosMemoryFreeClear(info->preLine.tags); + } (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 20f2d7dc25..6d4192280c 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -105,6 +105,44 @@ int smlProcess_telnet_Test() { return code; } +int smlProcess_telnet0_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql1[] = {"sysif.bytes.out 1479496100 1.3E0 host=web01 interface=eth0"}; + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + + const char *sql2[] = {"sysif.bytes.out 1479496700 1.6E0 host=web01 interface=eth0"}; + pRes = taos_schemaless_insert(taos, (char **)sql2, sizeof(sql2) / sizeof(sql2[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + + const char *sql3[] = {"sysif.bytes.out 1479496300 1.1E0 interface=eth0 host=web01"}; + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + + taos_close(taos); + + return code; +} + int smlProcess_json0_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -115,7 +153,7 @@ int smlProcess_json0_Test() { taos_free_result(pRes); const char *sql[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}}]"}; + "[{\"metric\":\"syscpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}}]"}; char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { @@ -141,8 +179,8 @@ int smlProcess_json0_Test() { const char *sql2[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":1}" - "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}" + "[{\"metric\":\"syscpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":1}" + "},{\"metric\":\"syscpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}" "}]", }; @@ -169,33 +207,6 @@ int smlProcess_json0_Test() { ASSERT(code == 0); - - // TD-22903 - const char *sql4[] = { - "[{\"metric\": \"test_us\", \"timestamp\": {\"value\": 1626006833639, \"type\": \"ms\"}, \"value\": true, \"tags\": {\"t0\": true}}, {\"metric\": \"test_us\", \"timestamp\": {\"value\": 1626006833638, \"type\": \"ms\"}, \"value\": false, \"tags\": {\"t0\": true}}]" - }; - char *sql5[1] = {0}; - for (int i = 0; i < 1; i++) { - sql5[i] = taosMemoryCalloc(1, 1024); - ASSERT(sql5[i] != NULL); - (void)strncpy(sql5[i], sql4[i], 1023); - } - - pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); - code = taos_errno(pRes); - if (code != 0) { - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - } else { - printf("%s result:success\n", __FUNCTION__); - } - taos_free_result(pRes); - - for (int i = 0; i < 1; i++) { - taosMemoryFree(sql5[i]); - } - ASSERT(code == 0); - taos_close(taos); return code; @@ -2231,8 +2242,9 @@ int main(int argc, char *argv[]) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } -// int ret = smlProcess_json0_Test(); - int ret = sml_ts5528_test(); + int ret = smlProcess_json0_Test(); + ASSERT(!ret); + ret = sml_ts5528_test(); ASSERT(!ret); ret = sml_td29691_Test(); ASSERT(ret); @@ -2252,8 +2264,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_ts3116_Test(); ASSERT(!ret); - ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file - ASSERT(!ret); +// ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file +// ASSERT(!ret); ret = sml_ts3303_Test(); ASSERT(!ret); ret = sml_ttl_Test(); @@ -2268,6 +2280,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = smlProcess_telnet_Test(); ASSERT(!ret); + ret = smlProcess_telnet0_Test(); + ASSERT(!ret); ret = smlProcess_json1_Test(); ASSERT(!ret); ret = smlProcess_json2_Test(); From 35d3d968ff7e582d11808cdc74c96539516ee0d6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 24 Oct 2024 18:55:37 +0800 Subject: [PATCH 12/43] enh:[TD-32166]refactor code in sml --- source/client/inc/clientSml.h | 1 - source/client/src/clientSml.c | 101 +++++++++++++------------- source/client/src/clientSmlJson.c | 33 ++++----- source/client/src/clientSmlLine.c | 29 +++----- source/client/src/clientSmlTelnet.c | 16 ++-- source/client/test/smlTest.cpp | 9 +++ source/libs/parser/src/parInsertSml.c | 6 +- utils/test/c/sml_test.c | 24 ++++-- 8 files changed, 120 insertions(+), 99 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 8558ec46dc..d94a671c45 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -211,7 +211,6 @@ typedef struct { cJSON *root; // for parse json int8_t offset[OTD_JSON_FIELDS_NUM]; SSmlLineInfo *lines; // element is SSmlLineInfo - bool parseJsonByLib; SArray *tagJsonArray; SArray *valueJsonArray; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 1b4643a958..f9ba8531f4 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -169,23 +169,23 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u } int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, SSmlTableInfo **tInfo) { + int32_t code = 0; + int32_t lino = 0; SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); - if (!tag) { - return terrno; - } + SML_CHECK_NULL(tag) tag->sTableName = measure; tag->sTableNameLen = measureLen; tag->cols = taosArrayInit(numRows, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - taosMemoryFree(tag); - return terrno; - } - + SML_CHECK_NULL(tag->cols) *tInfo = tag; - return TSDB_CODE_SUCCESS; + return code; + +END: + taosMemoryFree(tag); + uError("%s failed code:%d line:%d", __FUNCTION__ , code, lino); + return code; } void smlBuildTsKv(SSmlKv *kv, int64_t ts) { @@ -437,7 +437,7 @@ END: int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) { if (info->dataFormat) { - uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i); + uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); if (ret == TSDB_CODE_SUCCESS) { ret = smlBuildRow(info->currTableDataCtx); @@ -446,11 +446,11 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) clearColValArraySml(info->currTableDataCtx->pValues); taosArrayClearP(info->escapedStringList, taosMemoryFree); if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + uError("SML:0x%" PRIx64 " %s smlBuildCol error:%d", info->id, __FUNCTION__, ret); return ret; } } else { - uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i); + uDebug("SML:0x%" PRIx64 " %s format false, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); taosArraySet(elements->colArray, 0, kvTs); } info->preLine = *elements; @@ -1134,7 +1134,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { taosHashCleanup(hashTmp); hashTmp = NULL; } else { - uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); + uError("SML:0x%" PRIx64 " %s load table meta error: %s", info->id, __FUNCTION__, tstrerror(code)); goto END; } @@ -1145,11 +1145,11 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { taosMemoryFreeClear(sTableData->tableMeta); sTableData->tableMeta = pTableMeta; - uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, + uDebug("SML:0x%" PRIx64 " %s modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, __FUNCTION__, pTableMeta->uid, pTableMeta->sversion, pTableMeta->tversion); tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s end success, format:%d, needModifySchema:%d", info->id, __FUNCTION__, info->dataFormat, info->needModifySchema); return TSDB_CODE_SUCCESS; @@ -1159,7 +1159,7 @@ END: taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error - uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, + uError("SML:0x%" PRIx64 " %s end failed:%d:%s, format:%d, needModifySchema:%d", info->id, __FUNCTION__, code, tstrerror(code), info->dataFormat, info->needModifySchema); return code; @@ -1331,7 +1331,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { SML_CHECK_NULL(info->escapedStringList); *handle = info; - return code; + info = NULL; END: smlDestroyInfo(info); @@ -1362,8 +1362,8 @@ END: RETURN } -static int32_t smlParseLineBottom(SSmlHandle *info) { - uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, +static int32_t smlParseEnd(SSmlHandle *info) { + uDebug("SML:0x%" PRIx64 " %s start, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); int32_t code = 0; int32_t lino = 0; @@ -1403,28 +1403,28 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); if (tableMeta) { // update meta - uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s update meta, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); SML_CHECK_CODE(smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, (*tableMeta)->tagHash)); SML_CHECK_CODE(smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, (*tableMeta)->colHash)); } else { - uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s add meta, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = NULL; SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, &meta)); code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { smlDestroySTableMeta(&meta); - uError("SML:0x%" PRIx64 " put measuer to hash failed", info->id); + uError("SML:0x%" PRIx64 " put measure to hash failed", info->id); goto END; } SML_CHECK_CODE(smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL)); SML_CHECK_CODE(smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash)); } } - uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); + uDebug("SML:0x%" PRIx64 " %s end, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); END: RETURN @@ -1435,7 +1435,7 @@ static int32_t smlInsertData(SSmlHandle *info) { int32_t lino = 0; char *measure = NULL; SSmlTableInfo **oneTable = NULL; - uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat); + uDebug("SML:0x%" PRIx64 " %s start, format:%d", info->id, __FUNCTION__, info->dataFormat); if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); @@ -1484,7 +1484,7 @@ static int32_t smlInsertData(SSmlHandle *info) { SSmlSTableMeta **pMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); if (unlikely(NULL == pMeta || NULL == *pMeta || NULL == (*pMeta)->tableMeta)) { - uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName); + uError("SML:0x%" PRIx64 " %s NULL == pMeta. table name: %s", info->id, __FUNCTION__, tableData->childTableName); code = TSDB_CODE_SML_INTERNAL_ERROR; goto END; } @@ -1492,7 +1492,7 @@ static int32_t smlInsertData(SSmlHandle *info) { // use tablemeta of stable to save vgid and uid of child table (*pMeta)->tableMeta->vgId = vg.vgId; (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid - uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, + uDebug("SML:0x%" PRIx64 " %s table:%s, uid:%" PRIu64 ", format:%d", info->id, __FUNCTION__, pName.tname, tableData->uid, info->dataFormat); SML_CHECK_CODE(smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, @@ -1510,7 +1510,7 @@ static int32_t smlInsertData(SSmlHandle *info) { launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code - uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, + uDebug("SML:0x%" PRIx64 " %s end, format:%d, code:%d,%s", info->id, __FUNCTION__, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code)); return info->pRequest->code; @@ -1605,20 +1605,25 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi return true; } -static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { - uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); + +static int32_t smlParseJson(SSmlHandle *info, char *lines[], char *rawLine) { + int32_t code = TSDB_CODE_SUCCESS; + if (lines) { + code = smlParseJSONExt(info, *lines); + } else if (rawLine) { + code = smlParseJSONExt(info, rawLine); + } + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed code:%d", __FUNCTION__ , code); + } + return code; +} + +static int32_t smlParseStart(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { + uDebug("SML:0x%" PRIx64 " %s start", info->id, __FUNCTION__); int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - if (lines) { - code = smlParseJSONExt(info, *lines); - } else if (rawLine) { - code = smlParseJSONExt(info, rawLine); - } - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseJSON failed:%s", info->id, lines ? *lines : rawLine); - return code; - } - return code; + return smlParseJson(info, lines, rawLine); } char *oldRaw = rawLine; @@ -1644,19 +1649,17 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } else { code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i); } - } else { - code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { if (rawLine != NULL) { printRaw(info->id, i, numLines, DEBUG_ERROR, tmp, len); } else { - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); + uError("SML:0x%" PRIx64 " %s failed. line %d : %s", info->id, __FUNCTION__, i, tmp); } return code; } if (info->reRun) { - uDebug("SML:0x%" PRIx64 " smlParseLine re run", info->id); + uDebug("SML:0x%" PRIx64 " %s re run", info->id, __FUNCTION__); i = 0; rawLine = oldRaw; code = smlClearForRerun(info); @@ -1667,7 +1670,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } i++; } - uDebug("SML:0x%" PRIx64 " smlParseLine end", info->id); + uDebug("SML:0x%" PRIx64 " %s end", info->id, __FUNCTION__); return code; } @@ -1679,8 +1682,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL info->cost.parseTime = taosGetTimestampUs(); - SML_CHECK_CODE(smlParseLine(info, lines, rawLine, rawLineEnd, numLines)); - SML_CHECK_CODE(smlParseLineBottom(info)); + SML_CHECK_CODE(smlParseStart(info, lines, rawLine, rawLineEnd, numLines)); + SML_CHECK_CODE(smlParseEnd(info)); info->cost.lineNum = info->lineNum; info->cost.numOfSTables = taosHashGetSize(info->superTables); @@ -1752,7 +1755,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, int cnt = 0; while (1) { SML_CHECK_CODE(createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid, &request)); - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + SSmlMsgBuf msg = {request->msgBufLen, request->msgBuf}; request->code = smlBuildSmlInfo(taos, &info); SML_CHECK_CODE(request->code); @@ -1822,7 +1825,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } - END: +END: smlDestroyInfo(info); return (TAOS_RES *)request; } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index feb0b4645a..2dcc36eada 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -24,7 +24,7 @@ static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { elements->measureLen = strlen(metric->valuestring); if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - uError("OTD:0x%" PRIx64 " Metric length is 0 or large than 192", info->id); + uError("SML:0x%" PRIx64 " Metric length is 0 or large than 192", info->id); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -44,7 +44,7 @@ static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) { child = child->next; } if (*marks[i] == NULL) { - uError("smlGetJsonElements error, not find mark:%d:%s", i, jsonName[i]); + uError("SML %s error, not find mark:%d:%s", __FUNCTION__, i, jsonName[i]); return TSDB_CODE_TSC_INVALID_JSON; } } @@ -53,7 +53,7 @@ static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) { static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { if (strcasecmp(typeStr, "bool") != 0) { - uError("OTD:invalid type(%s) for JSON Bool", typeStr); + uError("SML:invalid type(%s) for JSON Bool", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } pVal->type = TSDB_DATA_TYPE_BOOL; @@ -67,7 +67,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // tinyint if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { if (!IS_VALID_TINYINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_TINYINT; @@ -78,7 +78,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // smallint if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { if (!IS_VALID_SMALLINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_SMALLINT; @@ -89,7 +89,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // int if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { if (!IS_VALID_INT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(int)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_INT; @@ -113,7 +113,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // float if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { if (!IS_VALID_FLOAT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(float)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_FLOAT; @@ -130,7 +130,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { } // if reach here means type is unsupported - uError("OTD:invalid type(%s) for JSON Number", typeStr); + uError("SML:invalid type(%s) for JSON Number", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } @@ -142,7 +142,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { } else if (strcasecmp(typeStr, "nchar") == 0) { pVal->type = TSDB_DATA_TYPE_NCHAR; } else { - uError("OTD:invalid type(%s) for JSON String", typeStr); + uError("SML:invalid type(%s) for JSON String", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } pVal->length = strlen(value->valuestring); @@ -225,7 +225,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { case cJSON_String: { int32_t ret = smlConvertJSONString(kv, "binary", root); if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse binary value from JSON Obj"); + uError("SML:Failed to parse binary value from JSON Obj"); return ret; } break; @@ -233,7 +233,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { case cJSON_Object: { int32_t ret = smlParseValueFromJSONObj(root, kv); if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse value from JSON Obj"); + uError("SML:Failed to parse value from JSON Obj"); return ret; } break; @@ -262,7 +262,7 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ } size_t keyLen = strlen(tag->string); if (unlikely(IS_INVALID_COL_LEN(keyLen))) { - uError("OTD:Tag key length is 0 or too large than 64"); + uError("SML:Tag key length is 0 or too large than 64"); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -436,7 +436,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo int32_t size = cJSON_GetArraySize(root); // outmost json fields has to be exactly 4 if (size != OTD_JSON_FIELDS_NUM) { - uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); + uError("SML:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); return TSDB_CODE_TSC_INVALID_JSON; } @@ -465,9 +465,9 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo if (unlikely(ts < 0)) { char* tmp = cJSON_PrintUnformatted(tsJson); if (tmp == NULL) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %" PRId64, info->id, info->msgBuf.buf, ts); + uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %" PRId64, info->id, info->msgBuf.buf, ts); } else { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); + uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); taosMemoryFree(tmp); } code = TSDB_CODE_INVALID_TIMESTAMP; @@ -506,7 +506,7 @@ int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { } else if (cJSON_IsObject(info->root)) { payloadNum = 1; } else { - uError("SML:0x%" PRIx64 " Invalid JSON Payload 3:%s", info->id, payload); + uError("SML:0x%" PRIx64 " Invalid JSON type:%s", info->id, payload); return TSDB_CODE_TSC_INVALID_JSON; } @@ -520,7 +520,6 @@ int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { SSmlLineInfo element = {0}; ret = smlParseJSONStringExt(info, dataPoint, &element); if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); - } else { ret = smlParseJSONStringExt(info, dataPoint, info->lines + cnt); } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e2e60ee7c4..b54f4e0beb 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -63,7 +63,7 @@ static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t le int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); if (unlikely(ts == -1)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid timestamp", data); return TSDB_CODE_SML_INVALID_DATA; } return ts; @@ -84,7 +84,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 'l' || pVal->value[0] == 'L') { // nchar - if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) { + if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') { pVal->type = TSDB_DATA_TYPE_NCHAR; pVal->length -= NCHAR_ADD_LEN; if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { @@ -97,7 +97,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 'g' || pVal->value[0] == 'G') { // geometry - if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= sizeof("POINT")+3) { + if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') { int32_t code = initCtxGeomFromText(); if (code != TSDB_CODE_SUCCESS) { return code; @@ -124,7 +124,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 'b' || pVal->value[0] == 'B') { // varbinary - if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) { + if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') { pVal->type = TSDB_DATA_TYPE_VARBINARY; if(isHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){ if(!isValidateHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){ @@ -346,7 +346,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL const char *escapeChar = NULL; while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid data", *sql); return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(IS_EQUAL(*sql,escapeChar))) { @@ -363,7 +363,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid key or key is too long than 64", key); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -397,18 +397,18 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL valueLen = *sql - value; if (unlikely(quoteNum != 0 && quoteNum != 2)) { - smlBuildInvalidDataMsg(&info->msgBuf, "unbalanced quotes", value); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line unbalanced quotes", value); return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(valueLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid value", value); return TSDB_CODE_SML_INVALID_DATA; } SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; int32_t ret = smlParseValue(&kv, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + uError("SML:0x%" PRIx64 " %s parse value error:%d.", info->id, __FUNCTION__, ret); return ret; } @@ -430,11 +430,6 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } (void)memcpy(tmp, kv.value, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); - if(kv.type == TSDB_DATA_TYPE_GEOMETRY) { - uError("SML:0x%" PRIx64 " smlParseColLine error, invalid GEOMETRY type.", info->id); - taosMemoryFree((void*)kv.value); - return TSDB_CODE_TSC_INVALID_VALUE; - } if(kv.type == TSDB_DATA_TYPE_VARBINARY){ taosMemoryFree((void*)kv.value); } @@ -503,7 +498,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } elements->measureLen = sql - elements->measure; if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) { - smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line measure is empty or too large than 192", NULL); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -550,7 +545,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->colsLen = sql - elements->cols; if (unlikely(elements->colsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "cols is empty", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line cols is empty", NULL); return TSDB_CODE_SML_INVALID_DATA; } @@ -567,7 +562,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine int64_t ts = smlParseInfluxTime(info, elements->timestamp, elements->timestampLen); if (unlikely(ts <= 0)) { - uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); + uError("SML:0x%" PRIx64 " %s error:%" PRId64, info->id, __FUNCTION__, ts); return TSDB_CODE_INVALID_TIMESTAMP; } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 4deb6a34cd..dd264da11e 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -172,14 +172,14 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine // parse metric smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen); if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid measure", sql); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } // parse timestamp smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen); if (unlikely(!elements->timestamp || elements->timestampLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid timestamp", sql); return TSDB_CODE_SML_INVALID_DATA; } @@ -189,19 +189,21 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); if (unlikely(ts < 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet parse timestamp failed", sql); return TSDB_CODE_INVALID_TIMESTAMP; } // parse value smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); if (unlikely(!elements->cols || elements->colsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid value", sql); return TSDB_CODE_TSC_INVALID_VALUE; } SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; - if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + int ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " %s parse value error:%d.", info->id, __FUNCTION__, ret); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -210,11 +212,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->tags = sql; elements->tagsLen = sqlEnd - sql; if (unlikely(!elements->tags || elements->tagsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid tag value", sql); return TSDB_CODE_TSC_INVALID_VALUE; } - int ret = smlParseTelnetTags(info, sql, sqlEnd, elements); + ret = smlParseTelnetTags(info, sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index bcd12a393a..338457bec4 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -68,6 +68,15 @@ TEST(testCase, smlParseInfluxString_Test) { taosArrayDestroy(elements.colArray); elements.colArray = nullptr; + // case 0 false + tmp = "st,t1=3 c3=\""; + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); + ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; + // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; (void)memcpy(sql, tmp, strlen(tmp) + 1); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index b1ff8cb733..b5cdf1e4ee 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -472,7 +472,7 @@ int32_t smlInitHandle(SQuery** query) { int32_t code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pQuery); if (code != 0) { - uError("create pQuery error"); + uError("SML create pQuery error"); goto END; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; @@ -480,12 +480,12 @@ int32_t smlInitHandle(SQuery** query) { pQuery->msgType = TDMT_VND_SUBMIT; code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&stmt); if (code != 0) { - uError("create SVnodeModifyOpStmt error"); + uError("SML create SVnodeModifyOpStmt error"); goto END; } stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (stmt->pTableBlockHashObj == NULL){ - uError("create pTableBlockHashObj error"); + uError("SML create pTableBlockHashObj error"); code = terrno; goto END; } diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 6d4192280c..a76dccbbc0 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1882,6 +1882,20 @@ int sml_td24559_Test() { pRes = taos_query(taos, "create database if not exists td24559"); taos_free_result(pRes); + const char *sql1[] = { + "sttb,t1=1 f1=283i32,f2=g\"\" 1632299372000", + "sttb,t1=1 f2=G\"Point(4.343 89.342)\",f1=106i32 1632299373000", + }; + + pRes = taos_query(taos, "use td24559"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + int code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code); + const char *sql[] = { "stb,t1=1 f1=283i32,f2=g\"Point(4.343 89.342)\" 1632299372000", "stb,t1=1 f2=G\"Point(4.343 89.342)\",f1=106i32 1632299373000", @@ -1895,7 +1909,7 @@ int sml_td24559_Test() { pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); - int code = taos_errno(pRes); + code = taos_errno(pRes); printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); @@ -2254,8 +2268,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_td18789_Test(); ASSERT(!ret); -// ret = sml_td24070_Test(); -// ASSERT(!ret); + ret = sml_td24070_Test(); + ASSERT(!ret); ret = sml_td23881_Test(); ASSERT(ret); ret = sml_escape_Test(); @@ -2264,8 +2278,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_ts3116_Test(); ASSERT(!ret); -// ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file -// ASSERT(!ret); + ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file + ASSERT(!ret); ret = sml_ts3303_Test(); ASSERT(!ret); ret = sml_ttl_Test(); From 6960a34cfbea562e98aaad277c0ffa003770423a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 24 Oct 2024 19:52:39 +0800 Subject: [PATCH 13/43] enh:[TD-32166]refactor code in sml --- source/client/src/clientSml.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index f9ba8531f4..82e58f1467 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -698,8 +698,8 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; if (index) { if (colField[*index].type != kv->type) { - snprintf(info->msgBuf.buf, info->msgBuf.len, "SML:0x%" PRIx64 " %s point type and db type mismatch. db type: %d, point type: %d, key: %s", - info->id, __FUNCTION__, colField[*index].type, kv->type, kv->key); + snprintf(info->msgBuf.buf, info->msgBuf.len, "SML:0x%" PRIx64 " %s point type and db type mismatch. db type: %s, point type: %s, key: %s", + info->id, __FUNCTION__, tDataTypes[colField[*index].type].name, tDataTypes[kv->type].name, kv->key); uError("%s", info->msgBuf.buf); return TSDB_CODE_SML_INVALID_DATA; } From bd81eeccd09b66ef0d60cae7a1b80b5be8c6e3f8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 24 Oct 2024 23:54:12 +0800 Subject: [PATCH 14/43] enh:[TD-32166]refactor code in sml --- utils/test/c/sml_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index a76dccbbc0..3387447059 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1895,6 +1895,7 @@ int sml_td24559_Test() { int code = taos_errno(pRes); printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); ASSERT(code); + taos_free_result(pRes); const char *sql[] = { "stb,t1=1 f1=283i32,f2=g\"Point(4.343 89.342)\" 1632299372000", From a062a0cb8453037b3924cf7a04760a44f742a93d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 25 Oct 2024 09:41:35 +0800 Subject: [PATCH 15/43] enh:[TD-32166]refactor code in sml --- source/client/src/clientSml.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 82e58f1467..9d3b28e105 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1851,7 +1851,7 @@ END: TAOS_RES *taos_schemaless_insert_ttl_with_reqid_tbname_key(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { - if (taos == NULL || lines == NULL || numLines <= 0) { + if (taos == NULL || lines == NULL || numLines < 0) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -1901,7 +1901,7 @@ static int32_t getRawLineLen(char *lines, int len, int protocol) { TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { - if (taos == NULL || lines == NULL || len <= 0) { + if (taos == NULL || lines == NULL || len < 0) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } From e9feec94b3e5067a9421d2a0014e5818119bdb4b Mon Sep 17 00:00:00 2001 From: Pengrongkun Date: Sun, 27 Oct 2024 11:04:39 +0800 Subject: [PATCH 16/43] TD-32120:add api taos_stmt2_get_all_fields --- include/client/taos.h | 16 ++++--- source/client/src/clientMain.c | 10 ++++ source/client/src/clientStmt2.c | 45 ++++++++++++++++++ source/libs/parser/src/parInsertSql.c | 27 +++++++++-- source/libs/parser/src/parInsertStmt.c | 7 +-- tests/script/api/makefile | 2 + tests/script/api/stmt2-get-fields.c | 64 ++++++++++++++++++++++++++ 7 files changed, 156 insertions(+), 15 deletions(-) create mode 100644 tests/script/api/stmt2-get-fields.c diff --git a/include/client/taos.h b/include/client/taos.h index 80dbe27c47..00fec66a71 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -81,6 +81,13 @@ typedef enum { TSDB_SML_TIMESTAMP_NANO_SECONDS, } TSDB_SML_TIMESTAMP_TYPE; +typedef enum TAOS_FIELD_T { + TAOS_FIELD_COL = 1, + TAOS_FIELD_TAG, + TAOS_FIELD_QUERY, + TAOS_FIELD_TBNAME, +} TAOS_FIELD_T; + typedef struct taosField { char name[65]; int8_t type; @@ -93,6 +100,7 @@ typedef struct TAOS_FIELD_E { uint8_t precision; uint8_t scale; int32_t bytes; + TAOS_FIELD_T field_type; } TAOS_FIELD_E; #ifdef WINDOWS @@ -195,13 +203,6 @@ DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); typedef void TAOS_STMT2; -typedef enum TAOS_FIELD_T { - TAOS_FIELD_COL = 1, - TAOS_FIELD_TAG, - TAOS_FIELD_QUERY, - TAOS_FIELD_TBNAME, -} TAOS_FIELD_T; - typedef struct TAOS_STMT2_OPTION { int64_t reqid; bool singleStbInsert; @@ -232,6 +233,7 @@ DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows); DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt); DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert); DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields); +DLL_EXPORT int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_E **fields); DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields); DLL_EXPORT TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt); DLL_EXPORT char *taos_stmt2_error(TAOS_STMT2 *stmt); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 64631fd754..43f8c9e1f9 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -2103,6 +2103,16 @@ int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, } } +int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_E **fields) { + if (stmt == NULL || NULL == count) { + tscError("NULL parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return stmtGetColFields2(stmt, count, fields); +} + void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields) { (void)stmt; if (!fields) return; diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 2f046b61d6..2d76d91e38 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1068,6 +1068,48 @@ static int stmtFetchColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E return TSDB_CODE_SUCCESS; } + +static int stmtFetchFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields) { + SBoundColInfo* tags = (SBoundColInfo*)pStmt->bInfo.boundTags; + STableMeta* meta = ((SVnodeModifyOpStmt*)(pStmt->sql.pQuery->pRoot))->pTableMeta; + if (tags == NULL || meta == NULL || (meta->schema == NULL && tags->numOfBound != 0)) { + return TSDB_CODE_INVALID_PARA; + } + + if (fields) { + *fields = taosMemoryCalloc(tags->numOfBound, sizeof(TAOS_FIELD_E)); + if (NULL == *fields) { + return terrno; + } + + SSchema* pSchema = meta->schema; + int32_t tbnameIdx = meta->tableInfo.numOfTags + meta->tableInfo.numOfColumns; + for (int32_t i = 0; i < tags->numOfBound; ++i) { + int16_t idx = tags->pColIndex[i]; + if (idx == tbnameIdx) { + (*fields)[i].field_type = TAOS_FIELD_TBNAME; + strcpy((*fields)[i].name, "tbname"); + continue; + } else if (idx < meta->tableInfo.numOfColumns) { + (*fields)[i].field_type = TAOS_FIELD_COL; + } else { + (*fields)[i].field_type = TAOS_FIELD_TAG; + } + SSchema schema = pSchema[tags->pColIndex[i]]; + if (TSDB_DATA_TYPE_TIMESTAMP == schema.type) { + (*fields)[i].precision = meta->tableInfo.precision; + } + + tstrncpy((*fields)[i].name, schema.name, sizeof((*fields)[i].name)); + (*fields)[i].type = schema.type; + (*fields)[i].bytes = schema.bytes; + } + } + + *fieldNum = tags->numOfBound; + + return TSDB_CODE_SUCCESS; +} /* SArray* stmtGetFreeCol(STscStmt2* pStmt, int32_t* idx) { while (true) { @@ -1847,6 +1889,9 @@ int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) { _return: pStmt->errCode = preCode; + if (code == TSDB_CODE_TSC_INVALID_OPERATION) { + return stmtFetchFields2(stmt, nums, fields); + } return code; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 1c26a7c70e..77617ad3b3 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -780,6 +780,9 @@ static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* p } int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type) { + if (pToken->type == TK_NK_QUESTION) { + return buildInvalidOperationMsg(pMsgBuf, "insert into super table syntax is not supported for stmt"); + } if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && pToken->type != TK_NK_BIN && @@ -2422,9 +2425,6 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - if (pStmt->stbSyntax && TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { - return buildInvalidOperationMsg(&pCxt->msg, "insert into super table syntax is not supported for stmt"); - } if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); @@ -3066,8 +3066,9 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal .isStmtBind = pCxt->isStmtBind}; int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)((*pQuery)->pRoot); if (TSDB_CODE_SUCCESS == code) { - code = parseInsertSqlImpl(&context, (SVnodeModifyOpStmt*)(*pQuery)->pRoot); + code = parseInsertSqlImpl(&context, pStmt); } if (TSDB_CODE_SUCCESS == code) { code = setNextStageInfo(&context, *pQuery, pCatalogReq); @@ -3076,8 +3077,24 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { code = setRefreshMeta(*pQuery); } - insDestroyBoundColInfo(&context.tags); + if (pStmt->stbSyntax && TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && + code == TSDB_CODE_TSC_INVALID_OPERATION) { + context.tags.numOfBound = pStmt->pStbRowsCxt->boundColsInfo.numOfBound; + context.tags.numOfCols = pStmt->pStbRowsCxt->boundColsInfo.numOfCols; + context.tags. hasBoundCols= pStmt->pStbRowsCxt->boundColsInfo.hasBoundCols; + context.tags.pColIndex = taosMemoryMalloc(sizeof(int16_t) * context.tags.numOfBound); + memcpy(context.tags.pColIndex, pStmt->pStbRowsCxt->boundColsInfo.pColIndex, + sizeof(int16_t) * pStmt->pStbRowsCxt->boundColsInfo.numOfBound); + code = setStmtInfo(&context, pStmt); + if (TSDB_CODE_SUCCESS == code) { + insDestroyBoundColInfo(&context.tags); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + insDestroyBoundColInfo(&context.tags); + // if no data to insert, set emptyMode to avoid request server if (!context.needRequest) { (*pQuery)->execMode = QUERY_EXEC_MODE_EMPTY_RESULT; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index ee61611bf2..f18b7b817b 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -885,7 +885,7 @@ _return: } int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, - TAOS_FIELD_E** fields, uint8_t timePrec) { + TAOS_FIELD_E** fields, uint8_t timePrec, TAOS_FIELD_T fieldType) { if (fields) { *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { @@ -900,6 +900,7 @@ int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSc for (int32_t i = 0; i < numOfBound; ++i) { schema = &pSchema[boundColumns[i]]; strcpy((*fields)[i].name, schema->name); + (*fields)[i].field_type = schema->type; (*fields)[i].type = schema->type; (*fields)[i].bytes = schema->bytes; } @@ -929,7 +930,7 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(tags->numOfBound, tags->pColIndex, pSchema, fieldNum, fields, 0)); + CHECK_CODE(buildBoundFields(tags->numOfBound, tags->pColIndex, pSchema, fieldNum, fields, 0, TAOS_FIELD_TAG)); return TSDB_CODE_SUCCESS; } @@ -947,7 +948,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel } CHECK_CODE(buildBoundFields(pDataBlock->boundColsInfo.numOfBound, pDataBlock->boundColsInfo.pColIndex, pSchema, - fieldNum, fields, pDataBlock->pMeta->tableInfo.precision)); + fieldNum, fields, pDataBlock->pMeta->tableInfo.precision, TAOS_FIELD_COL)); return TSDB_CODE_SUCCESS; } diff --git a/tests/script/api/makefile b/tests/script/api/makefile index d8a4e19218..9c2bb6be3d 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -25,6 +25,7 @@ exe: gcc $(CFLAGS) ./stmt.c -o $(ROOT)stmt $(LFLAGS) gcc $(CFLAGS) ./stmt2.c -o $(ROOT)stmt2 $(LFLAGS) gcc $(CFLAGS) ./stmt2-example.c -o $(ROOT)stmt2-example $(LFLAGS) + gcc $(CFLAGS) ./stmt2-get-fields.c -o $(ROOT)stmt2-get-fields $(LFLAGS) gcc $(CFLAGS) ./stmt2-nohole.c -o $(ROOT)stmt2-nohole $(LFLAGS) gcc $(CFLAGS) ./stmt-crash.c -o $(ROOT)stmt-crash $(LFLAGS) @@ -42,5 +43,6 @@ clean: rm $(ROOT)stmt rm $(ROOT)stmt2 rm $(ROOT)stmt2-example + rm $(ROOT)stmt2-get-fields rm $(ROOT)stmt2-nohole rm $(ROOT)stmt-crash diff --git a/tests/script/api/stmt2-get-fields.c b/tests/script/api/stmt2-get-fields.c new file mode 100644 index 0000000000..60bee16873 --- /dev/null +++ b/tests/script/api/stmt2-get-fields.c @@ -0,0 +1,64 @@ +// TAOS standard API example. The same syntax as MySQL, but only a subet +// to compile: gcc -o stmt2-get-fields stmt2-get-fields.c -ltaos + +#include +#include +#include +#include "taos.h" + +void do_query(TAOS *taos, const char *sql) { + TAOS_RES *result = taos_query(taos, sql); + int code = taos_errno(result); + if (code) { + printf("failed to query: %s, reason:%s\n", sql, taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); +} + +void do_stmt(TAOS *taos) { + do_query(taos, "drop database if exists db"); + do_query(taos, "create database db"); + do_query(taos, + "create table db.stb (ts timestamp, b binary(10)) tags(t1 " + "int, t2 binary(10))"); + + TAOS_STMT2_OPTION option = {0}; + TAOS_STMT2 *stmt = taos_stmt2_init(taos, &option); + const char *sql = "insert into db.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; + + int code = taos_stmt2_prepare(stmt, sql, 0); + if (code != 0) { + printf("failed to execute taos_stmt2_prepare. error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + return; + } + + int fieldNum = 0; + TAOS_FIELD_E *pFields = NULL; + code = taos_stmt2_get_all_fields(stmt, &fieldNum, &pFields); + if (code != 0) { + printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); + } else { + printf("col nums:%d\n", fieldNum); + for(int i = 0; i < fieldNum; i++) { + printf("field[%d]: %s,type:%d\n", i, pFields[i].name,pFields[i].field_type); + } + } + + + taos_stmt2_close(stmt); +} + +int main() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", "", 0); + if (!taos) { + printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + exit(1); + } + + do_stmt(taos); + taos_close(taos); + taos_cleanup(); +} \ No newline at end of file From 419c2e2974dc1cb6cb1d6e12675833d6d2b625d1 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Tue, 29 Oct 2024 11:13:50 +0800 Subject: [PATCH 17/43] fix:[TS-5567] fix bug when partition/group by const value's alias name. --- source/libs/parser/src/parTranslater.c | 114 +++++++++++-------- tests/system-test/2-query/group_partition.py | 19 +++- 2 files changed, 83 insertions(+), 50 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 636be7c5cc..7ce8d87b18 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1603,25 +1603,6 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p } } if (*pFound) { - if (QUERY_NODE_FUNCTION == nodeType(pFoundNode) && (SQL_CLAUSE_GROUP_BY == pCxt->currClause || SQL_CLAUSE_PARTITION_BY == pCxt->currClause)) { - pCxt->errCode = getFuncInfo(pCxt, (SFunctionNode*)pFoundNode); - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - if (fmIsVectorFunc(((SFunctionNode*)pFoundNode)->funcId)) { - pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, (*pCol)->colName); - return DEAL_RES_ERROR; - } else if (fmIsPseudoColumnFunc(((SFunctionNode*)pFoundNode)->funcId)) { - if ('\0' != (*pCol)->tableAlias[0]) { - return translateColumnWithPrefix(pCxt, pCol); - } else { - return translateColumnWithoutPrefix(pCxt, pCol); - } - } else { - /* Do nothing and replace old node with found node. */ - } - } else { - return DEAL_RES_ERROR; - } - } SNode* pNew = NULL; int32_t code = nodesCloneNode(pFoundNode, &pNew); if (NULL == pNew) { @@ -1630,14 +1611,6 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p } nodesDestroyNode(*(SNode**)pCol); *(SNode**)pCol = (SNode*)pNew; - if (QUERY_NODE_COLUMN == nodeType(pFoundNode)) { - pCxt->errCode = TSDB_CODE_SUCCESS; - if ('\0' != (*pCol)->tableAlias[0]) { - return translateColumnWithPrefix(pCxt, pCol); - } else { - return translateColumnWithoutPrefix(pCxt, pCol); - } - } } return DEAL_RES_CONTINUE; } @@ -1882,6 +1855,39 @@ static bool clauseSupportAlias(ESqlClause clause) { SQL_CLAUSE_ORDER_BY == clause; } +static EDealRes translateColumnInGroupByClause(STranslateContext* pCxt, SColumnNode** pCol, bool *translateAsAlias) { + *translateAsAlias = false; + // count(*)/first(*)/last(*) and so on + if (0 == strcmp((*pCol)->colName, "*")) { + return DEAL_RES_CONTINUE; + } + + if (pCxt->pParseCxt->biMode) { + SNode** ppNode = (SNode**)pCol; + bool ret; + pCxt->errCode = biRewriteToTbnameFunc(pCxt, ppNode, &ret); + if (TSDB_CODE_SUCCESS != pCxt->errCode) return DEAL_RES_ERROR; + if (ret) { + return translateFunction(pCxt, (SFunctionNode**)ppNode); + } + } + + EDealRes res = DEAL_RES_CONTINUE; + if ('\0' != (*pCol)->tableAlias[0]) { + res = translateColumnWithPrefix(pCxt, pCol); + } else { + bool found = false; + res = translateColumnWithoutPrefix(pCxt, pCol); + if (!(*pCol)->node.asParam && + res != DEAL_RES_CONTINUE && + res != DEAL_RES_END && pCxt->errCode != TSDB_CODE_PAR_AMBIGUOUS_COLUMN) { + res = translateColumnUseAlias(pCxt, pCol, &found); + *translateAsAlias = true; + } + } + return res; +} + static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { if (NULL == pCxt->pCurrStmt || (isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { @@ -5472,12 +5478,13 @@ typedef struct SReplaceGroupByAliasCxt { SNodeList* pProjectionList; } SReplaceGroupByAliasCxt; -static EDealRes replaceGroupByAliasImpl(SNode** pNode, void* pContext) { +static EDealRes translateGroupPartitionByImpl(SNode** pNode, void* pContext) { SReplaceGroupByAliasCxt* pCxt = pContext; SNodeList* pProjectionList = pCxt->pProjectionList; SNode* pProject = NULL; + int32_t code = TSDB_CODE_SUCCESS; + STranslateContext* pTransCxt = pCxt->pTranslateCxt; if (QUERY_NODE_VALUE == nodeType(*pNode)) { - STranslateContext* pTransCxt = pCxt->pTranslateCxt; SValueNode* pVal = (SValueNode*) *pNode; if (DEAL_RES_ERROR == translateValue(pTransCxt, pVal)) { return DEAL_RES_CONTINUE; @@ -5488,43 +5495,58 @@ static EDealRes replaceGroupByAliasImpl(SNode** pNode, void* pContext) { int32_t pos = getPositionValue(pVal); if (0 < pos && pos <= LIST_LENGTH(pProjectionList)) { SNode* pNew = NULL; - int32_t code = nodesCloneNode(nodesListGetNode(pProjectionList, pos - 1), (SNode**)&pNew); + code = nodesCloneNode(nodesListGetNode(pProjectionList, pos - 1), (SNode**)&pNew); if (TSDB_CODE_SUCCESS != code) { pCxt->pTranslateCxt->errCode = code; return DEAL_RES_ERROR; } nodesDestroyNode(*pNode); *pNode = pNew; - return DEAL_RES_CONTINUE; - } else { + } + code = translateExpr(pTransCxt, pNode); + if (TSDB_CODE_SUCCESS != code) { + pTransCxt->errCode = code; + return DEAL_RES_ERROR; + } + return DEAL_RES_CONTINUE; + } else if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + bool asAlias = false; + EDealRes res = translateColumnInGroupByClause(pTransCxt, (SColumnNode**)pNode, &asAlias); + if (DEAL_RES_ERROR == res) { + return DEAL_RES_ERROR; + } + pTransCxt->errCode = TSDB_CODE_SUCCESS; + if (nodeType(*pNode) == QUERY_NODE_COLUMN && !asAlias) { return DEAL_RES_CONTINUE; } - } else if (QUERY_NODE_COLUMN == nodeType(*pNode)) { - STranslateContext* pTransCxt = pCxt->pTranslateCxt; - return translateColumn(pTransCxt, (SColumnNode**)pNode); + code = translateExpr(pTransCxt, pNode); + if (TSDB_CODE_SUCCESS != code) { + pTransCxt->errCode = code; + return DEAL_RES_ERROR; + } + return DEAL_RES_CONTINUE; } - - return DEAL_RES_CONTINUE; + return doTranslateExpr(pNode, pTransCxt); } -static int32_t replaceGroupByAlias(STranslateContext* pCxt, SSelectStmt* pSelect) { +static int32_t translateGroupByList(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; } SReplaceGroupByAliasCxt cxt = { .pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList}; - nodesRewriteExprsPostOrder(pSelect->pGroupByList, replaceGroupByAliasImpl, &cxt); + nodesRewriteExprsPostOrder(pSelect->pGroupByList, translateGroupPartitionByImpl, &cxt); return pCxt->errCode; } -static int32_t replacePartitionByAlias(STranslateContext* pCxt, SSelectStmt* pSelect) { +static int32_t translatePartitionByList(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pPartitionByList) { return TSDB_CODE_SUCCESS; } SReplaceGroupByAliasCxt cxt = { .pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList}; - nodesRewriteExprsPostOrder(pSelect->pPartitionByList, replaceGroupByAliasImpl, &cxt); + nodesRewriteExprsPostOrder(pSelect->pPartitionByList, translateGroupPartitionByImpl, &cxt); return pCxt->errCode; } @@ -5588,11 +5610,8 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { NODES_DESTORY_LIST(pSelect->pGroupByList); return TSDB_CODE_SUCCESS; } - code = replaceGroupByAlias(pCxt, pSelect); - } - if (TSDB_CODE_SUCCESS == code) { pSelect->timeLineResMode = TIME_LINE_NONE; - code = translateExprList(pCxt, pSelect->pGroupByList); + code = translateGroupByList(pCxt, pSelect); } return code; } @@ -6287,10 +6306,7 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec (QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) { pSelect->timeLineResMode = TIME_LINE_MULTI; } - code = replacePartitionByAlias(pCxt, pSelect); - if (TSDB_CODE_SUCCESS == code) { - code = translateExprList(pCxt, pSelect->pPartitionByList); - } + code = translatePartitionByList(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { code = translateExprList(pCxt, pSelect->pTags); diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 384df02e8d..7ee528841c 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -420,7 +420,23 @@ class TDTestCase: tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 interval(1d)") - + def test_TS5567(self): + tdSql.query(f"select const_col from (select 1 as const_col from {self.dbname}.{self.stable}) t group by const_col") + tdSql.checkRows(50) + tdSql.query(f"select const_col from (select 1 as const_col from {self.dbname}.{self.stable}) t partition by const_col") + tdSql.checkRows(50) + tdSql.query(f"select const_col from (select 1 as const_col, count(c1) from {self.dbname}.{self.stable} t group by c1) group by const_col") + tdSql.checkRows(10) + tdSql.query(f"select const_col from (select 1 as const_col, count(c1) from {self.dbname}.{self.stable} t group by c1) partition by const_col") + tdSql.checkRows(10) + tdSql.query(f"select const_col as c_c from (select 1 as const_col from {self.dbname}.{self.stable}) t group by c_c") + tdSql.checkRows(50) + tdSql.query(f"select const_col as c_c from (select 1 as const_col from {self.dbname}.{self.stable}) t partition by c_c") + tdSql.checkRows(50) + tdSql.query(f"select const_col from (select 1 as const_col, count(c1) from {self.dbname}.{self.stable} t group by c1) group by 1") + tdSql.checkRows(10) + tdSql.query(f"select const_col from (select 1 as const_col, count(c1) from {self.dbname}.{self.stable} t group by c1) partition by 1") + tdSql.checkRows(10) def run(self): tdSql.prepare() self.prepare_db() @@ -453,6 +469,7 @@ class TDTestCase: self.test_window(nonempty_tb_num) self.test_event_window(nonempty_tb_num) + self.test_TS5567() ## test old version before changed # self.test_groupby('group', 0, 0) From e6e36bcd2dd0c53d271105189ff3e8d720d94a2a Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 29 Oct 2024 12:17:14 +0800 Subject: [PATCH 18/43] fix PARITION (INTERVAL) HAVING() caused planner internal err --- source/libs/parser/src/parTranslater.c | 2 +- tests/system-test/2-query/fill_with_group.py | 32 ++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 395801e3dd..34b8290b67 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3752,7 +3752,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { bool partionByTbname = hasTbnameFunction(pSelect->pPartitionByList); FOREACH(pPartKey, pSelect->pPartitionByList) { if (nodesEqualNode(pPartKey, *pNode)) { - return pCxt->currClause == SQL_CLAUSE_HAVING ? DEAL_RES_IGNORE_CHILD : rewriteExprToGroupKeyFunc(pCxt, pNode); + return pSelect->hasAggFuncs ? rewriteExprToGroupKeyFunc(pCxt, pNode) : DEAL_RES_IGNORE_CHILD; } if ((partionByTbname) && QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index 3b98ec30ce..4a40338504 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -344,9 +344,35 @@ class TDTestCase: tdSql.query(sql, queryTimes=1) tdSql.checkRows(55) - # TODO Fix Me! - sql = "explain SELECT count(*), timediff(_wend, last(ts)), timediff('2018-09-20 01:00:00', _wstart) FROM meters WHERE ts >= '2018-09-20 00:00:00.000' AND ts < '2018-09-20 01:00:00.000' PARTITION BY concat(tbname, 'asd') INTERVAL(5m) having(concat(tbname, 'asd') like '%asd');" - tdSql.error(sql, -2147473664) # Error: Planner internal error + sql = "SELECT count(*), timediff(_wend, last(ts)), timediff('2018-09-20 01:00:00', _wstart) FROM meters WHERE ts >= '2018-09-20 00:00:00.000' AND ts < '2018-09-20 01:00:00.000' PARTITION BY concat(tbname, 'asd') INTERVAL(5m) having(concat(tbname, 'asd') like '%asd');" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(60) + + sql = "SELECT count(*), timediff(_wend, last(ts)), timediff('2018-09-20 01:00:00', _wstart) FROM meters WHERE ts >= '2018-09-20 00:00:00.000' AND ts < '2018-09-20 01:00:00.000' PARTITION BY concat(tbname, 'asd') INTERVAL(5m) having(concat(tbname, 'asd') like 'asd%');" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(0) + + sql = "SELECT c1 FROM meters PARTITION BY c1 HAVING c1 > 0 slimit 2 limit 10" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(20) + + sql = "SELECT t1 FROM meters PARTITION BY t1 HAVING(t1 = 1)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(20000) + + sql = "SELECT concat(t2, 'asd') FROM meters PARTITION BY t2 HAVING(t2 like '%5')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10000) + tdSql.checkData(0, 0, 'tb5asd') + + sql = "SELECT concat(t2, 'asd') FROM meters PARTITION BY concat(t2, 'asd') HAVING(concat(t2, 'asd')like '%5%')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10000) + tdSql.checkData(0, 0, 'tb5asd') + + sql = "SELECT avg(c1) FROM meters PARTITION BY tbname, t1 HAVING(t1 = 1)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(2) def run(self): self.prepareTestEnv() From 14ddaf7c3932216f5e7c0366ad3498eba6c1baab Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 30 Oct 2024 11:08:41 +0800 Subject: [PATCH 19/43] add parition having tests and fix merge align interval + limit --- source/libs/executor/src/timewindowoperator.c | 7 +++- tests/system-test/2-query/fill_with_group.py | 37 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 34ecda6ce7..8c5f628034 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2018,7 +2018,12 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { cleanupAfterGroupResultGen(pMiaInfo, pRes); code = doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); QUERY_CHECK_CODE(code, lino, _end); - break; + if (pRes->info.rows == 0) { + // After filtering for last group, the result is empty, so we need to continue to process next group + continue; + } else { + break; + } } else { // continue pRes->info.id.groupId = pMiaInfo->groupId; diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index 4a40338504..3847f1dec4 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -374,6 +374,43 @@ class TDTestCase: tdSql.query(sql, queryTimes=1) tdSql.checkRows(2) + sql = "SELECT count(*) FROM meters PARTITION BY concat(tbname, 'asd') HAVING(concat(tbname, 'asd') like '%asd')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10) + + sql = "SELECT count(*), concat(tbname, 'asd') FROM meters PARTITION BY concat(tbname, 'asd') HAVING(concat(tbname, 'asd') like '%asd')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10) + + sql = "SELECT count(*) FROM meters PARTITION BY t1 HAVING(t1 < 4) order by t1 +1" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(4) + + sql = "SELECT count(*), t1 + 100 FROM meters PARTITION BY t1 HAVING(t1 < 4) order by t1 +1" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(4) + + sql = "SELECT count(*), t1 + 100 FROM meters PARTITION BY t1 INTERVAL(1d) HAVING(t1 < 4) order by t1 +1 desc" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(280) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd' and count(*) = 118)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(1) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd' and count(*) != 118)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(69) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd') order by count(*) asc limit 10" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd' or concat(t3, 'asd') like '%3asd') order by count(*) asc limit 10000" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(140) + + def run(self): self.prepareTestEnv() self.test_partition_by_with_interval_fill_prev_new_group_fill_error() From 19f85afc11314d07e18e63c7754c3616d3b6d8b6 Mon Sep 17 00:00:00 2001 From: dengbin40 <46433701+dengbin40@users.noreply.github.com> Date: Sun, 29 Sep 2024 09:58:18 +0800 Subject: [PATCH 20/43] =?UTF-8?q?fix:=20=E5=A2=9E=E5=8A=A0pod=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E7=9A=84=E5=88=86=E5=B8=83=E5=BC=8F=E9=83=A8=E7=BD=B2?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/zh/08-operation/03-deployment.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/zh/08-operation/03-deployment.md b/docs/zh/08-operation/03-deployment.md index 2e0c2a7989..e549e8613d 100644 --- a/docs/zh/08-operation/03-deployment.md +++ b/docs/zh/08-operation/03-deployment.md @@ -368,6 +368,18 @@ spec: labels: app: "tdengine" spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - tdengine + topologyKey: kubernetes.io/hostname containers: - name: "tdengine" image: "tdengine/tdengine:3.2.3.0" @@ -837,4 +849,4 @@ Helm 管理下，清理操作也变得简单： helm uninstall tdengine ``` -但 Helm 也不会自动移除 PVC，需要手动获取 PVC 然后删除掉。 \ No newline at end of file +但 Helm 也不会自动移除 PVC，需要手动获取 PVC 然后删除掉。 From aa3c79100bb55ad5fd296f88de21c4bc978e4b51 Mon Sep 17 00:00:00 2001 From: Pengrongkun Date: Wed, 30 Oct 2024 14:24:10 +0800 Subject: [PATCH 21/43] fix some format --- include/client/taos.h | 32 ++++++------- source/client/src/clientMain.c | 11 ++--- source/client/src/clientStmt2.c | 16 +++---- source/libs/parser/src/parInsertSql.c | 63 ++++++++++++++------------ source/libs/parser/src/parInsertStmt.c | 8 ++-- 5 files changed, 66 insertions(+), 64 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 00fec66a71..6797dfee5f 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -95,11 +95,11 @@ typedef struct taosField { } TAOS_FIELD; typedef struct TAOS_FIELD_E { - char name[65]; - int8_t type; - uint8_t precision; - uint8_t scale; - int32_t bytes; + char name[65]; + int8_t type; + uint8_t precision; + uint8_t scale; + int32_t bytes; TAOS_FIELD_T field_type; } TAOS_FIELD_E; @@ -253,17 +253,17 @@ DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); -DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); -DLL_EXPORT void taos_stop_query(TAOS_RES *res); -DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); -DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows); -DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); -DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); -DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); -DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); -DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex); -DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); -DLL_EXPORT void taos_reset_current_db(TAOS *taos); +DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); +DLL_EXPORT void taos_stop_query(TAOS_RES *res); +DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); +DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows); +DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); +DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); +DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); +DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); +DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex); +DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); +DLL_EXPORT void taos_reset_current_db(TAOS *taos); DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 43f8c9e1f9..54da1c013c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -84,7 +84,7 @@ void taos_cleanup(void) { taosCloseRef(id); nodesDestroyAllocatorSet(); -// cleanupAppInfo(); + // cleanupAppInfo(); rpcCleanup(); tscDebug("rpc cleanup"); @@ -388,7 +388,6 @@ void taos_free_result(TAOS_RES *res) { tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); } taosMemoryFree(pRsp); - } void taos_kill_query(TAOS *taos) { @@ -484,7 +483,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { return taos_print_row_with_size(str, INT32_MAX, row, fields, num_fields); } -int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields){ +int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { int32_t len = 0; for (int i = 0; i < num_fields; ++i) { if (i > 0 && len < size - 1) { @@ -589,7 +588,7 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD break; } } - if (len < size){ + if (len < size) { str[len] = 0; } @@ -2082,7 +2081,7 @@ int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) { } int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields) { - if (stmt == NULL || NULL == count) { + if (stmt == NULL || count == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -2104,7 +2103,7 @@ int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, } int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_E **fields) { - if (stmt == NULL || NULL == count) { + if (stmt == NULL || count == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 2d76d91e38..5d04006f06 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1076,33 +1076,33 @@ static int stmtFetchFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E** return TSDB_CODE_INVALID_PARA; } - if (fields) { + if (fields != NULL) { *fields = taosMemoryCalloc(tags->numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { return terrno; } SSchema* pSchema = meta->schema; - int32_t tbnameIdx = meta->tableInfo.numOfTags + meta->tableInfo.numOfColumns; + int32_t tbnameIdx = meta->tableInfo.numOfTags + meta->tableInfo.numOfColumns; for (int32_t i = 0; i < tags->numOfBound; ++i) { int16_t idx = tags->pColIndex[i]; if (idx == tbnameIdx) { (*fields)[i].field_type = TAOS_FIELD_TBNAME; - strcpy((*fields)[i].name, "tbname"); + tstrncpy((*fields)[i].name, "tbname", sizeof((*fields)[i].name)); continue; } else if (idx < meta->tableInfo.numOfColumns) { (*fields)[i].field_type = TAOS_FIELD_COL; } else { (*fields)[i].field_type = TAOS_FIELD_TAG; } - SSchema schema = pSchema[tags->pColIndex[i]]; - if (TSDB_DATA_TYPE_TIMESTAMP == schema.type) { + + if (TSDB_DATA_TYPE_TIMESTAMP == pSchema[tags->pColIndex[i]].type) { (*fields)[i].precision = meta->tableInfo.precision; } - tstrncpy((*fields)[i].name, schema.name, sizeof((*fields)[i].name)); - (*fields)[i].type = schema.type; - (*fields)[i].bytes = schema.bytes; + tstrncpy((*fields)[i].name, pSchema[tags->pColIndex[i]].name, sizeof((*fields)[i].name)); + (*fields)[i].type = pSchema[tags->pColIndex[i]].type; + (*fields)[i].bytes = pSchema[tags->pColIndex[i]].bytes; } } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 77617ad3b3..f4ef099515 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -30,7 +30,7 @@ typedef struct SInsertParseContext { bool forceUpdate; bool needTableTagVal; bool needRequest; // whether or not request server - bool isStmtBind; // whether is stmt bind + bool isStmtBind; // whether is stmt bind } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -757,7 +757,7 @@ int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SS STagVal val = {0}; int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf); if (TSDB_CODE_SUCCESS == code) { - if (NULL == taosArrayPush(pTagVals, &val)){ + if (NULL == taosArrayPush(pTagVals, &val)) { code = terrno; } } @@ -775,8 +775,8 @@ static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* p return terrno; } return insBuildCreateTbReq(pStmt->pCreateTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, - pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, - TSDB_DEFAULT_TABLE_TTL); + pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, + TSDB_DEFAULT_TABLE_TTL); } int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type) { @@ -813,7 +813,7 @@ typedef struct SRewriteTagCondCxt { static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { SValueNode* pValue = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pValue); + int32_t code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pValue); if (NULL == pValue) { return code; } @@ -1044,7 +1044,7 @@ static int32_t storeChildTableMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt return TSDB_CODE_OUT_OF_MEMORY; } - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(&pStmt->targetTableName, tbFName); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree(pBackup); @@ -1239,7 +1239,7 @@ static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModi } static int32_t collectUseTable(const SName* pName, SHashObj* pTable) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1385,7 +1385,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false); } - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(&pStmt->targetTableName, tbFName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1926,8 +1926,8 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod } if (code == TSDB_CODE_SUCCESS) { code = insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, - pStbRowsCxt->pStbMeta->uid, pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, - getNumOfTags(pStbRowsCxt->pStbMeta), TSDB_DEFAULT_TABLE_TTL); + pStbRowsCxt->pStbMeta->uid, pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, + getNumOfTags(pStbRowsCxt->pStbMeta), TSDB_DEFAULT_TABLE_TTL); pStbRowsCxt->pTag = NULL; } @@ -1936,9 +1936,9 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod code = tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); SVgroupInfo vg; SRequestConnInfo conn = {.pTrans = pCxt->pComCxt->pTransporter, - .requestId = pCxt->pComCxt->requestId, - .requestObjRefId = pCxt->pComCxt->requestRid, - .mgmtEps = pCxt->pComCxt->mgmtEpSet}; + .requestId = pCxt->pComCxt->requestId, + .requestObjRefId = pCxt->pComCxt->requestRid, + .mgmtEps = pCxt->pComCxt->mgmtEpSet}; if (TSDB_CODE_SUCCESS == code) { code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStbRowsCxt->ctbName, &vg); } @@ -2179,8 +2179,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt if (code == TSDB_CODE_SUCCESS) { SStbRowsDataContext* pStbRowsCxt = rowsDataCxt.pStbRowsCxt; void* pData = pTableDataCxt; - code = taosHashPut(pStmt->pTableCxtHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), &pData, - POINTER_BYTES); + code = taosHashPut(pStmt->pTableCxtHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), + &pData, POINTER_BYTES); if (TSDB_CODE_SUCCESS != code) { break; } @@ -2252,7 +2252,7 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt if (!pStmt->stbSyntax && numOfRows > 0) { void* pData = rowsDataCxt.pTableDataCxt; code = taosHashPut(pStmt->pTableCxtHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), &pData, - POINTER_BYTES); + POINTER_BYTES); } return code; @@ -2366,8 +2366,7 @@ static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDa if (TSDB_CODE_SUCCESS == code) { // col values and bound cols info of STableDataContext is not used pStbRowsCxt->aColVals = taosArrayInit(getNumOfColumns(pStbRowsCxt->pStbMeta), sizeof(SColVal)); - if (!pStbRowsCxt->aColVals) - code = terrno; + if (!pStbRowsCxt->aColVals) code = terrno; } if (TSDB_CODE_SUCCESS == code) { code = insInitColValues(pStbRowsCxt->pStbMeta, pStbRowsCxt->aColVals); @@ -2511,9 +2510,9 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif } // db.? situation，ensure that the only thing following the '.' mark is '?' - char *tbNameAfterDbName = strnchr(pTbName->z, '.', pTbName->n, true); + char* tbNameAfterDbName = strnchr(pTbName->z, '.', pTbName->n, true); if ((tbNameAfterDbName != NULL) && (*(tbNameAfterDbName + 1) == '?')) { - char *tbName = NULL; + char* tbName = NULL; if (NULL == pCxt->pComCxt->pStmtCb) { return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pTbName->z); } @@ -2528,7 +2527,8 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif if (pCxt->isStmtBind) { if (TK_NK_ID == pTbName->type || (tbNameAfterDbName != NULL && *(tbNameAfterDbName + 1) != '?')) { // In SQL statements, the table name has already been specified. - parserWarn("0x%" PRIx64 " table name is specified in sql, ignore the table name in bind param", pCxt->pComCxt->requestId); + parserWarn("0x%" PRIx64 " table name is specified in sql, ignore the table name in bind param", + pCxt->pComCxt->requestId); } } @@ -2614,7 +2614,7 @@ static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, SNode** pOutput) { SVnodeModifyOpStmt* pStmt = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&pStmt); + int32_t code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&pStmt); if (NULL == pStmt) { return code; } @@ -2729,7 +2729,7 @@ static int32_t buildTagNameFromMeta(STableMeta* pMeta, SArray** pTagName) { return terrno; } SSchema* pSchema = getTableTagSchema(pMeta); - int32_t code = 0; + int32_t code = 0; for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) { if (NULL == taosArrayPush(*pTagName, pSchema[i].name)) { code = terrno; @@ -2834,7 +2834,7 @@ static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) } if (NULL == pStmt->pTableBlockHashObj) { pStmt->pTableBlockHashObj = - taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } if (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -2866,7 +2866,7 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR static int32_t setRefreshMeta(SQuery* pQuery) { SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; - int32_t code = 0; + int32_t code = 0; if (taosHashGetSize(pStmt->pTableNameHashObj) > 0) { taosArrayDestroy(pQuery->pTableList); @@ -3065,7 +3065,7 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal .forceUpdate = (NULL != pCatalogReq ? pCatalogReq->forceUpdate : false), .isStmtBind = pCxt->isStmtBind}; - int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); + int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)((*pQuery)->pRoot); if (TSDB_CODE_SUCCESS == code) { code = parseInsertSqlImpl(&context, pStmt); @@ -3082,10 +3082,14 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal code == TSDB_CODE_TSC_INVALID_OPERATION) { context.tags.numOfBound = pStmt->pStbRowsCxt->boundColsInfo.numOfBound; context.tags.numOfCols = pStmt->pStbRowsCxt->boundColsInfo.numOfCols; - context.tags. hasBoundCols= pStmt->pStbRowsCxt->boundColsInfo.hasBoundCols; + context.tags.hasBoundCols = pStmt->pStbRowsCxt->boundColsInfo.hasBoundCols; context.tags.pColIndex = taosMemoryMalloc(sizeof(int16_t) * context.tags.numOfBound); - memcpy(context.tags.pColIndex, pStmt->pStbRowsCxt->boundColsInfo.pColIndex, - sizeof(int16_t) * pStmt->pStbRowsCxt->boundColsInfo.numOfBound); + if (NULL == context.tags.pColIndex) { + return terrno; + } + + (void)memcpy(context.tags.pColIndex, pStmt->pStbRowsCxt->boundColsInfo.pColIndex, + sizeof(int16_t) * pStmt->pStbRowsCxt->boundColsInfo.numOfBound); code = setStmtInfo(&context, pStmt); if (TSDB_CODE_SUCCESS == code) { insDestroyBoundColInfo(&context.tags); @@ -3094,7 +3098,6 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal } insDestroyBoundColInfo(&context.tags); - // if no data to insert, set emptyMode to avoid request server if (!context.needRequest) { (*pQuery)->execMode = QUERY_EXEC_MODE_EMPTY_RESULT; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index f18b7b817b..8c17689ea7 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -242,7 +242,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch } code = insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, - pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); pTag = NULL; end: @@ -594,7 +594,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c } code = insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, - pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); pTag = NULL; end: @@ -886,7 +886,7 @@ _return: int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, uint8_t timePrec, TAOS_FIELD_T fieldType) { - if (fields) { + if (fields != NULL) { *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { return terrno; @@ -940,7 +940,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); if (pDataBlock->boundColsInfo.numOfBound <= 0) { *fieldNum = 0; - if (fields) { + if (fields != NULL) { *fields = NULL; } From ef50e5d407dd3ae39929cd5761e9e6646b35e06b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Oct 2024 15:21:11 +0800 Subject: [PATCH 22/43] enh: optimize the error message for the case of the drop column --- include/util/taoserror.h | 1 + source/dnode/mnode/impl/src/mndStb.c | 8 ++++---- source/util/src/terror.c | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a53923b904..dd9b90d390 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -392,6 +392,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_MND_INVALID_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x03C6) #define TSDB_CODE_MND_STABLE_UID_NOT_MATCH TAOS_DEF_ERROR_CODE(0, 0x03C7) #define TSDB_CODE_MND_FIELD_CONFLICT_WITH_TSMA TAOS_DEF_ERROR_CODE(0, 0x03C8) +#define TSDB_CODE_MND_ONLY_TS_SCHEMA_NOT_ALLOW TAOS_DEF_ERROR_CODE(0, 0x03C9) // mnode-trans #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 2500d1ef40..2986bdc117 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1928,7 +1928,7 @@ static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStb } if (pOld->numOfColumns == 2) { - code = TSDB_CODE_MND_INVALID_STB_ALTER_OPTION; + code = TSDB_CODE_MND_ONLY_TS_SCHEMA_NOT_ALLOW; TAOS_RETURN(code); } @@ -4295,9 +4295,9 @@ static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupI return 0; } -int vgInfoCmp(const void* lp, const void* rp) { - SVgroupInfo* pLeft = (SVgroupInfo*)lp; - SVgroupInfo* pRight = (SVgroupInfo*)rp; +int vgInfoCmp(const void *lp, const void *rp) { + SVgroupInfo *pLeft = (SVgroupInfo *)lp; + SVgroupInfo *pRight = (SVgroupInfo *)rp; if (pLeft->hashBegin < pRight->hashBegin) { return -1; } else if (pLeft->hashBegin > pRight->hashBegin) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 52a3be120d..0a9eab43f8 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -307,6 +307,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_STABLE_UID_NOT_MATCH, "Invalid stable uid") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_CONFLICT_WITH_TSMA, "Field used by tsma") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_IN_CREATING, "Dnode in creating status") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_IN_DROPPING, "Dnode in dropping status") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ONLY_TS_SCHEMA_NOT_ALLOW, "Schema with only ts field is not allowed") // mnode-trans TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") From 4b21fd866df6ea3a95659a9e724cac5ce747218e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Oct 2024 15:21:32 +0800 Subject: [PATCH 23/43] doc: update os support info --- docs/zh/14-reference/07-supported.md | 47 ++++++++++++++++++---------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/docs/zh/14-reference/07-supported.md b/docs/zh/14-reference/07-supported.md index 10ca237653..828c8717e2 100644 --- a/docs/zh/14-reference/07-supported.md +++ b/docs/zh/14-reference/07-supported.md @@ -6,12 +6,25 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 服务端支持的平台列表 -| | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 以上** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** | -| ------------ | ---------------------------- | ----------------- | ---------------- | ------------------ | ------------ | ----------------- | ---------------- | --------- | -| X64 | ●/E | ●/E | ● | ● | ●/E | ●/E | ●/E | ● | -| 树莓派 ARM64 | | | ● | | | | | | -| 华为云 ARM64 | | | | ● | | | | | -| M1 | | | | | | | | ● | +| | **版本** | **X64 64bit** | **ARM64** | **M1** | +| ----------------------|----------------| ------------- | --------- | ------ | +| **CentOS** | **7.9 以上** | ● | ● | | +| **Ubuntu** | **18 以上** | ● | ● | | +| **RedHat** | **RHEL 6 以上** | ● | ● | | +| **Debian** | **6.0 以上** | ● | ● | | +| **FreeBSD** | **12 以上** | ● | ● | | +| **OpenSUSE** | **全部版本** | ● | ● | | +| **SUSE Linux** | **11 以上** | ● | ● | | +| **Fedora** | **21 以上** | ● | ● | | +| **Windows Server** | **2016/2019** | ●/E | | | +| **Windows** | **10/11** | ●/E | | | +| **银河麒麟** | **V10 以上** | ●/E | | | +| **中标麒麟** | **V7.0 以上** | ●/E | | | +| **通信 UOS** | **V20 以上** | ●/E | | | +| **凝思磐石** | **V6.0 以上** | ●/E | | | +| **华为欧拉 openEuler** | **V20.03 以上** | ●/E | | | +| **龙蜥 Anolis OS** | **V8.6 以上** | ●/E | | | +| **macOS** | **11.0 以上** | | | ● | 注：1) ● 表示经过官方测试验证， ○ 表示非官方测试验证，E 表示仅企业版支持。 2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/RedHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。 @@ -22,16 +35,16 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" 对照矩阵如下： -| **CPU** | **X64 64bit** | **X64 64bit** | **ARM64** | **X64 64bit** | **ARM64** | -| ----------- | ------------- | ------------- | --------- | ------------- | --------- | -| **OS** | **Linux** | **Win64** | **Linux** | **macOS** | **macOS** | -| **C/C++** | ● | ● | ● | ● | ● | -| **JDBC** | ● | ● | ● | ○ | ○ | -| **Python** | ● | ● | ● | ● | ● | -| **Go** | ● | ● | ● | ● | ● | -| **NodeJs** | ● | ● | ● | ○ | ○ | -| **C#** | ● | ● | ○ | ○ | ○ | -| **Rust** | ● | ● | ○ | ● | ● | -| **RESTful** | ● | ● | ● | ● | ● | +| **CPU** | **X64 64bit** | **X64 64bit** | **X64 64bit** | **ARM64** | **ARM64** | +| ----------- | ------------- | ------------- | ------------- | --------- | --------- | +| **OS** | **Linux** | **Win64** | **macOS** | **Linux** | **macOS** | +| **C/C++** | ● | ● | ● | ● | ● | +| **JDBC** | ● | ● | ○ | ● | ○ | +| **Python** | ● | ● | ● | ● | ● | +| **Go** | ● | ● | ● | ● | ● | +| **NodeJs** | ● | ● | ○ | ● | ○ | +| **C#** | ● | ● | ○ | ○ | ○ | +| **Rust** | ● | ● | ● | ○ | ● | +| **RESTful** | ● | ● | ● | ● | ● | 注：● 表示官方测试验证通过，○ 表示非官方测试验证通过，-- 表示未经验证。 From 83642604034258298de85652cc30f8478a7e8a21 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Oct 2024 15:30:32 +0800 Subject: [PATCH 24/43] enh: change the error code --- include/util/taoserror.h | 1 - source/dnode/mnode/impl/src/mndStb.c | 2 +- source/util/src/terror.c | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index dd9b90d390..a53923b904 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -392,7 +392,6 @@ int32_t taosGetErrSize(); #define TSDB_CODE_MND_INVALID_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x03C6) #define TSDB_CODE_MND_STABLE_UID_NOT_MATCH TAOS_DEF_ERROR_CODE(0, 0x03C7) #define TSDB_CODE_MND_FIELD_CONFLICT_WITH_TSMA TAOS_DEF_ERROR_CODE(0, 0x03C8) -#define TSDB_CODE_MND_ONLY_TS_SCHEMA_NOT_ALLOW TAOS_DEF_ERROR_CODE(0, 0x03C9) // mnode-trans #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 2986bdc117..3725d3a3fc 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1928,7 +1928,7 @@ static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStb } if (pOld->numOfColumns == 2) { - code = TSDB_CODE_MND_ONLY_TS_SCHEMA_NOT_ALLOW; + code = TSDB_CODE_PAR_INVALID_DROP_COL; TAOS_RETURN(code); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 0a9eab43f8..52a3be120d 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -307,7 +307,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_STABLE_UID_NOT_MATCH, "Invalid stable uid") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_CONFLICT_WITH_TSMA, "Field used by tsma") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_IN_CREATING, "Dnode in creating status") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_IN_DROPPING, "Dnode in dropping status") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ONLY_TS_SCHEMA_NOT_ALLOW, "Schema with only ts field is not allowed") // mnode-trans TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") From 935d118f42c295f0e3d7a5ea81b55151b37232fd Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 30 Oct 2024 07:29:13 +0000 Subject: [PATCH 25/43] fix/TD-32766-check-ref-before-free-main --- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 6 +++--- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index d07fe9c5ac..7566b69c02 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -38,7 +38,7 @@ int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnod SVnodeObj *pVnode = *ppVnode; if (pVnode && num < size) { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - // dTrace("vgId:%d, acquire vnode list, ref:%d", pVnode->vgId, refCount); + dTrace("vgId:%d,acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); pVnodes[num++] = (*ppVnode); pIter = taosHashIterate(pMgmt->hash, pIter); } else { @@ -52,7 +52,7 @@ int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnod SVnodeObj *pVnode = *ppVnode; if (pVnode && num < size) { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - // dTrace("vgId:%d, acquire vnode list, ref:%d", pVnode->vgId, refCount); + dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); pVnodes[num++] = (*ppVnode); pIter = taosHashIterate(pMgmt->closedHash, pIter); } else { @@ -84,7 +84,7 @@ int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeOb SVnodeObj *pVnode = *ppVnode; if (pVnode && num < size) { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - // dTrace("vgId:%d, acquire vnode list, ref:%d", pVnode->vgId, refCount); + dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); pVnodes[num++] = (*ppVnode); pIter = taosHashIterate(pMgmt->hash, pIter); } else { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index a285043382..0913d7539b 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -103,7 +103,7 @@ SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict) { pVnode = NULL; } else { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - // dTrace("vgId:%d, acquire vnode, ref:%d", pVnode->vgId, refCount); + dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); } (void)taosThreadRwlockUnlock(&pMgmt->lock); @@ -115,16 +115,24 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { return vmAcquireVno void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { if (pVnode == NULL) return; - (void)taosThreadRwlockRdlock(&pMgmt->lock); + //(void)taosThreadRwlockRdlock(&pMgmt->lock); int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - // dTrace("vgId:%d, release vnode, ref:%d", pVnode->vgId, refCount); - (void)taosThreadRwlockUnlock(&pMgmt->lock); + dTrace("vgId:%d, release vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); + //(void)taosThreadRwlockUnlock(&pMgmt->lock); } static void vmFreeVnodeObj(SVnodeObj **ppVnode) { if (!ppVnode || !(*ppVnode)) return; SVnodeObj *pVnode = *ppVnode; + + int32_t refCount = 1; + while (refCount > 0) { + dWarn("vgId:%d, vnode is refenced, retry to free in 200ms, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); + taosMsleep(200); + refCount = atomic_load_32(&pVnode->refCount); + } + taosMemoryFree(pVnode->path); taosMemoryFree(pVnode); ppVnode[0] = NULL; From 65f11a24633d7457e41aec32c6a81be5e79b0c09 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Oct 2024 15:38:29 +0800 Subject: [PATCH 26/43] doc: minor changes --- docs/zh/14-reference/07-supported.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/14-reference/07-supported.md b/docs/zh/14-reference/07-supported.md index 828c8717e2..96d12311be 100644 --- a/docs/zh/14-reference/07-supported.md +++ b/docs/zh/14-reference/07-supported.md @@ -18,10 +18,10 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" | **Fedora** | **21 以上** | ● | ● | | | **Windows Server** | **2016/2019** | ●/E | | | | **Windows** | **10/11** | ●/E | | | -| **银河麒麟** | **V10 以上** | ●/E | | | -| **中标麒麟** | **V7.0 以上** | ●/E | | | +| **银河麒麟** | **V10 以上** | ●/E | ●/E | | +| **中标麒麟** | **V7.0 以上** | ●/E | ●/E | | | **通信 UOS** | **V20 以上** | ●/E | | | -| **凝思磐石** | **V6.0 以上** | ●/E | | | +| **凝思磐石** | **V8.0 以上** | ●/E | | | | **华为欧拉 openEuler** | **V20.03 以上** | ●/E | | | | **龙蜥 Anolis OS** | **V8.6 以上** | ●/E | | | | **macOS** | **11.0 以上** | | | ● | From eed49788c0cfac34a3acc431d209732747937993 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Oct 2024 15:50:39 +0800 Subject: [PATCH 27/43] doc: update index --- docs/zh/01-index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/zh/01-index.md b/docs/zh/01-index.md index 32ea117fbb..1c9ad0b2e1 100644 --- a/docs/zh/01-index.md +++ b/docs/zh/01-index.md @@ -6,7 +6,7 @@ slug: / TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的时序数据库（Time Series Database, TSDB）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。如果你对时序数据的基本概念、价值以及其所能带来的业务价值尚不了解，请参考[时序数据基础](./concept) -TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论如何，请您仔细阅读[数据模型](./basic/model)一章。 +TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论你在工作中是什么角色，请您仔细阅读[数据模型](./basic/model)一章。 如果你是开发工程师，请一定仔细阅读[开发指南](./develop)一章，该部分对数据库连接、建模、插入数据、查询、流式计算、缓存、数据订阅、用户自定义函数等功能都做了详细介绍，并配有各种编程语言的示例代码。大部分情况下，你只要复制粘贴示例代码，针对自己的应用稍作改动，就能跑起来。对 REST API、各种编程语言的连接器（Connector）想做更多详细了解的话，请看[连接器](./reference/connector)一章。 @@ -16,6 +16,8 @@ TDengine 采用 SQL 作为查询语言，大大降低学习成本、降低迁移 如果你是系统管理员，关心安装、升级、容错灾备、关心数据导入、导出、配置参数，如何监测 TDengine 是否健康运行，如何提升系统运行的性能，请仔细参考[运维指南](./operation)一章。 +如果你对数据库内核设计感兴趣，或是开源爱好者，建议仔细阅读[技术内幕](./tdinterna)一章。该章从分布式架构到存储引擎、查询引擎、数据订阅，再到流计算引擎都做了详细阐述。建议对照文档，查看TDengine在GitHub的源代码，对TDengine的设计和编码做深入了解，更欢迎加入开源社区，贡献代码。 + 最后，作为一个开源软件，欢迎大家的参与。如果发现文档有任何错误、描述不清晰的地方，请在每个页面的最下方，点击“编辑本文档”直接进行修改。 Together, we make a difference! From 1cdad6572410aa034f909e85589b5189cca3a5e6 Mon Sep 17 00:00:00 2001 From: Chait Diwadkar <94201190+cdiwadkar16@users.noreply.github.com> Date: Fri, 6 Sep 2024 14:01:09 -0700 Subject: [PATCH 28/43] docs/Update 01-docker.md Corrected spelling mistake and have a statement on how to login to taos-explorer using root/taosdata. --- docs/en/04-get-started/01-docker.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/04-get-started/01-docker.md b/docs/en/04-get-started/01-docker.md index 882e2ef194..f361e5a10f 100644 --- a/docs/en/04-get-started/01-docker.md +++ b/docs/en/04-get-started/01-docker.md @@ -75,9 +75,9 @@ taos> ## TDegnine Graphic User Interface -From TDengine 3.3.0.0, there is a new componenet called `taos-explorer` added in the TDengine docker image. You can use it to manage the databases, super tables, child tables, and data in your TDengine system. There are also some features only available in TDengine Enterprise Edition, please contact TDengine sales team in case you need these features. +From TDengine 3.3.0.0, there is a new component called `taos-explorer` added in the TDengine docker image. You can use it to manage the databases, super tables, child tables, and data in your TDengine system. There are also some features only available in TDengine Enterprise Edition, please contact TDengine sales team in case you need these features. -To use taos-explorer in the container, you need to access the host port mapped from container port 6060. Assuming the host name is abc.com, and the port used on host is 6060, you need to access `http://abc.com:6060`. taos-explorer uses port 6060 by default in the container. When you use it the first time, you need to register with your enterprise email, then can logon using your user name and password in the TDengine database management system. +To use taos-explorer in the container, you need to access the host port mapped from container port 6060. Assuming the host name is abc.com, and the port used on host is 6060, you need to access `http://abc.com:6060`. taos-explorer uses port 6060 by default in the container. The default username and password to log in to the TDengine Database Management System is "root/taosdata". ## Test data insert performance From 016c4fa9d7009b00e5231cb9c55da2d53e28e62d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Oct 2024 16:14:14 +0800 Subject: [PATCH 29/43] doc: update minor errors --- docs/zh/14-reference/07-supported.md | 38 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/zh/14-reference/07-supported.md b/docs/zh/14-reference/07-supported.md index 96d12311be..a062257143 100644 --- a/docs/zh/14-reference/07-supported.md +++ b/docs/zh/14-reference/07-supported.md @@ -6,25 +6,25 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 服务端支持的平台列表 -| | **版本** | **X64 64bit** | **ARM64** | **M1** | -| ----------------------|----------------| ------------- | --------- | ------ | -| **CentOS** | **7.9 以上** | ● | ● | | -| **Ubuntu** | **18 以上** | ● | ● | | -| **RedHat** | **RHEL 6 以上** | ● | ● | | -| **Debian** | **6.0 以上** | ● | ● | | -| **FreeBSD** | **12 以上** | ● | ● | | -| **OpenSUSE** | **全部版本** | ● | ● | | -| **SUSE Linux** | **11 以上** | ● | ● | | -| **Fedora** | **21 以上** | ● | ● | | -| **Windows Server** | **2016/2019** | ●/E | | | -| **Windows** | **10/11** | ●/E | | | -| **银河麒麟** | **V10 以上** | ●/E | ●/E | | -| **中标麒麟** | **V7.0 以上** | ●/E | ●/E | | -| **通信 UOS** | **V20 以上** | ●/E | | | -| **凝思磐石** | **V8.0 以上** | ●/E | | | -| **华为欧拉 openEuler** | **V20.03 以上** | ●/E | | | -| **龙蜥 Anolis OS** | **V8.6 以上** | ●/E | | | -| **macOS** | **11.0 以上** | | | ● | +| | **版本** | **X64 64bit** | **ARM64** | +| ----------------------|----------------| ------------- | --------- | +| **CentOS** | **7.9 以上** | ● | ● | +| **Ubuntu** | **18 以上** | ● | ● | +| **RedHat** | **RHEL 7 以上** | ● | ● | +| **Debian** | **6.0 以上** | ● | ● | +| **FreeBSD** | **12 以上** | ● | ● | +| **OpenSUSE** | **全部版本** | ● | ● | +| **SUSE Linux** | **11 以上** | ● | ● | +| **Fedora** | **21 以上** | ● | ● | +| **Windows Server** | **2016 以上** | ●/E | | +| **Windows** | **10/11** | ●/E | | +| **银河麒麟** | **V10 以上** | ●/E | ●/E | +| **中标麒麟** | **V7.0 以上** | ●/E | ●/E | +| **统信 UOS** | **V20 以上** | ●/E | | +| **凝思磐石** | **V8.0 以上** | ●/E | | +| **华为欧拉 openEuler** | **V20.03 以上** | ●/E | | +| **龙蜥 Anolis OS** | **V8.6 以上** | ●/E | | +| **macOS** | **11.0 以上** | | ● | 注：1) ● 表示经过官方测试验证， ○ 表示非官方测试验证，E 表示仅企业版支持。 2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/RedHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。 From 6aba3e9cac2a2de018161c82376e78a2bceea996 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Oct 2024 16:17:53 +0800 Subject: [PATCH 30/43] doc: minor changes --- docs/zh/14-reference/07-supported.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/07-supported.md b/docs/zh/14-reference/07-supported.md index a062257143..f98f96755a 100644 --- a/docs/zh/14-reference/07-supported.md +++ b/docs/zh/14-reference/07-supported.md @@ -43,7 +43,7 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" | **Python** | ● | ● | ● | ● | ● | | **Go** | ● | ● | ● | ● | ● | | **NodeJs** | ● | ● | ○ | ● | ○ | -| **C#** | ● | ● | ○ | ○ | ○ | +| **C#** | ● | ● | ○ | ● | ○ | | **Rust** | ● | ● | ● | ○ | ● | | **RESTful** | ● | ● | ● | ● | ● | From 25064ce77cbdea4d95cffc1cb64d26fdb0600d4d Mon Sep 17 00:00:00 2001 From: RuiLiang51666 <130521128+RuiLiang51666@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:35:19 +0800 Subject: [PATCH 31/43] Update 01-index.md add the missing period. --- docs/zh/01-index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/01-index.md b/docs/zh/01-index.md index 1c9ad0b2e1..1fda72024c 100644 --- a/docs/zh/01-index.md +++ b/docs/zh/01-index.md @@ -4,7 +4,7 @@ sidebar_label: 文档首页 slug: / --- -TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的时序数据库（Time Series Database, TSDB）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。如果你对时序数据的基本概念、价值以及其所能带来的业务价值尚不了解，请参考[时序数据基础](./concept) +TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的时序数据库（Time Series Database, TSDB）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。如果你对时序数据的基本概念、价值以及其所能带来的业务价值尚不了解，请参考[时序数据基础](./concept)。 TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论你在工作中是什么角色，请您仔细阅读[数据模型](./basic/model)一章。 From fbb8585593968ba5194ba699613b0ee361628490 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 30 Oct 2024 09:00:42 +0000 Subject: [PATCH 32/43] fix/TD-32766-check-ref-before-free-check-first --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 0913d7539b..682c179270 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -126,7 +126,7 @@ static void vmFreeVnodeObj(SVnodeObj **ppVnode) { SVnodeObj *pVnode = *ppVnode; - int32_t refCount = 1; + int32_t refCount = atomic_load_32(&pVnode->refCount); while (refCount > 0) { dWarn("vgId:%d, vnode is refenced, retry to free in 200ms, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); taosMsleep(200); From 296eb9a14665247838a9166a43651507924a1d50 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:11:20 +0800 Subject: [PATCH 33/43] =?UTF-8?q?docs=EF=BC=9AUpdate=20get-started=20descr?= =?UTF-8?q?iption?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/zh/04-get-started/03-package.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/zh/04-get-started/03-package.md b/docs/zh/04-get-started/03-package.md index 2a1f594b4f..cef3cb6943 100644 --- a/docs/zh/04-get-started/03-package.md +++ b/docs/zh/04-get-started/03-package.md @@ -14,7 +14,9 @@ TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc） 为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 Lite 版本的安装包。 -在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTools 包获得。TDengine 也提供 Windows x64 平台和 macOS x64/m1 平台的安装包。 +在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统，用户可以根据自己的运行环境自行选择。同时我们也提供了 tar.gz 格式安装包，以及 `apt-get` 工具从线上进行安装。 + +此外，TDengine 也提供 macOS x64/m1 平台的 pkg 安装包。 ## 运行环境要求 在linux系统中，运行环境最低要求如下: @@ -317,4 +319,4 @@ SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 1 SELECT _wstart, AVG(current), MAX(voltage), MIN(phase) FROM test.d1001 INTERVAL(10s); ``` -在上面的查询中，使用系统提供的伪列_wstart 来给出每个窗口的开始时间。 \ No newline at end of file +在上面的查询中，使用系统提供的伪列_wstart 来给出每个窗口的开始时间。 From ae369cdeb873d25f0841ffa8e78649210040db5a Mon Sep 17 00:00:00 2001 From: beliefstar Date: Tue, 13 Aug 2024 16:50:41 +0800 Subject: [PATCH 34/43] Update 02-cache.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wal_level设置为2时，每次写入请求都会执行fsync，不需要控制执行频率，只有当wal_level设置为1时，才需要控制fsync的执行频率。 --- docs/zh/06-advanced/02-cache.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/02-cache.md b/docs/zh/06-advanced/02-cache.md index ca1da30dbf..b83509c2dd 100644 --- a/docs/zh/06-advanced/02-cache.md +++ b/docs/zh/06-advanced/02-cache.md @@ -54,7 +54,7 @@ TDengine 利用这些日志文件实现故障前的状态恢复。在写入 WAL 数据库参数 wal_level 和 wal_fsync_period 共同决定了 WAL 的保存行为。。 - wal_level：此参数控制 WAL 的保存级别。级别 1 表示仅将数据写入 WAL，但不立即执行 fsync 函数；级别 2 则表示在写入 WAL 的同时执行 fsync 函数。默认情况下，wal_level 设为 1。虽然执行 fsync 函数可以提高数据的持久性，但相应地也会降低写入性能。 -- wal_fsync_period：当 wal_level 设置为 2 时，这个参数控制执行 fsync 的频率。设置为 0 表示每次写入后立即执行 fsync，这可以确保数据的安全性，但可能会牺牲一些性能。当设置为大于 0 的数值时，表示 fsync 周期，默认为 3000，范围是[1， 180000]，单位毫秒。 +- wal_fsync_period：当 wal_level 设置为 1 时，这个参数控制执行 fsync 的频率。设置为 0 表示每次写入后立即执行 fsync，这可以确保数据的安全性，但可能会牺牲一些性能。当设置为大于 0 的数值时，表示 fsync 周期，默认为 3000，范围是[1， 180000]，单位毫秒。 ```sql CREATE DATABASE POWER WAL_LEVEL 1 WAL_FSYNC_PERIOD 3000; @@ -119,4 +119,4 @@ taos> select last_row(ts,current) from meters; Query OK, 1 row(s) in set (0.046682s) ``` -可以看到查询的时延从 353/344ms 缩短到了 44ms，提升约 8 倍。 \ No newline at end of file +可以看到查询的时延从 353/344ms 缩短到了 44ms，提升约 8 倍。 From 68be4a7c3baab3969dd63bd1d0e414b6a2556880 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:13:04 +0800 Subject: [PATCH 35/43] Update 14-stream.md --- docs/zh/14-reference/03-taos-sql/14-stream.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/14-reference/03-taos-sql/14-stream.md b/docs/zh/14-reference/03-taos-sql/14-stream.md index cd5c76a4ad..6baad76e23 100644 --- a/docs/zh/14-reference/03-taos-sql/14-stream.md +++ b/docs/zh/14-reference/03-taos-sql/14-stream.md @@ -212,11 +212,11 @@ TDengine 对于修改数据提供两种处理方式，由 IGNORE UPDATE 选项 ```sql [field1_name,...] ``` -用来指定stb_name的列与subquery输出结果的对应关系。如果stb_name的列与subquery输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果stb_name的列与subquery输出结果的数据类型不匹配，会把subquery输出结果的类型转换成对应的stb_name的列的类型。 +在本页文档顶部的 [field1_name,...] 是用来指定 stb_name 的列与 subquery 输出结果的对应关系的。如果 stb_name 的列与 subquery 输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果 stb_name 的列与 subquery 输出结果的数据类型不匹配，会把 subquery 输出结果的类型转换成对应的 stb_name 的列的类型。 对于已经存在的超级表，检查列的schema信息 -1. 检查列的schema信息是否匹配，对于不匹配的，则自动进行类型转换，当前只有数据长度大于4096byte时才报错，其余场景都能进行类型转换。 -2. 检查列的个数是否相同，如果不同，需要显示的指定超级表与subquery的列的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。 +1. 检查列的 schema 信息是否匹配，对于不匹配的，则自动进行类型转换，当前只有数据长度大于 4096byte 时才报错，其余场景都能进行类型转换。 +2. 检查列的个数是否相同，如果不同，需要显示的指定超级表与 subquery 的列的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。 ## 自定义TAG From 411bcd04e1b8b43bb23a9d0340d3407ddf8c9dde Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:22:30 +0800 Subject: [PATCH 36/43] docs:Update README-CN.md --- README-CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-CN.md b/README-CN.md index 06ac087859..1f785eb458 100644 --- a/README-CN.md +++ b/README-CN.md @@ -348,7 +348,7 @@ TDengine 提供了丰富的应用程序开发接口，其中包括 C/C++、Java # 成为社区贡献者 -点击 [这里](https://www.taosdata.com/cn/contributor/)，了解如何成为 TDengine 的贡献者。 +点击 [这里](https://www.taosdata.com/contributor)，了解如何成为 TDengine 的贡献者。 # 加入技术交流群 From 3895a7707af617ee8acf9689ce51ca7bb4c00b6b Mon Sep 17 00:00:00 2001 From: Pengrongkun Date: Wed, 30 Oct 2024 16:51:59 +0800 Subject: [PATCH 37/43] convert TAOS_FIELD_E to TAOS_FIELD_ALL, to prevent modifications to the original API --- include/client/taos.h | 13 ++++++-- source/client/inc/clientStmt2.h | 1 + source/client/src/clientMain.c | 10 ++++-- source/client/src/clientStmt2.c | 45 ++++++++++++++++++++++++-- source/libs/parser/src/parInsertStmt.c | 7 ++-- tests/script/api/stmt2-get-fields.c | 13 ++++---- 6 files changed, 72 insertions(+), 17 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 6797dfee5f..266e0e192d 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -95,13 +95,21 @@ typedef struct taosField { } TAOS_FIELD; typedef struct TAOS_FIELD_E { + char name[65]; + int8_t type; + uint8_t precision; + uint8_t scale; + int32_t bytes; +} TAOS_FIELD_E; + +typedef struct TAOS_FIELD_ALL { char name[65]; int8_t type; uint8_t precision; uint8_t scale; int32_t bytes; TAOS_FIELD_T field_type; -} TAOS_FIELD_E; +} TAOS_FIELD_ALL; #ifdef WINDOWS #define DLL_EXPORT __declspec(dllexport) @@ -233,8 +241,9 @@ DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows); DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt); DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert); DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields); -DLL_EXPORT int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_E **fields); +DLL_EXPORT int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields); DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields); +DLL_EXPORT void taos_stmt2_free_all_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields); DLL_EXPORT TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt); DLL_EXPORT char *taos_stmt2_error(TAOS_STMT2 *stmt); diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h index 4e9a09c082..e54057e72f 100644 --- a/source/client/inc/clientStmt2.h +++ b/source/client/inc/clientStmt2.h @@ -222,6 +222,7 @@ int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags); int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx); int stmtGetTagFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); int stmtGetColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); +int stmtGetColFieldsALL2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_ALL **fields); int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums); int stmtGetParamTbName(TAOS_STMT2 *stmt, int *nums); int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 54da1c013c..f2fc2b8bdc 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -2102,14 +2102,14 @@ int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, } } -int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_E **fields) { +int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields) { if (stmt == NULL || count == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - return stmtGetColFields2(stmt, count, fields); + return stmtGetColFieldsALL2(stmt, count, fields); } void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields) { @@ -2118,6 +2118,12 @@ void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields) { taosMemoryFree(fields); } +DLL_EXPORT void taos_stmt2_free_all_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields) { + (void)stmt; + if (!fields) return; + taosMemoryFree(fields); +} + TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 5d04006f06..67de878ee5 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1069,7 +1069,7 @@ static int stmtFetchColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E return TSDB_CODE_SUCCESS; } -static int stmtFetchFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields) { +static int stmtFetchFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_ALL** fields) { SBoundColInfo* tags = (SBoundColInfo*)pStmt->bInfo.boundTags; STableMeta* meta = ((SVnodeModifyOpStmt*)(pStmt->sql.pQuery->pRoot))->pTableMeta; if (tags == NULL || meta == NULL || (meta->schema == NULL && tags->numOfBound != 0)) { @@ -1077,7 +1077,7 @@ static int stmtFetchFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E** } if (fields != NULL) { - *fields = taosMemoryCalloc(tags->numOfBound, sizeof(TAOS_FIELD_E)); + *fields = taosMemoryCalloc(tags->numOfBound, sizeof(TAOS_FIELD_ALL)); if (NULL == *fields) { return terrno; } @@ -1886,6 +1886,47 @@ int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) { STMT_ERRI_JRET(stmtFetchColFields2(stmt, nums, fields)); +_return: + + pStmt->errCode = preCode; + + return code; +} + +int stmtGetColFieldsALL2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_ALL** fields) { + int32_t code = 0; + STscStmt2* pStmt = (STscStmt2*)stmt; + int32_t preCode = pStmt->errCode; + + STMT_DLOG_E("start to get col fields"); + + if (pStmt->errCode != TSDB_CODE_SUCCESS) { + return pStmt->errCode; + } + + if (STMT_TYPE_QUERY == pStmt->sql.type) { + STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR); + } + + STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && + STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + + if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + STMT_ERR_RET(stmtCreateRequest(pStmt)); + } + + STMT_ERRI_JRET(stmtCreateRequest(pStmt)); + + if (pStmt->bInfo.needParse) { + STMT_ERRI_JRET(stmtParseSql(pStmt)); + } + _return: pStmt->errCode = preCode; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 8c17689ea7..1355d9e09f 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -885,7 +885,7 @@ _return: } int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, - TAOS_FIELD_E** fields, uint8_t timePrec, TAOS_FIELD_T fieldType) { + TAOS_FIELD_E** fields, uint8_t timePrec) { if (fields != NULL) { *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { @@ -900,7 +900,6 @@ int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSc for (int32_t i = 0; i < numOfBound; ++i) { schema = &pSchema[boundColumns[i]]; strcpy((*fields)[i].name, schema->name); - (*fields)[i].field_type = schema->type; (*fields)[i].type = schema->type; (*fields)[i].bytes = schema->bytes; } @@ -930,7 +929,7 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(tags->numOfBound, tags->pColIndex, pSchema, fieldNum, fields, 0, TAOS_FIELD_TAG)); + CHECK_CODE(buildBoundFields(tags->numOfBound, tags->pColIndex, pSchema, fieldNum, fields, 0)); return TSDB_CODE_SUCCESS; } @@ -948,7 +947,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel } CHECK_CODE(buildBoundFields(pDataBlock->boundColsInfo.numOfBound, pDataBlock->boundColsInfo.pColIndex, pSchema, - fieldNum, fields, pDataBlock->pMeta->tableInfo.precision, TAOS_FIELD_COL)); + fieldNum, fields, pDataBlock->pMeta->tableInfo.precision)); return TSDB_CODE_SUCCESS; } diff --git a/tests/script/api/stmt2-get-fields.c b/tests/script/api/stmt2-get-fields.c index 60bee16873..23b91b56c9 100644 --- a/tests/script/api/stmt2-get-fields.c +++ b/tests/script/api/stmt2-get-fields.c @@ -25,8 +25,8 @@ void do_stmt(TAOS *taos) { "int, t2 binary(10))"); TAOS_STMT2_OPTION option = {0}; - TAOS_STMT2 *stmt = taos_stmt2_init(taos, &option); - const char *sql = "insert into db.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; + TAOS_STMT2 *stmt = taos_stmt2_init(taos, &option); + const char *sql = "insert into db.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; int code = taos_stmt2_prepare(stmt, sql, 0); if (code != 0) { @@ -35,19 +35,18 @@ void do_stmt(TAOS *taos) { return; } - int fieldNum = 0; - TAOS_FIELD_E *pFields = NULL; + int fieldNum = 0; + TAOS_FIELD_ALL *pFields = NULL; code = taos_stmt2_get_all_fields(stmt, &fieldNum, &pFields); if (code != 0) { printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); } else { printf("col nums:%d\n", fieldNum); - for(int i = 0; i < fieldNum; i++) { - printf("field[%d]: %s,type:%d\n", i, pFields[i].name,pFields[i].field_type); + for (int i = 0; i < fieldNum; i++) { + printf("field[%d]: %s,type:%d\n", i, pFields[i].name, pFields[i].field_type); } } - taos_stmt2_close(stmt); } From 7d05ccbfa03ea43055e507cc01fe4831e9566b7c Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Tue, 29 Oct 2024 11:13:50 +0800 Subject: [PATCH 38/43] fix:[TS-5567] fix bug when partition/group by const value's alias name. --- source/libs/parser/src/parTranslater.c | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7b1896d6a9..1faa010663 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1607,26 +1607,6 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p } } if (*pFound) { - if (QUERY_NODE_FUNCTION == nodeType(pFoundNode) && - (SQL_CLAUSE_GROUP_BY == pCxt->currClause || SQL_CLAUSE_PARTITION_BY == pCxt->currClause)) { - pCxt->errCode = getFuncInfo(pCxt, (SFunctionNode*)pFoundNode); - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - if (fmIsVectorFunc(((SFunctionNode*)pFoundNode)->funcId)) { - pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, (*pCol)->colName); - return DEAL_RES_ERROR; - } else if (fmIsPseudoColumnFunc(((SFunctionNode*)pFoundNode)->funcId)) { - if ('\0' != (*pCol)->tableAlias[0]) { - return translateColumnWithPrefix(pCxt, pCol); - } else { - return translateColumnWithoutPrefix(pCxt, pCol); - } - } else { - /* Do nothing and replace old node with found node. */ - } - } else { - return DEAL_RES_ERROR; - } - } SNode* pNew = NULL; int32_t code = nodesCloneNode(pFoundNode, &pNew); if (NULL == pNew) { @@ -5496,8 +5476,7 @@ static EDealRes translateGroupPartitionByImpl(SNode** pNode, void* pContext) { int32_t code = TSDB_CODE_SUCCESS; STranslateContext* pTransCxt = pCxt->pTranslateCxt; if (QUERY_NODE_VALUE == nodeType(*pNode)) { - STranslateContext* pTransCxt = pCxt->pTranslateCxt; - SValueNode* pVal = (SValueNode*)*pNode; + SValueNode* pVal = (SValueNode*) *pNode; if (DEAL_RES_ERROR == translateValue(pTransCxt, pVal)) { return DEAL_RES_CONTINUE; } @@ -5506,7 +5485,7 @@ static EDealRes translateGroupPartitionByImpl(SNode** pNode, void* pContext) { } int32_t pos = getPositionValue(pVal); if (0 < pos && pos <= LIST_LENGTH(pProjectionList)) { - SNode* pNew = NULL; + SNode* pNew = NULL; code = nodesCloneNode(nodesListGetNode(pProjectionList, pos - 1), (SNode**)&pNew); if (TSDB_CODE_SUCCESS != code) { pCxt->pTranslateCxt->errCode = code; From af03868864db5aec6a2cf831f3fd90159cf91d02 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Oct 2024 19:42:09 +0800 Subject: [PATCH 39/43] fix unit test --- source/dnode/mnode/impl/test/stb/stb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index aa12c107a1..e92231907f 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -782,7 +782,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) { { void* pReq = BuildAlterStbDropColumnReq(stbname, "col1", &contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_ALTER_OPTION); + ASSERT_EQ(pRsp->code, TSDB_CODE_PAR_INVALID_DROP_COL); rpcFreeCont(pRsp->pCont); } From d5f6f9aacb0035a18ea33a44ac047493f8521549 Mon Sep 17 00:00:00 2001 From: yingzhao Date: Wed, 30 Oct 2024 19:53:00 +0800 Subject: [PATCH 40/43] fix(dosc): br compile eror --- docs/zh/06-advanced/05-data-in/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/05-data-in/index.md b/docs/zh/06-advanced/05-data-in/index.md index 50c66cc377..0dfa04db56 100644 --- a/docs/zh/06-advanced/05-data-in/index.md +++ b/docs/zh/06-advanced/05-data-in/index.md @@ -22,7 +22,7 @@ TDengine Enterprise 配备了一个强大的可视化数据管理工具—taosEx | Aveva Historian | AVEVA Historian 2020 RS SP1 | 工业大数据分析软件，前身为 Wonderware Historian，专为工业环境设计，用于存储、管理和分析来自各种工业设备、传感器的实时和历史数据 | | OPC DA | Matrikon OPC version: 1.7.2.7433 | OPC 是 Open Platform Communications 的缩写，是一种开放式、标准化的通信协议，用于不同厂商的自动化设备之间进行数据交换。它最初由微软公司开发，旨在解决工业控制领域中不同设备之间互操作性差的问题；OPC 协议最初于 1996 年发布，当时称为 OPC DA （Data Access），主要用于实时数据采集和控制。 | | OPC UA | KeepWare KEPServerEx 6.5 | 2006 年，OPC 基金会发布了 OPC UA （Unified Architecture） 标准，它是一种基于服务的面向对象的协议，具有更高的灵活性和可扩展性，已成为 OPC 协议的主流版本 | -| MQTT | emqx: 3.0.0 到 5.7.1
hivemq: 4.0.0 到 4.31.0
mosquitto: 1.4.4 到 2.0.18 | Message Queuing Telemetry Transport 的缩写，一种基于发布/订阅模式的轻量级通讯协议，专为低开销、低带宽占用的即时通讯设计，广泛适用于物联网、小型设备、移动应用等领域。 | +| MQTT | emqx: 3.0.0 到 5.7.1
hivemq: 4.0.0 到 4.31.0
mosquitto: 1.4.4 到 2.0.18 | Message Queuing Telemetry Transport 的缩写，一种基于发布/订阅模式的轻量级通讯协议，专为低开销、低带宽占用的即时通讯设计，广泛适用于物联网、小型设备、移动应用等领域。 | | Kafka | 2.11 ~ 3.8.0 | 由 Apache 软件基金会开发的一个开源流处理平台，主要用于处理实时数据，并提供一个统一、高通量、低延迟的消息系统。它具备高速度、可伸缩性、持久性和分布式设计等特点，使得它能够在每秒处理数十万次的读写操作，支持上千个客户端，同时保持数据的可靠性和可用性。 | | InfluxDB | 1.7、1.8、2.0-2.7 | InfluxDB 是一种流行的开源时间序列数据库，它针对处理大量时间序列数据进行了优化。| | OpenTSDB | 2.4.1 | 基于 HBase 的分布式、可伸缩的时序数据库。它主要用于存储、索引和提供从大规模集群（包括网络设备、操作系统、应用程序等）中收集的指标数据，使这些数据更易于访问和图形化展示。 | From 2d310221b34327b2d702c56b11e76409f67484f2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 31 Oct 2024 10:03:43 +0800 Subject: [PATCH 41/43] enh:[TD-32166]refactor code in sml --- source/client/src/clientSml.c | 72 +++++++++++++------------------ source/client/src/clientSmlJson.c | 3 +- 2 files changed, 30 insertions(+), 45 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 9d3b28e105..e8221c8b8d 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -138,7 +138,7 @@ void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2 if (pBuf->buf == NULL) { return; } - (void)memset(pBuf->buf, 0, pBuf->len); + pBuf->buf[0] = 0; if (msg1) { (void)strncat(pBuf->buf, msg1, pBuf->len - 1); } @@ -475,7 +475,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam } } if (autoChildName) { - (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + childTableName[0] = '\0'; for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); SML_CHECK_NULL(tag); @@ -499,7 +499,6 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam SML_CHECK_NULL(tag); // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { - (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); tstrncpy(childTableName, tag->value, TMIN(TSDB_TABLE_NAME_LEN, tag->length + 1)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); @@ -524,8 +523,7 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { dst = taosArrayDup(oneTable->tags, NULL); SML_CHECK_NULL(dst); if (oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN) { - code = TSDB_CODE_SML_INTERNAL_ERROR; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INTERNAL_ERROR); } char superName[TSDB_TABLE_NAME_LEN] = {0}; RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; @@ -676,8 +674,9 @@ int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, ST conn.requestId = info->pRequest->requestId; conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(measureLen, TSDB_TABLE_NAME_LEN - 1); (void)memcpy(pName.tname, measure, measureLen); + pName.tname[len] = 0; return catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); } @@ -766,8 +765,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); SML_CHECK_NULL(kv); if (taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL) { - code = TSDB_CODE_PAR_DUPLICATED_COLUMN; - goto END; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } END: @@ -788,8 +786,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); SML_CHECK_NULL(kv); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - code = TSDB_CODE_SML_INVALID_DATA; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } } @@ -810,7 +807,7 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = TSDB_CODE_SUCCESS; + int32_t lino = 0; for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); SML_CHECK_NULL(kv); @@ -825,9 +822,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); if (index == NULL) { - uError("smlBuildFieldsList get error, key:%s", kv->key); - code = TSDB_CODE_SML_INVALID_DATA; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; @@ -894,17 +889,14 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; smlBuildCreateStbReq(&pReq, pTableMeta->sversion + 1, pTableMeta->tversion, pTableMeta->uid, TD_REQ_FROM_TAOX); } else { - uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); - code = TSDB_CODE_SML_INVALID_DATA; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } SML_CHECK_CODE(buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0)); pRequest->syncQuery = true; if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto END; + SML_CHECK_CODE(TSDB_CODE_PAR_DB_NOT_SPECIFIED); } if (pReq.numOfTags == 0) { @@ -924,15 +916,16 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pCmdMsg.msgType = TDMT_MND_CREATE_STB; pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); if (pCmdMsg.msgLen < 0) { - code = pCmdMsg.msgLen; - goto END; + uError("failed to serialize create stable request1, code:%d, terrno:%d", pCmdMsg.msgLen, terrno); + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); SML_CHECK_NULL(pCmdMsg.pMsg); code = tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); if (code < 0) { taosMemoryFree(pCmdMsg.pMsg); - goto END; + uError("failed to serialize create stable request2, code:%d, terrno:%d", code, terrno); + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } SQuery pQuery = {0}; @@ -1108,8 +1101,9 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); } smlStrReplace(measure, superTableLen); - (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - (void)memcpy(pName.tname, measure, TMIN(superTableLen, TSDB_TABLE_NAME_LEN - 1)); + size_t nameLen = TMIN(superTableLen, TSDB_TABLE_NAME_LEN - 1); + (void)memcpy(pName.tname, measure, nameLen); + pName.tname[nameLen] = '\0'; taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1118,8 +1112,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { SML_CHECK_CODE(smlCreateTable(info, &conn, sTableData, &pName, &pTableMeta)); } else if (code == TSDB_CODE_SUCCESS) { if (smlIsPKTable(pTableMeta)) { - code = TSDB_CODE_SML_NOT_SUPPORT_PK; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_NOT_SUPPORT_PK); } hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); @@ -1176,8 +1169,7 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols if (ret == 0) { SML_CHECK_NULL(taosArrayPush(metaArray, kv)); if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { - code = TSDB_CODE_PAR_DUPLICATED_COLUMN; - goto END; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } else if (terrno == TSDB_CODE_DUP_KEY) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; @@ -1208,8 +1200,7 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } if (kv->type != value->type) { smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - code = TSDB_CODE_SML_NOT_SAME_TYPE; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_NOT_SAME_TYPE); } if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger @@ -1219,16 +1210,13 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols size_t tmp = taosArrayGetSize(metaArray); if (tmp > INT16_MAX) { smlBuildInvalidDataMsg(msg, "too many cols or tags", kv->key); - uError("too many cols or tags"); - code = TSDB_CODE_SML_INVALID_DATA; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } int16_t size = tmp; SML_CHECK_CODE(taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES)); SML_CHECK_NULL(taosArrayPush(metaArray, kv)); if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { - code = TSDB_CODE_PAR_DUPLICATED_COLUMN; - goto END; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } } @@ -1349,8 +1337,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { terrno = 0; code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { - code = TSDB_CODE_PAR_DUPLICATED_COLUMN; - goto END; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } SML_CHECK_CODE(code); } @@ -1417,8 +1404,7 @@ static int32_t smlParseEnd(SSmlHandle *info) { code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { smlDestroySTableMeta(&meta); - uError("SML:0x%" PRIx64 " put measure to hash failed", info->id); - goto END; + SML_CHECK_CODE(code); } SML_CHECK_CODE(smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL)); SML_CHECK_CODE(smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash)); @@ -1459,8 +1445,8 @@ static int32_t smlInsertData(SSmlHandle *info) { PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); } smlStrReplace(measure, measureLen); - (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); (void)memcpy(pName.tname, measure, measureLen); + pName.tname[measureLen] = '\0'; if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); @@ -1485,8 +1471,7 @@ static int32_t smlInsertData(SSmlHandle *info) { (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); if (unlikely(NULL == pMeta || NULL == *pMeta || NULL == (*pMeta)->tableMeta)) { uError("SML:0x%" PRIx64 " %s NULL == pMeta. table name: %s", info->id, __FUNCTION__, tableData->childTableName); - code = TSDB_CODE_SML_INTERNAL_ERROR; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INTERNAL_ERROR); } // use tablemeta of stable to save vgid and uid of child table @@ -1565,12 +1550,13 @@ END: } static void printRaw(int64_t id, int lineNum, int numLines, ELogLevel level, char* data, int32_t len){ - char *print = taosMemoryCalloc(len + 1, 1); + char *print = taosMemoryMalloc(len + 1); if (print == NULL) { uError("SML:0x%" PRIx64 " smlParseLine failed. code : %d", id, terrno); return; } (void)memcpy(print, data, len); + print[len] = '\0'; if (level == DEBUG_DEBUG){ uDebug("SML:0x%" PRIx64 " smlParseLine is raw, line %d/%d : %s", id, lineNum, numLines, print); }else if (level == DEBUG_ERROR){ diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 2dcc36eada..9a27a30d84 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -470,8 +470,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); taosMemoryFree(tmp); } - code = TSDB_CODE_INVALID_TIMESTAMP; - goto END; + SML_CHECK_CODE(TSDB_CODE_INVALID_TIMESTAMP); } SSmlKv kvTs = {0}; smlBuildTsKv(&kvTs, ts); From 40cf21b15ce07e56a0c2d43b4567675ddf852b82 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 31 Oct 2024 11:24:31 +0800 Subject: [PATCH 42/43] fix firstVer incorrect while romove wal dir --- source/libs/wal/src/walWrite.c | 2 +- tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index c8c37b11bc..1c53baf360 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -664,7 +664,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy // set status if (pWal->vers.firstVer == -1) { - pWal->vers.firstVer = 0; + pWal->vers.firstVer = index; } pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalCkHead) + cyptedBodyLen; diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py index e0632686d4..40879d5c66 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py @@ -17,8 +17,6 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - - updatecfgDict = {'sDebugFlag':143, 'wDebugFlag':143} def __init__(self): self.vgroups = 1 self.ctbNum = 10 From e260eb7a07d6e6176b9052ed22a27d8f067aa305 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 31 Oct 2024 13:35:42 +0800 Subject: [PATCH 43/43] Revert "enh(stmt2):add api taos_stmt2_get_all_fields" --- include/client/taos.h | 47 ++++++-------- source/client/inc/clientStmt2.h | 1 - source/client/src/clientMain.c | 25 ++------ source/client/src/clientStmt2.c | 86 -------------------------- source/libs/parser/src/parInsertSql.c | 82 ++++++++++-------------- source/libs/parser/src/parInsertStmt.c | 8 +-- tests/script/api/makefile | 2 - tests/script/api/stmt2-get-fields.c | 63 ------------------- 8 files changed, 58 insertions(+), 256 deletions(-) delete mode 100644 tests/script/api/stmt2-get-fields.c diff --git a/include/client/taos.h b/include/client/taos.h index 266e0e192d..80dbe27c47 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -81,13 +81,6 @@ typedef enum { TSDB_SML_TIMESTAMP_NANO_SECONDS, } TSDB_SML_TIMESTAMP_TYPE; -typedef enum TAOS_FIELD_T { - TAOS_FIELD_COL = 1, - TAOS_FIELD_TAG, - TAOS_FIELD_QUERY, - TAOS_FIELD_TBNAME, -} TAOS_FIELD_T; - typedef struct taosField { char name[65]; int8_t type; @@ -102,15 +95,6 @@ typedef struct TAOS_FIELD_E { int32_t bytes; } TAOS_FIELD_E; -typedef struct TAOS_FIELD_ALL { - char name[65]; - int8_t type; - uint8_t precision; - uint8_t scale; - int32_t bytes; - TAOS_FIELD_T field_type; -} TAOS_FIELD_ALL; - #ifdef WINDOWS #define DLL_EXPORT __declspec(dllexport) #else @@ -211,6 +195,13 @@ DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); typedef void TAOS_STMT2; +typedef enum TAOS_FIELD_T { + TAOS_FIELD_COL = 1, + TAOS_FIELD_TAG, + TAOS_FIELD_QUERY, + TAOS_FIELD_TBNAME, +} TAOS_FIELD_T; + typedef struct TAOS_STMT2_OPTION { int64_t reqid; bool singleStbInsert; @@ -241,9 +232,7 @@ DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows); DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt); DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert); DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields); -DLL_EXPORT int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields); DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields); -DLL_EXPORT void taos_stmt2_free_all_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields); DLL_EXPORT TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt); DLL_EXPORT char *taos_stmt2_error(TAOS_STMT2 *stmt); @@ -262,17 +251,17 @@ DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); -DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); -DLL_EXPORT void taos_stop_query(TAOS_RES *res); -DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); -DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows); -DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); -DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); -DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); -DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); -DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex); -DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); -DLL_EXPORT void taos_reset_current_db(TAOS *taos); +DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); +DLL_EXPORT void taos_stop_query(TAOS_RES *res); +DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); +DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows); +DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); +DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); +DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); +DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); +DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex); +DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); +DLL_EXPORT void taos_reset_current_db(TAOS *taos); DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h index e54057e72f..4e9a09c082 100644 --- a/source/client/inc/clientStmt2.h +++ b/source/client/inc/clientStmt2.h @@ -222,7 +222,6 @@ int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags); int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx); int stmtGetTagFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); int stmtGetColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); -int stmtGetColFieldsALL2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_ALL **fields); int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums); int stmtGetParamTbName(TAOS_STMT2 *stmt, int *nums); int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index f2fc2b8bdc..64631fd754 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -84,7 +84,7 @@ void taos_cleanup(void) { taosCloseRef(id); nodesDestroyAllocatorSet(); - // cleanupAppInfo(); +// cleanupAppInfo(); rpcCleanup(); tscDebug("rpc cleanup"); @@ -388,6 +388,7 @@ void taos_free_result(TAOS_RES *res) { tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); } taosMemoryFree(pRsp); + } void taos_kill_query(TAOS *taos) { @@ -483,7 +484,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { return taos_print_row_with_size(str, INT32_MAX, row, fields, num_fields); } -int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { +int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields){ int32_t len = 0; for (int i = 0; i < num_fields; ++i) { if (i > 0 && len < size - 1) { @@ -588,7 +589,7 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD break; } } - if (len < size) { + if (len < size){ str[len] = 0; } @@ -2081,7 +2082,7 @@ int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) { } int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields) { - if (stmt == NULL || count == NULL) { + if (stmt == NULL || NULL == count) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -2102,28 +2103,12 @@ int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, } } -int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields) { - if (stmt == NULL || count == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - return stmtGetColFieldsALL2(stmt, count, fields); -} - void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields) { (void)stmt; if (!fields) return; taosMemoryFree(fields); } -DLL_EXPORT void taos_stmt2_free_all_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields) { - (void)stmt; - if (!fields) return; - taosMemoryFree(fields); -} - TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 67de878ee5..2f046b61d6 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1068,48 +1068,6 @@ static int stmtFetchColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E return TSDB_CODE_SUCCESS; } - -static int stmtFetchFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_ALL** fields) { - SBoundColInfo* tags = (SBoundColInfo*)pStmt->bInfo.boundTags; - STableMeta* meta = ((SVnodeModifyOpStmt*)(pStmt->sql.pQuery->pRoot))->pTableMeta; - if (tags == NULL || meta == NULL || (meta->schema == NULL && tags->numOfBound != 0)) { - return TSDB_CODE_INVALID_PARA; - } - - if (fields != NULL) { - *fields = taosMemoryCalloc(tags->numOfBound, sizeof(TAOS_FIELD_ALL)); - if (NULL == *fields) { - return terrno; - } - - SSchema* pSchema = meta->schema; - int32_t tbnameIdx = meta->tableInfo.numOfTags + meta->tableInfo.numOfColumns; - for (int32_t i = 0; i < tags->numOfBound; ++i) { - int16_t idx = tags->pColIndex[i]; - if (idx == tbnameIdx) { - (*fields)[i].field_type = TAOS_FIELD_TBNAME; - tstrncpy((*fields)[i].name, "tbname", sizeof((*fields)[i].name)); - continue; - } else if (idx < meta->tableInfo.numOfColumns) { - (*fields)[i].field_type = TAOS_FIELD_COL; - } else { - (*fields)[i].field_type = TAOS_FIELD_TAG; - } - - if (TSDB_DATA_TYPE_TIMESTAMP == pSchema[tags->pColIndex[i]].type) { - (*fields)[i].precision = meta->tableInfo.precision; - } - - tstrncpy((*fields)[i].name, pSchema[tags->pColIndex[i]].name, sizeof((*fields)[i].name)); - (*fields)[i].type = pSchema[tags->pColIndex[i]].type; - (*fields)[i].bytes = pSchema[tags->pColIndex[i]].bytes; - } - } - - *fieldNum = tags->numOfBound; - - return TSDB_CODE_SUCCESS; -} /* SArray* stmtGetFreeCol(STscStmt2* pStmt, int32_t* idx) { while (true) { @@ -1893,50 +1851,6 @@ _return: return code; } -int stmtGetColFieldsALL2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_ALL** fields) { - int32_t code = 0; - STscStmt2* pStmt = (STscStmt2*)stmt; - int32_t preCode = pStmt->errCode; - - STMT_DLOG_E("start to get col fields"); - - if (pStmt->errCode != TSDB_CODE_SUCCESS) { - return pStmt->errCode; - } - - if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR); - } - - STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); - - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && - STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { - pStmt->bInfo.needParse = false; - } - - if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { - taos_free_result(pStmt->exec.pRequest); - pStmt->exec.pRequest = NULL; - STMT_ERR_RET(stmtCreateRequest(pStmt)); - } - - STMT_ERRI_JRET(stmtCreateRequest(pStmt)); - - if (pStmt->bInfo.needParse) { - STMT_ERRI_JRET(stmtParseSql(pStmt)); - } - -_return: - - pStmt->errCode = preCode; - if (code == TSDB_CODE_TSC_INVALID_OPERATION) { - return stmtFetchFields2(stmt, nums, fields); - } - - return code; -} - int stmtGetParamNum2(TAOS_STMT2* stmt, int* nums) { STscStmt2* pStmt = (STscStmt2*)stmt; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index f4ef099515..1c26a7c70e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -30,7 +30,7 @@ typedef struct SInsertParseContext { bool forceUpdate; bool needTableTagVal; bool needRequest; // whether or not request server - bool isStmtBind; // whether is stmt bind + bool isStmtBind; // whether is stmt bind } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -757,7 +757,7 @@ int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SS STagVal val = {0}; int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf); if (TSDB_CODE_SUCCESS == code) { - if (NULL == taosArrayPush(pTagVals, &val)) { + if (NULL == taosArrayPush(pTagVals, &val)){ code = terrno; } } @@ -775,14 +775,11 @@ static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* p return terrno; } return insBuildCreateTbReq(pStmt->pCreateTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, - pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, - TSDB_DEFAULT_TABLE_TTL); + pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, + TSDB_DEFAULT_TABLE_TTL); } int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type) { - if (pToken->type == TK_NK_QUESTION) { - return buildInvalidOperationMsg(pMsgBuf, "insert into super table syntax is not supported for stmt"); - } if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && pToken->type != TK_NK_BIN && @@ -813,7 +810,7 @@ typedef struct SRewriteTagCondCxt { static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { SValueNode* pValue = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pValue); + int32_t code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pValue); if (NULL == pValue) { return code; } @@ -1044,7 +1041,7 @@ static int32_t storeChildTableMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt return TSDB_CODE_OUT_OF_MEMORY; } - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(&pStmt->targetTableName, tbFName); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree(pBackup); @@ -1239,7 +1236,7 @@ static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModi } static int32_t collectUseTable(const SName* pName, SHashObj* pTable) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1385,7 +1382,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false); } - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(&pStmt->targetTableName, tbFName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1926,8 +1923,8 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod } if (code == TSDB_CODE_SUCCESS) { code = insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, - pStbRowsCxt->pStbMeta->uid, pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, - getNumOfTags(pStbRowsCxt->pStbMeta), TSDB_DEFAULT_TABLE_TTL); + pStbRowsCxt->pStbMeta->uid, pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, + getNumOfTags(pStbRowsCxt->pStbMeta), TSDB_DEFAULT_TABLE_TTL); pStbRowsCxt->pTag = NULL; } @@ -1936,9 +1933,9 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod code = tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); SVgroupInfo vg; SRequestConnInfo conn = {.pTrans = pCxt->pComCxt->pTransporter, - .requestId = pCxt->pComCxt->requestId, - .requestObjRefId = pCxt->pComCxt->requestRid, - .mgmtEps = pCxt->pComCxt->mgmtEpSet}; + .requestId = pCxt->pComCxt->requestId, + .requestObjRefId = pCxt->pComCxt->requestRid, + .mgmtEps = pCxt->pComCxt->mgmtEpSet}; if (TSDB_CODE_SUCCESS == code) { code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStbRowsCxt->ctbName, &vg); } @@ -2179,8 +2176,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt if (code == TSDB_CODE_SUCCESS) { SStbRowsDataContext* pStbRowsCxt = rowsDataCxt.pStbRowsCxt; void* pData = pTableDataCxt; - code = taosHashPut(pStmt->pTableCxtHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - &pData, POINTER_BYTES); + code = taosHashPut(pStmt->pTableCxtHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), &pData, + POINTER_BYTES); if (TSDB_CODE_SUCCESS != code) { break; } @@ -2252,7 +2249,7 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt if (!pStmt->stbSyntax && numOfRows > 0) { void* pData = rowsDataCxt.pTableDataCxt; code = taosHashPut(pStmt->pTableCxtHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), &pData, - POINTER_BYTES); + POINTER_BYTES); } return code; @@ -2366,7 +2363,8 @@ static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDa if (TSDB_CODE_SUCCESS == code) { // col values and bound cols info of STableDataContext is not used pStbRowsCxt->aColVals = taosArrayInit(getNumOfColumns(pStbRowsCxt->pStbMeta), sizeof(SColVal)); - if (!pStbRowsCxt->aColVals) code = terrno; + if (!pStbRowsCxt->aColVals) + code = terrno; } if (TSDB_CODE_SUCCESS == code) { code = insInitColValues(pStbRowsCxt->pStbMeta, pStbRowsCxt->aColVals); @@ -2424,6 +2422,9 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + if (pStmt->stbSyntax && TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + return buildInvalidOperationMsg(&pCxt->msg, "insert into super table syntax is not supported for stmt"); + } if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); @@ -2510,9 +2511,9 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif } // db.? situation，ensure that the only thing following the '.' mark is '?' - char* tbNameAfterDbName = strnchr(pTbName->z, '.', pTbName->n, true); + char *tbNameAfterDbName = strnchr(pTbName->z, '.', pTbName->n, true); if ((tbNameAfterDbName != NULL) && (*(tbNameAfterDbName + 1) == '?')) { - char* tbName = NULL; + char *tbName = NULL; if (NULL == pCxt->pComCxt->pStmtCb) { return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pTbName->z); } @@ -2527,8 +2528,7 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif if (pCxt->isStmtBind) { if (TK_NK_ID == pTbName->type || (tbNameAfterDbName != NULL && *(tbNameAfterDbName + 1) != '?')) { // In SQL statements, the table name has already been specified. - parserWarn("0x%" PRIx64 " table name is specified in sql, ignore the table name in bind param", - pCxt->pComCxt->requestId); + parserWarn("0x%" PRIx64 " table name is specified in sql, ignore the table name in bind param", pCxt->pComCxt->requestId); } } @@ -2614,7 +2614,7 @@ static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, SNode** pOutput) { SVnodeModifyOpStmt* pStmt = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&pStmt); + int32_t code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&pStmt); if (NULL == pStmt) { return code; } @@ -2729,7 +2729,7 @@ static int32_t buildTagNameFromMeta(STableMeta* pMeta, SArray** pTagName) { return terrno; } SSchema* pSchema = getTableTagSchema(pMeta); - int32_t code = 0; + int32_t code = 0; for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) { if (NULL == taosArrayPush(*pTagName, pSchema[i].name)) { code = terrno; @@ -2834,7 +2834,7 @@ static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) } if (NULL == pStmt->pTableBlockHashObj) { pStmt->pTableBlockHashObj = - taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } if (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -2866,7 +2866,7 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR static int32_t setRefreshMeta(SQuery* pQuery) { SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; - int32_t code = 0; + int32_t code = 0; if (taosHashGetSize(pStmt->pTableNameHashObj) > 0) { taosArrayDestroy(pQuery->pTableList); @@ -3065,10 +3065,9 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal .forceUpdate = (NULL != pCatalogReq ? pCatalogReq->forceUpdate : false), .isStmtBind = pCxt->isStmtBind}; - int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); - SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)((*pQuery)->pRoot); + int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); if (TSDB_CODE_SUCCESS == code) { - code = parseInsertSqlImpl(&context, pStmt); + code = parseInsertSqlImpl(&context, (SVnodeModifyOpStmt*)(*pQuery)->pRoot); } if (TSDB_CODE_SUCCESS == code) { code = setNextStageInfo(&context, *pQuery, pCatalogReq); @@ -3077,27 +3076,8 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { code = setRefreshMeta(*pQuery); } - - if (pStmt->stbSyntax && TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && - code == TSDB_CODE_TSC_INVALID_OPERATION) { - context.tags.numOfBound = pStmt->pStbRowsCxt->boundColsInfo.numOfBound; - context.tags.numOfCols = pStmt->pStbRowsCxt->boundColsInfo.numOfCols; - context.tags.hasBoundCols = pStmt->pStbRowsCxt->boundColsInfo.hasBoundCols; - context.tags.pColIndex = taosMemoryMalloc(sizeof(int16_t) * context.tags.numOfBound); - if (NULL == context.tags.pColIndex) { - return terrno; - } - - (void)memcpy(context.tags.pColIndex, pStmt->pStbRowsCxt->boundColsInfo.pColIndex, - sizeof(int16_t) * pStmt->pStbRowsCxt->boundColsInfo.numOfBound); - code = setStmtInfo(&context, pStmt); - if (TSDB_CODE_SUCCESS == code) { - insDestroyBoundColInfo(&context.tags); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - insDestroyBoundColInfo(&context.tags); + // if no data to insert, set emptyMode to avoid request server if (!context.needRequest) { (*pQuery)->execMode = QUERY_EXEC_MODE_EMPTY_RESULT; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 1355d9e09f..ee61611bf2 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -242,7 +242,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch } code = insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, - pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); pTag = NULL; end: @@ -594,7 +594,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c } code = insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, - pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); pTag = NULL; end: @@ -886,7 +886,7 @@ _return: int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, uint8_t timePrec) { - if (fields != NULL) { + if (fields) { *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { return terrno; @@ -939,7 +939,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); if (pDataBlock->boundColsInfo.numOfBound <= 0) { *fieldNum = 0; - if (fields != NULL) { + if (fields) { *fields = NULL; } diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 9c2bb6be3d..d8a4e19218 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -25,7 +25,6 @@ exe: gcc $(CFLAGS) ./stmt.c -o $(ROOT)stmt $(LFLAGS) gcc $(CFLAGS) ./stmt2.c -o $(ROOT)stmt2 $(LFLAGS) gcc $(CFLAGS) ./stmt2-example.c -o $(ROOT)stmt2-example $(LFLAGS) - gcc $(CFLAGS) ./stmt2-get-fields.c -o $(ROOT)stmt2-get-fields $(LFLAGS) gcc $(CFLAGS) ./stmt2-nohole.c -o $(ROOT)stmt2-nohole $(LFLAGS) gcc $(CFLAGS) ./stmt-crash.c -o $(ROOT)stmt-crash $(LFLAGS) @@ -43,6 +42,5 @@ clean: rm $(ROOT)stmt rm $(ROOT)stmt2 rm $(ROOT)stmt2-example - rm $(ROOT)stmt2-get-fields rm $(ROOT)stmt2-nohole rm $(ROOT)stmt-crash diff --git a/tests/script/api/stmt2-get-fields.c b/tests/script/api/stmt2-get-fields.c deleted file mode 100644 index 23b91b56c9..0000000000 --- a/tests/script/api/stmt2-get-fields.c +++ /dev/null @@ -1,63 +0,0 @@ -// TAOS standard API example. The same syntax as MySQL, but only a subet -// to compile: gcc -o stmt2-get-fields stmt2-get-fields.c -ltaos - -#include -#include -#include -#include "taos.h" - -void do_query(TAOS *taos, const char *sql) { - TAOS_RES *result = taos_query(taos, sql); - int code = taos_errno(result); - if (code) { - printf("failed to query: %s, reason:%s\n", sql, taos_errstr(result)); - taos_free_result(result); - return; - } - taos_free_result(result); -} - -void do_stmt(TAOS *taos) { - do_query(taos, "drop database if exists db"); - do_query(taos, "create database db"); - do_query(taos, - "create table db.stb (ts timestamp, b binary(10)) tags(t1 " - "int, t2 binary(10))"); - - TAOS_STMT2_OPTION option = {0}; - TAOS_STMT2 *stmt = taos_stmt2_init(taos, &option); - const char *sql = "insert into db.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; - - int code = taos_stmt2_prepare(stmt, sql, 0); - if (code != 0) { - printf("failed to execute taos_stmt2_prepare. error:%s\n", taos_stmt2_error(stmt)); - taos_stmt2_close(stmt); - return; - } - - int fieldNum = 0; - TAOS_FIELD_ALL *pFields = NULL; - code = taos_stmt2_get_all_fields(stmt, &fieldNum, &pFields); - if (code != 0) { - printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); - } else { - printf("col nums:%d\n", fieldNum); - for (int i = 0; i < fieldNum; i++) { - printf("field[%d]: %s,type:%d\n", i, pFields[i].name, pFields[i].field_type); - } - } - - taos_stmt2_close(stmt); -} - -int main() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", "", 0); - if (!taos) { - printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); - exit(1); - } - - do_stmt(taos); - taos_close(taos); - taos_cleanup(); -} \ No newline at end of file