diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index 5e8c9aaa92..a1c46875aa 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -75,10 +75,10 @@ database_option: { - TABLE_PREFIX:The prefix length in the table name that is ignored when distributing table to vnode based on table name. - TABLE_SUFFIX:The suffix length in the table name that is ignored when distributing table to vnode based on table name. - TSDB_PAGESIZE: The page size of the data storage engine in a vnode. The unit is KB. The default is 4 KB. The range is 1 to 16384, that is, 1 KB to 16 MB. -- WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription. Enter a time in seconds. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is 4 days. -- WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription. Enter a size in KB. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is -1. -- WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value of single copy is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk. The default values of multiple copy is 1 day. -- WAL_SEGMENT_SIZE: specifies the maximum size of a WAL file. After the current WAL file reaches this size, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk. +- WAL_RETENTION_PERIOD: specifies the maximum time of which WAL files are to be kept after consumption. This parameter is used for data subscription. Enter a time in seconds. The default value 0. A value of 0 indicates that WAL files are not required to keep after consumption. -1: the time of WAL files to keep has no upper limit. +- WAL_RETENTION_SIZE: specifies the maximum total size of which WAL files are to be kept after consumption. This parameter is used for data subscription. Enter a size in KB. The default value is 0. A value of 0 indicates that WAL files are not required to keep after consumption. -1: the total size of WAL files to keep has no upper limit. +- WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after TSDB data in memory are flushed to disk. +- WAL_SEGMENT_SIZE: specifies the maximum size of a WAL file. After the current WAL file reaches this size, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after TSDB data in memory are flushed to disk. ### Example Statement diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index d9ef3b51aa..9e346fdca8 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -75,10 +75,10 @@ database_option: { - TABLE_PREFIX:内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。 - TABLE_SUFFIX:内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。 - TSDB_PAGESIZE:一个 VNODE 中时序数据存储引擎的页大小,单位为 KB,默认为 4 KB。范围为 1 到 16384,即 1 KB到 16 MB。 -- WAL_RETENTION_PERIOD:wal 文件的额外保留策略,用于数据订阅。wal 的保存时长,单位为 s。单副本默认为 0,即落盘后立即删除。-1 表示不删除。多副本默认为 4 天。 -- WAL_RETENTION_SIZE:wal 文件的额外保留策略,用于数据订阅。wal 的保存的最大上限,单位为 KB。单副本默认为 0,即落盘后立即删除。多副本默认为-1,表示不删除。 -- WAL_ROLL_PERIOD:wal 文件切换时长,单位为 s。当 wal 文件创建并写入后,经过该时间,会自动创建一个新的 wal 文件。单副本默认为 0,即仅在落盘时创建新文件。多副本默认为 1 天。 -- WAL_SEGMENT_SIZE:wal 单个文件大小,单位为 KB。当前写入文件大小超过上限后会自动创建一个新的 wal 文件。默认为 0,即仅在落盘时创建新文件。 +- WAL_RETENTION_PERIOD:数据订阅已消费WAL日志,WAL文件的最大额外保留的时长策略。单位为 s。默认为 0,表示无需额外保留。-1, 表示额外保留,时间无上限。 +- WAL_RETENTION_SIZE:数据订阅已消费WAL日志,WAL文件的最大额外保留的累计大小策略。单位为 KB。默认为 0,表示无需额外保留。-1, 表示额外保留,累计大小无上限。 +- WAL_ROLL_PERIOD:wal 文件切换时长,单位为 s。当WAL文件创建并写入后,经过该时间,会自动创建一个新的WAL文件。默认为 0,即仅在TSDB落盘时创建新文件。 +- WAL_SEGMENT_SIZE:wal 单个文件大小,单位为 KB。当前写入文件大小超过上限后会自动创建一个新的WAL文件。默认为 0,即仅在TSDB落盘时创建新文件。 ### 创建数据库示例 diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 3dcba673bf..d8d41d8be6 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -190,7 +190,7 @@ typedef struct { // SArray *preLineTagKV; SArray *maxTagKVs; - SArray *preLineColKV; + SArray *masColKVs; SSmlLineInfo preLine; STableMeta *currSTableMeta; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index b83e7ee976..c438196e71 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1073,8 +1073,6 @@ void smlDestroyInfo(SSmlHandle *info) { taosArrayDestroy(info->valueJsonArray); taosArrayDestroy(info->preLineTagKV); - taosArrayDestroy(info->maxTagKVs); - taosArrayDestroy(info->preLineColKV); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { @@ -1117,8 +1115,6 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - info->maxTagKVs = taosArrayInit(8, sizeof(SSmlKv)); - info->preLineColKV = taosArrayInit(8, sizeof(SSmlKv)); if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables) { uError("create SSmlHandle failed"); @@ -1141,6 +1137,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + if(terrno == TSDB_CODE_DUP_KEY){return terrno;} } taosArrayPush(colsArray, &kvHash); @@ -1204,6 +1201,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); + if(terrno == TSDB_CODE_DUP_KEY){return terrno;} smlInsertMeta(meta->colHash, meta->cols, elements->colArray); taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index e89227d412..da82d43950 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -683,13 +683,10 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo int cnt = 0; SArray *preLineKV = info->preLineTagKV; - SArray *maxKVs = info->maxTagKVs; - bool isSuperKVInit = true; - SArray *superKV = NULL; if (info->dataFormat) { if (unlikely(!isSameMeasure)) { SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); - SSmlSTableMeta *sMeta = NULL; + SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); if (pTableMeta == NULL) { @@ -700,18 +697,16 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo sMeta = smlBuildSTableMeta(info->dataFormat); sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); + for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ + SSchema *tag = pTableMeta->schema + i; + SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE }; + taosArrayPush(sMeta->tags, &kv); + } tmp = &sMeta; } info->currSTableMeta = (*tmp)->tableMeta; - superKV = (*tmp)->tags; - - if (unlikely(taosArrayGetSize(superKV) == 0)) { - isSuperKVInit = false; - } - taosArrayClear(maxKVs); + info->maxTagKVs = (*tmp)->tags; } - } else { - taosArrayClear(maxKVs); } taosArrayClear(preLineKV); @@ -747,58 +742,21 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo return TSDB_CODE_SUCCESS; } - if (isSameMeasure) { - if (unlikely(cnt >= taosArrayGetSize(maxKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - SSmlSTableMeta **tableMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); - if (unlikely(NULL == tableMeta)) { - uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); - return TSDB_CODE_SML_INTERNAL_ERROR; - } - - SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt); - oldKV->length = kv.length; - info->needModifySchema = true; - } - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - } else { - if (isSuperKVInit) { - if (unlikely(cnt >= taosArrayGetSize(superKV))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - } else { - kv.length = maxKV->length; - } - info->needModifySchema = true; - - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - } else { - taosArrayPush(superKV, &kv); - } - taosArrayPush(maxKVs, &kv); + if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + if (unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + info->needModifySchema = true; } - } else { - taosArrayPush(maxKVs, &kv); } taosArrayPush(preLineKV, &kv); cnt++; @@ -1214,7 +1172,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_INVALID_TIMESTAMP; } else if (elements->timestamp[0] == '{') { char tmp = elements->timestamp[elements->timestampLen]; - elements->cols[elements->timestampLen] = '\0'; + elements->timestamp[elements->timestampLen] = '\0'; cJSON *tsJson = cJSON_Parse(elements->timestamp); ts = smlParseTSFromJSON(info, tsJson); if (unlikely(ts < 0)) { diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index a2f752edb9..66f1316cd5 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -144,13 +144,11 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin int cnt = 0; SArray *preLineKV = info->preLineTagKV; - SArray *maxKVs = info->maxTagKVs; - bool isSuperKVInit = true; - SArray *superKV = NULL; if (info->dataFormat) { if (unlikely(!isSameMeasure)) { SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); + SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); @@ -162,18 +160,16 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin sMeta = smlBuildSTableMeta(info->dataFormat); sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); + for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ + SSchema *tag = pTableMeta->schema + i; + SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE }; + taosArrayPush(sMeta->tags, &kv); + } tmp = &sMeta; } info->currSTableMeta = (*tmp)->tableMeta; - superKV = (*tmp)->tags; - - if (unlikely(taosArrayGetSize(superKV) == 0)) { - isSuperKVInit = false; - } - taosArrayClear(maxKVs); + info->maxTagKVs = (*tmp)->tags; } - } else { - taosArrayClear(maxKVs); } taosArrayClear(preLineKV); @@ -252,58 +248,23 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SUCCESS; } - if (isSameMeasure) { - if (unlikely(cnt >= taosArrayGetSize(maxKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - SSmlSTableMeta **tableMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); - if (unlikely(NULL == tableMeta)) { - uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); - return TSDB_CODE_SML_INTERNAL_ERROR; - } - - SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt); - oldKV->length = kv.length; - info->needModifySchema = true; - } - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - } else { - if (isSuperKVInit) { - if (unlikely(cnt >= taosArrayGetSize(superKV))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - } else { - kv.length = maxKV->length; - } - info->needModifySchema = true; - - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - } else { - taosArrayPush(superKV, &kv); - } - taosArrayPush(maxKVs, &kv); + if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); + + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if (unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + info->needModifySchema = true; } - } else { - taosArrayPush(maxKVs, &kv); } taosArrayPush(preLineKV, &kv); @@ -344,9 +305,6 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure, bool isSameCTable) { int cnt = 0; - SArray *preLineKV = info->preLineColKV; - bool isSuperKVInit = true; - SArray *superKV = NULL; if (info->dataFormat) { if (unlikely(!isSameCTable)) { SSmlTableInfo **oneTable = @@ -361,7 +319,6 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (unlikely(!isSameMeasure)) { SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); - SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); if (pTableMeta == NULL) { @@ -369,17 +326,23 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->reRun = true; return TSDB_CODE_SUCCESS; } - sMeta = smlBuildSTableMeta(info->dataFormat); - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); - tmp = &sMeta; + *tmp = smlBuildSTableMeta(info->dataFormat); + (*tmp)->tableMeta = pTableMeta; + taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES); + + for(int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++){ + SSchema *tag = pTableMeta->schema + i; + SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type }; + if(tag->type == TSDB_DATA_TYPE_NCHAR){ + kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + }else if(tag->type == TSDB_DATA_TYPE_BINARY){ + kv.length = tag->bytes - VARSTR_HEADER_SIZE; + } + taosArrayPush((*tmp)->cols, &kv); + } } info->currSTableMeta = (*tmp)->tableMeta; - superKV = (*tmp)->cols; - if (unlikely(taosArrayGetSize(superKV) == 0)) { - isSuperKVInit = false; - } - taosArrayClear(preLineKV); + info->masColKVs = (*tmp)->cols; } } @@ -478,69 +441,26 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->reRun = true; return TSDB_CODE_SUCCESS; } + if (cnt >= taosArrayGetSize(info->masColKVs)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt); + if (kv.type != maxKV->type) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } - if (isSameMeasure) { - if (cnt >= taosArrayGetSize(preLineKV)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); - if (kv.type != maxKV->type) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - - if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) { - maxKV->length = kv.length; - SSmlSTableMeta **tableMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); - if (unlikely(NULL == tableMeta)) { - uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); - return TSDB_CODE_SML_INTERNAL_ERROR; - } - - SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->cols, cnt); - oldKV->length = kv.length; - info->needModifySchema = true; - } - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - } else { - if (isSuperKVInit) { - if (unlikely(cnt >= taosArrayGetSize(superKV))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); - if (unlikely(kv.type != maxKV->type)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - - if (IS_VAR_DATA_TYPE(kv.type)) { - if (kv.length > maxKV->length) { - maxKV->length = kv.length; - } else { - kv.length = maxKV->length; - } - info->needModifySchema = true; - } - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - } else { - taosArrayPush(superKV, &kv); - } - taosArrayPush(preLineKV, &kv); + if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) { + maxKV->length = kv.length; + info->needModifySchema = true; } } else { if (currElement->colArray == NULL) { diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index ab071305fa..ccf79cfc64 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -79,13 +79,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS int cnt = 0; SArray *preLineKV = info->preLineTagKV; - SArray *maxKVs = info->maxTagKVs; - bool isSuperKVInit = true; - SArray *superKV = NULL; if (info->dataFormat) { if (!isSameMeasure) { SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); - SSmlSTableMeta *sMeta = NULL; + SSmlSTableMeta *sMeta = NULL; if (unlikely(tmp == NULL)) { STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); if (pTableMeta == NULL) { @@ -96,18 +93,16 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS sMeta = smlBuildSTableMeta(info->dataFormat); sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); + for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ + SSchema *tag = pTableMeta->schema + i; + SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE }; + taosArrayPush(sMeta->tags, &kv); + } tmp = &sMeta; } info->currSTableMeta = (*tmp)->tableMeta; - superKV = (*tmp)->tags; - - if (unlikely(taosArrayGetSize(superKV) == 0)) { - isSuperKVInit = false; - } - taosArrayClear(maxKVs); + info->maxTagKVs = (*tmp)->tags; } - } else { - taosArrayClear(maxKVs); } taosArrayClear(preLineKV); @@ -175,59 +170,21 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS info->reRun = true; return TSDB_CODE_SUCCESS; } - - if (isSameMeasure) { - if (unlikely(cnt >= taosArrayGetSize(maxKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - SSmlSTableMeta **tableMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); - if (unlikely(NULL == tableMeta)) { - uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); - return TSDB_CODE_SML_INTERNAL_ERROR; - } - - SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt); - oldKV->length = kv.length; - info->needModifySchema = true; - } - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - } else { - if (isSuperKVInit) { - if (unlikely(cnt >= taosArrayGetSize(superKV))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - } else { - kv.length = maxKV->length; - } - info->needModifySchema = true; - - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - } else { - taosArrayPush(superKV, &kv); - } - taosArrayPush(maxKVs, &kv); + if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + if (unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + info->needModifySchema = true; } - } else { - taosArrayPush(maxKVs, &kv); } taosArrayPush(preLineKV, &kv); cnt++; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d71e4b33c2..b29d87f31b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1249,6 +1249,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false); if (taosMulModeMkDir(tsLogDir, 0777) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to create dir:%s since %s", tsLogDir, terrstr()); cfgCleanup(pCfg); return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 1e96a76170..e71b03d2af 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -292,7 +292,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) code = vnodePreProcessWriteMsg(pVnode, pMsg); if (code != 0) { - vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr()); + vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, tstrerror(code)); if (terrno != 0) code = terrno; vnodeHandleProposeError(pVnode, pMsg, code); rpcFreeCont(pMsg->pCont); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 1016b50542..b6f995b11e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -881,6 +881,8 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { QUEUE_REMOVE(&conn->q); QUEUE_INIT(&conn->q); + conn->broken = true; + if (conn->list != NULL) { SConnList* connList = conn->list; connList->list->numOfConn--; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index c36ab38877..3d2c08149b 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -137,6 +137,8 @@ int smlProcess_json1_Test() { for (int i = 0; i < 1; i++) { taosMemoryFree(sql1[i]); } + ASSERT(code == 0); + const char *sql2[] = { "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":\"lga\"}" @@ -164,6 +166,34 @@ int smlProcess_json1_Test() { taosMemoryFree(sql3[i]); } + ASSERT(code == 0); + + + // TD-22903 + const char *sql4[] = { + "[{\"metric\": \"test_us\", \"timestamp\": {\"value\": 1626006833639, \"type\": \"ms\"}, \"value\": true, \"tags\": {\"t0\": true}}, {\"metric\": \"test_us\", \"timestamp\": {\"value\": 1626006833638, \"type\": \"ms\"}, \"value\": false, \"tags\": {\"t0\": true}}]" + }; + char *sql5[1] = {0}; + for (int i = 0; i < 1; i++) { + sql5[i] = taosMemoryCalloc(1, 1024); + strncpy(sql5[i], sql4[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql5[i]); + } + ASSERT(code == 0); + taos_close(taos); return code; @@ -927,6 +957,71 @@ int sml_ts2164_Test() { return code; } +int sml_td22898_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = + taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); + taos_free_result(pRes); + + const char *sql[] = { + "svlzxdfutx,id=nyavpjyfas,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639" + }; + + pRes = taos_query(taos, "use line_test"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + + const char *sql1[] = { + "svlzxdfutx,id=nyavpjyfas,t0=True,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"tgqkvsws\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"htvnnldm\",c8=L\"ncharColValue\",c9=7u64 1626006833639" + }; + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from svlzxdfutx"); + taos_free_result(pRes); + + taos_close(taos); + + return code; +} + +int sml_td22900_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = + taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); + taos_free_result(pRes); + + const char *sql[] = { + "qddkgilwfu,id=qddkgilwfu_42383_49198,t0=t,t1=127i8 c4=9223372036854775807i64,c6=11.12345f32,c6=22.123456789f64 1626006833639" + }; + + pRes = taos_query(taos, "use line_test"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_ttl_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1106,6 +1201,10 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_ts2164_Test(); ASSERT(!ret); + ret = sml_td22898_Test(); + ASSERT(!ret); + ret = sml_td22900_Test(); + ASSERT(ret); ret = smlProcess_influx_Test(); ASSERT(!ret); ret = smlProcess_telnet_Test();