From 38f0eccaadf8070c7f68512c0d302cc34ce2e99a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 Aug 2022 20:31:28 +0800 Subject: [PATCH 01/55] opti: group by tag --- source/libs/executor/inc/executil.h | 1 + source/libs/executor/src/executil.c | 221 +++++++++++++++++++++++- source/libs/executor/src/executorimpl.c | 46 ++--- 3 files changed, 232 insertions(+), 36 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 52c73f85f5..4da4747108 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -115,6 +115,7 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); +int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo); size_t getTableTagsBufLen(const SNodeList* pGroups); SArray* createSortInfo(SNodeList* pNodeList); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index baeb972e05..7f61eb678f 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -428,8 +428,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray // int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { int64_t* uid = taosArrayGet(uidList, i); - void* tag = taosHashGet(tags, uid, sizeof(int64_t)); - ASSERT(tag); for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); @@ -441,6 +439,8 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray qDebug("tagfilter uid:%ld, tbname:%s", *uid, str+2); #endif }else{ + void* tag = taosHashGet(tags, uid, sizeof(int64_t)); + ASSERT(tag); STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); @@ -501,6 +501,223 @@ end: return output.columnData; } +static void releaseColInfoData(void* pCol) { + if(pCol){ + SColumnInfoData* col = (SColumnInfoData*) pCol; + colDataDestroy(col); + taosMemoryFree(col); + } +} + +int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo){ + int32_t code = TSDB_CODE_SUCCESS; + SArray *pBlockList = NULL; + SSDataBlock *pResBlock = NULL; + SHashObj *tags = NULL; + SArray *uidList = NULL; + void *keyBuf = NULL; + SArray *groupData = NULL; + + int32_t rows = taosArrayGetSize(pTableListInfo->pTableList); + if(rows == 0){ + return TDB_CODE_SUCCESS; + } + + tagFilterAssist ctx = {0}; + ctx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + if(ctx.colHash == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + ctx.index = 0; + ctx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo)); + if(ctx.cInfoList == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + SNode* pNode = NULL; + FOREACH(pNode, group) { + nodesRewriteExprPostOrder(&pNode, getColumn, (void *)&ctx); + REPLACE_NODE(pNode); + } + + pResBlock = createDataBlock(); + if (pResBlock == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) { + SColumnInfoData colInfo = {{0}, 0}; + colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i); + blockDataAppendColInfo(pResBlock, &colInfo); + } + + uidList = taosArrayInit(rows, sizeof(uint64_t)); + for (int32_t i = 0; i < rows; ++i) { + STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); + taosArrayPush(uidList, &pkeyInfo->uid); + } + +// int64_t stt = taosGetTimestampUs(); + tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + code = metaGetTableTags(metaHandle, pTableListInfo->suid, uidList, tags); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + +// int64_t stt1 = taosGetTimestampUs(); +// qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); + + code = blockDataEnsureCapacity(pResBlock, rows); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + +// int64_t st = taosGetTimestampUs(); + for (int32_t i = 0; i < rows; i++) { + int64_t* uid = taosArrayGet(uidList, i); + for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ + SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); + + if(pColInfo->info.colId == -1){ // tbname + char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + metaGetTableNameByUid(metaHandle, *uid, str); + colDataAppend(pColInfo, i, str, false); +#if TAG_FILTER_DEBUG + qDebug("tagfilter uid:%ld, tbname:%s", *uid, str+2); +#endif + }else{ + void* tag = taosHashGet(tags, uid, sizeof(int64_t)); + ASSERT(tag); + STagVal tagVal = {0}; + tagVal.cid = pColInfo->info.colId; + const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); + + if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)){ + colDataAppend(pColInfo, i, p, true); + } else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) { + colDataAppend(pColInfo, i, p, false); + } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { + char *tmp = taosMemoryCalloc(tagVal.nData + VARSTR_HEADER_SIZE + 1, 1); + varDataSetLen(tmp, tagVal.nData); + memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData); + colDataAppend(pColInfo, i, tmp, false); +#if TAG_FILTER_DEBUG + qDebug("tagfilter varch:%s", tmp+2); +#endif + taosMemoryFree(tmp); + } else { + colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false); +#if TAG_FILTER_DEBUG + if(pColInfo->info.type == TSDB_DATA_TYPE_INT){ + qDebug("tagfilter int:%d", *(int*)(&tagVal.i64)); + }else if(pColInfo->info.type == TSDB_DATA_TYPE_DOUBLE){ + qDebug("tagfilter double:%f", *(double *)(&tagVal.i64)); + } +#endif + } + } + } + } + pResBlock->info.rows = rows; + +// int64_t st1 = taosGetTimestampUs(); +// qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st); + + pBlockList = taosArrayInit(2, POINTER_BYTES); + taosArrayPush(pBlockList, &pResBlock); + + groupData = taosArrayInit(2, POINTER_BYTES); + FOREACH(pNode, group) { + SScalarParam output = {0}; + + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_OPERATOR: + case QUERY_NODE_FUNCTION: + case QUERY_NODE_LOGIC_CONDITION:{ + SExprNode* expNode = (SExprNode*)pNode; + code = createResultData(&expNode->resType, rows, &output); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + break; + } + default: + ASSERT(0); + } + code = scalarCalculate(pNode, pBlockList, &output); + if(code != TSDB_CODE_SUCCESS){ + releaseColInfoData(output.columnData); + goto end; + } + taosArrayPush(groupData, &output.columnData); + } + + int32_t keyLen = 0; + SNode* node; + FOREACH(node, group) { + SExprNode* pExpr = (SExprNode*)node; + keyLen += pExpr->resType.bytes; + } + + int32_t nullFlagSize = sizeof(int8_t) * LIST_LENGTH(group); + keyLen += nullFlagSize; + + keyBuf = taosMemoryCalloc(1, keyLen); + if (keyBuf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + for(int i = 0; i < rows; i++){ + STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); + + char* isNull = (char*)keyBuf; + char* pStart = (char*)keyBuf + sizeof(int8_t) * LIST_LENGTH(group); + for(int j = 0; j < taosArrayGetSize(groupData); j++){ + SColumnInfoData* pValue = (SColumnInfoData*)taosArrayGetP(groupData, j); + + ASSERT(pValue->info.type != TSDB_DATA_TYPE_JSON); + + if (pValue->info.type == TSDB_DATA_TYPE_NULL || colDataIsNull_s(pValue, i)) { + isNull[j] = 1; + continue; + } else { + isNull[j] = 0; + char* data = colDataGetData(pValue, i); + if (IS_VAR_DATA_TYPE(pValue->info.type)) { + memcpy(pStart, data, varDataTLen(data)); + pStart += varDataTLen(data); + } else { + memcpy(pStart, data, pValue->info.bytes); + pStart += pValue->info.type; + } + } + } + + int32_t len = (int32_t)(pStart - (char*)keyBuf); + info->groupId = calcGroupId(keyBuf, len); + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t)); + } + +// int64_t st2 = taosGetTimestampUs(); +// qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); + + end: + taosMemoryFreeClear(keyBuf); + taosHashCleanup(tags); + taosHashCleanup(ctx.colHash); + taosArrayDestroy(ctx.cInfoList); + blockDataDestroy(pResBlock); + taosArrayDestroy(pBlockList); + taosArrayDestroy(uidList); + taosArrayDestroyP(groupData, releaseColInfoData); + return code; +} + int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9ff5b5d759..9a512b3c2b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3874,9 +3874,9 @@ static void cleanupTableSchemaInfo(SSchemaInfo* pSchemaInfo) { tDeleteSSchemaWrapper(pSchemaInfo->qsw); } -static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum) { +static int32_t sortTableGroup(STableListInfo* pTableListInfo) { taosArrayClear(pTableListInfo->pGroupList); - SArray* sortSupport = taosArrayInit(groupNum, sizeof(uint64_t)); + SArray* sortSupport = taosArrayInit(16, sizeof(uint64_t)); if (sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY; for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); @@ -3954,48 +3954,26 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, if (pTableListInfo->map == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - int32_t keyLen = 0; - void* keyBuf = NULL; - - SNode* node; - FOREACH(node, group) { - SExprNode* pExpr = (SExprNode*)node; - keyLen += pExpr->resType.bytes; - } - - int32_t nullFlagSize = sizeof(int8_t) * LIST_LENGTH(group); - keyLen += nullFlagSize; - - keyBuf = taosMemoryCalloc(1, keyLen); - if (keyBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } bool assignUid = groupbyTbname(group); - int32_t groupNum = 0; size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); - for (int32_t i = 0; i < numOfTables; i++) { - STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); - - if (assignUid) { + if(assignUid){ + for (int32_t i = 0; i < numOfTables; i++) { + STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); info->groupId = info->uid; - } else { - int32_t code = getGroupIdFromTagsVal(pHandle->meta, info->uid, group, keyBuf, &info->groupId); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t)); + } + }else{ + int32_t code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; } - - taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t)); - groupNum++; } - taosMemoryFree(keyBuf); - if (pTableListInfo->needSortTableByGroupId) { - return sortTableGroup(pTableListInfo, groupNum); + return sortTableGroup(pTableListInfo); } return TDB_CODE_SUCCESS; From 2743ac61933f3562bb5687e7c896ff14304d970d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 20 Aug 2022 15:45:56 +0800 Subject: [PATCH 02/55] opti: group by tag --- source/libs/executor/src/executil.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 7f61eb678f..8ec1e9b584 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -680,15 +680,24 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis for(int j = 0; j < taosArrayGetSize(groupData); j++){ SColumnInfoData* pValue = (SColumnInfoData*)taosArrayGetP(groupData, j); - ASSERT(pValue->info.type != TSDB_DATA_TYPE_JSON); - - if (pValue->info.type == TSDB_DATA_TYPE_NULL || colDataIsNull_s(pValue, i)) { + if (colDataIsNull_s(pValue, i)) { isNull[j] = 1; - continue; } else { isNull[j] = 0; char* data = colDataGetData(pValue, i); - if (IS_VAR_DATA_TYPE(pValue->info.type)) { + if (pValue->info.type == TSDB_DATA_TYPE_JSON) { + if (tTagIsJson(data)) { + code = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; + goto end; + } + if(tTagIsJsonNull(data)){ + isNull[j] = 1; + continue; + } + int32_t len = getJsonValueLen(data); + memcpy(pStart, data, len); + pStart += len; + } else if (IS_VAR_DATA_TYPE(pValue->info.type)) { memcpy(pStart, data, varDataTLen(data)); pStart += varDataTLen(data); } else { From 73e98d4f3d0839dbff3b1d87c4404c73d302a6e6 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 22 Aug 2022 09:00:16 +0800 Subject: [PATCH 03/55] docs(stream) --- docs/zh/12-taos-sql/14-stream.md | 35 ++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 1a056e278c..94f34ea1ff 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -3,9 +3,6 @@ sidebar_label: 流式计算 title: 流式计算 --- -在时序数据的处理中，经常要对原始数据进行清洗、预处理，再使用时序数据库进行长久的储存。用户通常需要在时序数据库之外再搭建 Kafka、Flink、Spark 等流计算处理引擎，增加了用户的开发成本和维护成本。 - -使用 TDengine 3.0 的流式计算引擎能够最大限度的减少对这些额外中间件的依赖，真正将数据的写入、预处理、长期存储、复杂分析、实时计算、实时报警触发等功能融为一体，并且，所有这些任务只需要使用 SQL 完成，极大降低了用户的学习成本、使用成本。 ## 创建流式计算 @@ -40,17 +37,27 @@ window_clause: { 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 +窗口的定义与时序数据特色查询中的定义完全相同。 + 例如，如下语句创建流式计算，同时自动创建名为 avg_vol 的超级表，此流计算以一分钟为时间窗口、30 秒为前向增量统计这些电表的平均电压，并将来自 meters 表的数据的计算结果写入 avg_vol 表，不同 partition 的数据会分别创建子表并写入不同子表。 ```sql CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + +## 流式计算的 partition + +可以使用 PARTITION BY TBNAME 或 PARTITION BY tag，对一个流进行多分区的计算，每个分区的时间线与时间窗口是独立的，会各自聚合，并写入到目的表中的不同子表。 + +不带 PARTITION BY 选项时，所有的数据将写入到一张子表。 + +流式计算创建的超级表有唯一的 tag 列 groupId，每个 partition 会被分配唯一 groupId。与 schemaless 写入一致，我们通过 MD5 计算子表名，并自动创建它。 ``` ## 删除流式计算 ```sql -DROP STREAM [IF NOT EXISTS] stream_name +DROP STREAM [IF NOT EXISTS] stream_name; ``` 仅删除流式计算任务，由流式计算写入的数据不会被删除。 @@ -61,6 +68,12 @@ DROP STREAM [IF NOT EXISTS] stream_name SHOW STREAMS; ``` +若要展示更详细的信息，可以使用： + +```sql +SELECT * from performance_schema.`perf_streams`; +``` + ## 流式计算的触发模式 在创建流时，可以通过 TRIGGER 指令指定流式计算的触发模式。 @@ -87,13 +100,17 @@ MAX_DELAY 模式在窗口关闭时会立即触发计算。此外，当数据写 T = 最新事件时间 - watermark -每批到来的数据都会以上述公式更新窗口关闭时间，并将窗口结束时间 < T 的所有打开的窗口关闭，若触发模式为 WINDOW_CLOSE 或 MAX_DELAY，则推送窗口聚合结果。 +每次写入的数据都会以上述公式更新窗口关闭时间，并将窗口结束时间 < T 的所有打开的窗口关闭，若触发模式为 WINDOW_CLOSE 或 MAX_DELAY，则推送窗口聚合结果。 -流式计算的过期数据处理策略 -对于已关闭的窗口，再次落入该窗口中的数据被标记为过期数据，对于过期数据，流式计算提供两种处理方式： +## 流式计算的过期数据处理策略 -1. 直接丢弃：这是常见流式计算引擎提供的默认（甚至是唯一）计算模式 +对于已关闭的窗口，再次落入该窗口中的数据被标记为过期数据. + +TDengine 对于过期数据提供两种处理方式，由 IGNORE EXPIRED 选项指定： + +1. 重新计算，即 IGNORE EXPIRED 0：默认配置，从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果 + +2. 直接丢弃, 即 IGNORE EXPIRED 1：忽略过期数据 -2. 重新计算：从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果 无论在哪种模式下，watermark 都应该被妥善设置，来得到正确结果（直接丢弃模式）或避免频繁触发重算带来的性能开销（重新计算模式）。 From 0b6122084617006d829397a10300ab9aebd5898e Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 09:13:38 +0800 Subject: [PATCH 04/55] docs: add some files from zh to en --- docs/en/07-develop/01-connect/_connect_php.mdx | 3 +++ docs/en/07-develop/03-insert-data/_php_sql.mdx | 3 +++ docs/en/07-develop/03-insert-data/_php_stmt.mdx | 3 +++ docs/en/07-develop/04-query-data/_php.mdx | 3 +++ docs/examples/rust/Cargo.toml | 2 -- 5 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 docs/en/07-develop/01-connect/_connect_php.mdx create mode 100644 docs/en/07-develop/03-insert-data/_php_sql.mdx create mode 100644 docs/en/07-develop/03-insert-data/_php_stmt.mdx create mode 100644 docs/en/07-develop/04-query-data/_php.mdx delete mode 100644 docs/examples/rust/Cargo.toml diff --git a/docs/en/07-develop/01-connect/_connect_php.mdx b/docs/en/07-develop/01-connect/_connect_php.mdx new file mode 100644 index 0000000000..dbad72bc19 --- /dev/null +++ b/docs/en/07-develop/01-connect/_connect_php.mdx @@ -0,0 +1,3 @@ +```php title="原生连接" +{{#include docs/examples/php/connect.php}} +``` diff --git a/docs/en/07-develop/03-insert-data/_php_sql.mdx b/docs/en/07-develop/03-insert-data/_php_sql.mdx new file mode 100644 index 0000000000..78cd663ec2 --- /dev/null +++ b/docs/en/07-develop/03-insert-data/_php_sql.mdx @@ -0,0 +1,3 @@ +```php +{{#include docs/examples/php/insert.php}} +``` diff --git a/docs/en/07-develop/03-insert-data/_php_stmt.mdx b/docs/en/07-develop/03-insert-data/_php_stmt.mdx new file mode 100644 index 0000000000..3bb7b2f8da --- /dev/null +++ b/docs/en/07-develop/03-insert-data/_php_stmt.mdx @@ -0,0 +1,3 @@ +```php +{{#include docs/examples/php/insert_stmt.php}} +``` diff --git a/docs/en/07-develop/04-query-data/_php.mdx b/docs/en/07-develop/04-query-data/_php.mdx new file mode 100644 index 0000000000..bcafd1cfbc --- /dev/null +++ b/docs/en/07-develop/04-query-data/_php.mdx @@ -0,0 +1,3 @@ +```go +{{#include docs/examples/php/query.php}} +``` diff --git a/docs/examples/rust/Cargo.toml b/docs/examples/rust/Cargo.toml deleted file mode 100644 index 136d09ffbb..0000000000 --- a/docs/examples/rust/Cargo.toml +++ /dev/null @@ -1,2 +0,0 @@ -[workspace] -members = ["restexample", "nativeexample"] From 8078642eaa098518fbe65fa4b36823f054fe90f8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 22 Aug 2022 09:14:55 +0800 Subject: [PATCH 05/55] docs(stream) --- docs/zh/12-taos-sql/14-stream.md | 21 +++++++++++++++++++-- docs/zh/12-taos-sql/watermark.webp | Bin 0 -> 17808 bytes 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 docs/zh/12-taos-sql/watermark.webp diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 94f34ea1ff..24e695bcdc 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -92,9 +92,9 @@ SELECT * from performance_schema.`perf_streams`; MAX_DELAY 模式在窗口关闭时会立即触发计算。此外，当数据写入后，计算触发的时间超过 max delay 指定的时间，则立即触发计算 -## 流式计算的乱序数据容忍策略 +## 流式计算的窗口关闭 -在创建流时，可以在 stream_option 中指定 watermark。 +在创建流时，可以在 stream_option 中指定 watermark，它定义了数据乱序的容忍上界。 流式计算通过 watermark 来度量对乱序数据的容忍程度，watermark 默认为 0。 @@ -102,6 +102,23 @@ T = 最新事件时间 - watermark 每次写入的数据都会以上述公式更新窗口关闭时间，并将窗口结束时间 < T 的所有打开的窗口关闭，若触发模式为 WINDOW_CLOSE 或 MAX_DELAY，则推送窗口聚合结果。 + +![TDengine 流式计算窗口关闭示意图](./watermark.webp) + + +图中，纵轴表示不同时刻，对于不同时刻，我们画出其对应的 TDengine 收到的数据，即为横轴。 + +横轴上的数据点表示已经收到的数据，其中蓝色的点表示事件时间(即数据中的时间戳主键)最后的数据，该数据点减去定义的 watermark 时间，得到乱序容忍的上界 T。 + +所有结束时间小于 T 的窗口都将被关闭（图中以灰色方框标记）。 + +T2 时刻，乱序数据（黄色的点）到达 TDengine，由于有 watermark 的存在，这些数据进入的窗口并未被关闭，因此可以被正确处理。 + +T3 时刻，最新事件到达，T 向后推移超过了第二个窗口关闭的时间，该窗口被关闭，乱序数据被正确处理。 + +在 window_close 或 max_delay 模式下，窗口关闭直接影响推送结果。在 at_once 模式下，窗口关闭只与内存占用有关。 + + ## 流式计算的过期数据处理策略 对于已关闭的窗口，再次落入该窗口中的数据被标记为过期数据. diff --git a/docs/zh/12-taos-sql/watermark.webp b/docs/zh/12-taos-sql/watermark.webp new file mode 100644 index 0000000000000000000000000000000000000000..3307faccffdaaec6dddf5cad8b7c11016fd28bd4 GIT binary patch literal 17808 zcmV*cKvTa`Nk&G9MF0R-MM6+kP&iC{MF0RV*+OXmRSSc*ksQVRVPE|MAR;E9xplv2 z+_DTRn?v-#SZ!6-oe!qHa(L3q&TT&g(Mv7gQVl(AT0yo~rn9-KNtND;-uy3K_DvI& z4VNq<1sOMNeH$<9x=ZeHAx)K&s}vdgGG6=tj%}MWw(XhceqP%T#xt+2`?YO*V%xTj zEl8ieNuO@+`#KN)kN}-&4x|6nXxqx#TJy9u*e1ok_U1!^=E(bl`~JW4mgM|bWgV64psZ6m zWmlb3ZI;@Fc9*-mYHUw?Y}>YNJ-_GoJm3ACUB5c}+>UuS72B$Gxa%tRn0+i#J26E*BYC#ZF6kySUtJyv1>dN+fG+( z=fv#p@Vc*J+qRwjV%zF9727s?$F^-Jz4~q#+qgl7%~-_m)H>M?%w7O*kRJlw?g84vKZA&6J*`H2 zv8)+n)MCwI%65zj-4SpVYH_Hn&UzdcgK8ZsSF#`bK^ZDm+KG8R{#%d#*5kh{LT$=x z5#>56x2OSxN=gQ{8i|deqsnA@=-5Oj5~K>bPEOvHR1X}7heToljb@d}*+O2EHEFZR zo;e~#`i%{uE2(Z9Mk(Dg(iWA3KWO{vpei=#Gq#!7JY>JT`p{J5iB1^H$fk`V`w(E? z9CX@fI7rxT5hjv#L^@enFC3+k+9ndQ{!?tbh`t0^vi&M7p|nY4A03`pUCOQnWz}Ys zn(vcCH^g$u^*aD7C|I4CUO9?SAt;ne+4e6~F(DBB3?QedpN@p7D@M9%1hG9TqyvaU zUSm7RNzwN`fZjT+Ra6+r^XQGE)`>!SwOOP7hpJxzkVL@rCNFzch#gT@hjJ`&#i_|5 zV%Lpq5YZjOh*$>IWAzgoxNaIo#5$Efq!ICSyI_=2Z6-O}4X>mw$Ce^GC6X%i7 zj8mD|rHr--%y!BgEx0S#rqjRPa;f7Ue;m4=n|O`|=|7dQOhz+JEd%%Z@_h{b|I+03y9i3NMJq8{ ztr3yeUqY8y+|(GOsI|`Dc9p1&MoL@fWdS)DBnvF=n(){W4<&E7oX2W_}sxfR0fhS-rsg>*HsluowWPkur$`|e_E2@=pRQiKP6OFW7w#rJCR<5)n z)HK~kjKLWT!>ao0c zSyUtLRk3oZ#!xYLg|VqIy2C`?B?3d0xQ<;kN_0|1-S;cJq?~!jF3$EUE5E+S(RNyd zhxS?PP;x#k=MF&6NUZ@xBgbYAdG++HWOEn{6W8(-U{>`PW{}|zG%jxy=*m3PD4T=8 z#@=Oj$hx}W&}+nad7kq6{%|(J3oUFQn+_Ne@%hI3?1=b9``cTG-Bp}rjrPJ|V|0g! zyh{XzDslPWvBAc6vFlGwCoa#F^Qv@vq@Rj&#;9{NB2(Rpo*B=PFdv9iX;jZnO$Li{ z)Y}55s?0`O1)4*(hFPU;@<>1?w@D%PRD0@K$sMXFRBNC_1A1l-Q0h8}S%LPoWosQ- zOS0kTsfnG}C^Wtb%Mxylc(IdoeYL;GleNy|R_S;?_SN;AY}n!$97XAQX@H|Fk=&|Y z9(b+(lU-UfVbKff`wW4pmH_nZ)aupl!#Ro4b21lU(%M%?!9DHKMgz>sc%tB4sCko3m*H&=iebWB85M)bTq8(2t; zt?+ao89M1xi!CP4&vTpjUnP{^r%3c_NiPVf}U<0u=4KqC(z|}#dGsd?HexVW3C{rcC*n#T3 zFuqm-LDB4(B;uzV0z90iRf%!X50GGc1oI{F$Ie}g^}8(gvo>7F%HQ*)r|Jv2kP8`@ z=KkiV5iYi?BQ5z!I>fD|dWeIH56d$|*$oZOCmq*G&R3LsZv zejnr3{w`tTFXYLYngPg`o}M2+er>bK?sYn>34gdX$(fo1$eEg*A1oDD z{2L-|`Ts!W#jpBj+)$}%BIJ&n6EcoF{o2lHb9jDQmQ#HQqa)qm2WtCmb%xP52T#so zUIA`lzz`uYA)T^C0yK}T>m*#3sw+z!u}}MElNWb#pF&?eSIln7&V? zhbcwzC1s5=#sXXqwJ7!)kyq#14N&LjLiqB8Y7Ts3Mznyl0vIp?Q@K&TOhynS2-QSL zHi(FRo>!dasR+pj z2vLx)Qe()sMj?x=A-a36yGN+46Vt{Z zV{>x3_HDuMcrXOUBAhPtkeHB_q@p>>6=CEsUL{A_DuKhWvN+c(kYKiq0?Dg*#cJ`W#%UuXtfuh>r7L_-CI!)%$$e(WY zS?ofs^C#}tF8mYEn=$yC|804%o=rYFcJC09$mDAy$r!PpVO*`oHw^5z_e4}FgLO|*ntW5F*j)KQ)dVz0)Y!RcSu z!DXnS}L#jR=b|9EF!pudFho&v=M2a5JA~72w0Dfi!H4WYqq_}y< z!H9^6D6k+zL}x8^$db=G3X>+DKx9LTBi1IcshBjPL8%WNYCz3 zm(BY_%cB}GyXdLPUa#>@+A8w*`WZkso%}ixIF>ex?1iJo-@W6;42*&hSw$^>PYh8y zsdTt*tU9Y>T|5YscB{DJ{9QBhr;O_$|6HDwCx_3PwOB+VAG&1pYtxghfLs@i6pMoa zFC&X+Q)Oh|=534^5SoSe&S7S;_fU^AQyO*y@=D9L70gpM)`G+U-y}H6s$!PR^$b)xqcqe!kZM!#R~D6=tnMd z9_nQv;>P`+hyU0Vn=>LJqaaRE!6A>e^&*MeG{wTmDg_VEG;~CafXQ^ah|iRU3p3~# zd^W5JOY#DH^`|j#;n`Q*KH&@x+EpX>M)L5caFFnZir*g_uc*8FJrBgs>9HebDMS(q zh=?aPbYW)DvaF+EEn4EN4YyO=xZVeN$c>w3z;MVWI`hErn0~<2BgeNF2pR@nQQK`w zgaJcAE>bSVH*sY?3o zPI+qM7Cu{1ggFIqZfTT+Wf}V6kyOJ5AGFsFm*K@LVf&E1cSr<*^0GJdJC(8>A%0m8 z43G7XD`)%Bf7}UU`0O`Sv#8YUw)>Aww{0!Lzv@9lBXCFRNC3nPLE1QC@wDbK753SU?iL;y?z z$E!RaE}ka~5fGAM%Q_0-R1p3#xz06y5WaBAi2#sX=T35W@P$`S_`~cvcR}9-Uj$`@ zf5@(LJw!W(FM@KyFBZk_)z3N#5tR}CAiK^rYU9=nexQgPJ&TL2XC2RYh-gf#Za;~O z4QCyNNX7_1vAE7P{?rVb?ynHkGf=1*s_~7FCvX0qN zj1nHMbBl8`_=MJVuJK>+MKwaW3ml0xj$h+qdugv=26UZkTn~KqM#O4gO+!=yE_Rjj zypSP8zVZ_;hReCEqY%xos>TVGD_vuk@I`a1s&N7k=$1y0SY)=a>PE6S+&_t;6$d~% z_@WzD)z|@`j`*tB-|QUu*@^C$&}AF-T0>B<%;9AhG7R(vaBv{dMNVOnxcC$I;2IIb zk*Ycbh>G3!nezOJ5n?%Xr8}s?7t@id#tKlm7pFXpMu_Q9xzCpJw7@r7LRVamUc*l~ zM`7tz^(}P7Moq^Ax_ZFp;E+)6@hU0LuHf5lRgLaQM;v3gA@?Z6c0jn&HMR%egjMzI zP_eh=NqGpqI9{phC!k^{>nL2bs@gktrE5GNe5$J28%D)`o|CMjpsH$ACpvV)u6^dY z6PJOC{d|HM$7~}J#i=XZ;thl6K}0cviv7IZDbLo8uoI}*ueFeI6gExbO4ryEc)p<_ zA}dA3eoeEK=iH1C$tWuJ>riGKg^wU2DRrf5bOGpC7H>>MjzU|V>ZT;jFG7rA2j^$3f-%Qw*?C0?VVLii_?f|N7p|9G( z7a^jiDr^4IQ!4fr4=JvtmzyOOx^g9{{k|b(!`G2Y{FD=)zIwAF8zhCn9FPX_^Q{j z*cllXu5t9)ZJ2y4QANw=B!QGy+WppJu6?N|&&%S8iRX>`q{tvWz~Ht?-f?lGS~usixd%frS6rdZuoFU()*TR?_pfrgKd^ z-WO!FP92T>;=hK>121nXe0B%78ltwP-gI5>Z0GB>6?J@YD(*qUeES!M+VgJAvF1la z6)m47{IPBG++DMFYW~U~s~dWOAyMs*7&WEDb)RD))9LUyIBW)Zg}w!zU24btI+UgL zGY?FZ4dYb(n%PL*`94F$7h4fAVf9Z8PmfsC@RWdR2*aWmiinl>zRA#(8AhV2)=v?4 zQP=YAg%L5=I53cs7V>4IG&se4NM}loMLv9~!=Pyk6Ia+56-gYTvsN2)|ed z2@53qj%}UWJbEO2LRt0fV^5*T%VAU_xpE|qa?yAML{)8{Bos*was%u@AMJMxP)N z>)nRmB?2dw3`E?@UojUideMl8rP}*$bpNQufNB^XB+vB2XhDI<)bU}W&GUCH8tF(} z=^91vzy0rq>@o)=A(JO&Ev-i{v*q)KFxfqXQ{z`8(aQg3aH#wpLpiOVGDLp)U+QPn zl#O4f0IFfAvhQ_<&?WmU(G!_FK1fv9{$ay>S(NitL{cU^lViS~^D3+PbK9P2;1yqY zv{x&g^?F)A{Xo>*zdkxu)0f)&ZuIMjnO)5NAKlZhHq9TpU8-%VcO;2xe{h!9Q|r{D z9zup&4MXt6%!Rk_?TAy&kBLkjA0qs-Y18aIO;_3b`sDTB`m{wHq{fCRcx1Nxls4hW z?qZiZurP4pi>{#kBlLKcm22qoZz=28)5DbP%{}!<0%KMDp)K=U>Rj$#J~LrRXSFZr zL2hmND(?8e@1hL18rH4GCUZE;=reMSJda4<{Kcn~irV0b-r9O@+#&09jT(MHn3m`K ziP`IjudrIvKH(%nu~zICqHfGM+!Ty*HZlRm$>dO)?kaapzNfH2U6~{8GgJ4Qi^#R7=n={f|FJ5_2`BlZvm&q+|+FJ z_fOgK4_OB0)@QFcv%P5VyU|;EmgwbwMa~v&!PpY_rrZs~kGFtRV{U48v4*gj-GsT& z>-kFd)@OCE-1>$!jyxx2!`D+aetG$}JzRDdXN$JrWQoD^;n`jVrQ7%1L4`m5U1lU;HKD{nr-k~c+r!)XENbpw zh=`Xzd5*tPRrSY}yMfd37I0JSP0iw0{oJr}7}%G#z3i6H@U}h+k(iRq`n&+G-%+xGwGZhZ!_{;H}zfy;7#3%Duvre;`W|LZ=)7v$Vi zkIXUtBz7rmcgb^8%H6=}cni2G_NHby6T?UiM@N^!$JwGSq!=JX`#Hhrcni2G_NHdI z?+@}I-?!S#QkXbfwC$Fvs^_od4UCSrfSax%Vs2_iUG8(SOJU+{(H2TH9HjmPw|Hl8 z`xDGK+8EBBu)BJn^1_m4xjpQMw(#D+5tHlm0HiQ7L8+nE`f zQ4nM*}A9v_Dcu3#&Qu)rW}|2UBTy(Gs@N%oHe z(Vj98ODt~xI21P}xc5r3e;kO;E%h-XJv+koj{{mmHf8>gIm7;O@bkQUboBml(8(HZ z3eQdyVTMuI{&B#6GgWk;MDHJm01u}`{Csnw%?Yxv1v+|pR_wij_7<{M?jHxDE)Hl2 z$yirntZKs-pH5j~)`1~I1c8)N)WzR|{uUPE$P&79C+6@kcc0bKtJml#y|H*~c0?;J zPdM~Zs9qv7D14R<<5m5l+}2N4*zuwJic#_B)rL(o?))P{_y1lnJv$IK>BARvF1J+A z$}R1AHR@S|o)=xEUkmqeeD6 zEQP;j&1$&7!nMfq27&I+l(zMG5CbbU3_rIh@^Y{#{tzfKF%CMs8DI~5b)Gh|UP}7= z+8S2a&@Fj{EO```t805d?BLou&g|N6@*}eNhBlKCJv%qSyU{G_FE~}DHSP}d;a|)= zb`--vuS`VpMf9w^wKW8p3omR^0IQ1T9gfU@#;e3`*!fSCy@2euFq$jh_<@YhU505TeZtlRD` zP`QzzQ=w|d0O^bo?OzxI7#`iu9RR?LSO`eDJtC8MbE3?RQ3F7mJ729;EU(YZw={mK z`stV3We?>A2F7J2U)i#}P5-kKLDewC1dfP>+bV505LZ^vM2~a8 zDTyFTRR43Q)^CwiR`oZCG_}8Pf=>3#F@T)usU}#-omc?KmO0!6Kl?ZTp@@T=`HEk+ z*_YkvbQlxAb8E6@068+pn`k0c+c)a|=~jTYzS=@nSzhDMAP!;L8_r@Re6g2E+3-^* zop#z=MR7X3v5fOm>R!1GpiO-*bUHbklT^oNZ;7}#9bFILv@!0~^6e(HEpsluUH^4+ zt-Z2u>n_N~9ILNCE!3!TfiRctG$m z4wVtPLqBm+{DV?+GLIp$Mn9WF$gZDIVF3a4Yd_D9i5|2vGw?pp=>A>2lp4t(IYFe) zH(~QlbN$o}k-qTvbjqpljFa3#t{ZJz*S#lsAL^oviKxVhvaVi5#aSPKJ2{!lXRR&g zuX>SOwe)*p$A^jUKqMp;!XZkvSY`7((+_i{?tCA-tLh;oQ)-mH=j}X~H1_ltT6^#b zdU*k7(OtE#EqB~KhROY=Y7|5A@_59e?yP!1?se@`ldr93 znK-k%a@Oz-A=ZT!aWv=-k=2Z2H8w2t#EWCzS;8T+T4huz`7-VpTi{2z9)zGE2g>*web*C>|pklD=69{)z( zYCk9yI%aysyw`SIJ3X!kefW~Z;^y9TOJ~2ceLm#l{0&LObQuF;9)DUQy~1Za3=YC| zSFT&_{GjZo^BhOw+8?~LWEXRcJ$b(Sj-1$&ll9Y{|1%E?qMDQ%1z>e__xUV~zMU%? zcgE9pAR*WXto7#<^>Y+Z62W@{7>zW%5Kn1P_$Qx~;%BK{{#eL-{L??xyaMeONA?U+ zbN~8n|7<{CduxmhKl63}>-b2YD3Kr!Kxn+8Th~ktmf~A*2YeS3?pcvbOQsjm)B3Vqz{f=QQe%XqtJrTS` zY@VvBVqb_Fmt#NCXz5dB)J5S9eQU5cED_o;>DW)WNujm=E7a3)BDVy4$I`?D34Qk& zCLQ|;C)xXqh8Q(c{q3ya6I3;hS12@i9Qz47pR0_fh_P(2fwNQ935z|z<=9XBiGA;| zYNg|Q;4g+DRgLXH>|%F6_7g>nhKXf?zUYobPdrc~F2{bN(XQvxO5;rA#Wbj@4h~W1 zdURZl{X~mB)t$>oF&4Os;Yd}NUTg!GV?R-~=w%u!29LSXs_JM_=y(;Aj{Sr?v{z|} z(O|I*h>Ph^wC~2{*iY=rmU98nJu^q!(yQu{3B91%c#6xhpQu{$yBRMAk9Z-bL7^x4 zFL6sNeuY9lL`1~UinEnM-6Xz@sEEPi>!MZl?1Y}gp_UF1Bv7cces*uBb+IkA{zEi5^q#qBXkls}c7l zV#*v*qb8yl71~&5f5gS0eZgMuz<_(hlZQmdX!sbRf%gUpDk(LhTbj_VaM8t5YdoIx}Kjg*IkVF~zLBp43^17c!kDM2l=mc99hiD8B=?bbqP9TUxi@HIl zCtPR$%$!*K*4VN5Gi#2;Z%u1#4!nREiR;h>p24RC46^`l0Yqq#I>6!(!Fu6~<;6Hi zkA>`5$gty>FIIOP^8$L~=zt4^k(h!munY6 zkR04!a)H@3{o3dR4IjbKFa_OT6$0CUSz^Lg6uR`^S4l8P0q`$5Xk|jdj-^Xq5zWyA za)CWw*Yp+?2pdr@{6=dNAR5YsPg!(g@X=glG=1_XCZuQQXN30ZPiYpo_!B!^3Mi_}2=onMLK16c z@51l)1VvNX;K$Otn&5q0^rL43EXt?GPpT%QXD1XJm;7QMVbL%({L>qTE(q1c3%KOB zQVZtFO9X5kx*wOO)W{{jWpNWQI?85WaMc9C*}C8xMKnk$a3y8_WKU4B6Au)EtqcAc ze1b^|{uF`9xj`3c z^~+^w@I@%4&GPu`eet*3@FJAV%`UkU$Rlpe2 zvvX5I8~bHLTvTmc4ql)GS!ucW6jE2T1wLy*CM{aGF@_B;&JB4bq!o>!LL2p@xWHXt z-XcJm8Sq(iegAa^Lq_RZ{RzuGF7{;Qa`2>_>{r7%QEDhwjToOjj$SB0{oY17QZ2#* z2}NsB8uE#)8!@Jalolo}C|qnE`OiRlMKA7-{su5#q2=RhlKT{Sm*E*5}o}LEw4V`Dt-i>OvG*g)4qN`N} zFLOshmX};!PZB4Df{3v*z?t zTs)sWzq0UB)t9yokmQ%Ql=^Fh{fVj(V`|9FXO}R})lfzS{!zt^uO=X83Zy)Y;N%6b z^d76+ue%B#&bx|Izsu%D3HPaL+e2kc4H4baG)aMr?GfV1#q|*8h7dLIIu_42f&IcN zbfXYn5Ay=z$;I{hPx&%NCFbMObz4i7T(w=HGNy)z8pj?mxY!jLo?Ki5ltnIeyIYZc z@<`rXm-wD}fO*Cq(4B~K^dQ8OiyL&$jZr0$zhkgxYEWK4WlRkbeb#||dzC3N5*ICmQ*v>kC~~PAPzp707a>73(1#cjpTwJg1-G``F zAdXV|13?~`KS5^_Q%KOvlwiwi;_)5t|J&*1^AlnTIJ>kU96>Df{8zE7Nz ziwi-JSjDbdQ_fKUXh@JZ$==si{lNrBs^AAmB^MV0FMiKaxq>)Aw5uv);WB$K3BQmP zY4H{(r{E*800_eEMl|poUVTG!4Gn5lYczs0olRjra*5#PBe&63yl7iL4FYk9Jv&Fk z?ezkZF~i!3F*PK5Jvt`iX1v8xN)aJOEj)(@(({CuV|+m*@jxHpObvzk$kjV;#`CES z$qu75$vGM{^~(cV4_;i#QAt>+ z%tvk|YCdv(#4fArbL@Ux9pTFj4`y?6#dtON&6X`WyB#Q{CIxXsK{x#LJ3VX$oeqnG z%euEAKoprXD-nNRk)6%O2ZK?pa6TJ;zMgvhEzB|@n}D(5J;O1K7(y(viP$6(b3H*Z zi843{Vc>T9{-AVB4V~-g$S(SRJ2_F=b)Ty8k^5C2f0&F^=4)zJ!arO|K605R zhiz6_JjU_mN!QoV^Mm#AAabF9~|&9A3;?K%Sk7SMO9Ak3S0$&&SbGEcf{M zjGS;yOJpwa1B**`ljT)SdAgbi^WNGQf5q`%_e%$KSmoQD0k{U&|UIUOsa7AIn2xH7gfc3#iCP zE{EmAsXdN$w;sAu$IR_)BMcY#Gs&ggH_CpN;Q?3~2K?W9P&1~6 ze)a&Rj`h=ede2|0Kc|;FJVgtR2vNF}$i`kppu7H)d&77!@{e#Eo#K z?sAm%%0Gv@%$!}GOmd*2hrl>x_U-zERitMpAd~`nW**yJU}NpWlcVM~6F-)RH&bx8 zV2e@gCult3)|eM3zhTqPAc9~`gJL0P5TEitE8tG77=lCMPTikpD2(g@xR2Nj&v#r4 z6ZnHP{w}_u74w{V#s>79?3-+67hj!|WWd%C5phSdIOLT0hod4VFs6pM>Dh5VZw==c z_0S#n1en>~8RNya%3OAFyF89;%4xH89e(F1ltm;)vHyIyl-rV ztH@;J#shfmQFf<(8s+?|c9k&_OOsDO^2>{P413PNPm}iAL3}%2Wk%@%`}B3+Od&m{ zhIU-flRH$ciCO$1G4@wYY_Y`zwC1Vj<}C#J;bd#8`6i0&u;}dAL?n13sN`9M7xPac zA7*ixiElDr?obtFo!cyyX80b;4ZwXNT+LWQ7V_NG3jxk^;oik^9{I5s2AIZ3+4Srf z4~k#t7KwP?ZnN9d2XqQAFCqT|nQl>mtgg@T;+x5Frh_mR>9^Zj;xfZ^BG<`%Z4NaL z%yBMx44!G(j;5zumU$*+0tbcDlLwfQR{V>#y>I$mn)+QDS6wfcK;eDZ zYtl>ataQ;O97I8d~ zxpWm7c|emeNCMsj^`EF=BhdZnF%CMf<|WIto@0w~BU>>5wtQo3z37bOPy5F^F z;ZX@2f8k6=RIQ2(ti1Vt`(}H(V>sVaiIO-99y8TpN)Ah@B`g8tNGwtHgw7ta%^psk zJ^_&`JzcP1LA&hX(AmROU1g`HAyTAIEE$!C3o?=+IR{W{sCK3xHM?#LFBnSTIv^9#YMylRwSPiHT+Z*KHD6#jHm8=^0r0%!JMybSxawSDBn= zma#MTso7|WIR`L_PtTW|!FP}^d(i{rcI3QjN5@0T{_|t}fuH|kbfjqcUTwcm&aReM zPc>7n#o}t@GO7PLY31+u($n<-(=rj~gaU!5iO0%l)ZA7n*^>(ZS(5Xlnkcz#k;}G4 z&TTD}taL9#s_gzkg(8=2oz4YPDZAtVM2gbAbVbuC#7MQNSx9mAx$T9xnWXMDzar(8 z9D*n#d!SIE)FnqnIA-@3DpYl9M!*%Dr0!pH^Q5x>%xa|09Dzt$x|dyMVj?J5=rh?&*I4A`Aw^#H>Y}8f;Em7nU7UL(K|l0DhNo--YG2 z$5$s^UoDv;aA1i3S#G2%7Jzi>3k*c5^#=6?fG%T+wa(f{0ZA-$b& zEB(NiJOX|0c8VTUuW6ZJ2suX)#l&L^5R+U9BY1Y4t24wb-A`ORRUDw zSj20Z|7|OLwFFQv@OCx_#CU8VA3<~3^z2w;)c7QCknBK#Ao?n<2=o1#Qvr8D7=L>d z@adwr%=X^B3XCVy9OsIEFcOGD5^+R9Bl|zDDc7sPH|^ML*##$N!xw`b#9IIi14U%z zVYBsH^aJ^-cZ`S@kO6iIGi@9cU zoEA%yvgI)TIF@5>SNxNaP!uvehxj5UV8gZJtS?HXO1XCi!(m&;7I@xeIFM&IKL4@s zA1Q^vB;>t=93}sQB@ zt1p5k2}_|?uh>`m^iZs;j-VREbQuF zV;eu1X&yfJ?x56>7X&bmW8Yz~C}vA#5c^z1Bd zv8?azFK$@rThuxlWxATjc)p481UsKM1bMK&PObr>Ae6iGR z^GwDQ3L_$N8$Tl)qA?hz3!XsB1NJ~?iqB$Kp!?GUU!5QL>Re6!v?}^rScNf@0-xu( z6Rn#})4he&)4cKnp?K(wbYJ+gp8oq(?-k2qeA>f5{QTUXM)5dX+S%iGKH+(4k>lPD z90vOFk$!?JdZ@oqI;MuM>X@J4*f)CLf_n@j!qxT|#sk_11|K*|;?lQq1^C;0-xI+J zaodJ?p@H`4O$^7a4Ym^o$f^aGMKZ^Xb%Gzh>L=d*qb^8 z#?+9Cb)PPt)t3h0*&=1-@?g{sxz3M zzBkz1+D?v$^sRuG+YN+upanXa&FM?C>u8K;f zOe|B?1eh0`1OG9|N@z`~#fX#|iG77tT(l7WMQFGUVoXx#(NQSKppj}G7mbAf1T-HF zQtByF)r2uMY#*ojj1hC6)PbNFZwkJj^5%rQt5ep?<_ zxR5{`!5{@`3?ou%r09L0?l38MqB)v!ysqgj&`T7(;SQ5_VyRc79vFY32Jw3P@-Tpn zXeEE16L**ds#DWEFa(3N5?-+&iJ~{0VbZ|iBL`lM3%i<-Qlosq^9kBN%O0L1m?P-E z4Qm2rU+fCAh%GP8;Zf_G`gl}ToxPGjZ?{!uLE@t^8gSd5fE9Vs)3Za3ljS5b+gfmk z%y4?PV2q&unjV!gHAHkv`)x&6Ef9G)U*e-Ny#DQWnJ}h?l>If$Twzw}S%NVG(|w~nm!2E()8>od&3=O1z*93XpB|>x&Xx)Q$u18rjH7Sd~Aj< z_F^wq=O$n5#YQ2_6*CH6Xl@wZ)V66jLm!%=F+8CaoG~@zY<-`@64jiHvmhc$0jogI z_6+MPBjU1t7iR1OLLBM{*$3<>-w;urW1)2(IK@K~2um>$M-s-Q=#H7Y_a#Wyjw(Y% zR23@GrCvYrCuiZ1IqCNR?Q2L7Va8x64hAEF5n2o}QqiTZ9Yo?z?$|App$8AtFGB6! znXWN4MAU6Umm(tW#dp9K_-sFzWz91{TA*eudEy&+c zoN-phzRvVZIX5e5Efr6J<(lgkP*wE_#P;fsn6|{7+_8He@g1*p0-D$Kq1Z@7H@K6t zAlEDq8A~XdfM*NQ_fb{#roCt@l>>bNBDe`QS*`zBU zbvmO$@q}jhg)(8QFSgkkoB^(<$%Nc`s5gZ=LRhbGod0H`KIte47x76)M!0DTbbIji5BVfi)!wnY z*!Ae+1T-$Tha8o!ZCg~U8N_zONM8cm9SF=0pl%Y_`|HeIEvNHsYa86hS#>uGw0F~AyF4WJGFzs>V7}*Z4VKv zbBysEzvH})mQ`Q&t`+9_oJ9-~JkDD;8t{Td8#eDmxXYHGbiB50jt1K)g8ma}w>Rcz zvV&X(*1dC>TU4o1*;w~Afb`@dLrRK13MNU}&Ql&~2Zrn09{$kq~2i3Eo8211)72T#X30(XM>O{u86qiLTD_s0!?J>>85$k23Dlx9; zB+k9bKkaQmZBkXL)O$^$B68s%$HR~agCqd7-7@m&ZdxvQv{$p)WK=S4@y?8=`~S`C z44$|^b2jeFVsZQ;yHl{&Mmtb6XLbgvy1rY>krO5TKaYryK? zLw*jy#0aK!gk6o`2hl2WYM^~oGCtrhEB|k@#XEC=R)_7TjoLO2n`)z(XrrCD=Rp>Z zfL$h5_&EfVbP|G2LX>C~xfvqS>?HKC|A{^xu5P27*uVyQf417T=X&`M5iba*bp%&> z!G}EDM;#GFMk2xz6=EeVOeSYxi5e(OSlBrZ9ntL?x@?44;X}kSu>vqz_r$^_5=kPM zWRhg1l7r99@?B}Rn>{+~HoA$M9eBkU)HlJ^$T0qJMZ9K`cv?q7K$+@RzDZa}R~ai) z|HP~iBWYnWISZ?=f!c(H?a5IYF**AE9qTj!AiS=EvPyQ4tF&AzJw=bS#l8Q>P2A`Yl&-6<*iU}aoGVw?Z?gjC*j$rfN zk9iMn3G50bk`^YCvoHow0CYWF=N& zB$?#4E&FCHq0PrEP!J#PZMKl>8_Ed(t>kki6A{MBHdtlS!X$GRR#^j;2@4zHP#dg( z9O563yqivt@l#0DJ+W|NWF>Ok{`ns@#lphFkwwco@I8?|Fjl#7hkIL-oin=mjW2xl zDVc~ccDBJ@buYIfnX|A&WtAo@tcylo_IwbJ(R9Gb_$egno>(|B@)FtTRX^vDiL~+H zM+#u(1W)NF+-yehlvxIM8-1Ea_f3X*!WTi)Z6EI`3k~l>Xet$Mjx`jXveoeQXO;cZ z5Q3U(2+OqyPqb22)9kxGZ|6{-w}3B4Rcvp(p8;%;3+_)52#ek5X?69 zl^2t@h=9rTOn36!f(E^(_F#2&)L0CR_wf2!0Z~i68Q-}#E^N?y+BfIYm%+TD7;kTq zFZ+;d>>(cm&KbAO?I}s10XM0KH6$D-{uNieR{L0-~}SnR*bo9s;#j&lJA|v_mYnRXOG+F z_mm{kfV=)BWey2H;Fd1tNIrM-y^*mj>?%@Z!^lCk%C9!y3)e+Nkn^uYVY13xXW578RzAMmE z62AdA!5R`i1nL^xrZ=*dg%ID|Q3&Iw{Pr&=wbFE=0Cac=sh` zYL2odrm4urfHTK!i}aMlZ@?|9hqjKUw?V|h-pE=ORuw7oVdS7%>Ih3*@oo}n*p+^E*%)xg7mM0q1?S=d!X zWDFyR)XErwPFsFrDyF%x>YOS2+SFk#L{>nkt0VmA&0pyuT+Fily?hKfbKJH-Pf7d+ z+&s8rr{3@oR=Rf%y?w-%u`HY_A~J@NgKCvuZCc}>FyOg#!^kT!>zpb3+Rzq{k+~3A z0l}`0_GIBNJMt`|OUcH7v&U@<^pwPHz|A-D-6+0R^(ae>-WwUq!l@#%hLMA6m0xYz nOyBIpT52XpEX+D*%Dgrb=0Y&72ZI>q+Q$q##(i}+yN)vgi^8x& literal 0 HcmV?d00001 From a47c6ff2813092c9d3d6f3bb80395fecb13dacd3 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 09:32:13 +0800 Subject: [PATCH 06/55] doc: copy _preparition.mdx from zh to en --- docs/en/14-reference/03-connector/_preparition.mdx | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/en/14-reference/03-connector/_preparition.mdx diff --git a/docs/en/14-reference/03-connector/_preparition.mdx b/docs/en/14-reference/03-connector/_preparition.mdx new file mode 100644 index 0000000000..87538ebfd8 --- /dev/null +++ b/docs/en/14-reference/03-connector/_preparition.mdx @@ -0,0 +1,10 @@ +- 已安装客户端驱动（使用原生连接必须安装，使用 REST 连接无需安装） + +:::info + +由于 TDengine 的客户端驱动使用 C 语言编写，使用原生连接时需要加载系统对应安装在本地的客户端驱动共享库文件，通常包含在 TDengine 安装包。TDengine Linux 服务端安装包附带了 TDengine 客户端，也可以单独安装 [Linux 客户端](/get-started/) 。在 Windows 环境开发时需要安装 TDengine 对应的 [Windows 客户端](https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client) 。 + +- libtaos.so: 在 Linux 系统中成功安装 TDengine 后，依赖的 Linux 版客户端驱动 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。 +- taos.dll: 在 Windows 系统中安装完客户端之后，依赖的 Windows 版客户端驱动 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。 + +::: From 41bc062d7c9f2c0571b8e579a93fdc6d7ff79f4b Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 09:35:49 +0800 Subject: [PATCH 07/55] doc: fix syntax error in udf.md --- docs/en/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 172a11f341..f8170d0d63 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -15,7 +15,7 @@ When you create a user-defined function, you must implement standard interface f - For aggregate functions, implement the `aggfn_start`, `aggfn`, and `aggfn_finish` interface functions. - To initialize your function, implement the `udf_init` function. To terminate your function, implement the `udf_destroy` function. -There are strict naming conventions for these interface functions. The names of the start, finish, init, and destroy interfaces must be _start, _finish, _init, and _destroy, respectively. Replace `scalarfn`, `aggfn`, and `udf` with the name of your user-defined function. +There are strict naming conventions for these interface functions. The names of the start, finish, init, and destroy interfaces must be _start, _finish, _init, and _destroy, respectively. Replace `scalarfn`, `aggfn`, and `udf` with the name of your user-defined function. ## Implementing a Scalar Function The implementation of a scalar function is described as follows: From b58cce905e5531cfaa824d811517caddedca6200 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 22 Aug 2022 09:56:32 +0800 Subject: [PATCH 08/55] docs(stream) --- docs/zh/12-taos-sql/14-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 24e695bcdc..27725dd6dc 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -44,6 +44,7 @@ window_clause: { ```sql CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); +``` ## 流式计算的 partition @@ -52,7 +53,6 @@ SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVA 不带 PARTITION BY 选项时，所有的数据将写入到一张子表。 流式计算创建的超级表有唯一的 tag 列 groupId，每个 partition 会被分配唯一 groupId。与 schemaless 写入一致，我们通过 MD5 计算子表名，并自动创建它。 -``` ## 删除流式计算 From 36ff7e4242fa7f60dfb66fdc2ff3842d0fb92f2e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 Aug 2022 10:14:40 +0800 Subject: [PATCH 09/55] opti:grou by tag --- source/libs/executor/src/executil.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 8ec1e9b584..0ba669c34e 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -637,8 +637,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis case QUERY_NODE_COLUMN: case QUERY_NODE_VALUE: case QUERY_NODE_OPERATOR: - case QUERY_NODE_FUNCTION: - case QUERY_NODE_LOGIC_CONDITION:{ + case QUERY_NODE_FUNCTION:{ SExprNode* expNode = (SExprNode*)pNode; code = createResultData(&expNode->resType, rows, &output); if (code != TSDB_CODE_SUCCESS) { From d6db745dd06ae7318b46fc6ae5571d8d86592db4 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 22 Aug 2022 10:29:54 +0800 Subject: [PATCH 10/55] Update 14-stream.md --- docs/zh/12-taos-sql/14-stream.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 27725dd6dc..a967299e40 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -82,7 +82,7 @@ SELECT * from performance_schema.`perf_streams`; 1. AT_ONCE：写入立即触发 -2. WINDOW_CLOSE：窗口关闭时触发（窗口关闭由事件时间决定，可配合 watermark 使用，详见《流式计算的乱序数据容忍策略》） +2. WINDOW_CLOSE：窗口关闭时触发（窗口关闭由事件时间决定，可配合 watermark 使用） 3. MAX_DELAY time：若窗口关闭，则触发计算。若窗口未关闭，且未关闭时长超过 max delay 指定的时间，则触发计算。 @@ -94,6 +94,8 @@ MAX_DELAY 模式在窗口关闭时会立即触发计算。此外，当数据写 ## 流式计算的窗口关闭 +流式计算以事件时间（插入记录中的时间戳主键）为基准计算窗口关闭，而非以 TDengine 服务器的时间，以事件时间为基准，可以避免客户端与服务器时间不一致带来的问题，能够解决乱序数据写入等等问题。流式计算还提供了 watermark 来定义容忍的乱序程度。 + 在创建流时，可以在 stream_option 中指定 watermark，它定义了数据乱序的容忍上界。 流式计算通过 watermark 来度量对乱序数据的容忍程度，watermark 默认为 0。 From 4b94c7662151b2c3fca331b4ad8728327b346ebb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 22 Aug 2022 10:40:23 +0800 Subject: [PATCH 11/55] test: valgrind case --- tests/script/tsim/valgrind/checkError6.sim | 4 ++-- tests/script/tsim/valgrind/checkError7.sim | 2 +- tests/script/tsim/valgrind/checkError8.sim | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 00de00f71d..fcc5b04c90 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -114,8 +114,8 @@ sql select tbcol5 - tbcol3 from stb sql select spread( tbcol2 )/44, spread(tbcol2), 0.204545455 * 44 from stb; sql select min(tbcol) * max(tbcol) /4, sum(tbcol2) * apercentile(tbcol2, 20), apercentile(tbcol2, 33) + 52/9 from stb; sql select distinct(tbname), tgcol from stb; -#sql select sum(tbcol) from stb partition by tbname interval(1s) slimit 1 soffset 1; -#sql select sum(tbcol) from stb partition by tbname interval(1s) slimit 2 soffset 4 limit 10 offset 1; +sql select sum(tbcol) from stb partition by tbname interval(1s) slimit 1 soffset 1; +sql select sum(tbcol) from stb partition by tbname interval(1s) slimit 2 soffset 4 limit 10 offset 1; print =============== step5: explain sql explain analyze select ts from stb where -2; diff --git a/tests/script/tsim/valgrind/checkError7.sim b/tests/script/tsim/valgrind/checkError7.sim index a66ddb30df..af42d1e76b 100644 --- a/tests/script/tsim/valgrind/checkError7.sim +++ b/tests/script/tsim/valgrind/checkError7.sim @@ -66,7 +66,7 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 2 then +if $system_content > 0 then return -1 endi diff --git a/tests/script/tsim/valgrind/checkError8.sim b/tests/script/tsim/valgrind/checkError8.sim index 7ca01bc3d0..2f204768eb 100644 --- a/tests/script/tsim/valgrind/checkError8.sim +++ b/tests/script/tsim/valgrind/checkError8.sim @@ -143,7 +143,7 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 2 then +if $system_content > 0 then return -1 endi From d4d9275980fe1a424a07c17da9b9f948091828e8 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 11:17:45 +0800 Subject: [PATCH 12/55] doc: fix syntax error in 09-udf.md --- docs/en/07-develop/09-udf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index f8170d0d63..deb9c4cdb5 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -102,7 +102,7 @@ Replace `aggfn` with the name of your function. ## Interface Functions -There are strict naming conventions for interface functions. The names of the start, finish, init, and destroy interfaces must be _start, _finish, _init, and _destroy, respectively. Replace `scalarfn`, `aggfn`, and `udf` with the name of your user-defined function. +There are strict naming conventions for interface functions. The names of the start, finish, init, and destroy interfaces must be _start, _finish, _init, and _destroy, respectively. Replace `scalarfn`, `aggfn`, and `udf` with the name of your user-defined function. Interface functions return a value that indicates whether the operation was successful. If an operation fails, the interface function returns an error code. Otherwise, it returns TSDB_CODE_SUCCESS. The error codes are defined in `taoserror.h` and in the common API error codes in `taos.h`. For example, TSDB_CODE_UDF_INVALID_INPUT indicates invalid input. TSDB_CODE_OUT_OF_MEMORY indicates insufficient memory. From 36b60ab082ef9aa3b0a4365fe9f8bb50c61b691a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 Aug 2022 11:28:58 +0800 Subject: [PATCH 13/55] opti:grou by tag --- source/libs/executor/src/executil.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 0ba669c34e..66da50f2d5 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -634,8 +634,9 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis SScalarParam output = {0}; switch (nodeType(pNode)) { - case QUERY_NODE_COLUMN: case QUERY_NODE_VALUE: + break; + case QUERY_NODE_COLUMN: case QUERY_NODE_OPERATOR: case QUERY_NODE_FUNCTION:{ SExprNode* expNode = (SExprNode*)pNode; @@ -646,9 +647,18 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis break; } default: - ASSERT(0); + code = TSDB_CODE_OPS_NOT_SUPPORT; + goto end; + } + if(nodeType(pNode) == QUERY_NODE_COLUMN){ + SColumnNode* pSColumnNode = (SColumnNode*)pNode; + SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, pSColumnNode->slotId); + code = colDataAssign(output.columnData, pColInfo, rows, NULL); + }else if(nodeType(pNode) == QUERY_NODE_VALUE){ + continue; + }else{ + code = scalarCalculate(pNode, pBlockList, &output); } - code = scalarCalculate(pNode, pBlockList, &output); if(code != TSDB_CODE_SUCCESS){ releaseColInfoData(output.columnData); goto end; From ad8be92e0baed5596509c719a6cb6e10eec2ae98 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 11:51:18 +0800 Subject: [PATCH 14/55] doc: fix some broken links --- docs/en/07-develop/04-query-data/index.mdx | 6 +++--- docs/en/12-taos-sql/index.md | 2 +- docs/en/28-releases.md | 9 +++++++++ 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 docs/en/28-releases.md diff --git a/docs/en/07-develop/04-query-data/index.mdx b/docs/en/07-develop/04-query-data/index.mdx index d530c59185..38dc98d1ff 100644 --- a/docs/en/07-develop/04-query-data/index.mdx +++ b/docs/en/07-develop/04-query-data/index.mdx @@ -43,7 +43,7 @@ Query OK, 2 row(s) in set (0.001100s) To meet the requirements of varied use cases, some special functions have been added in TDengine. Some examples are `twa` (Time Weighted Average), `spread` (The difference between the maximum and the minimum), and `last_row` (the last row). -For detailed query syntax, see [Select](../../taos-sql././select). +For detailed query syntax, see [Select](../../taos-sql/select). ## Aggregation among Tables @@ -74,7 +74,7 @@ taos> SELECT count(*), max(current) FROM meters where groupId = 2; Query OK, 1 row(s) in set (0.002136s) ``` -In [Select](../../taos-sql././select), all query operations are marked as to whether they support STables or not. +In [Select](../../taos-sql/select), all query operations are marked as to whether they support STables or not. ## Down Sampling and Interpolation @@ -122,7 +122,7 @@ In many use cases, it's hard to align the timestamp of the data collected by eac Interpolation can be performed in TDengine if there is no data in a time range. -For more information, see [Aggregate by Window](../../taos-sql/interval). +For more information, see [Aggregate by Window](../../taos-sql/distinguished). ## Examples diff --git a/docs/en/12-taos-sql/index.md b/docs/en/12-taos-sql/index.md index f63de6308d..f78ed70e32 100644 --- a/docs/en/12-taos-sql/index.md +++ b/docs/en/12-taos-sql/index.md @@ -3,7 +3,7 @@ title: TDengine SQL description: "The syntax supported by TDengine SQL " --- -This section explains the syntax of SQL to perform operations on databases, tables and STables, insert data, select data and use functions. We also provide some tips that can be used in TDengine SQL. If you have previous experience with SQL this section will be fairly easy to understand. If you do not have previous experience with SQL, you'll come to appreciate the simplicity and power of SQL. TDengine SQL has been enhanced in version 3.0, and the query engine has been rearchitected. For information about how TDengine SQL has changed, see [Changes in TDengine 3.0](/taos-sql/changes). +This section explains the syntax of SQL to perform operations on databases, tables and STables, insert data, select data and use functions. We also provide some tips that can be used in TDengine SQL. If you have previous experience with SQL this section will be fairly easy to understand. If you do not have previous experience with SQL, you'll come to appreciate the simplicity and power of SQL. TDengine SQL has been enhanced in version 3.0, and the query engine has been rearchitected. For information about how TDengine SQL has changed, see [Changes in TDengine 3.0](../changes). TDengine SQL is the major interface for users to write data into or query from TDengine. It uses standard SQL syntax and includes extensions and optimizations for time-series data and services. The maximum length of a TDengine SQL statement is 1 MB. Note that keyword abbreviations are not supported. For example, DELETE cannot be entered as DEL. diff --git a/docs/en/28-releases.md b/docs/en/28-releases.md new file mode 100644 index 0000000000..a0c9eb1199 --- /dev/null +++ b/docs/en/28-releases.md @@ -0,0 +1,9 @@ +--- +sidebar_label: Releases +title: Released Versions +--- + +import Release from "/components/ReleaseV3"; + + + From 530c775d978cade8bfd5574c30b5c8b70fc92822 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Aug 2022 11:57:10 +0800 Subject: [PATCH 15/55] refactor(query): do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 58 ++++++++++++++++++-------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0caf10f391..947ab2e7ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -69,8 +69,10 @@ typedef struct SIOCostSummary { double buildmemBlock; int64_t headFileLoad; double headFileLoadTime; - int64_t smaData; + int64_t smaDataLoad; double smaLoadTime; + int64_t lastBlockLoad; + double lastBlockLoadTime; } SIOCostSummary; typedef struct SBlockLoadSuppInfo { @@ -98,10 +100,10 @@ typedef struct SLastBlockReader { } SLastBlockReader; typedef struct SFilesetIter { - int32_t numOfFiles; // number of total files - int32_t index; // current accessed index in the list - SArray* pFileList; // data file list - int32_t order; + int32_t numOfFiles; // number of total files + int32_t index; // current accessed index in the list + SArray* pFileList; // data file list + int32_t order; SLastBlockReader* pLastBlockReader; // last file block reader } SFilesetIter; @@ -1303,9 +1305,23 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); } - return (overlapWithNeighbor || hasDup || dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock) || - keyOverlapFileBlock(key, pBlock, &pReader->verRange) || (pBlock->nRow > pReader->capacity) || - overlapWithDel || overlapWithlastBlock); + bool moreThanOutputCapacity = pBlock->nRow > pReader->capacity; + bool partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock); + bool overlapWithKey = keyOverlapFileBlock(key, pBlock, &pReader->verRange); + + bool loadDataBlock = (overlapWithNeighbor || hasDup || partiallyRequired || overlapWithKey || + moreThanOutputCapacity || overlapWithDel || overlapWithlastBlock); + + // log the reason why load the datablock for profile + if (loadDataBlock) { + tsdbDebug("%p uid:%" PRIu64 + " need to load the datablock, reason overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, " + "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s", + pReader, pFBlock->uid, overlapWithNeighbor, hasDup, partiallyRequired, overlapWithKey, + moreThanOutputCapacity, overlapWithDel, overlapWithlastBlock, pReader->idStr); + } + + return loadDataBlock; } static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) { @@ -2383,7 +2399,6 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return TSDB_CODE_SUCCESS; } -// todo add elapsed time results static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo *pBlockScanInfo, STsdbReader* pReader) { SArray* pBlocks = pLastBlockReader->pBlockL; SBlockL* pBlock = NULL; @@ -2415,6 +2430,7 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STable return TSDB_CODE_SUCCESS; } + int64_t st = taosGetTimestampUs(); int32_t code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr); @@ -2422,17 +2438,23 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STable } code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); + + double el = (taosGetTimestampUs() - st) / 1000.0; if (code != TSDB_CODE_SUCCESS) { tsdbError("%p error occurs in loading last block into buffer, last block index:%d, total:%d code:%s %s", pReader, pLastBlockReader->currentBlockIndex, totalLastBlocks, tstrerror(code), pReader->idStr); } else { tsdbDebug("%p load last block completed, uid:%" PRIu64 - " last block index:%d, total:%d rows:%d, minVer:%d, maxVer:%d, brange:%" PRId64 " - %" PRId64 " %s", + " last block index:%d, total:%d rows:%d, minVer:%d, maxVer:%d, brange:%" PRId64 " - %" PRId64 + " elapsed time:%.2f ms, %s", pReader, uid, pLastBlockReader->currentBlockIndex, totalLastBlocks, pBlock->nRow, pBlock->minVer, - pBlock->maxVer, pBlock->minKey, pBlock->maxKey, pReader->idStr); + pBlock->maxVer, pBlock->minKey, pBlock->maxKey, el, pReader->idStr); } pLastBlockReader->currentBlockIndex = index; + pReader->cost.lastBlockLoad += 1; + pReader->cost.lastBlockLoadTime += el; + return TSDB_CODE_SUCCESS; } @@ -2627,7 +2649,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl // initialize the block iterator for a new fileset if (num.numOfBlocks > 0) { code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks); - } else { + } else { // no block data, only last block exists tBlockDataReset(&pReader->status.fileBlockData); resetDataBlockIterator(pBlockIter, pReader->order, pReader->status.pTableMap); } @@ -2700,7 +2722,6 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { if (hasNext) { // check for the next block in the block accessed order list initBlockDumpInfo(pReader, pBlockIter); } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { // data blocks in current file are exhausted, let's try the next file now - // todo dump all data in last block if exists. tBlockDataReset(&pReader->status.fileBlockData); resetDataBlockIterator(pBlockIter, pReader->order, pReader->status.pTableMap); goto _begin; @@ -3497,10 +3518,11 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-time:%.2f ms, " - "build in-memory-block-time:%.2f ms, STableBlockScanInfo size:%.2f Kb %s", - pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaData, pCost->smaLoadTime, - pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, - numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); + "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 + ", lastBlocks-time:%.2f ms, STableBlockScanInfo size:%.2f Kb %s", + pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, + pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, + pCost->lastBlockLoadTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); @@ -3662,7 +3684,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS double elapsed = (taosGetTimestampUs() - stime) / 1000.0; pReader->cost.smaLoadTime += elapsed; - pReader->cost.smaData += 1; + pReader->cost.smaDataLoad += 1; *pBlockStatis = pSup->plist; From b512c721f1603f8e1df02902531098ba9b55c59e Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 12:04:31 +0800 Subject: [PATCH 16/55] doc: fix broken links --- docs/en/12-taos-sql/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/index.md b/docs/en/12-taos-sql/index.md index f78ed70e32..e243cd2318 100644 --- a/docs/en/12-taos-sql/index.md +++ b/docs/en/12-taos-sql/index.md @@ -3,7 +3,7 @@ title: TDengine SQL description: "The syntax supported by TDengine SQL " --- -This section explains the syntax of SQL to perform operations on databases, tables and STables, insert data, select data and use functions. We also provide some tips that can be used in TDengine SQL. If you have previous experience with SQL this section will be fairly easy to understand. If you do not have previous experience with SQL, you'll come to appreciate the simplicity and power of SQL. TDengine SQL has been enhanced in version 3.0, and the query engine has been rearchitected. For information about how TDengine SQL has changed, see [Changes in TDengine 3.0](../changes). +This section explains the syntax of SQL to perform operations on databases, tables and STables, insert data, select data and use functions. We also provide some tips that can be used in TDengine SQL. If you have previous experience with SQL this section will be fairly easy to understand. If you do not have previous experience with SQL, you'll come to appreciate the simplicity and power of SQL. TDengine SQL has been enhanced in version 3.0, and the query engine has been rearchitected. For information about how TDengine SQL has changed, see [Changes in TDengine 3.0](../taos-sql/changes). TDengine SQL is the major interface for users to write data into or query from TDengine. It uses standard SQL syntax and includes extensions and optimizations for time-series data and services. The maximum length of a TDengine SQL statement is 1 MB. Note that keyword abbreviations are not supported. For example, DELETE cannot be entered as DEL. From aca0d0bf0b6fd34c1b0165487c3acee9a9d0038e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Aug 2022 13:25:45 +0800 Subject: [PATCH 17/55] fix: fix memory leak issue --- source/client/src/clientImpl.c | 3 +++ source/client/src/clientMain.c | 2 ++ source/libs/catalog/src/catalog.c | 2 +- source/libs/parser/src/parUtil.c | 10 ++++++++++ source/libs/qworker/src/qworker.c | 7 ++----- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 9c086fc83e..5f0af55d13 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -238,6 +238,9 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC TSWAP(pRequest->targetTableList, (*pQuery)->pTargetTableList); } + taosArrayDestroy(cxt.pTableMetaPos); + taosArrayDestroy(cxt.pTableVgroupPos); + return code; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 0e95cd4d99..f449641f10 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -674,6 +674,8 @@ static void destorySqlParseWrapper(SqlParseWrapper *pWrapper) { taosArrayDestroy(pWrapper->catalogReq.pIndex); taosArrayDestroy(pWrapper->catalogReq.pUser); taosArrayDestroy(pWrapper->catalogReq.pTableIndex); + taosArrayDestroy(pWrapper->pCtx->pTableMetaPos); + taosArrayDestroy(pWrapper->pCtx->pTableVgroupPos); taosMemoryFree(pWrapper->pCtx); taosMemoryFree(pWrapper); } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 933e65e582..b6e958e192 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -893,7 +893,7 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo *pConn, SArray* CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SName name; + SName name = {0}; int32_t sver = 0; int32_t tver = 0; int32_t tbNum = taosArrayGetSize(pTables); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 17e78e7806..32513fd0b6 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -1159,6 +1159,16 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { taosHashCleanup(pMetaCache->pTableMeta); taosHashCleanup(pMetaCache->pTableVgroup); } + SInsertTablesMetaReq* p = taosHashIterate(pMetaCache->pInsertTables, NULL); + while (NULL != p) { + taosArrayDestroy(p->pTableMetaPos); + taosArrayDestroy(p->pTableMetaReq); + taosArrayDestroy(p->pTableVgroupPos); + taosArrayDestroy(p->pTableVgroupReq); + + p = taosHashIterate(pMetaCache->pInsertTables, p); + } + taosHashCleanup(pMetaCache->pInsertTables); taosHashCleanup(pMetaCache->pDbVgroup); taosHashCleanup(pMetaCache->pDbCfg); taosHashCleanup(pMetaCache->pDbInfo); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 862d142100..f006096ce2 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -149,13 +149,10 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { } } +_return: + taosArrayDestroy(pResList); QW_RET(code); - -_return: - taosArrayDestroy(pResList); - - return code; } int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { From e61ee31de65bae4b29555e5f8046e6c8d8d85d04 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Aug 2022 13:26:09 +0800 Subject: [PATCH 18/55] refactor(query): do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 947ab2e7ff..12ac0c2f1e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -730,7 +730,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug("load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, size:%.2f Kb, elapsed time:%.2f ms %s", - numOfTables, total, numOfQTable, pBlockNum->numOfLastBlocks, sizeInDisk + numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastBlocks, sizeInDisk / 1000.0, el, pReader->idStr); pReader->cost.numOfBlocks += total; @@ -2445,10 +2445,10 @@ static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STable pLastBlockReader->currentBlockIndex, totalLastBlocks, tstrerror(code), pReader->idStr); } else { tsdbDebug("%p load last block completed, uid:%" PRIu64 - " last block index:%d, total:%d rows:%d, minVer:%d, maxVer:%d, brange:%" PRId64 " - %" PRId64 + " last block index:%d, total:%d rows:%d, minVer:%d, maxVer:%d, brange:%" PRId64 "-%" PRId64 " elapsed time:%.2f ms, %s", - pReader, uid, pLastBlockReader->currentBlockIndex, totalLastBlocks, pBlock->nRow, pBlock->minVer, - pBlock->maxVer, pBlock->minKey, pBlock->maxKey, el, pReader->idStr); + pReader, uid, index, totalLastBlocks, pBlock->nRow, pBlock->minVer, pBlock->maxVer, pBlock->minKey, + pBlock->maxKey, el, pReader->idStr); } pLastBlockReader->currentBlockIndex = index; From cd22be63a42083b154cbca129427a6a0856bac06 Mon Sep 17 00:00:00 2001 From: huolibo Date: Mon, 22 Aug 2022 13:26:59 +0800 Subject: [PATCH 19/55] fix(driver): jdbc sample for 3.0 (#16235) * fix(driver): jdbc sample for 3.0 * fix: drop table if exists * test: valgrind case Co-authored-by: Shengliang Guan Co-authored-by: Shuduo Sang --- examples/JDBC/JDBCDemo/pom.xml | 2 +- examples/JDBC/SpringJdbcTemplate/pom.xml | 2 +- examples/JDBC/SpringJdbcTemplate/readme.md | 4 +- .../taosdata/example/jdbcTemplate/App.java | 2 +- .../jdbcTemplate/BatcherInsertTest.java | 2 +- examples/JDBC/connectionPools/README-cn.md | 6 +- examples/JDBC/connectionPools/pom.xml | 2 +- examples/JDBC/mybatisplus-demo/pom.xml | 2 +- examples/JDBC/mybatisplus-demo/readme | 14 +++++ .../mybatisplusdemo/mapper/WeatherMapper.java | 10 ++++ .../src/main/resources/application.yml | 2 +- .../mapper/TemperatureMapperTest.java | 18 +----- .../mapper/WeatherMapperTest.java | 31 +++++++--- examples/JDBC/readme.md | 2 +- examples/JDBC/springbootdemo/pom.xml | 2 +- examples/JDBC/springbootdemo/readme.md | 3 +- .../controller/WeatherController.java | 1 - .../springbootdemo/dao/WeatherMapper.xml | 3 +- .../src/main/resources/application.properties | 2 +- examples/JDBC/taosdemo/pom.xml | 2 +- examples/JDBC/taosdemo/readme.md | 4 +- .../taosdemo/TaosDemoApplication.java | 23 +++++--- .../taosdemo/service/QueryService.java | 6 -- .../taosdata/taosdemo/utils/SqlSpeller.java | 57 +++++++++++-------- .../src/main/resources/application.properties | 4 +- .../taosdemo/service/TableServiceTest.java | 31 ---------- 26 files changed, 119 insertions(+), 118 deletions(-) create mode 100644 examples/JDBC/mybatisplus-demo/readme delete mode 100644 examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java diff --git a/examples/JDBC/JDBCDemo/pom.xml b/examples/JDBC/JDBCDemo/pom.xml index 8cf0356721..807ceb0f24 100644 --- a/examples/JDBC/JDBCDemo/pom.xml +++ b/examples/JDBC/JDBCDemo/pom.xml @@ -17,7 +17,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.34 + 3.0.0 diff --git a/examples/JDBC/SpringJdbcTemplate/pom.xml b/examples/JDBC/SpringJdbcTemplate/pom.xml index eac3dec0a9..6e4941b4f1 100644 --- a/examples/JDBC/SpringJdbcTemplate/pom.xml +++ b/examples/JDBC/SpringJdbcTemplate/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.18 + 3.0.0 diff --git a/examples/JDBC/SpringJdbcTemplate/readme.md b/examples/JDBC/SpringJdbcTemplate/readme.md index b70a6565f8..f59bcdbeb5 100644 --- a/examples/JDBC/SpringJdbcTemplate/readme.md +++ b/examples/JDBC/SpringJdbcTemplate/readme.md @@ -10,7 +10,7 @@ ```xml - + @@ -28,5 +28,5 @@ mvn clean package ``` 打包成功之后，进入 `target/` 目录下，执行以下命令就可运行测试： ```shell -java -jar SpringJdbcTemplate-1.0-SNAPSHOT-jar-with-dependencies.jar +java -jar target/SpringJdbcTemplate-1.0-SNAPSHOT-jar-with-dependencies.jar ``` \ No newline at end of file diff --git a/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java b/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java index 6942d62a83..ce26b7504a 100644 --- a/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java +++ b/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java @@ -28,7 +28,7 @@ public class App { //use database executor.doExecute("use test"); // create table - executor.doExecute("create table if not exists test.weather (ts timestamp, temperature int, humidity float)"); + executor.doExecute("create table if not exists test.weather (ts timestamp, temperature float, humidity int)"); WeatherDao weatherDao = ctx.getBean(WeatherDao.class); Weather weather = new Weather(new Timestamp(new Date().getTime()), random.nextFloat() * 50.0f, random.nextInt(100)); diff --git a/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/example/jdbcTemplate/BatcherInsertTest.java b/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/example/jdbcTemplate/BatcherInsertTest.java index 29d0f79fd4..782fcbe0eb 100644 --- a/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/example/jdbcTemplate/BatcherInsertTest.java +++ b/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/example/jdbcTemplate/BatcherInsertTest.java @@ -41,7 +41,7 @@ public class BatcherInsertTest { //use database executor.doExecute("use test"); // create table - executor.doExecute("create table if not exists test.weather (ts timestamp, temperature int, humidity float)"); + executor.doExecute("create table if not exists test.weather (ts timestamp, temperature float, humidity int)"); } @Test diff --git a/examples/JDBC/connectionPools/README-cn.md b/examples/JDBC/connectionPools/README-cn.md index 9b26df3c2e..6e589418b1 100644 --- a/examples/JDBC/connectionPools/README-cn.md +++ b/examples/JDBC/connectionPools/README-cn.md @@ -13,13 +13,13 @@ ConnectionPoolDemo的程序逻辑： ### 如何运行这个例子： ```shell script -mvn clean package assembly:single -java -jar target/connectionPools-1.0-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 +mvn clean package +java -jar target/ConnectionPoolDemo-jar-with-dependencies.jar -host 127.0.0.1 ``` 使用mvn运行ConnectionPoolDemo的main方法，可以指定参数 ```shell script Usage: -java -jar target/connectionPools-1.0-SNAPSHOT-jar-with-dependencies.jar +java -jar target/ConnectionPoolDemo-jar-with-dependencies.jar -host : hostname -poolType -poolSize diff --git a/examples/JDBC/connectionPools/pom.xml b/examples/JDBC/connectionPools/pom.xml index 99a7892a25..61717cf112 100644 --- a/examples/JDBC/connectionPools/pom.xml +++ b/examples/JDBC/connectionPools/pom.xml @@ -18,7 +18,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.18 + 3.0.0 diff --git a/examples/JDBC/mybatisplus-demo/pom.xml b/examples/JDBC/mybatisplus-demo/pom.xml index ad6a63e800..5555145958 100644 --- a/examples/JDBC/mybatisplus-demo/pom.xml +++ b/examples/JDBC/mybatisplus-demo/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.18 + 3.0.0 diff --git a/examples/JDBC/mybatisplus-demo/readme b/examples/JDBC/mybatisplus-demo/readme new file mode 100644 index 0000000000..b31b6c34bf --- /dev/null +++ b/examples/JDBC/mybatisplus-demo/readme @@ -0,0 +1,14 @@ +# 使用说明 + +## 创建使用db +```shell +$ taos + +> create database mp_test +``` + +## 执行测试用例 + +```shell +$ mvn clean test +``` \ No newline at end of file diff --git a/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapper.java b/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapper.java index 6733cbded9..1f0338db34 100644 --- a/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapper.java +++ b/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapper.java @@ -2,7 +2,17 @@ package com.taosdata.example.mybatisplusdemo.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.taosdata.example.mybatisplusdemo.domain.Weather; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Update; public interface WeatherMapper extends BaseMapper { + @Update("CREATE TABLE if not exists weather(ts timestamp, temperature float, humidity int, location nchar(100))") + int createTable(); + + @Insert("insert into weather (ts, temperature, humidity, location) values(#{ts}, #{temperature}, #{humidity}, #{location})") + int insertOne(Weather one); + + @Update("drop table if exists weather") + void dropTable(); } diff --git a/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml b/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml index 38180c6d75..985ed1675e 100644 --- a/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml +++ b/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml @@ -2,7 +2,7 @@ spring: datasource: driver-class-name: com.taosdata.jdbc.TSDBDriver url: jdbc:TAOS://localhost:6030/mp_test?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8 - user: root + username: root password: taosdata druid: diff --git a/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java b/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java index 4331d15d34..4d9dbf8d2f 100644 --- a/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java +++ b/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java @@ -82,27 +82,15 @@ public class TemperatureMapperTest { Assert.assertEquals(1, affectRows); } - /*** - * test SelectOne - * **/ - @Test - public void testSelectOne() { - QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.eq("location", "beijing"); - Temperature one = mapper.selectOne(wrapper); - System.out.println(one); - Assert.assertNotNull(one); - } - /*** * test select By map * ***/ @Test public void testSelectByMap() { Map map = new HashMap<>(); - map.put("location", "beijing"); + map.put("location", "北京"); List temperatures = mapper.selectByMap(map); - Assert.assertEquals(1, temperatures.size()); + Assert.assertTrue(temperatures.size() > 1); } /*** @@ -120,7 +108,7 @@ public class TemperatureMapperTest { @Test public void testSelectCount() { int count = mapper.selectCount(null); - Assert.assertEquals(5, count); + Assert.assertEquals(10, count); } /**** diff --git a/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java b/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java index 1699344552..dba8abd1ed 100644 --- a/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java +++ b/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapperTest.java @@ -6,6 +6,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.taosdata.example.mybatisplusdemo.domain.Weather; import org.junit.Assert; import org.junit.Test; +import org.junit.Before; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -26,6 +27,18 @@ public class WeatherMapperTest { @Autowired private WeatherMapper mapper; + @Before + public void createTable(){ + mapper.dropTable(); + mapper.createTable(); + Weather one = new Weather(); + one.setTs(new Timestamp(1605024000000l)); + one.setTemperature(12.22f); + one.setLocation("望京"); + one.setHumidity(100); + mapper.insertOne(one); + } + @Test public void testSelectList() { List weathers = mapper.selectList(null); @@ -46,20 +59,20 @@ public class WeatherMapperTest { @Test public void testSelectOne() { QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.eq("location", "beijing"); + wrapper.eq("location", "望京"); Weather one = mapper.selectOne(wrapper); System.out.println(one); Assert.assertEquals(12.22f, one.getTemperature(), 0.00f); - Assert.assertEquals("beijing", one.getLocation()); + Assert.assertEquals("望京", one.getLocation()); } - @Test - public void testSelectByMap() { - Map map = new HashMap<>(); - map.put("location", "beijing"); - List weathers = mapper.selectByMap(map); - Assert.assertEquals(1, weathers.size()); - } + // @Test + // public void testSelectByMap() { + // Map map = new HashMap<>(); + // map.put("location", "beijing"); + // List weathers = mapper.selectByMap(map); + // Assert.assertEquals(1, weathers.size()); + // } @Test public void testSelectObjs() { diff --git a/examples/JDBC/readme.md b/examples/JDBC/readme.md index 9a017f4fea..c7d7875308 100644 --- a/examples/JDBC/readme.md +++ b/examples/JDBC/readme.md @@ -10,4 +10,4 @@ | 6 | taosdemo | This is an internal tool for testing Our JDBC-JNI, JDBC-RESTful, RESTful interfaces | -more detail: https://www.taosdata.com/cn//documentation20/connector-java/ \ No newline at end of file +more detail: https://docs.taosdata.com/reference/connector/java/ \ No newline at end of file diff --git a/examples/JDBC/springbootdemo/pom.xml b/examples/JDBC/springbootdemo/pom.xml index 9126813b67..ee15f6013e 100644 --- a/examples/JDBC/springbootdemo/pom.xml +++ b/examples/JDBC/springbootdemo/pom.xml @@ -68,7 +68,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.34 + 3.0.0 diff --git a/examples/JDBC/springbootdemo/readme.md b/examples/JDBC/springbootdemo/readme.md index 67a28947d2..a3942a6a51 100644 --- a/examples/JDBC/springbootdemo/readme.md +++ b/examples/JDBC/springbootdemo/readme.md @@ -1,10 +1,11 @@ ## TDengine SpringBoot + Mybatis Demo +## 需要提前创建 test 数据库 ### 配置 application.properties ```properties # datasource config spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver -spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/log +spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test spring.datasource.username=root spring.datasource.password=taosdata diff --git a/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java b/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index ed720fe6c0..3ee5b597ab 100644 --- a/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java +++ b/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -6,7 +6,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import java.util.List; -import java.util.Map; @RequestMapping("/weather") @RestController diff --git a/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml b/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml index 91938ca24e..99d5893ec1 100644 --- a/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml +++ b/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -10,8 +10,7 @@ diff --git a/examples/JDBC/springbootdemo/src/main/resources/application.properties b/examples/JDBC/springbootdemo/src/main/resources/application.properties index 06daa81bbb..bf21047395 100644 --- a/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -5,7 +5,7 @@ #spring.datasource.password=taosdata # datasource config - JDBC-RESTful spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver -spring.datasource.url=jdbc:TAOS-RS://localhsot:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +spring.datasource.url=jdbc:TAOS-RS://localhost:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 spring.datasource.username=root spring.datasource.password=taosdata spring.datasource.druid.initial-size=5 diff --git a/examples/JDBC/taosdemo/pom.xml b/examples/JDBC/taosdemo/pom.xml index 07fd4a3576..724ecc7407 100644 --- a/examples/JDBC/taosdemo/pom.xml +++ b/examples/JDBC/taosdemo/pom.xml @@ -67,7 +67,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.20 + 3.0.0 diff --git a/examples/JDBC/taosdemo/readme.md b/examples/JDBC/taosdemo/readme.md index 451fa2960a..e5f4eb132b 100644 --- a/examples/JDBC/taosdemo/readme.md +++ b/examples/JDBC/taosdemo/readme.md @@ -2,9 +2,9 @@ cd tests/examples/JDBC/taosdemo mvn clean package -Dmaven.test.skip=true # 先建表，再插入的 -java -jar target/taosdemo-2.0-jar-with-dependencies.jar -host [hostname] -database [database] -doCreateTable true -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 +java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host [hostname] -database [database] -doCreateTable true -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 # 不建表，直接插入的 -java -jar target/taosdemo-2.0-jar-with-dependencies.jar -host [hostname] -database [database] -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 +java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host [hostname] -database [database] -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 ``` 需求： diff --git a/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java b/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java index d4f5ff2688..6854054703 100644 --- a/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java +++ b/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java @@ -32,8 +32,10 @@ public class TaosDemoApplication { System.exit(0); } // 初始化 - final DataSource dataSource = DataSourceFactory.getInstance(config.host, config.port, config.user, config.password); - if (config.executeSql != null && !config.executeSql.isEmpty() && !config.executeSql.replaceAll("\\s", "").isEmpty()) { + final DataSource dataSource = DataSourceFactory.getInstance(config.host, config.port, config.user, + config.password); + if (config.executeSql != null && !config.executeSql.isEmpty() + && !config.executeSql.replaceAll("\\s", "").isEmpty()) { Thread task = new Thread(new SqlExecuteTask(dataSource, config.executeSql)); task.start(); try { @@ -55,7 +57,7 @@ public class TaosDemoApplication { databaseParam.put("keep", Integer.toString(config.keep)); databaseParam.put("days", Integer.toString(config.days)); databaseParam.put("replica", Integer.toString(config.replica)); - //TODO: other database parameters + // TODO: other database parameters databaseService.createDatabase(databaseParam); databaseService.useDatabase(config.database); long end = System.currentTimeMillis(); @@ -70,11 +72,13 @@ public class TaosDemoApplication { if (config.database != null && !config.database.isEmpty()) superTableMeta.setDatabase(config.database); } else if (config.numOfFields == 0) { - String sql = "create table " + config.database + "." + config.superTable + " (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"; + String sql = "create table " + config.database + "." + config.superTable + + " (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"; superTableMeta = SuperTableMetaGenerator.generate(sql); } else { // create super table with specified field size and tag size - superTableMeta = SuperTableMetaGenerator.generate(config.database, config.superTable, config.numOfFields, config.prefixOfFields, config.numOfTags, config.prefixOfTags); + superTableMeta = SuperTableMetaGenerator.generate(config.database, config.superTable, config.numOfFields, + config.prefixOfFields, config.numOfTags, config.prefixOfTags); } /**********************************************************************************/ // 建表 @@ -84,7 +88,8 @@ public class TaosDemoApplication { superTableService.create(superTableMeta); if (!config.autoCreateTable) { // 批量建子表 - subTableService.createSubTable(superTableMeta, config.numOfTables, config.prefixOfTable, config.numOfThreadsForCreate); + subTableService.createSubTable(superTableMeta, config.numOfTables, config.prefixOfTable, + config.numOfThreadsForCreate); } } end = System.currentTimeMillis(); @@ -93,7 +98,7 @@ public class TaosDemoApplication { // 插入 long tableSize = config.numOfTables; int threadSize = config.numOfThreadsForInsert; - long startTime = getProperStartTime(config.startTime, config.keep); + long startTime = getProperStartTime(config.startTime, config.days); if (tableSize < threadSize) threadSize = (int) tableSize; @@ -101,13 +106,13 @@ public class TaosDemoApplication { start = System.currentTimeMillis(); // multi threads to insert - int affectedRows = subTableService.insertMultiThreads(superTableMeta, threadSize, tableSize, startTime, gap, config); + int affectedRows = subTableService.insertMultiThreads(superTableMeta, threadSize, tableSize, startTime, gap, + config); end = System.currentTimeMillis(); logger.info("insert " + affectedRows + " rows, time cost: " + (end - start) + " ms"); /**********************************************************************************/ // 查询 - /**********************************************************************************/ // 删除表 if (config.dropTable) { diff --git a/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java b/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java index efabff6afe..ab0a1125d2 100644 --- a/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java +++ b/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java @@ -1,7 +1,5 @@ package com.taosdata.taosdemo.service; -import com.taosdata.jdbc.utils.SqlSyntaxValidator; - import javax.sql.DataSource; import java.sql.*; import java.util.ArrayList; @@ -23,10 +21,6 @@ public class QueryService { Boolean[] ret = new Boolean[sqls.length]; for (int i = 0; i < sqls.length; i++) { ret[i] = true; - if (!SqlSyntaxValidator.isValidForExecuteQuery(sqls[i])) { - ret[i] = false; - continue; - } try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { stmt.executeQuery(sqls[i]); } catch (SQLException e) { diff --git a/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/SqlSpeller.java b/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/SqlSpeller.java index a60f0641d3..7651d1e318 100644 --- a/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/SqlSpeller.java +++ b/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/SqlSpeller.java @@ -15,9 +15,12 @@ public class SqlSpeller { StringBuilder sb = new StringBuilder(); sb.append("create database if not exists ").append(map.get("database")).append(" "); if (map.containsKey("keep")) - sb.append("keep ").append(map.get("keep")).append(" "); - if (map.containsKey("days")) - sb.append("days ").append(map.get("days")).append(" "); + sb.append("keep "); + if (map.containsKey("days")) { + sb.append(map.get("days")).append("d "); + } else { + sb.append(" "); + } if (map.containsKey("replica")) sb.append("replica ").append(map.get("replica")).append(" "); if (map.containsKey("cache")) @@ -29,7 +32,7 @@ public class SqlSpeller { if (map.containsKey("maxrows")) sb.append("maxrows ").append(map.get("maxrows")).append(" "); if (map.containsKey("precision")) - sb.append("precision ").append(map.get("precision")).append(" "); + sb.append("precision '").append(map.get("precision")).append("' "); if (map.containsKey("comp")) sb.append("comp ").append(map.get("comp")).append(" "); if (map.containsKey("walLevel")) @@ -46,11 +49,13 @@ public class SqlSpeller { // create table if not exists xx.xx using xx.xx tags(x,x,x) public static String createTableUsingSuperTable(SubTableMeta subTableMeta) { StringBuilder sb = new StringBuilder(); - sb.append("create table if not exists ").append(subTableMeta.getDatabase()).append(".").append(subTableMeta.getName()).append(" "); - sb.append("using ").append(subTableMeta.getDatabase()).append(".").append(subTableMeta.getSupertable()).append(" "); -// String tagStr = subTableMeta.getTags().stream().filter(Objects::nonNull) -// .map(tagValue -> tagValue.getName() + " '" + tagValue.getValue() + "' ") -// .collect(Collectors.joining(",", "(", ")")); + sb.append("create table if not exists ").append(subTableMeta.getDatabase()).append(".") + .append(subTableMeta.getName()).append(" "); + sb.append("using ").append(subTableMeta.getDatabase()).append(".").append(subTableMeta.getSupertable()) + .append(" "); + // String tagStr = subTableMeta.getTags().stream().filter(Objects::nonNull) + // .map(tagValue -> tagValue.getName() + " '" + tagValue.getValue() + "' ") + // .collect(Collectors.joining(",", "(", ")")); sb.append("tags ").append(tagValues(subTableMeta.getTags())); return sb.toString(); } @@ -63,7 +68,7 @@ public class SqlSpeller { return sb.toString(); } - //f1, f2, f3 + // f1, f2, f3 private static String fieldValues(List fields) { return IntStream.range(0, fields.size()).mapToObj(i -> { if (i == 0) { @@ -73,13 +78,13 @@ public class SqlSpeller { } }).collect(Collectors.joining(",", "(", ")")); -// return fields.stream() -// .filter(Objects::nonNull) -// .map(fieldValue -> "'" + fieldValue.getValue() + "'") -// .collect(Collectors.joining(",", "(", ")")); + // return fields.stream() + // .filter(Objects::nonNull) + // .map(fieldValue -> "'" + fieldValue.getValue() + "'") + // .collect(Collectors.joining(",", "(", ")")); } - //(f1, f2, f3),(f1, f2, f3) + // (f1, f2, f3),(f1, f2, f3) private static String rowValues(List rowValues) { return rowValues.stream().filter(Objects::nonNull) .map(rowValue -> fieldValues(rowValue.getFields())) @@ -89,8 +94,10 @@ public class SqlSpeller { // insert into xx.xxx using xx.xx tags(x,x,x) values(x,x,x),(x,x,x)... public static String insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue) { StringBuilder sb = new StringBuilder(); - sb.append("insert into ").append(subTableValue.getDatabase()).append(".").append(subTableValue.getName()).append(" "); - sb.append("using ").append(subTableValue.getDatabase()).append(".").append(subTableValue.getSupertable()).append(" "); + sb.append("insert into ").append(subTableValue.getDatabase()).append(".").append(subTableValue.getName()) + .append(" "); + sb.append("using ").append(subTableValue.getDatabase()).append(".").append(subTableValue.getSupertable()) + .append(" "); sb.append("tags ").append(tagValues(subTableValue.getTags()) + " "); sb.append("values ").append(rowValues(subTableValue.getValues())); return sb.toString(); @@ -126,7 +133,8 @@ public class SqlSpeller { // create table if not exists xx.xx (f1 xx,f2 xx...) tags(t1 xx, t2 xx...) public static String createSuperTable(SuperTableMeta tableMetadata) { StringBuilder sb = new StringBuilder(); - sb.append("create table if not exists ").append(tableMetadata.getDatabase()).append(".").append(tableMetadata.getName()); + sb.append("create table if not exists ").append(tableMetadata.getDatabase()).append(".") + .append(tableMetadata.getName()); String fields = tableMetadata.getFields().stream() .filter(Objects::nonNull).map(field -> field.getName() + " " + field.getType() + " ") .collect(Collectors.joining(",", "(", ")")); @@ -139,10 +147,10 @@ public class SqlSpeller { return sb.toString(); } - public static String createTable(TableMeta tableMeta) { StringBuilder sb = new StringBuilder(); - sb.append("create table if not exists ").append(tableMeta.getDatabase()).append(".").append(tableMeta.getName()).append(" "); + sb.append("create table if not exists ").append(tableMeta.getDatabase()).append(".").append(tableMeta.getName()) + .append(" "); String fields = tableMeta.getFields().stream() .filter(Objects::nonNull).map(field -> field.getName() + " " + field.getType() + " ") .collect(Collectors.joining(",", "(", ")")); @@ -179,16 +187,17 @@ public class SqlSpeller { public static String insertMultiTableMultiValuesWithColumns(List tables) { StringBuilder sb = new StringBuilder(); sb.append("insert into ").append(tables.stream().filter(Objects::nonNull) - .map(table -> table.getDatabase() + "." + table.getName() + " " + columnNames(table.getColumns()) + " values " + rowValues(table.getValues())) + .map(table -> table.getDatabase() + "." + table.getName() + " " + columnNames(table.getColumns()) + + " values " + rowValues(table.getValues())) .collect(Collectors.joining(" "))); return sb.toString(); } public static String insertMultiTableMultiValues(List tables) { StringBuilder sb = new StringBuilder(); - sb.append("insert into ").append(tables.stream().filter(Objects::nonNull).map(table -> - table.getDatabase() + "." + table.getName() + " values " + rowValues(table.getValues()) - ).collect(Collectors.joining(" "))); + sb.append("insert into ").append(tables.stream().filter(Objects::nonNull) + .map(table -> table.getDatabase() + "." + table.getName() + " values " + rowValues(table.getValues())) + .collect(Collectors.joining(" "))); return sb.toString(); } } diff --git a/examples/JDBC/taosdemo/src/main/resources/application.properties b/examples/JDBC/taosdemo/src/main/resources/application.properties index 488185196f..4f550f6523 100644 --- a/examples/JDBC/taosdemo/src/main/resources/application.properties +++ b/examples/JDBC/taosdemo/src/main/resources/application.properties @@ -1,5 +1,5 @@ -jdbc.driver=com.taosdata.jdbc.rs.RestfulDriver -#jdbc.driver=com.taosdata.jdbc.TSDBDriver +# jdbc.driver=com.taosdata.jdbc.rs.RestfulDriver +jdbc.driver=com.taosdata.jdbc.TSDBDriver hikari.maximum-pool-size=20 hikari.minimum-idle=20 hikari.max-lifetime=0 \ No newline at end of file diff --git a/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java b/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java deleted file mode 100644 index 1f52198d68..0000000000 --- a/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.taosdata.taosdemo.service; - -import com.taosdata.taosdemo.domain.TableMeta; -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -public class TableServiceTest { - private TableService tableService; - - private List tables; - - @Before - public void before() { - tables = new ArrayList<>(); - for (int i = 0; i < 1; i++) { - TableMeta tableMeta = new TableMeta(); - tableMeta.setDatabase("test"); - tableMeta.setName("weather" + (i + 1)); - tables.add(tableMeta); - } - } - - @Test - public void testCreate() { - tableService.create(tables); - } - -} \ No newline at end of file From 25b735f0e3c101fedd7c140f4e7bff9ebe9410ba Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 22 Aug 2022 14:05:19 +0800 Subject: [PATCH 20/55] feat(stream): delete result --- include/libs/function/function.h | 1 + source/libs/executor/src/executil.c | 1 + source/libs/executor/src/timewindowoperator.c | 43 +++++++++++-------- tests/script/tsim/stream/state0.sim | 6 +-- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index e708a2c42d..d5da306fd2 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -142,6 +142,7 @@ typedef struct SqlFunctionCtx { struct SSDataBlock *pDstBlock; // used by indifinite rows function to set selectivity int32_t curBufPage; bool increase; + bool isStream; char udfName[TSDB_FUNC_NAME_LEN]; } SqlFunctionCtx; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index bf969bf2e4..f3b395cc7c 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -987,6 +987,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->end.key = INT64_MIN; pCtx->numOfParams = pExpr->base.numOfParams; pCtx->increase = false; + pCtx->isStream = false; pCtx->param = pFunct->pParam; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9eaab69633..0594a727fc 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1793,6 +1793,12 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor pScanInfo->sessionSup.pIntervalAggSup = pSup; } +void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) { + for (int32_t i = 0; i < numOfExpr; i++) { + pCtx[i].isStream = true; + } +} + SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, @@ -1835,6 +1841,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* if (isStream) { ASSERT(numOfCols > 0); increaseTs(pSup->pCtx); + initStreamFunciton(pSup->pCtx, pSup->numOfExprs); } initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); @@ -3329,6 +3336,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs); initBasicInfo(&pInfo->binfo, pResBlock); ASSERT(numOfCols > 0); @@ -3470,6 +3478,7 @@ int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* if (code != TSDB_CODE_SUCCESS) { return code; } + initStreamFunciton(pSup->pCtx, pSup->numOfExprs); initBasicInfo(pBasicInfo, pResultBlock); @@ -4569,8 +4578,8 @@ SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_ return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol); } -int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, SColumnInfoData* pKeyCol, int32_t rows, - int32_t start, bool* allEqual, SHashObj* pSeDelete) { +int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, uint64_t groupId, + SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual, SHashObj* pSeDeleted) { *allEqual = true; SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex); for (int32_t i = start; i < rows; ++i) { @@ -4590,9 +4599,10 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, S } } if (pWinInfo->winInfo.win.skey > pTs[i]) { - if (pSeDelete && pWinInfo->winInfo.isOutput) { - taosHashPut(pSeDelete, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &pWinInfo->winInfo.win.skey, - sizeof(TSKEY)); + if (pSeDeleted && pWinInfo->winInfo.isOutput) { + SWinRes res = {.ts = pWinInfo->winInfo.win.skey, .groupId = groupId}; + taosHashPut(pSeDeleted, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &res, + sizeof(SWinRes)); pWinInfo->winInfo.isOutput = false; } pWinInfo->winInfo.win.skey = pTs[i]; @@ -4605,22 +4615,23 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, S return rows - start; } -static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int32_t tsIndex, SColumn* pCol, - int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) { +static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, + int32_t tsIndex, SColumn* pCol, int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) { SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); SColumnInfoData* pKeyColInfo = taosArrayGet(pBlock->pDataBlock, keyIndex); TSKEY* tsCol = (TSKEY*)pTsColInfo->pData; bool allEqual = false; int32_t step = 1; + uint64_t groupId = pBlock->info.groupId; for (int32_t i = 0; i < pBlock->info.rows; i += step) { char* pKeyData = colDataGetData(pKeyColInfo, i); int32_t winIndex = 0; - SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], pBlock->info.groupId, &winIndex); + SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], groupId, &winIndex); if (!pCurWin) { continue; } - step = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual, - pSeDeleted); + step = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCol, groupId, pKeyColInfo, + pBlock->info.rows, i, &allEqual, pSeDeleted); ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData)); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo); @@ -4659,12 +4670,12 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t winIndex = 0; bool allEqual = true; SStateWindowInfo* pCurWin = - getStateWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, pKeyData, &pInfo->stateCol, &winIndex); - winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo, pSDataBlock->info.rows, i, - &allEqual, pInfo->pSeDeleted); + getStateWindow(pAggSup, tsCols[i], groupId, pKeyData, &pInfo->stateCol, &winIndex); + winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, groupId, pKeyColInfo, + pSDataBlock->info.rows, i, &allEqual, pStDeleted); if (!allEqual) { appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, - &pSDataBlock->info.groupId); + &groupId); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo); continue; @@ -4828,9 +4839,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; - // pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); - pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false); // todo(liuyao) for delete - pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; // todo(liuyao) for delete + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pChildren = NULL; pInfo->ignoreExpiredData = pStateNode->window.igExpired; diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 4fa883b813..877a2877b9 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -5,15 +5,15 @@ sleep 50 sql connect print =============== create database -sql create database test vgroups 1 -sql select * from information_schema.ins_databases +sql create database test vgroups 1; +sql select * from information_schema.ins_databases; if $rows != 3 then return -1 endi print $data00 $data01 $data02 -sql use test +sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double, id int); sql create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); From dd212b5dbbf6c9f85de92d79a406c379b9996f62 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 22 Aug 2022 14:25:36 +0800 Subject: [PATCH 21/55] refactor(sync): modify config file len --- source/libs/sync/src/syncRaftCfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 5de21bceca..ab404d1b9a 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -171,7 +171,7 @@ SRaftCfg *raftCfgOpen(const char *path) { taosLSeekFile(pCfg->pFile, 0, SEEK_SET); - char buf[1024] = {0}; + char buf[CONFIG_FILE_LEN] = {0}; int len = taosReadFile(pCfg->pFile, buf, sizeof(buf)); ASSERT(len > 0); From 4322f59a221d032e21c429fcc29ba166f4844f88 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Aug 2022 14:28:13 +0800 Subject: [PATCH 22/55] fix(query): fix bug in descending order scan in lastblock. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 102 +++++++++++++++---------- 1 file changed, 60 insertions(+), 42 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 12ac0c2f1e..2e66cac21e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1315,7 +1315,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc // log the reason why load the datablock for profile if (loadDataBlock) { tsdbDebug("%p uid:%" PRIu64 - " need to load the datablock, reason overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, " + " need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, " "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s", pReader, pFBlock->uid, overlapWithNeighbor, hasDup, partiallyRequired, overlapWithKey, moreThanOutputCapacity, overlapWithDel, overlapWithlastBlock, pReader->idStr); @@ -2007,7 +2007,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI if (pBlockData->nRow > 0) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - // no last block + // no last block available, only data block exists if (pLastBlockReader->lastBlockData.nRow == 0 || (!hasDataInLastBlock(pLastBlockReader))) { if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { return TSDB_CODE_SUCCESS; @@ -2028,54 +2028,63 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI // row in last file block int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader); - if (ts < key) { // save rows in last block - SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + ASSERT(ts >= key); - STSRow* pTSRow = NULL; - SRowMerger merge = {0}; + if (ASCENDING_TRAVERSE(pReader->order)) { + if (key < ts) { + // imem & mem are all empty, only file exist + if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + return TSDB_CODE_SUCCESS; + } else { + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); - tRowMergerInit(&merge, &fRow1, pReader->pSchema); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); - tRowMergerGetRow(&merge, &pTSRow); - - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); - - taosMemoryFree(pTSRow); - tRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; - } else if (ts == key) { - STSRow* pTSRow = NULL; - SRowMerger merge = {0}; - - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); - - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); - - taosMemoryFree(pTSRow); - tRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; - } else { // ts > key, asc; todo handle desc - // imem & mem are all empty, only file exist - if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { - return TSDB_CODE_SUCCESS; - } else { + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; + } + } else if (key == ts) { STSRow* pTSRow = NULL; SRowMerger merge = {0}; tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); + tRowMergerGetRow(&merge, &pTSRow); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); taosMemoryFree(pTSRow); tRowMergerClear(&merge); return TSDB_CODE_SUCCESS; + } else { + ASSERT(0); + return TSDB_CODE_SUCCESS; } + } else { // desc order + SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + + STSRow* pTSRow = NULL; + SRowMerger merge = {0}; + tRowMergerInit(&merge, &fRow1, pReader->pSchema); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); + + if (ts == key) { + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + } + + tRowMergerGetRow(&merge, &pTSRow); + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + return TSDB_CODE_SUCCESS; } } else { // only last block exists SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; @@ -2575,13 +2584,22 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // todo rows in buffer should be less than the file block in asc, greater than file block in desc int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); - } else { // whole block is required, return it directly - SDataBlockInfo* pInfo = &pReader->pResBlock->info; - pInfo->rows = pBlock->nRow; - pInfo->uid = pScanInfo->uid; - pInfo->window = (STimeWindow){.skey = pBlock->minKey.ts, .ekey = pBlock->maxKey.ts}; - setComposedBlockFlag(pReader, false); - setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order); + } else { + if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { + // only return the rows in last block + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + ASSERT (tsLast >= pBlock->maxKey.ts); + tBlockDataReset(&pReader->status.fileBlockData); + + code = buildComposedDataBlock(pReader); + } else { // whole block is required, return it directly + SDataBlockInfo* pInfo = &pReader->pResBlock->info; + pInfo->rows = pBlock->nRow; + pInfo->uid = pScanInfo->uid; + pInfo->window = (STimeWindow){.skey = pBlock->minKey.ts, .ekey = pBlock->maxKey.ts}; + setComposedBlockFlag(pReader, false); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order); + } } return code; From 3d6aa3435ffe2cd87eb749d3d8c70c34abb729cb Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Mon, 22 Aug 2022 14:30:44 +0800 Subject: [PATCH 23/55] ci: if all changed files are in docs/en or docs/zh, make it a docs PR and skip tests --- Jenkinsfile2 | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 12e806c87a..c6b54b1174 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -1,6 +1,7 @@ import hudson.model.Result import hudson.model.*; import jenkins.model.CauseOfInterruption +docs_only=0 node { } @@ -29,6 +30,48 @@ def abort_previous(){ if (buildNumber > 1) milestone(buildNumber - 1) milestone(buildNumber) } +def check_docs() { + if (env.CHANGE_URL =~ /\/TDengine\//) { + sh ''' + hostname + date + env + ''' + sh ''' + cd ${WKC} + git reset --hard + git clean -fxd + rm -rf examples/rust/ + git remote prune origin + git fetch + ''' + script { + sh ''' + cd ${WKC} + git checkout ''' + env.CHANGE_TARGET + ''' + ''' + } + sh ''' + cd ${WKC} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + ''' + def file_changed = sh ( + script: ''' + cd ${WKC} + git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/" + ''', + returnStdout: true + ).trim() + if (file_changed == '') { + echo "docs PR" + docs_only=1 + } else { + echo file_changed + } + } +} def pre_test(){ sh ''' hostname @@ -307,10 +350,25 @@ pipeline { WKPY = '/var/lib/jenkins/workspace/taos-connector-python' } stages { + stage('check') { + when { + allOf { + not { expression { env.CHANGE_BRANCH =~ /docs\// }} + not { expression { env.CHANGE_URL =~ /\/TDinternal\// }} + } + } + parallel { + stage('check docs') { + agent{label " worker03 || slave215 || slave217 || slave219 || Mac_catalina "} + check_docs() + } + } + } stage('run test') { when { allOf { not { expression { env.CHANGE_BRANCH =~ /docs\// }} + expression { docs_only == 0 } } } parallel { From 9ad81089c1c1fe2cc7ef030268e38b264d3034f5 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Mon, 22 Aug 2022 14:43:22 +0800 Subject: [PATCH 24/55] ci: add steps to check docs stage --- Jenkinsfile2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index c6b54b1174..98d7a5a731 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -360,7 +360,9 @@ pipeline { parallel { stage('check docs') { agent{label " worker03 || slave215 || slave217 || slave219 || Mac_catalina "} - check_docs() + steps { + check_docs() + } } } } From 23d8ebf1298e7da3a51630e59b1b49659f99701a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 Aug 2022 15:15:03 +0800 Subject: [PATCH 25/55] fix:error in grou by tag --- source/libs/executor/src/executil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 66da50f2d5..39b2bcef4b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -711,7 +711,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis pStart += varDataTLen(data); } else { memcpy(pStart, data, pValue->info.bytes); - pStart += pValue->info.type; + pStart += pValue->info.bytes; } } } From 70df538180115b0f8f8c9e0d951083c61f2d1a04 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 22 Aug 2022 15:22:49 +0800 Subject: [PATCH 26/55] fix(query): fix pthread_create memleak in shell engine TD-17791 --- tools/shell/inc/shellInt.h | 1 + tools/shell/src/shellEngine.c | 15 ++++++++++++--- tools/shell/src/shellUtil.c | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 358377f804..26ca6895ac 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -95,6 +95,7 @@ typedef struct { TAOS* conn; TdThread pid; tsem_t cancelSem; + bool exit; #ifdef WEBSOCKET WS_TAOS* ws_conn; bool stop_query; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 724ac8fbfd..68e3a272c3 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -948,6 +948,10 @@ void shellCleanup(void *arg) { taosResetTerminalMode(); } void *shellCancelHandler(void *arg) { setThreadName("shellCancelHandler"); while (1) { + if (shell.exit == true) { + break; + } + if (tsem_wait(&shell.cancelSem) != 0) { taosMsleep(10); continue; @@ -961,7 +965,7 @@ void *shellCancelHandler(void *arg) { taos_kill_query(shell.conn); #ifdef WEBSOCKET } -#endif +#endif #ifdef WINDOWS printf("\n%s", shell.info.promptHeader); #endif @@ -1009,7 +1013,7 @@ int32_t shellExecute() { if (shell.args.restful || shell.args.cloud) { if (shell_conn_ws_server(1)) { return -1; - } + } } else { #endif if (shell.args.auth == NULL) { @@ -1043,7 +1047,7 @@ int32_t shellExecute() { if (shell.args.restful || shell.args.cloud) { ws_close(shell.ws_conn); } else { -#endif +#endif taos_close(shell.conn); #ifdef WEBSOCKET } @@ -1079,7 +1083,12 @@ int32_t shellExecute() { taosThreadCreate(&shell.pid, NULL, shellThreadLoop, NULL); taosThreadJoin(shell.pid, NULL); taosThreadClear(&shell.pid); + if (shell.exit) { + tsem_post(&shell.cancelSem); + break; + } } + taosThreadJoin(spid, NULL); shellCleanupHistory(); return 0; diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index e5e61e0b24..0430428c38 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -157,6 +157,6 @@ void shellExit() { taos_close(shell.conn); shell.conn = NULL; } + shell.exit = true; taos_cleanup(); - exit(EXIT_FAILURE); } From 6953c8a5e1e0a61d16aeedfbe3e824d7584c3120 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 15:31:39 +0800 Subject: [PATCH 27/55] doc: remove useless 2.4 version information --- docs/en/07-develop/01-connect/index.md | 2 +- docs/en/13-operation/01-pkg-install.md | 44 ++-- docs/en/14-reference/03-connector/php.mdx | 2 +- docs/en/14-reference/04-taosadapter.md | 2 +- docs/en/14-reference/12-config/index.md | 24 +- docs/en/27-train-faq/03-docker.md | 285 ---------------------- 6 files changed, 29 insertions(+), 330 deletions(-) delete mode 100644 docs/en/27-train-faq/03-docker.md diff --git a/docs/en/07-develop/01-connect/index.md b/docs/en/07-develop/01-connect/index.md index 017a1a0ee4..2053706421 100644 --- a/docs/en/07-develop/01-connect/index.md +++ b/docs/en/07-develop/01-connect/index.md @@ -223,7 +223,7 @@ phpize && ./configure && make -j && make install **Specify TDengine Location：** ```shell -phpize && ./configure --with-tdengine-dir=/usr/local/Cellar/tdengine/2.4.0.0 && make -j && make install +phpize && ./configure --with-tdengine-dir=/usr/local/Cellar/tdengine/3.0.0.0 && make -j && make install ``` > `--with-tdengine-dir=` is followed by the TDengine installation location. diff --git a/docs/en/13-operation/01-pkg-install.md b/docs/en/13-operation/01-pkg-install.md index c098002962..b0f607170d 100644 --- a/docs/en/13-operation/01-pkg-install.md +++ b/docs/en/13-operation/01-pkg-install.md @@ -13,16 +13,16 @@ TDengine community version provides deb and rpm packages for users to choose fro -1. Download deb package from official website, for example TDengine-server-2.4.0.7-Linux-x64.deb +1. Download deb package from official website, for example TDengine-server-3.0.0.0-Linux-x64.deb 2. In the directory where the package is located, execute the command below ```bash -$ sudo dpkg -i TDengine-server-2.4.0.7-Linux-x64.deb +$ sudo dpkg -i TDengine-server-3.0.0.0-Linux-x64.deb (Reading database ... 137504 files and directories currently installed.) -Preparing to unpack TDengine-server-2.4.0.7-Linux-x64.deb ... +Preparing to unpack TDengine-server-3.0.0.0-Linux-x64.deb ... TDengine is removed successfully! -Unpacking tdengine (2.4.0.7) over (2.4.0.7) ... -Setting up tdengine (2.4.0.7) ... +Unpacking tdengine (3.0.0.0) over (3.0.0.0) ... +Setting up tdengine (3.0.0.0) ... Start to install TDengine... System hostname is: ubuntu-1804 @@ -45,14 +45,14 @@ TDengine is installed successfully! -1. Download rpm package from official website, for example TDengine-server-2.4.0.7-Linux-x64.rpm； +1. Download rpm package from official website, for example TDengine-server-3.0.0.0-Linux-x64.rpm； 2. In the directory where the package is located, execute the command below ``` -$ sudo rpm -ivh TDengine-server-2.4.0.7-Linux-x64.rpm +$ sudo rpm -ivh TDengine-server-3.0.0.0-Linux-x64.rpm Preparing... ################################# [100%] Updating / installing... - 1:tdengine-2.4.0.7-3 ################################# [100%] + 1:tdengine-3.0.0.0-3 ################################# [100%] Start to install TDengine... System hostname is: centos7 @@ -76,27 +76,27 @@ TDengine is installed successfully! -1. Download the tar.gz package, for example TDengine-server-2.4.0.7-Linux-x64.tar.gz； -2. In the directory where the package is located, first decompress the file, then switch to the sub-directory generated in decompressing, i.e. "TDengine-enterprise-server-2.4.0.7/" in this example, and execute the `install.sh` script. +1. Download the tar.gz package, for example TDengine-server-3.0.0.0-Linux-x64.tar.gz； +2. In the directory where the package is located, first decompress the file, then switch to the sub-directory generated in decompressing, i.e. "TDengine-enterprise-server-3.0.0.0/" in this example, and execute the `install.sh` script. ```bash -$ tar xvzf TDengine-enterprise-server-2.4.0.7-Linux-x64.tar.gz -TDengine-enterprise-server-2.4.0.7/ -TDengine-enterprise-server-2.4.0.7/driver/ -TDengine-enterprise-server-2.4.0.7/driver/vercomp.txt -TDengine-enterprise-server-2.4.0.7/driver/libtaos.so.2.4.0.7 -TDengine-enterprise-server-2.4.0.7/install.sh -TDengine-enterprise-server-2.4.0.7/examples/ +$ tar xvzf TDengine-enterprise-server-3.0.0.0-Linux-x64.tar.gz +TDengine-enterprise-server-3.0.0.0/ +TDengine-enterprise-server-3.0.0.0/driver/ +TDengine-enterprise-server-3.0.0.0/driver/vercomp.txt +TDengine-enterprise-server-3.0.0.0/driver/libtaos.so.3.0.0.0 +TDengine-enterprise-server-3.0.0.0/install.sh +TDengine-enterprise-server-3.0.0.0/examples/ ... $ ll total 43816 drwxrwxr-x 3 ubuntu ubuntu 4096 Feb 22 09:31 ./ drwxr-xr-x 20 ubuntu ubuntu 4096 Feb 22 09:30 ../ -drwxrwxr-x 4 ubuntu ubuntu 4096 Feb 22 09:30 TDengine-enterprise-server-2.4.0.7/ --rw-rw-r-- 1 ubuntu ubuntu 44852544 Feb 22 09:31 TDengine-enterprise-server-2.4.0.7-Linux-x64.tar.gz +drwxrwxr-x 4 ubuntu ubuntu 4096 Feb 22 09:30 TDengine-enterprise-server-3.0.0.0/ +-rw-rw-r-- 1 ubuntu ubuntu 44852544 Feb 22 09:31 TDengine-enterprise-server-3.0.0.0-Linux-x64.tar.gz -$ cd TDengine-enterprise-server-2.4.0.7/ +$ cd TDengine-enterprise-server-3.0.0.0/ $ ll total 40784 @@ -146,7 +146,7 @@ Deb package of TDengine can be uninstalled as below: ```bash $ sudo dpkg -r tdengine (Reading database ... 137504 files and directories currently installed.) -Removing tdengine (2.4.0.7) ... +Removing tdengine (3.0.0.0) ... TDengine is removed successfully! ``` @@ -245,7 +245,7 @@ For example, if using `systemctl` , the commands to start, stop, restart and che - Check server status：`systemctl status taosd` -From version 2.4.0.0, a new independent component named as `taosAdapter` has been included in TDengine. `taosAdapter` should be started and stopped using `systemctl`. +Another component named as `taosAdapter` is to provide HTTP service for TDengine, it should be started and stopped using `systemctl`. If the server process is OK, the output of `systemctl status` is like below: diff --git a/docs/en/14-reference/03-connector/php.mdx b/docs/en/14-reference/03-connector/php.mdx index 69dcce91e8..9ee89d468a 100644 --- a/docs/en/14-reference/03-connector/php.mdx +++ b/docs/en/14-reference/03-connector/php.mdx @@ -61,7 +61,7 @@ phpize && ./configure && make -j && make install **Specify TDengine location：** ```shell -phpize && ./configure --with-tdengine-dir=/usr/local/Cellar/tdengine/2.4.0.0 && make -j && make install +phpize && ./configure --with-tdengine-dir=/usr/local/Cellar/tdengine/3.0.0.0 && make -j && make install ``` > `--with-tdengine-dir=` is followed by TDengine location. diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index dc47246e20..31310b0f3e 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -30,7 +30,7 @@ taosAdapter provides the following features. ### Install taosAdapter -taosAdapter has been part of TDengine server software since TDengine v2.4.0.0. If you use the TDengine server, you don't need additional steps to install taosAdapter. You can download taosAdapter from [TDengine official website](https://tdengine.com/all-downloads/) to download the TDengine server installation package (taosAdapter is included in v2.4.0.0 and later version). If you need to deploy taosAdapter separately on another server other than the TDengine server, you should install the full TDengine server package on that server to install taosAdapter. If you need to build taosAdapter from source code, you can refer to the [Building taosAdapter]( https://github.com/taosdata/taosadapter/blob/3.0/BUILD.md) documentation. +If you use the TDengine server, you don't need additional steps to install taosAdapter. You can download taosAdapter from [TDengine 3.0 released versions](../../releases) to download the TDengine server installation package. If you need to deploy taosAdapter separately on another server other than the TDengine server, you should install the full TDengine server package on that server to install taosAdapter. If you need to build taosAdapter from source code, you can refer to the [Building taosAdapter]( https://github.com/taosdata/taosadapter/blob/3.0/BUILD.md) documentation. ### Start/Stop taosAdapter diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index b6b535429b..cb7daf3c47 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -75,7 +75,6 @@ taos --dump-config | Applicable | Server Only | | Meaning | The port for external access after `taosd` is started | | Default Value | 6030 | -| Note | REST service is provided by `taosd` before 2.4.0.0 but by `taosAdapter` after 2.4.0.0, the default port of REST service is 6041 | :::note TDengine uses 13 continuous ports, both TCP and UDP, starting with the port specified by `serverPort`. You should ensure, in your firewall rules, that these ports are kept open. Below table describes the ports used by TDengine in details. @@ -87,11 +86,11 @@ TDengine uses 13 continuous ports, both TCP and UDP, starting with the port spec | TCP | 6030 | Communication between client and server | serverPort | | TCP | 6035 | Communication among server nodes in cluster | serverPort+5 | | TCP | 6040 | Data syncup among server nodes in cluster | serverPort+10 | -| TCP | 6041 | REST connection between client and server | Prior to 2.4.0.0: serverPort+11; After 2.4.0.0 refer to [taosAdapter](/reference/taosadapter/) | +| TCP | 6041 | REST connection between client and server | Please refer to [taosAdapter](../taosadapter/) | | TCP | 6042 | Service Port of Arbitrator | The parameter of Arbitrator | | TCP | 6043 | Service Port of TaosKeeper | The parameter of TaosKeeper | -| TCP | 6044 | Data access port for StatsD | refer to [taosAdapter](/reference/taosadapter/) | -| UDP | 6045 | Data access for statsd | refer to [taosAdapter](/reference/taosadapter/) | +| TCP | 6044 | Data access port for StatsD | refer to [taosAdapter](../taosadapter/) | +| UDP | 6045 | Data access for statsd | refer to [taosAdapter](../taosadapter/) | | TCP | 6060 | Port of Monitoring Service in Enterprise version | | | UDP | 6030-6034 | Communication between client and server | serverPort | | UDP | 6035-6039 | Communication among server nodes in cluster | serverPort | @@ -777,12 +776,6 @@ To prevent system resource from being exhausted by multiple concurrent streams, ## HTTP Parameters -:::note -HTTP service was provided by `taosd` prior to version 2.4.0.0 and is provided by `taosAdapter` after version 2.4.0.0. -The parameters described in this section are only application in versions prior to 2.4.0.0. If you are using any version from 2.4.0.0, please refer to [taosAdapter](/reference/taosadapter/). - -::: - ### http | Attribute | Description | @@ -980,16 +973,7 @@ The parameters described in this section are only application in versions prior | Applicable | Server and Client | | Meaning | Log level of common module | | Value Range | Same as debugFlag | -| Default Value | | - -### httpDebugFlag - -| Attribute | Description | -| ------------- | ------------------------------------------- | -| Applicable | Server Only | -| Meaning | Log level of http module (prior to 2.4.0.0) | -| Value Range | Same as debugFlag | -| Default Value | | +| Default Value | | | ### mqttDebugFlag diff --git a/docs/en/27-train-faq/03-docker.md b/docs/en/27-train-faq/03-docker.md deleted file mode 100644 index 0378fffb8b..0000000000 --- a/docs/en/27-train-faq/03-docker.md +++ /dev/null @@ -1,285 +0,0 @@ ---- -sidebar_label: TDengine in Docker -title: Deploy TDengine in Docker ---- - -We do not recommend deploying TDengine using Docker in a production system. However, Docker is still very useful in a development environment, especially when your host is not Linux. From version 2.0.14.0, the official image of TDengine can support X86-64, X86, arm64, and rm32 . - -In this chapter we introduce a simple step by step guide to use TDengine in Docker. - -## Install Docker - -To install Docker please refer to [Get Docker](https://docs.docker.com/get-docker/). - -After Docker is installed, you can check whether Docker is installed properly by displaying Docker version. - -```bash -$ docker -v -Docker version 20.10.3, build 48d30b5 -``` - -## Launch TDengine in Docker - -### Launch TDengine Server - -```bash -$ docker run -d -p 6030-6049:6030-6049 -p 6030-6049:6030-6049/udp tdengine/tdengine -526aa188da767ae94b244226a2b2eec2b5f17dd8eff592893d9ec0cd0f3a1ccd -``` - -In the above command, a docker container is started to run TDengine server, the port range 6030-6049 of the container is mapped to host port range 6030-6049. If port range 6030-6049 has been occupied on the host, please change to an available host port range. For port requirements on the host, please refer to [Port Configuration](/reference/config/#serverport). - -- **docker run**: Launch a docker container -- **-d**: the container will run in background mode -- **-p**: port mapping -- **tdengine/tdengine**: The image from which to launch the container -- **526aa188da767ae94b244226a2b2eec2b5f17dd8eff592893d9ec0cd0f3a1ccd**: the container ID if successfully launched. - -Furthermore, `--name` can be used with `docker run` to specify name for the container, `--hostname` can be used to specify hostname for the container, `-v` can be used to mount local volumes to the container so that the data generated inside the container can be persisted to disk on the host. - -```bash -docker run -d --name tdengine --hostname="tdengine-server" -v ~/work/taos/log:/var/log/taos -v ~/work/taos/data:/var/lib/taos -p 6030-6049:6030-6049 -p 6030-6049:6030-6049/udp tdengine/tdengine -``` - -- **--name tdengine**: specify the name of the container, the name can be used to specify the container later -- **--hostname=tdengine-server**: specify the hostname inside the container, the hostname can be used inside the container without worrying the container IP may vary -- **-v**: volume mapping between host and container - -### Check the container - -```bash -docker ps -``` - -The output is like below: - -``` -CONTAINER ID IMAGE COMMAND CREATED STATUS ··· -c452519b0f9b tdengine/tdengine "taosd" 14 minutes ago Up 14 minutes ··· -``` - -- **docker ps**: List all the containers -- **CONTAINER ID**: Container ID -- **IMAGE**: The image used for the container -- **COMMAND**: The command used when launching the container -- **CREATED**: When the container was created -- **STATUS**: Status of the container - -### Access TDengine inside container - -```bash -$ docker exec -it tdengine /bin/bash -root@tdengine-server:~/TDengine-server-2.4.0.4# -``` - -- **docker exec**: Attach to the container -- **-i**: Interactive mode -- **-t**: Use terminal -- **tdengine**: Container name, up to the output of `docker ps` -- **/bin/bash**: The command to execute once the container is attached - -Inside the container, start TDengine CLI `taos` - -```bash -root@tdengine-server:~/TDengine-server-2.4.0.4# taos - -Welcome to the TDengine shell from Linux, Client Version:2.4.0.4 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. - -taos> -``` - -The above example is for a successful connection. If `taos` fails to connect to the server side, error information would be shown. - -In TDengine CLI, SQL commands can be executed to create/drop databases, tables, STables, and insert or query data. For details please refer to [TAOS SQL](/taos-sql/). - -### Access TDengine from host - -If option `-p` used to map ports properly between host and container, it's also able to access TDengine in container from the host as long as `firstEp` is configured correctly for the client on host. - -``` -$ taos - -Welcome to the TDengine shell from Linux, Client Version:2.4.0.4 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. - -taos> -``` - -It's also able to access the REST interface provided by TDengine in container from the host. - -``` -curl -L -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql -``` - -Output is like below: - -``` -{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep0,keep1,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep0,keep1,keep(D)",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[["test","2021-08-18 06:01:11.021",10000,4,1,1,10,"3650,3650,3650",16,6,100,4096,1,3000,2,0,"ms",0,"ready"],["log","2021-08-18 05:51:51.065",4,1,1,1,10,"30,30,30",1,3,100,4096,1,3000,2,0,"us",0,"ready"]],"rows":2} -``` - -For details of REST API please refer to [REST API](/reference/rest-api/). - -### Run TDengine server and taosAdapter inside container - -From version 2.4.0.0, in the TDengine Docker image, `taosAdapter` is enabled by default, but can be disabled using environment variable `TAOS_DISABLE_ADAPTER=true` . `taosAdapter` can also be run alone without `taosd` when launching a container. - -For the port mapping of `taosAdapter`, please refer to [taosAdapter](/reference/taosadapter/). - -- Run both `taosd` and `taosAdapter` (by default) in docker container: - -```bash -docker run -d --name tdengine-all -p 6030-6049:6030-6049 -p 6030-6049:6030-6049/udp tdengine/tdengine:2.4.0.4 -``` - -- Run `taosAdapter` only in docker container, `TAOS_FIRST_EP` environment variable needs to be used to specify the container name in which `taosd` is running: - -```bash -docker run -d --name tdengine-taosa -p 6041-6049:6041-6049 -p 6041-6049:6041-6049/udp -e TAOS_FIRST_EP=tdengine-all tdengine/tdengine:2.4.0.4 taosadapter -``` - -- Run `taosd` only in docker container: - -```bash -docker run -d --name tdengine-taosd -p 6030-6042:6030-6042 -p 6030-6042:6030-6042/udp -e TAOS_DISABLE_ADAPTER=true tdengine/tdengine:2.4.0.4 -``` - -- Verify the REST interface: - -```bash -curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "show databases;" 127.0.0.1:6041/rest/sql -``` - -Below is an example output: - -``` -{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[["log","2021-12-28 09:18:55.765",10,1,1,1,10,"30",1,3,100,4096,1,3000,2,0,"us",0,"ready"]],"rows":1} -``` - -### Use taosBenchmark on host to access TDengine server in container - -1. Run `taosBenchmark`, named as `taosdemo` previously, on the host: - - ```bash - $ taosBenchmark - - taosBenchmark is simulating data generated by power equipments monitoring... - - host: 127.0.0.1:6030 - user: root - password: taosdata - configDir: - resultFile: ./output.txt - thread num of insert data: 10 - thread num of create table: 10 - top insert interval: 0 - number of records per req: 30000 - max sql length: 1048576 - database count: 1 - database[0]: - database[0] name: test - drop: yes - replica: 1 - precision: ms - super table count: 1 - super table[0]: - stbName: meters - autoCreateTable: no - childTblExists: no - childTblCount: 10000 - childTblPrefix: d - dataSource: rand - iface: taosc - insertRows: 10000 - interlaceRows: 0 - disorderRange: 1000 - disorderRatio: 0 - maxSqlLen: 1048576 - timeStampStep: 1 - startTimestamp: 2017-07-14 10:40:00.000 - sampleFormat: - sampleFile: - tagsFile: - columnCount: 3 - column[0]:FLOAT column[1]:INT column[2]:FLOAT - tagCount: 2 - tag[0]:INT tag[1]:BINARY(16) - - Press enter key to continue or Ctrl-C to stop - ``` - - Once the execution is finished, a database `test` is created, a STable `meters` is created in database `test`, 10,000 sub tables are created using `meters` as template, named as "d0" to "d9999", while 10,000 rows are inserted into each table, so totally 100,000,000 rows are inserted. - -2. Check the data - - - **Check database** - - ```bash - $ taos> show databases; - name | created_time | ntables | vgroups | ··· - test | 2021-08-18 06:01:11.021 | 10000 | 6 | ··· - log | 2021-08-18 05:51:51.065 | 4 | 1 | ··· - - ``` - - - **Check STable** - - ```bash - $ taos> use test; - Database changed. - - $ taos> show stables; - name | created_time | columns | tags | tables | - ============================================================================================ - meters | 2021-08-18 06:01:11.116 | 4 | 2 | 10000 | - Query OK, 1 row(s) in set (0.003259s) - - ``` - - - **Check Tables** - - ```bash - $ taos> select * from test.t0 limit 10; - - DB error: Table does not exist (0.002857s) - taos> select * from test.d0 limit 10; - ts | current | voltage | phase | - ====================================================================================== - 2017-07-14 10:40:00.000 | 10.12072 | 223 | 0.34167 | - 2017-07-14 10:40:00.001 | 10.16103 | 224 | 0.34445 | - 2017-07-14 10:40:00.002 | 10.00204 | 220 | 0.33334 | - 2017-07-14 10:40:00.003 | 10.00030 | 220 | 0.33333 | - 2017-07-14 10:40:00.004 | 9.84029 | 216 | 0.32222 | - 2017-07-14 10:40:00.005 | 9.88028 | 217 | 0.32500 | - 2017-07-14 10:40:00.006 | 9.88110 | 217 | 0.32500 | - 2017-07-14 10:40:00.007 | 10.08137 | 222 | 0.33889 | - 2017-07-14 10:40:00.008 | 10.12063 | 223 | 0.34167 | - 2017-07-14 10:40:00.009 | 10.16086 | 224 | 0.34445 | - Query OK, 10 row(s) in set (0.016791s) - - ``` - - - **Check tag values of table d0** - - ```bash - $ taos> select groupid, location from test.d0; - groupid | location | - ================================= - 0 | California.SanDiego | - Query OK, 1 row(s) in set (0.003490s) - ``` - -### Access TDengine from 3rd party tools - -A lot of 3rd party tools can be used to write data into TDengine through `taosAdapter`, for details please refer to [3rd party tools](/third-party/). - -There is nothing different from the 3rd party side to access TDengine server inside a container, as long as the end point is specified correctly, the end point should be the FQDN and the mapped port of the host. - -## Stop TDengine inside container - -```bash -docker stop tdengine -``` - -- **docker stop**: stop a container -- **tdengine**: container name From da75122b33c1aacf47c652ced2d75d1599bdb1d3 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 15:32:07 +0800 Subject: [PATCH 28/55] doc: remove useless 2.4 version information --- docs/zh/27-train-faq/01-faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index 59e0d7cae0..04ee011b93 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -187,7 +187,7 @@ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端 ### 17. 为什么 RESTful 接口无响应、Grafana 无法添加 TDengine 为数据源、TDengineGUI 选了 6041 端口还是无法连接成功？ -taosAdapter 从 TDengine 2.4.0.0 版本开始成为 TDengine 服务端软件的组成部分，是 TDengine 集群和应用程序之间的桥梁和适配器。在此之前 RESTful 接口等功能是由 taosd 内置的 HTTP 服务提供的，而如今要实现上述功能需要执行：```systemctl start taosadapter``` 命令来启动 taosAdapter 服务。 +这个现象可能是因为 taosAdapter 没有被正确启动引起的，需要执行：```systemctl start taosadapter``` 命令来启动 taosAdapter 服务。 需要说明的是，taosAdapter 的日志路径 path 需要单独配置，默认路径是 /var/log/taos ；日志等级 logLevel 有 8 个等级，默认等级是 info ，配置成 panic 可关闭日志输出。请注意操作系统 / 目录的空间大小，可通过命令行参数、环境变量或配置文件来修改配置，默认配置文件是 /etc/taos/taosadapter.toml 。 From cb49ec48d0bb6f804a1b6d2ca7de9a55aa6e491d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 22 Aug 2022 15:22:49 +0800 Subject: [PATCH 29/55] fix(query): fix pthread_create memleak in shell engine TD-17791 --- tools/shell/src/shellMain.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 703533f8a9..964082f3c3 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -19,6 +19,7 @@ SShellObj shell = {0}; int main(int argc, char *argv[]) { + shell.exit = false; #ifdef WEBSOCKET shell.args.timeout = 10; shell.args.cloud = true; @@ -46,7 +47,7 @@ int main(int argc, char *argv[]) { shellPrintHelp(); return 0; } -#ifdef WEBSOCKET +#ifdef WEBSOCKET shellCheckConnectMode(); #endif taos_init(); From 364b849ea1319e8505fe2fcfaf8a1bb14aec95c5 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 15:34:13 +0800 Subject: [PATCH 30/55] doc: remove useless 2.4 version information --- docs/en/12-taos-sql/18-escape.md | 5 ----- docs/en/14-reference/12-directory.md | 5 ----- 2 files changed, 10 deletions(-) diff --git a/docs/en/12-taos-sql/18-escape.md b/docs/en/12-taos-sql/18-escape.md index 46ab35a276..a2ae40de98 100644 --- a/docs/en/12-taos-sql/18-escape.md +++ b/docs/en/12-taos-sql/18-escape.md @@ -15,11 +15,6 @@ title: Escape Characters | `\%` | % see below for details | | `\_` | \_ see below for details | -:::note -Escape characters are available from version 2.4.0.4 . - -::: - ## Restrictions 1. If there are escape characters in identifiers (database name, table name, column name) diff --git a/docs/en/14-reference/12-directory.md b/docs/en/14-reference/12-directory.md index 0eaa7843ec..19b036418f 100644 --- a/docs/en/14-reference/12-directory.md +++ b/docs/en/14-reference/12-directory.md @@ -29,11 +29,6 @@ All executable files of TDengine are in the _/usr/local/taos/bin_ directory by d - _set_core.sh_: script for setting up the system to generate core dump files for easy debugging - _taosd-dump-cfg.gdb_: script to facilitate debugging of taosd's gdb execution. -:::note -taosdump after version 2.4.0.0 require taosTools as a standalone installation. A new version of taosBenchmark is include in taosTools too. - -::: - :::tip You can configure different data directories and log directories by modifying the system configuration file `taos.cfg`. From 8da795028242fa837f17139b1c735192eaac7027 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 22 Aug 2022 15:40:28 +0800 Subject: [PATCH 31/55] refactor(sync): close receiver when become leader --- source/libs/sync/inc/syncSnapshot.h | 25 +++++++++++++------------ source/libs/sync/src/syncMain.c | 5 +++++ source/libs/sync/src/syncSnapshot.c | 13 ++++++------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 0dc67cf150..6fb558e45c 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -28,10 +28,10 @@ extern "C" { #include "syncMessage.h" #include "taosdef.h" -#define SYNC_SNAPSHOT_SEQ_INVALID -1 +#define SYNC_SNAPSHOT_SEQ_INVALID -1 #define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -2 -#define SYNC_SNAPSHOT_SEQ_BEGIN 0 -#define SYNC_SNAPSHOT_SEQ_END 0x7FFFFFFF +#define SYNC_SNAPSHOT_SEQ_BEGIN 0 +#define SYNC_SNAPSHOT_SEQ_END 0x7FFFFFFF #define SYNC_SNAPSHOT_RETRY_MS 5000 @@ -40,14 +40,14 @@ typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void * pReader; - void * pCurrentBlock; + void *pReader; + void *pCurrentBlock; int32_t blockLen; SSnapshotParam snapshotParam; SSnapshot snapshot; SSyncCfg lastConfig; int64_t sendingMS; - SSyncNode * pSyncNode; + SSyncNode *pSyncNode; int32_t replicaIndex; SyncTerm term; SyncTerm privateTerm; @@ -64,20 +64,20 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char * snapshotSender2Str(SSyncSnapshotSender *pSender); -char * snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); +char *snapshotSender2Str(SSyncSnapshotSender *pSender); +char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); //--------------------------------------------------- typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void * pWriter; + void *pWriter; SyncTerm term; SyncTerm privateTerm; SSnapshotParam snapshotParam; SSnapshot snapshot; SRaftId fromId; - SSyncNode * pSyncNode; + SSyncNode *pSyncNode; } SSyncSnapshotReceiver; @@ -86,10 +86,11 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver); cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); //--------------------------------------------------- // on message diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3fe600ecbb..51098374b0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2181,6 +2181,11 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { (pMySender->privateTerm) += 100; } + // close receiver + if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { + snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); + } + // stop elect timer syncNodeStopElectTimer(pSyncNode); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 702e9f01dc..5489a107e7 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -24,7 +24,6 @@ //---------------------------------- static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg); static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); -static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver); static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); @@ -374,14 +373,14 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) { int32_t len = 256; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; char host[64]; @@ -480,7 +479,7 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapsh } // force stop -static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { +void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { // force close, abandon incomplete data if (pReceiver->pWriter != NULL) { int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, @@ -653,7 +652,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { cJSON_AddStringToObject(pFromId, "addr", u64buf); { uint64_t u64 = pReceiver->fromId.addr; - cJSON * pTmp = pFromId; + cJSON *pTmp = pFromId; char host[128] = {0}; uint16_t port; syncUtilU642Addr(u64, host, sizeof(host), &port); @@ -686,14 +685,14 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event) { int32_t len = 256; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); SRaftId fromId = pReceiver->fromId; char host[128]; From 63fd6f4ceea5f41bf3e8a703e58dfa011dd6bce5 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Mon, 22 Aug 2022 16:05:46 +0800 Subject: [PATCH 32/55] ci: ignore return value while check changed files if it is a docs PR --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 98d7a5a731..bc309ff66c 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -60,7 +60,7 @@ def check_docs() { def file_changed = sh ( script: ''' cd ${WKC} - git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/" + git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/" || : ''', returnStdout: true ).trim() From 05db189a17165d3d30228edfd75eba6c4bd9fc93 Mon Sep 17 00:00:00 2001 From: Yang Zhao Date: Mon, 22 Aug 2022 16:46:03 +0800 Subject: [PATCH 33/55] fix: remove python subscribe demo (#15987) Co-authored-by: Shuduo Sang --- docs/examples/python/subscribe_demo.py | 38 -------------------------- 1 file changed, 38 deletions(-) delete mode 100644 docs/examples/python/subscribe_demo.py diff --git a/docs/examples/python/subscribe_demo.py b/docs/examples/python/subscribe_demo.py deleted file mode 100644 index db9d49c3f4..0000000000 --- a/docs/examples/python/subscribe_demo.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -Python asynchronous subscribe demo. -run on Linux system with: python3 subscribe_demo.py -""" - -from ctypes import c_void_p - -import taos -import time - - -def query_callback(p_sub, p_result, p_param, code): - """ - :param p_sub: pointer returned by native API -- taos_subscribe - :param p_result: pointer to native TAOS_RES - :param p_param: None - :param code: error code - :return: None - """ - print("in callback") - result = taos.TaosResult(c_void_p(p_result)) - # raise exception if error occur - result.check_error(code) - for row in result.rows_iter(): - print(row) - print(f"{result.row_count} rows consumed.") - - -if __name__ == '__main__': - conn = taos.connect() - restart = True - topic = "topic-meter-current-bg" - sql = "select * from power.meters where current > 10" # Error sql - interval = 2000 # consumption interval in microseconds. - _ = conn.subscribe(restart, topic, sql, interval, query_callback) - # Note: we received the return value as _ above, to avoid the TaosSubscription object to be deleted by gc. - while True: - time.sleep(10) # use Ctrl + C to interrupt From 2ed38aad93c84f1c023484b49f192e98b2fd9dde Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 22 Aug 2022 17:08:19 +0800 Subject: [PATCH 34/55] fix: use new API tBlockDataInit to init pBlockData --- source/dnode/vnode/src/tsdb/tsdbCache.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ed25783e9f..b614b813d1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -422,6 +422,8 @@ typedef struct { STsdb *pTsdb; // [input] SBlockIdx *pBlockIdxExp; // [input] STSchema *pTSchema; // [input] + tb_uid_t suid; + tb_uid_t uid; int32_t nFileSet; int32_t iFileSet; SArray *aDFileSet; @@ -593,6 +595,9 @@ typedef struct SFSNextRowIter { SFSNEXTROWSTATES state; // [input] STsdb *pTsdb; // [input] SBlockIdx *pBlockIdxExp; // [input] + STSchema *pTSchema; // [input] + tb_uid_t suid; + tb_uid_t uid; int32_t nFileSet; int32_t iFileSet; SArray *aDFileSet; @@ -685,6 +690,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetBlock); /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ + tBlockDataReset(state->pBlockData); + code = tBlockDataInit(state->pBlockData, state->suid, state->uid, state->pTSchema); + if (code) goto _err; + code = tsdbReadDataBlock(state->pDataFReader, &block, state->pBlockData); if (code) goto _err; @@ -958,16 +967,21 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; - pIter->fsLastState.state = (SFSLASTNEXTROWSTATES) SFSNEXTROW_FS; + pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS; pIter->fsLastState.pTsdb = pTsdb; pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; pIter->fsLastState.pBlockIdxExp = &pIter->idx; pIter->fsLastState.pTSchema = pTSchema; + pIter->fsLastState.suid = suid; + pIter->fsLastState.uid = uid; pIter->fsState.state = SFSNEXTROW_FS; pIter->fsState.pTsdb = pTsdb; pIter->fsState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; pIter->fsState.pBlockIdxExp = &pIter->idx; + pIter->fsState.pTSchema = pTSchema; + pIter->fsState.suid = suid; + pIter->fsState.uid = uid; pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, &pIter->memState, getNextRowFromMem, NULL}; pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, &pIter->imemState, getNextRowFromMem, NULL}; From 6c32df7adf09c8cfb80c76ea9ecd4232f089dc47 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 22 Aug 2022 17:21:03 +0800 Subject: [PATCH 35/55] fix(query): fix stream session window out of order data agg function result error TD-18287 --- source/libs/function/src/builtinsimpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5051dcd65c..0481c4c7c8 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -468,7 +468,7 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - pResInfo->isNullRes = (pResInfo->isNullRes == 1) ? 1 : (pResInfo->numOfRes == 0); + pResInfo->isNullRes = pResInfo->numOfRes == 0; char* in = GET_ROWCELL_INTERBUF(pResInfo); colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); @@ -1613,7 +1613,7 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - pEntryInfo->isNullRes = (pEntryInfo->isNullRes == 1) ? 1 : (pEntryInfo->numOfRes == 0); + pEntryInfo->isNullRes = pEntryInfo->numOfRes == 0; if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) { float v = *(double*)&pRes->v; From 3e7699cc2fe2fce0ce6331f8ba5a62b0903d1ac1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Aug 2022 17:38:50 +0800 Subject: [PATCH 36/55] enh: async launch tasks --- source/libs/scheduler/inc/schInt.h | 3 +-- source/libs/scheduler/src/schTask.c | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 1b3d75f33b..ce841ed83c 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -60,8 +60,7 @@ typedef enum { #define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000 #define SCH_MAX_TASK_TIMEOUT_USEC 60000000 #define SCH_DEFAULT_MAX_RETRY_NUM 6 - -#define SCH_ASYNC_LAUNCH_TASK 0 +#define SCH_MIN_AYSNC_EXEC_NUM 3 typedef struct SSchDebug { bool lockEnable; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 612b908d41..d16d15c119 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -871,14 +871,14 @@ _return: taosMemoryFree(param); -#if SCH_ASYNC_LAUNCH_TASK - if (code) { - code = schProcessOnTaskFailure(pJob, pTask, code); + if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { + if (code) { + code = schProcessOnTaskFailure(pJob, pTask, code); + } + if (code) { + code = schHandleJobFailure(pJob, code); + } } - if (code) { - code = schHandleJobFailure(pJob, code); - } -#endif SCH_RET(code); } @@ -893,12 +893,12 @@ int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { param->pJob = pJob; param->pTask = pTask; -#if SCH_ASYNC_LAUNCH_TASK - taosAsyncExec(schLaunchTaskImpl, param, NULL); -#else - SCH_ERR_RET(schLaunchTaskImpl(param)); -#endif - + if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { + taosAsyncExec(schLaunchTaskImpl, param, NULL); + } else { + SCH_ERR_RET(schLaunchTaskImpl(param)); + } + return TSDB_CODE_SUCCESS; } From e7b5972c68e75e386ae3b7bef77a4bc7989acb91 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 17:39:09 +0800 Subject: [PATCH 37/55] doc: remove 2 obsolte files --- docs/en/07-develop/06-continuous-query.mdx | 83 ------- docs/en/07-develop/07-subscribe.mdx | 259 --------------------- 2 files changed, 342 deletions(-) delete mode 100644 docs/en/07-develop/06-continuous-query.mdx delete mode 100644 docs/en/07-develop/07-subscribe.mdx diff --git a/docs/en/07-develop/06-continuous-query.mdx b/docs/en/07-develop/06-continuous-query.mdx deleted file mode 100644 index 1aea5783fc..0000000000 --- a/docs/en/07-develop/06-continuous-query.mdx +++ /dev/null @@ -1,83 +0,0 @@ ---- -sidebar_label: Continuous Query -description: "Continuous query is a query that's executed automatically at a predefined frequency to provide aggregate query capability by time window. It is essentially simplified, time driven, stream computing." -title: "Continuous Query" ---- - -A continuous query is a query that's executed automatically at a predefined frequency to provide aggregate query capability by time window. It is essentially simplified, time driven, stream computing. A continuous query can be performed on a table or STable in TDengine. The results of a continuous query can be pushed to clients or written back to TDengine. Each query is executed on a time window, which moves forward with time. The size of time window and the forward sliding time need to be specified with parameter `INTERVAL` and `SLIDING` respectively. - -A continuous query in TDengine is time driven, and can be defined using TAOS SQL directly without any extra operations. With a continuous query, the result can be generated based on a time window to achieve down sampling of the original data. Once a continuous query is defined using TAOS SQL, the query is automatically executed at the end of each time window and the result is pushed back to clients or written to TDengine. - -There are some differences between continuous query in TDengine and time window computation in stream computing： - -- The computation is performed and the result is returned in real time in stream computing, but the computation in continuous query is only started when a time window closes. For example, if the time window is 1 day, then the result will only be generated at 23:59:59. -- If a historical data row is written in to a time window for which the computation has already finished, the computation will not be performed again and the result will not be pushed to client applications again. If the results have already been written into TDengine, they will not be updated. -- In continuous query, if the result is pushed to a client, the client status is not cached on the server side and Exactly-once is not guaranteed by the server. If the client program crashes, a new time window will be generated from the time where the continuous query is restarted. If the result is written into TDengine, the data written into TDengine can be guaranteed as valid and continuous. - -## Syntax - -```sql -[CREATE TABLE AS] SELECT select_expr [, select_expr ...] - FROM {tb_name_list} - [WHERE where_condition] - [INTERVAL(interval_val [, interval_offset]) [SLIDING sliding_val]] - -``` - -INTERVAL: The time window for which continuous query is performed - -SLIDING: The time step for which the time window moves forward each time - -## How to Use - -In this section the use case of meters will be used to introduce how to use continuous query. Assume the STable and subtables have been created using the SQL statements below. - -```sql -create table meters (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupId int); -create table D1001 using meters tags ("California.SanFrancisco", 2); -create table D1002 using meters tags ("California.LosAngeles", 2); -``` - -The SQL statement below retrieves the average voltage for a one minute time window, with each time window moving forward by 30 seconds. - -```sql -select avg(voltage) from meters interval(1m) sliding(30s); -``` - -Whenever the above SQL statement is executed, all the existing data will be computed again. If the computation needs to be performed every 30 seconds automatically to compute on the data in the past one minute, the above SQL statement needs to be revised as below, in which `{startTime}` stands for the beginning timestamp in the latest time window. - -```sql -select avg(voltage) from meters where ts > {startTime} interval(1m) sliding(30s); -``` - -An easier way to achieve this is to prepend `create table {tableName} as` before the `select`. - -```sql -create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s); -``` - -A table named as `avg_vol` will be created automatically, then every 30 seconds the `select` statement will be executed automatically on the data in the past 1 minute, i.e. the latest time window, and the result is written into table `avg_vol`. The client program just needs to query from table `avg_vol`. For example: - -```sql -taos> select * from avg_vol; - ts | avg_voltage_ | -=================================================== - 2020-07-29 13:37:30.000 | 222.0000000 | - 2020-07-29 13:38:00.000 | 221.3500000 | - 2020-07-29 13:38:30.000 | 220.1700000 | - 2020-07-29 13:39:00.000 | 223.0800000 | -``` - -Please note that the minimum allowed time window is 10 milliseconds, and there is no upper limit. - -It's possible to specify the start and end time of a continuous query. If the start time is not specified, the timestamp of the first row will be considered as the start time; if the end time is not specified, the continuous query will be performed indefinitely, otherwise it will be terminated once the end time is reached. For example, the continuous query in the SQL statement below will be started from now and terminated one hour later. - -```sql -create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s); -``` - -`now` in the above SQL statement stands for the time when the continuous query is created, not the time when the computation is actually performed. To avoid the trouble caused by a delay in receiving data as much as possible, the actual computation in a continuous query is started after a little delay. That means, once a time window closes, the computation is not started immediately. Normally, the result are available after a little time, normally within one minute, after the time window closes. - -## How to Manage - -`show streams` command can be used in the TDengine CLI `taos` to show all the continuous queries in the system, and `kill stream` can be used to terminate a continuous query. diff --git a/docs/en/07-develop/07-subscribe.mdx b/docs/en/07-develop/07-subscribe.mdx deleted file mode 100644 index 782fcdbaf2..0000000000 --- a/docs/en/07-develop/07-subscribe.mdx +++ /dev/null @@ -1,259 +0,0 @@ ---- -sidebar_label: Data Subscription -description: "Lightweight service for data subscription and publishing. Time series data inserted into TDengine continuously can be pushed automatically to subscribing clients." -title: Data Subscription ---- - -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; -import Java from "./_sub_java.mdx"; -import Python from "./_sub_python.mdx"; -import Go from "./_sub_go.mdx"; -import Rust from "./_sub_rust.mdx"; -import Node from "./_sub_node.mdx"; -import CSharp from "./_sub_cs.mdx"; -import CDemo from "./_sub_c.mdx"; - -## Introduction - -Due to the nature of time series data, data insertion into TDengine is similar to data publishing in message queues. Data is stored in ascending order of timestamp inside TDengine, and so each table in TDengine can essentially be considered as a message queue. - -A lightweight service for data subscription and publishing is built into TDengine. With the API provided by TDengine, client programs can use `select` statements to subscribe to data from one or more tables. The subscription and state maintenance is performed on the client side. The client programs poll the server to check whether there is new data, and if so the new data will be pushed back to the client side. If the client program is restarted, where to start retrieving new data is up to the client side. - -There are 3 major APIs related to subscription provided in the TDengine client driver. - -```c -taos_subscribe -taos_consume -taos_unsubscribe -``` - -For more details about these APIs please refer to [C/C++ Connector](/reference/connector/cpp). Their usage will be introduced below using the use case of meters, in which the schema of STable and subtables from the previous section [Continuous Query](/develop/continuous-query) are used. Full sample code can be found [here](https://github.com/taosdata/TDengine/blob/master/examples/c/subscribe.c). - -If we want to get a notification and take some actions if the current exceeds a threshold, like 10A, from some meters, there are two ways: - -The first way is to query each sub table and record the last timestamp matching the criteria. Then after some time, query the data later than the recorded timestamp, and repeat this process. The SQL statements for this way are as below. - -```sql -select * from D1001 where ts > {last_timestamp1} and current > 10; -select * from D1002 where ts > {last_timestamp2} and current > 10; -... -``` - -The above way works, but the problem is that the number of `select` statements increases with the number of meters. Additionally, the performance of both client side and server side will be unacceptable once the number of meters grows to a big enough number. - -A better way is to query on the STable, only one `select` is enough regardless of the number of meters, like below: - -```sql -select * from meters where ts > {last_timestamp} and current > 10; -``` - -However, this presents a new problem in how to choose `last_timestamp`. First, the timestamp when the data is generated is different from the timestamp when the data is inserted into the database, sometimes the difference between them may be very big. Second, the time when the data from different meters arrives at the database may be different too. If the timestamp of the "slowest" meter is used as `last_timestamp` in the query, the data from other meters may be selected repeatedly; but if the timestamp of the "fastest" meter is used as `last_timestamp`, some data from other meters may be missed. - -All the problems mentioned above can be resolved easily using the subscription functionality provided by TDengine. - -The first step is to create subscription using `taos_subscribe`. - -```c -TAOS_SUB* tsub = NULL; -if (async) { -　　// create an asynchronous subscription, the callback function will be called every 1s -　　tsub = taos_subscribe(taos, restart, topic, sql, subscribe_callback, &blockFetch, 1000); -} else { -　　// create an synchronous subscription, need to call 'taos_consume' manually -　　tsub = taos_subscribe(taos, restart, topic, sql, NULL, NULL, 0); -} -``` - -The subscription in TDengine can be either synchronous or asynchronous. In the above sample code, the value of variable `async` is determined from the CLI input, then it's used to create either an async or sync subscription. Sync subscription means the client program needs to invoke `taos_consume` to retrieve data, and async subscription means another thread created by `taos_subscribe` internally invokes `taos_consume` to retrieve data and pass the data to `subscribe_callback` for processing. `subscribe_callback` is a callback function provided by the client program. You should not perform time consuming operations in the callback function. - -The parameter `taos` is an established connection. Nothing special needs to be done for thread safety for synchronous subscription. For asynchronous subscription, the taos_subscribe function should be called exclusively by the current thread, to avoid unpredictable errors. - -The parameter `sql` is a `select` statement in which the `where` clause can be used to specify filter conditions. In our example, we can subscribe to the records in which the current exceeds 10A, with the following SQL statement: - -```sql -select * from meters where current > 10; -``` - -Please note that, all the data will be processed because no start time is specified. If we only want to process data for the past day, a time related condition can be added: - -```sql -select * from meters where ts > now - 1d and current > 10; -``` - -The parameter `topic` is the name of the subscription. The client application must guarantee that the name is unique. However, it doesn't have to be globally unique because subscription is implemented in the APIs on the client side. - -If the subscription named as `topic` doesn't exist, the parameter `restart` will be ignored. If the subscription named as `topic` has been created before by the client program, when the client program is restarted with the subscription named `topic`, parameter `restart` is used to determine whether to retrieve data from the beginning or from the last point where the subscription was broken. - -If the value of `restart` is **true** (i.e. a non-zero value), data will be retrieved from the beginning. If it is **false** (i.e. zero), the data already consumed before will not be processed again. - -The last parameter of `taos_subscribe` is the polling interval in units of millisecond. In sync mode, if the time difference between two continuous invocations to `taos_consume` is smaller than the interval specified by `taos_subscribe`, `taos_consume` will be blocked until the interval is reached. In async mode, this interval is the minimum interval between two invocations to the call back function. - -The second to last parameter of `taos_subscribe` is used to pass arguments to the call back function. `taos_subscribe` doesn't process this parameter and simply passes it to the call back function. This parameter is simply ignored in sync mode. - -After a subscription is created, its data can be consumed and processed. Shown below is the sample code to consume data in sync mode, in the else condition of `if (async)`. - -```c -if (async) { -　　getchar(); -} else while(1) { -　　TAOS_RES* res = taos_consume(tsub); -　　if (res == NULL) { -　　　　printf("failed to consume data."); -　　　　break; -　　} else { -　　　　print_result(res, blockFetch); -　　　　getchar(); -　　} -} -``` - -In the above sample code in the else condition, there is an infinite loop. Each time carriage return is entered `taos_consume` is invoked. The return value of `taos_consume` is the selected result set. In the above sample, `print_result` is used to simplify the printing of the result set. It is similar to `taos_use_result`. Below is the implementation of `print_result`. - -```c -void print_result(TAOS_RES* res, int blockFetch) { -　　TAOS_ROW row = NULL; -　　int num_fields = taos_num_fields(res); -　　TAOS_FIELD* fields = taos_fetch_fields(res); -　　int nRows = 0; -　　if (blockFetch) { -　　　　nRows = taos_fetch_block(res, &row); -　　　　for (int i = 0; i < nRows; i++) { -　　　　　　char temp[256]; -　　　　　　taos_print_row(temp, row + i, fields, num_fields); -　　　　　　puts(temp); -　　　　} -　　} else { -　　　　while ((row = taos_fetch_row(res))) { -　　　　　　char temp[256]; -　　　　　　taos_print_row(temp, row, fields, num_fields); -　　　　　　puts(temp); -　　　　　　nRows++; -　　　　} -　　} -　　printf("%d rows consumed.\n", nRows); -} -``` - -In the above code `taos_print_row` is used to process the data consumed. All matching rows are printed. - -In async mode, consuming data is simpler as shown below. - -```c -void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { -　　print_result(res, *(int*)param); -} -``` - -`taos_unsubscribe` can be invoked to terminate a subscription. - -```c -taos_unsubscribe(tsub, keep); -``` - -The second parameter `keep` is used to specify whether to keep the subscription progress on the client sde. If it is **false**, i.e. **0**, then subscription will be restarted from beginning regardless of the `restart` parameter's value when `taos_subscribe` is invoked again. The subscription progress information is stored in _{DataDir}/subscribe/_ , under which there is a file with the same name as `topic` for each subscription(Note: The default value of `DataDir` in the `taos.cfg` file is **/var/lib/taos/**. However, **/var/lib/taos/** does not exist on the Windows server. So you need to change the `DataDir` value to the corresponding existing directory."), the subscription will be restarted from the beginning if the corresponding progress file is removed. - -Now let's see the effect of the above sample code, assuming below prerequisites have been done. - -- The sample code has been downloaded to local system -- TDengine has been installed and launched properly on same system -- The database, STable, and subtables required in the sample code are ready - -Launch the command below in the directory where the sample code resides to compile and start the program. - -```bash -make -./subscribe -sql='select * from meters where current > 10;' -``` - -After the program is started, open another terminal and launch TDengine CLI `taos`, then use the below SQL commands to insert a row whose current is 12A into table **D1001**. - -```sql -use test; -insert into D1001 values(now, 12, 220, 1); -``` - -Then, this row of data will be shown by the example program on the first terminal because its current exceeds 10A. More data can be inserted for you to observe the output of the example program. - -## Examples - -The example program below demonstrates how to subscribe, using connectors, to data rows in which current exceeds 10A. - -### Prepare Data - -```bash -# create database "power" -taos> create database power; -# use "power" as the database in following operations -taos> use power; -# create super table "meters" -taos> create table meters(ts timestamp, current float, voltage int, phase int) tags(location binary(64), groupId int); -# create tabes using the schema defined by super table "meters" -taos> create table d1001 using meters tags ("California.SanFrancisco", 2); -taos> create table d1002 using meters tags ("California.LoSangeles", 2); -# insert some rows -taos> insert into d1001 values("2020-08-15 12:00:00.000", 12, 220, 1),("2020-08-15 12:10:00.000", 12.3, 220, 2),("2020-08-15 12:20:00.000", 12.2, 220, 1); -taos> insert into d1002 values("2020-08-15 12:00:00.000", 9.9, 220, 1),("2020-08-15 12:10:00.000", 10.3, 220, 1),("2020-08-15 12:20:00.000", 11.2, 220, 1); -# filter out the rows in which current is bigger than 10A -taos> select * from meters where current > 10; - ts | current | voltage | phase | location | groupid | -=========================================================================================================== - 2020-08-15 12:10:00.000 | 10.30000 | 220 | 1 | California.LoSangeles | 2 | - 2020-08-15 12:20:00.000 | 11.20000 | 220 | 1 | California.LoSangeles | 2 | - 2020-08-15 12:00:00.000 | 12.00000 | 220 | 1 | California.SanFrancisco | 2 | - 2020-08-15 12:10:00.000 | 12.30000 | 220 | 2 | California.SanFrancisco | 2 | - 2020-08-15 12:20:00.000 | 12.20000 | 220 | 1 | California.SanFrancisco | 2 | -Query OK, 5 row(s) in set (0.004896s) -``` - -### Example Programs - - - - - - - - - {/* - - */} - - - - {/* - - - - - */} - - - - - -### Run the Examples - -The example programs first consume all historical data matching the criteria. - -```bash -ts: 1597464000000 current: 12.0 voltage: 220 phase: 1 location: California.SanFrancisco groupid : 2 -ts: 1597464600000 current: 12.3 voltage: 220 phase: 2 location: California.SanFrancisco groupid : 2 -ts: 1597465200000 current: 12.2 voltage: 220 phase: 1 location: California.SanFrancisco groupid : 2 -ts: 1597464600000 current: 10.3 voltage: 220 phase: 1 location: California.LoSangeles groupid : 2 -ts: 1597465200000 current: 11.2 voltage: 220 phase: 1 location: California.LoSangeles groupid : 2 -``` - -Next, use TDengine CLI to insert a new row. - -``` -# taos -taos> use power; -taos> insert into d1001 values(now, 12.4, 220, 1); -``` - -Because the current in the inserted row exceeds 10A, it will be consumed by the example program. - -``` -ts: 1651146662805 current: 12.4 voltage: 220 phase: 1 location: California.SanFrancisco groupid: 2 -``` From 254d436d75b3882c8627a5d4fc0e577732eb8f68 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 17:43:13 +0800 Subject: [PATCH 38/55] doc: move 2 files to internal --- docs/en/{10-cluster => 21-tdinternal}/03-high-availability.md | 0 docs/en/{10-cluster => 21-tdinternal}/04-load-balance.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename docs/en/{10-cluster => 21-tdinternal}/03-high-availability.md (100%) rename docs/en/{10-cluster => 21-tdinternal}/04-load-balance.md (100%) diff --git a/docs/en/10-cluster/03-high-availability.md b/docs/en/21-tdinternal/03-high-availability.md similarity index 100% rename from docs/en/10-cluster/03-high-availability.md rename to docs/en/21-tdinternal/03-high-availability.md diff --git a/docs/en/10-cluster/04-load-balance.md b/docs/en/21-tdinternal/04-load-balance.md similarity index 100% rename from docs/en/10-cluster/04-load-balance.md rename to docs/en/21-tdinternal/04-load-balance.md From 9cdf2bb3a576a96fbf91d2f64a4543768d18c78f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Aug 2022 17:43:55 +0800 Subject: [PATCH 39/55] fix(query): init the blockdata before load data. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2e66cac21e..75b2e1fcb4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3043,7 +3043,12 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn // 3. load the neighbor block, and set it to be the currently accessed file data block tBlockDataReset(&pStatus->fileBlockData); - int32_t code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); + int32_t code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pFBlock->uid, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { return code; } From e4a1b87070e9b4335cfc68c46657e79745ba9326 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 17:45:37 +0800 Subject: [PATCH 40/55] doc: remove obsolete cluster files --- docs/en/10-cluster/01-deploy.md | 126 ----------------------- docs/en/10-cluster/02-cluster-mgmt.md | 138 -------------------------- docs/en/10-cluster/_category_.yml | 1 - docs/en/10-cluster/index.md | 15 --- 4 files changed, 280 deletions(-) delete mode 100644 docs/en/10-cluster/01-deploy.md delete mode 100644 docs/en/10-cluster/02-cluster-mgmt.md delete mode 100644 docs/en/10-cluster/_category_.yml delete mode 100644 docs/en/10-cluster/index.md diff --git a/docs/en/10-cluster/01-deploy.md b/docs/en/10-cluster/01-deploy.md deleted file mode 100644 index d998fd6ad0..0000000000 --- a/docs/en/10-cluster/01-deploy.md +++ /dev/null @@ -1,126 +0,0 @@ ---- -title: Deployment ---- - -## Prerequisites - -### Step 1 - -The FQDN of all hosts must be setup properly. All FQDNs need to be configured in the /etc/hosts file on each host. You must confirm that each FQDN can be accessed from any other host, you can do this by using the `ping` command. - -The command `hostname -f` can be executed to get the hostname on any host. `ping ` command can be executed on each host to check whether any other host is accessible from it. If any host is not accessible, the network configuration, like /etc/hosts or DNS configuration, needs to be checked and revised, to make any two hosts accessible to each other. - -:::note - -- The host where the client program runs also needs to be configured properly for FQDN, to make sure all hosts for client or server can be accessed from any other. In other words, the hosts where the client is running are also considered as a part of the cluster. - -- Please ensure that your firewall rules do not block TCP/UDP on ports 6030-6042 on all hosts in the cluster. - -::: - -### Step 2 - -If any previous version of TDengine has been installed and configured on any host, the installation needs to be removed and the data needs to be cleaned up. For details about uninstalling please refer to [Install and Uninstall](/operation/pkg-install). To clean up the data, please use `rm -rf /var/lib/taos/\*` assuming the `dataDir` is configured as `/var/lib/taos`. - -:::note - -As a best practice, before cleaning up any data files or directories, please ensure that your data has been backed up correctly, if required by your data integrity, backup, security, or other standard operating protocols (SOP). - -::: - -### Step 3 - -Now it's time to install TDengine on all hosts but without starting `taosd`. Note that the versions on all hosts should be same. If you are prompted to input the existing TDengine cluster, simply press carriage return to ignore the prompt. `install.sh -e no` can also be used to disable this prompt. For details please refer to [Install and Uninstall](/operation/pkg-install). - -### Step 4 - -Now each physical node (referred to, hereinafter, as `dnode` which is an abbreviation for "data node") of TDengine needs to be configured properly. Please note that one dnode doesn't stand for one host. Multiple TDengine dnodes can be started on a single host as long as they are configured properly without conflicting. More specifically each instance of the configuration file `taos.cfg` stands for a dnode. Assuming the first dnode of TDengine cluster is "h1.taosdata.com:6030", its `taos.cfg` is configured as following. - -```c -// firstEp is the end point to connect to when any dnode starts -firstEp h1.taosdata.com:6030 - -// must be configured to the FQDN of the host where the dnode is launched -fqdn h1.taosdata.com - -// the port used by the dnode, default is 6030 -serverPort 6030 - -// only necessary when replica is configured to an even number -#arbitrator ha.taosdata.com:6042 -``` - -`firstEp` and `fqdn` must be configured properly. In `taos.cfg` of all dnodes in TDengine cluster, `firstEp` must be configured to point to same address, i.e. the first dnode of the cluster. `fqdn` and `serverPort` compose the address of each node itself. If you want to start multiple TDengine dnodes on a single host, please make sure all other configurations like `dataDir`, `logDir`, and other resources related parameters are not conflicting. - -For all the dnodes in a TDengine cluster, the below parameters must be configured exactly the same, any node whose configuration is different from dnodes already in the cluster can't join the cluster. - -| **#** | **Parameter** | **Definition** | -| ----- | -------------- | ------------------------------------------------------------- | -| 1 | statusInterval | The time interval for which dnode reports its status to mnode | -| 2 | timezone | Time Zone where the server is located | -| 3 | locale | Location code of the system | -| 4 | charset | Character set of the system | - -## Start Cluster - -In the following example we assume that first dnode has FQDN h1.taosdata.com and the second dnode has FQDN h2.taosdata.com. - -### Start The First DNODE - -Start the first dnode following the instructions in [Get Started](/get-started/). Then launch TDengine CLI `taos` and execute command `show dnodes`, the output is as following for example: - -``` -Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 -Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. - -Server is Enterprise trial Edition, ver:3.0.0.0 and will never expire. - -taos> show dnodes; - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | h1.taosdata.com:6030 | 0 | 1024 | ready | 2022-07-16 10:50:42.673 | | -Query OK, 1 rows affected (0.007984s) - -taos> -``` - -From the above output, it is shown that the end point of the started dnode is "h1.taosdata.com:6030", which is the `firstEp` of the cluster. - -### Start Other DNODEs - -There are a few steps necessary to add other dnodes in the cluster. - -Let's assume we are starting the second dnode with FQDN, h2.taosdata.com. Firstly we make sure the configuration is correct. - -```c -// firstEp is the end point to connect to when any dnode starts -firstEp h1.taosdata.com:6030 - -// must be configured to the FQDN of the host where the dnode is launched -fqdn h2.taosdata.com - -// the port used by the dnode, default is 6030 -serverPort 6030 - -``` - -Secondly, we can start `taosd` as instructed in [Get Started](/get-started/). - -Then, on the first dnode i.e. h1.taosdata.com in our example, use TDengine CLI `taos` to execute the following command to add the end point of the dnode in the cluster. In the command "fqdn:port" should be quoted using double quotes. - -```sql -CREATE DNODE "h2.taos.com:6030"; -``` - -Then on the first dnode h1.taosdata.com, execute `show dnodes` in `taos` to show whether the second dnode has been added in the cluster successfully or not. - -```sql -SHOW DNODES; -``` - -If the status of the newly added dnode is offline, please check: - -- Whether the `taosd` process is running properly or not -- In the log file `taosdlog.0` to see whether the fqdn and port are correct - -The above process can be repeated to add more dnodes in the cluster. diff --git a/docs/en/10-cluster/02-cluster-mgmt.md b/docs/en/10-cluster/02-cluster-mgmt.md deleted file mode 100644 index 19ee034127..0000000000 --- a/docs/en/10-cluster/02-cluster-mgmt.md +++ /dev/null @@ -1,138 +0,0 @@ ---- -sidebar_label: Operation -title: Manage DNODEs ---- - -The previous section, [Deployment],(/cluster/deploy) showed you how to deploy and start a cluster from scratch. Once a cluster is ready, the status of dnode(s) in the cluster can be shown at any time. Dnodes can be managed from the TDengine CLI. New dnode(s) can be added to scale out the cluster, an existing dnode can be removed and you can even perform load balancing manually, if necessary. - -:::note -All the commands introduced in this chapter must be run in the TDengine CLI - `taos`. Note that sometimes it is necessary to use root privilege. - -::: - -## Show DNODEs - -The below command can be executed in TDengine CLI `taos` to list all dnodes in the cluster, including ID, end point (fqdn:port), status (ready, offline), number of vnodes, number of free vnodes and so on. We recommend executing this command after adding or removing a dnode. - -```sql -SHOW DNODES; -``` - -Below is the example output of this command. - -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.008298s) -``` - -## Show VGROUPs - -To utilize system resources efficiently and provide scalability, data sharding is required. The data of each database is divided into multiple shards and stored in multiple vnodes. These vnodes may be located on different dnodes. One way of scaling out is to add more vnodes on dnodes. Each vnode can only be used for a single DB, but one DB can have multiple vnodes. The allocation of vnode is scheduled automatically by mnode based on system resources of the dnodes. - -Launch TDengine CLI `taos` and execute below command: - -```sql -USE SOME_DATABASE; -SHOW VGROUPS; -``` - -The output is like below: - -taos> use db; -Database changed. - -taos> show vgroups; -vgId | tables | status | onlines | v1_dnode | v1_status | compacting | -========================================================================================== -14 | 38000 | ready | 1 | 1 | leader | 0 | -15 | 38000 | ready | 1 | 1 | leader | 0 | -16 | 38000 | ready | 1 | 1 | leader | 0 | -17 | 38000 | ready | 1 | 1 | leader | 0 | -18 | 37001 | ready | 1 | 1 | leader | 0 | -19 | 37000 | ready | 1 | 1 | leader | 0 | -20 | 37000 | ready | 1 | 1 | leader | 0 | -21 | 37000 | ready | 1 | 1 | leader | 0 | -Query OK, 8 row(s) in set (0.001154s) - -```` - -## Add DNODE - -Launch TDengine CLI `taos` and execute the command below to add the end point of a new dnode into the EPI (end point) list of the cluster. "fqdn:port" must be quoted using double quotes. - -```sql -CREATE DNODE "fqdn:port"; -```` - -The example output is as below: - -``` -taos> create dnode "localhost:7030"; -Query OK, 0 of 0 row(s) in database (0.008203s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 0 | 0 | offline | any | 2022-04-19 08:11:42.158 | status not received | -Query OK, 2 row(s) in set (0.001017s) -``` - -It can be seen that the status of the new dnode is "offline". Once the dnode is started and connects to the firstEp of the cluster, you can execute the command again and get the example output below. As can be seen, both dnodes are in "ready" status. - -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 3 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 6 | 8 | ready | any | 2022-04-19 08:14:59.165 | | -Query OK, 2 row(s) in set (0.001316s) -``` - -## Drop DNODE - -Launch TDengine CLI `taos` and execute the command below to drop or remove a dnode from the cluster. In the command, you can get `dnodeId` from `show dnodes`. - -```sql -DROP DNODE "fqdn:port"; -``` - -or - -```sql -DROP DNODE dnodeId; -``` - -The example output is below： - -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 0 | 0 | offline | any | 2022-04-19 08:11:42.158 | status not received | -Query OK, 2 row(s) in set (0.001017s) - -taos> drop dnode 2; -Query OK, 0 of 0 row(s) in database (0.000518s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.001137s) -``` - -In the above example, when `show dnodes` is executed the first time, two dnodes are shown. After `drop dnode 2` is executed, you can execute `show dnodes` again and it can be seen that only the dnode with ID 1 is still in the cluster. - -:::note - -- Once a dnode is dropped, it can't rejoin the cluster. To rejoin, the dnode needs to deployed again after cleaning up the data directory. Before dropping a dnode, the data belonging to the dnode MUST be migrated/backed up according to your data retention, data security or other SOPs. -- Please note that `drop dnode` is different from stopping `taosd` process. `drop dnode` just removes the dnode out of TDengine cluster. Only after a dnode is dropped, can the corresponding `taosd` process be stopped. -- Once a dnode is dropped, other dnodes in the cluster will be notified of the drop and will not accept the request from the dropped dnode. -- dnodeID is allocated automatically and can't be manually modified. dnodeID is generated in ascending order without duplication. - -::: diff --git a/docs/en/10-cluster/_category_.yml b/docs/en/10-cluster/_category_.yml deleted file mode 100644 index 141fd78326..0000000000 --- a/docs/en/10-cluster/_category_.yml +++ /dev/null @@ -1 +0,0 @@ -label: Cluster diff --git a/docs/en/10-cluster/index.md b/docs/en/10-cluster/index.md deleted file mode 100644 index 5a45a2ce7b..0000000000 --- a/docs/en/10-cluster/index.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Cluster -keywords: ["cluster", "high availability", "load balance", "scale out"] ---- - -TDengine has a native distributed design and provides the ability to scale out. A few nodes can form a TDengine cluster. If you need higher processing power, you just need to add more nodes into the cluster. TDengine uses virtual node technology to virtualize a node into multiple virtual nodes to achieve load balancing. At the same time, TDengine can group virtual nodes on different nodes into virtual node groups, and use the replication mechanism to ensure the high availability of the system. The cluster feature of TDengine is completely open source. - -This chapter mainly introduces cluster deployment, maintenance, and how to achieve high availability and load balancing. - -```mdx-code-block -import DocCardList from '@theme/DocCardList'; -import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; - - -``` From 4cf9bd6ac1eb6e92b2e53f084205d552f2568fb4 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 22 Aug 2022 17:46:57 +0800 Subject: [PATCH 41/55] fix: the syntax problem of querying the state value in the state window --- include/libs/nodes/querynodes.h | 1 + source/libs/executor/src/executil.c | 3 +++ source/libs/parser/src/parInsert.c | 5 +++++ source/libs/parser/src/parTranslater.c | 12 +++++++++--- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index e1f86bae58..cec6f1a691 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -276,6 +276,7 @@ typedef struct SSelectStmt { bool hasLastRowFunc; bool hasTimeLineFunc; bool hasUdaf; + bool hasStateKey; bool onlyHasKeepOrderFunc; bool groupSort; } SSelectStmt; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f3b395cc7c..a6ccceccef 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -408,6 +408,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); code = metaGetTableTags(metaHandle, suid, uidList, tags); if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid); terrno = code; goto end; } @@ -484,11 +485,13 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; code = createResultData(&type, rows, &output); if (code != TSDB_CODE_SUCCESS) { + qError("failed to create result, reason:%s", tstrerror(code)); goto end; } code = scalarCalculate(pTagCond, pBlockList, &output); if(code != TSDB_CODE_SUCCESS){ + qError("failed to calculate scalar, reason:%s", tstrerror(code)); terrno = code; } // int64_t st2 = taosGetTimestampUs(); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 0922cdb6b9..de9f815618 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -681,6 +681,11 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* break; } + char tmpTokenBuf[TSDB_COL_NAME_LEN + 2] = {0}; // used for deleting Escape character backstick(`) + strncpy(tmpTokenBuf, sToken.z, sToken.n); + sToken.z = tmpTokenBuf; + sToken.n = strdequote(sToken.z); + col_id_t t = lastColIdx + 1; col_id_t index = findCol(&sToken, t, nCols, pSchema); if (index < 0 && t > 0) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3c0d9a5f63..09847feb4d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1881,6 +1881,12 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { return rewriteExprToGroupKeyFunc(pCxt, pNode); } } + if (NULL != pSelect->pWindow && QUERY_NODE_STATE_WINDOW == nodeType(pSelect->pWindow)) { + if (nodesEqualNode(((SStateWindowNode*)pSelect->pWindow)->pExpr, *pNode)) { + pSelect->hasStateKey = true; + return rewriteExprToGroupKeyFunc(pCxt, pNode); + } + } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) { return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); @@ -1973,7 +1979,7 @@ static int32_t checkWindowFuncCoexist(STranslateContext* pCxt, SSelectStmt* pSel if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } - if (NULL != pSelect->pWindow && !pSelect->hasAggFuncs) { + if (NULL != pSelect->pWindow && !pSelect->hasAggFuncs && !pSelect->hasStateKey) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN); } return TSDB_CODE_SUCCESS; @@ -2825,7 +2831,7 @@ static int32_t createDefaultFillNode(STranslateContext* pCxt, SNode** pOutput) { static int32_t checkEvery(STranslateContext* pCxt, SValueNode* pInterval) { int32_t len = strlen(pInterval->literal); - char *unit = &pInterval->literal[len - 1]; + char* unit = &pInterval->literal[len - 1]; if (*unit == 'n' || *unit == 'y') { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Unsupported time unit in EVERY clause"); @@ -2837,7 +2843,7 @@ static int32_t checkEvery(STranslateContext* pCxt, SValueNode* pInterval) { static int32_t translateInterpEvery(STranslateContext* pCxt, SNode** pEvery) { int32_t code = TSDB_CODE_SUCCESS; - code = checkEvery(pCxt, (SValueNode *)(*pEvery)); + code = checkEvery(pCxt, (SValueNode*)(*pEvery)); if (TSDB_CODE_SUCCESS == code) { code = translateExpr(pCxt, pEvery); } From f06b7bfdc25ec9555002a3515e44c68033fa1737 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 17:48:04 +0800 Subject: [PATCH 42/55] doc: remove obsolete installation part in operation guide --- docs/en/13-operation/01-pkg-install.md | 126 +------------------------ 1 file changed, 1 insertion(+), 125 deletions(-) diff --git a/docs/en/13-operation/01-pkg-install.md b/docs/en/13-operation/01-pkg-install.md index b0f607170d..ef841321a1 100644 --- a/docs/en/13-operation/01-pkg-install.md +++ b/docs/en/13-operation/01-pkg-install.md @@ -10,131 +10,7 @@ TDengine community version provides deb and rpm packages for users to choose fro ## Install - - - -1. Download deb package from official website, for example TDengine-server-3.0.0.0-Linux-x64.deb -2. In the directory where the package is located, execute the command below - -```bash -$ sudo dpkg -i TDengine-server-3.0.0.0-Linux-x64.deb -(Reading database ... 137504 files and directories currently installed.) -Preparing to unpack TDengine-server-3.0.0.0-Linux-x64.deb ... -TDengine is removed successfully! -Unpacking tdengine (3.0.0.0) over (3.0.0.0) ... -Setting up tdengine (3.0.0.0) ... -Start to install TDengine... - -System hostname is: ubuntu-1804 - -Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join -OR leave it blank to build one: - -Enter your email address for priority support or enter empty to skip: -Created symlink /etc/systemd/system/multi-user.target.wants/taosd.service → /etc/systemd/system/taosd.service. - -To configure TDengine : edit /etc/taos/taos.cfg -To start TDengine : sudo systemctl start taosd -To access TDengine : taos -h ubuntu-1804 to login into TDengine server - - -TDengine is installed successfully! -``` - - - - - -1. Download rpm package from official website, for example TDengine-server-3.0.0.0-Linux-x64.rpm； -2. In the directory where the package is located, execute the command below - -``` -$ sudo rpm -ivh TDengine-server-3.0.0.0-Linux-x64.rpm -Preparing... ################################# [100%] -Updating / installing... - 1:tdengine-3.0.0.0-3 ################################# [100%] -Start to install TDengine... - -System hostname is: centos7 - -Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join -OR leave it blank to build one: - -Enter your email address for priority support or enter empty to skip: - -Created symlink from /etc/systemd/system/multi-user.target.wants/taosd.service to /etc/systemd/system/taosd.service. - -To configure TDengine : edit /etc/taos/taos.cfg -To start TDengine : sudo systemctl start taosd -To access TDengine : taos -h centos7 to login into TDengine server - - -TDengine is installed successfully! -``` - - - - - -1. Download the tar.gz package, for example TDengine-server-3.0.0.0-Linux-x64.tar.gz； -2. In the directory where the package is located, first decompress the file, then switch to the sub-directory generated in decompressing, i.e. "TDengine-enterprise-server-3.0.0.0/" in this example, and execute the `install.sh` script. - -```bash -$ tar xvzf TDengine-enterprise-server-3.0.0.0-Linux-x64.tar.gz -TDengine-enterprise-server-3.0.0.0/ -TDengine-enterprise-server-3.0.0.0/driver/ -TDengine-enterprise-server-3.0.0.0/driver/vercomp.txt -TDengine-enterprise-server-3.0.0.0/driver/libtaos.so.3.0.0.0 -TDengine-enterprise-server-3.0.0.0/install.sh -TDengine-enterprise-server-3.0.0.0/examples/ -... - -$ ll -total 43816 -drwxrwxr-x 3 ubuntu ubuntu 4096 Feb 22 09:31 ./ -drwxr-xr-x 20 ubuntu ubuntu 4096 Feb 22 09:30 ../ -drwxrwxr-x 4 ubuntu ubuntu 4096 Feb 22 09:30 TDengine-enterprise-server-3.0.0.0/ --rw-rw-r-- 1 ubuntu ubuntu 44852544 Feb 22 09:31 TDengine-enterprise-server-3.0.0.0-Linux-x64.tar.gz - -$ cd TDengine-enterprise-server-3.0.0.0/ - - $ ll -total 40784 -drwxrwxr-x 4 ubuntu ubuntu 4096 Feb 22 09:30 ./ -drwxrwxr-x 3 ubuntu ubuntu 4096 Feb 22 09:31 ../ -drwxrwxr-x 2 ubuntu ubuntu 4096 Feb 22 09:30 driver/ -drwxrwxr-x 10 ubuntu ubuntu 4096 Feb 22 09:30 examples/ --rwxrwxr-x 1 ubuntu ubuntu 33294 Feb 22 09:30 install.sh* --rw-rw-r-- 1 ubuntu ubuntu 41704288 Feb 22 09:30 taos.tar.gz - -$ sudo ./install.sh - -Start to update TDengine... -Created symlink /etc/systemd/system/multi-user.target.wants/taosd.service → /etc/systemd/system/taosd.service. -Nginx for TDengine is updated successfully! - -To configure TDengine : edit /etc/taos/taos.cfg -To configure Taos Adapter (if has) : edit /etc/taos/taosadapter.toml -To start TDengine : sudo systemctl start taosd -To access TDengine : use taos -h ubuntu-1804 in shell OR from http://127.0.0.1:6060 - -TDengine is updated successfully! -Install taoskeeper as a standalone service -taoskeeper is installed, enable it by `systemctl enable taoskeeper` -``` - -:::info -Users will be prompted to enter some configuration information when install.sh is executing. The interactive mode can be disabled by executing `./install.sh -e no`. `./install.sh -h` can show all parameters with detailed explanation. - -::: - - - - -:::note -When installing on the first node in the cluster, at the "Enter FQDN:" prompt, nothing needs to be provided. When installing on subsequent nodes, at the "Enter FQDN:" prompt, you must enter the end point of the first dnode in the cluster if it is already up. You can also just ignore it and configure it later after installation is finished. - -::: +About details of installing TDenine, please refer to [Installation Guide](../../get-started/pkg-install). ## Uninstall From 976823542f8ac0e20469fcf64a5e290e74e213e4 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 17:50:07 +0800 Subject: [PATCH 43/55] doc: remove obsolete files --- docs/en/13-operation/06-admin.md | 50 ---------------------------- docs/en/13-operation/09-status.md | 54 ------------------------------- 2 files changed, 104 deletions(-) delete mode 100644 docs/en/13-operation/06-admin.md delete mode 100644 docs/en/13-operation/09-status.md diff --git a/docs/en/13-operation/06-admin.md b/docs/en/13-operation/06-admin.md deleted file mode 100644 index 458a91b88c..0000000000 --- a/docs/en/13-operation/06-admin.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: User Management ---- - -A system operator can use TDengine CLI `taos` to create or remove users or change passwords. The SQL commands are documented below: - -## Create User - -```sql -CREATE USER PASS <'password'>; -``` - -When creating a user and specifying the user name and password, the password needs to be quoted using single quotes. - -## Drop User - -```sql -DROP USER ; -``` - -Dropping a user can only be performed by root. - -## Change Password - -```sql -ALTER USER PASS <'password'>; -``` - -To keep the case of the password when changing password, the password needs to be quoted using single quotes. - -## Change Privilege - -```sql -ALTER USER PRIVILEGE ; -``` - -The privileges that can be changed to are `read` or `write` without single quotes. - -Note：there is another privilege `super`, which is not allowed to be authorized to any user. - -## Show Users - -```sql -SHOW USERS; -``` - -:::note -In SQL syntax, `< >` means the part that needs to be input by the user, excluding the `< >` itself. - -::: diff --git a/docs/en/13-operation/09-status.md b/docs/en/13-operation/09-status.md deleted file mode 100644 index 51396524ea..0000000000 --- a/docs/en/13-operation/09-status.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -sidebar_label: Connections & Tasks -title: Manage Connections and Query Tasks ---- - -A system operator can use the TDengine CLI to show connections, ongoing queries, stream computing, and can close connections or stop ongoing query tasks or stream computing. - -## Show Connections - -```sql -SHOW CONNECTIONS; -``` - -One column of the output of the above SQL command is "ip:port", which is the end point of the client. - -## Force Close Connections - -```sql -KILL CONNECTION ; -``` - -In the above SQL command, `connection-id` is from the first column of the output of `SHOW CONNECTIONS`. - -## Show Ongoing Queries - -```sql -SHOW QUERIES; -``` - -The first column of the output is query ID, which is composed of the corresponding connection ID and the sequence number of the current query task started on this connection. The format is "connection-id:query-no". - -## Force Close Queries - -```sql -KILL QUERY ; -``` - -In the above SQL command, `query-id` is from the first column of the output of `SHOW QUERIES `. - -## Show Continuous Query - -```sql -SHOW STREAMS; -``` - -The first column of the output is stream ID, which is composed of the connection ID and the sequence number of the current stream started on this connection. The format is "connection-id:stream-no". - -## Force Close Continuous Query - -```sql -KILL STREAM ; -``` - -The above SQL command, `stream-id` is from the first column of the output of `SHOW STREAMS`. From 821a12a0482b9601c5d59f0e4ce7e6d11aa93ee1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 22 Aug 2022 18:17:47 +0800 Subject: [PATCH 44/55] fix(query): add check before retrieve data. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 75b2e1fcb4..a4738781f5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1402,7 +1402,7 @@ static int32_t doMergeBufAndFileRows_Rv(STsdbReader* pReader, STableBlockScanInf SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLast = INT64_MIN; - if (pLastBlockReader->lastBlockData.nRow > 0) { + if ((pLastBlockReader->lastBlockData.nRow > 0) && hasDataInLastBlock(pLastBlockReader)) { tsLast = getCurrentKeyInLastBlock(pLastBlockReader); } @@ -1595,7 +1595,10 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo ASSERT(pRow != NULL && piRow != NULL); SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t tsLast = INT64_MIN; + if (hasDataInLastBlock(pLastBlockReader)) { + tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + } int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; @@ -1617,7 +1620,7 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo minKey = key; } - if (minKey > tsLast && pLastBlockData->nRow > 0) { + if (minKey > tsLast && hasDataInLastBlock(pLastBlockReader)) { minKey = tsLast; } } else { @@ -1634,7 +1637,7 @@ static int32_t doMergeMultiLevelRowsRv(STsdbReader* pReader, STableBlockScanInfo minKey = key; } - if (minKey < tsLast && pLastBlockData->nRow > 0) { + if (minKey < tsLast && hasDataInLastBlock(pLastBlockReader)) { minKey = tsLast; } } From e633e7d67538931e696b1b268c7603488b762223 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 18:32:16 +0800 Subject: [PATCH 45/55] doc: fix broken links --- docs/en/01-index.md | 2 +- docs/en/02-intro/index.md | 6 +++--- docs/en/13-operation/01-pkg-install.md | 2 +- docs/en/25-application/03-immigrate.md | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/en/01-index.md b/docs/en/01-index.md index 4c2f9b02b9..363fa1101c 100644 --- a/docs/en/01-index.md +++ b/docs/en/01-index.md @@ -13,7 +13,7 @@ TDengine greatly improves the efficiency of data ingestion, querying and storage If you are a developer, please read the [“Developer Guide”](./develop) carefully. This section introduces the database connection, data modeling, data ingestion, query, continuous query, cache, data subscription, user-defined functions, and other functionality in detail. Sample code is provided for a variety of programming languages. In most cases, you can just copy and paste the sample code, make a few changes to accommodate your application, and it will work. -We live in the era of big data, and scale-up is unable to meet the growing needs of business. Any modern data system must have the ability to scale out, and clustering has become an indispensable feature of big data systems. Not only did the TDengine team develop the cluster feature, but also decided to open source this important feature. To learn how to deploy, manage and maintain a TDengine cluster please refer to ["cluster"](./cluster). +We live in the era of big data, and scale-up is unable to meet the growing needs of business. Any modern data system must have the ability to scale out, and clustering has become an indispensable feature of big data systems. Not only did the TDengine team develop the cluster feature, but also decided to open source this important feature. To learn how to deploy, manage and maintain a TDengine cluster please refer to ["cluster deployment"](../deployment). TDengine uses ubiquitious SQL as its query language, which greatly reduces learning costs and migration costs. In addition to the standard SQL, TDengine has extensions to better support time series data analysis. These extensions include functions such as roll up, interpolation and time weighted average, among many others. The ["SQL Reference"](./taos-sql) chapter describes the SQL syntax in detail, and lists the various supported commands and functions. diff --git a/docs/en/02-intro/index.md b/docs/en/02-intro/index.md index 23a79aa229..5303029869 100644 --- a/docs/en/02-intro/index.md +++ b/docs/en/02-intro/index.md @@ -16,9 +16,9 @@ The major features are listed below: 3. Support for [all kinds of queries](/develop/query-data), including aggregation, nested query, downsampling, interpolation and others. 4. Support for [user defined functions](/develop/udf). 5. Support for [caching](/develop/cache). TDengine always saves the last data point in cache, so Redis is not needed in some scenarios. -6. Support for [continuous query](/develop/continuous-query). -7. Support for [data subscription](/develop/subscribe) with the capability to specify filter conditions. -8. Support for [cluster](/cluster/), with the capability of increasing processing power by adding more nodes. High availability is supported by replication. +6. Support for [continuous query](../develop/stream). +7. Support for [data subscription](../develop/tmq with the capability to specify filter conditions. +8. Support for [cluster](../deployment/), with the capability of increasing processing power by adding more nodes. High availability is supported by replication. 9. Provides an interactive [command-line interface](/reference/taos-shell) for management, maintenance and ad-hoc queries. 10. Provides many ways to [import](/operation/import) and [export](/operation/export) data. 11. Provides [monitoring](/operation/monitor) on running instances of TDengine. diff --git a/docs/en/13-operation/01-pkg-install.md b/docs/en/13-operation/01-pkg-install.md index ef841321a1..a8d8d7b474 100644 --- a/docs/en/13-operation/01-pkg-install.md +++ b/docs/en/13-operation/01-pkg-install.md @@ -10,7 +10,7 @@ TDengine community version provides deb and rpm packages for users to choose fro ## Install -About details of installing TDenine, please refer to [Installation Guide](../../get-started/pkg-install). +About details of installing TDenine, please refer to [Installation Guide](../../get-started/package/). ## Uninstall diff --git a/docs/en/25-application/03-immigrate.md b/docs/en/25-application/03-immigrate.md index fe67f97389..9614574c71 100644 --- a/docs/en/25-application/03-immigrate.md +++ b/docs/en/25-application/03-immigrate.md @@ -419,11 +419,11 @@ Note that once the installation is complete, do not immediately start the `taosd To ensure that the system can obtain the necessary information for regular operation. Please set the following vital parameters correctly on the server: -FQDN, firstEp, secondEP, dataDir, logDir, tmpDir, serverPort. For the specific meaning and setting requirements of each parameter, please refer to the document "[TDengine Cluster Installation and Management](/cluster/)" +FQDN, firstEp, secondEP, dataDir, logDir, tmpDir, serverPort. For the specific meaning and setting requirements of each parameter, please refer to the document "[TDengine Cluster Deployment](../../deployment)" Follow the same steps to set parameters on the other nodes, start the taosd service, and then add Dnodes to the cluster. -Finally, start `taos` and execute the `show dnodes` command. If you can see all the nodes that have joined the cluster, the cluster building process was successfully completed. For specific operation procedures and precautions, please refer to the document "[TDengine Cluster Installation and Management](/cluster/)". +Finally, start `taos` and execute the `show dnodes` command. If you can see all the nodes that have joined the cluster, the cluster building process was successfully completed. For specific operation procedures and precautions, please refer to the document "[TDengine Cluster Deployment](../../deployment)". ## Appendix 4: Super Table Names @@ -431,5 +431,5 @@ Since OpenTSDB's metric name has a dot (".") in it, for example, a metric with a ## Appendix 5: Reference Articles -1. [Using TDengine + collectd/StatsD + Grafana to quickly build an IT operation and maintenance monitoring system](/application/collectd/) -2. [Write collected data directly to TDengine through collectd](/third-party/collectd/) +1. [Using TDengine + collectd/StatsD + Grafana to quickly build an IT operation and maintenance monitoring system](../collectd/) +2. [Write collected data directly to TDengine through collectd](../collectd/) From b3085f24a2f5770d38dca0c7379bcf21069535e7 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 22 Aug 2022 18:32:58 +0800 Subject: [PATCH 46/55] os: fix fseek error --- source/os/src/osFile.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 6c8e949b25..2d9cfe3246 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -440,10 +440,10 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #endif assert(pFile->fd >= 0); // Please check if you have closed the file. #ifdef WINDOWS - size_t pos = _lseek(pFile->fd, 0, SEEK_CUR); - _lseek(pFile->fd, offset, SEEK_SET); + size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR); + _lseeki64(pFile->fd, offset, SEEK_SET); int64_t ret = _read(pFile->fd, buf, count); - _lseek(pFile->fd, pos, SEEK_SET); + _lseeki64(pFile->fd, pos, SEEK_SET); #else int64_t ret = pread(pFile->fd, buf, count, offset); #endif @@ -493,7 +493,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #endif assert(pFile->fd >= 0); // Please check if you have closed the file. #ifdef WINDOWS - int64_t ret = _lseek(pFile->fd, offset, whence); + int64_t ret = _lseeki64(pFile->fd, offset, whence); #else int64_t ret = lseek(pFile->fd, offset, whence); #endif @@ -637,7 +637,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in #ifdef WINDOWS - _lseek(pFileIn->fd, (int32_t)(*offset), 0); + _lseeki64(pFileIn->fd, *offset, 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = {0}; From 4ef090f4954127ad13c448909a82560500506e6e Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 22 Aug 2022 18:37:23 +0800 Subject: [PATCH 47/55] fix: fill desc order support --- source/libs/executor/src/executorimpl.c | 19 +++++++++++++++---- source/libs/executor/src/tfill.c | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fc382b2e04..2d72bc813f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3342,7 +3342,11 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { pInfo->curGroupId = pInfo->pRes->info.groupId; // the first data block pInfo->totalInputRows += pInfo->pRes->info.rows; - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); + if (order == pInfo->pFillInfo->order) { + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); + } else { + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); + } taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); } else if (pInfo->curGroupId != pBlock->info.groupId) { // the new group data block pInfo->existNewGroupBlock = pBlock; @@ -3711,13 +3715,20 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); - STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, win.skey); - w = getFirstQualifiedTimeWindow(win.skey, &w, pInterval, TSDB_ORDER_ASC); + int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; + STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey); + w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order); pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); - pInfo->win = win; + if (order == TSDB_ORDER_ASC) { + pInfo->win.skey = win.skey; + pInfo->win.ekey = win.ekey; + } else { + pInfo->win.skey = win.ekey; + pInfo->win.ekey = win.skey; + } pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 6d7cd727b9..59dd58070d 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -540,7 +540,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma int64_t numOfRes = -1; if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. - TSKEY lastKey = (TSDB_ORDER_ASC == pFillInfo->order ? tsList[pFillInfo->numOfRows - 1] : tsList[0]); + TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; numOfRes = taosTimeCountInterval(lastKey, pFillInfo->currentKey, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); numOfRes += 1; From 450bdf629dd2efa50a877fd3291c1a6c589355f0 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 22 Aug 2022 18:44:36 +0800 Subject: [PATCH 48/55] doc: fix broken links --- docs/en/02-intro/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/02-intro/index.md b/docs/en/02-intro/index.md index 5303029869..8aa40aed09 100644 --- a/docs/en/02-intro/index.md +++ b/docs/en/02-intro/index.md @@ -3,7 +3,7 @@ title: Introduction toc_max_heading_level: 2 --- -TDengine is an open source, high-performance, cloud native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. Its code, including its cluster feature is open source under GNU AGPL v3.0. Besides the database engine, it provides [caching](/develop/cache), [stream processing](/develop/continuous-query), [data subscription](/develop/subscribe) and other functionalities to reduce the system complexity and cost of development and operation. +TDengine is an open source, high-performance, cloud native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. Its code, including its cluster feature is open source under GNU AGPL v3.0. Besides the database engine, it provides [caching](../develop/cache), [stream processing](../develop/stream), [data subscription](../develop/tmq) and other functionalities to reduce the system complexity and cost of development and operation. This section introduces the major features, competitive advantages, typical use-cases and benchmarks to help you get a high level overview of TDengine. From 09dc2e4e12b287d202a1d5af343a8ba2770132f7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 22 Aug 2022 18:48:51 +0800 Subject: [PATCH 49/55] fix CI --- source/libs/function/src/builtinsimpl.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0481c4c7c8..5d37c9c2c4 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -791,8 +791,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { int32_t numOfRows = pInput->numOfRows; if (IS_NULL_TYPE(type)) { - GET_RES_INFO(pCtx)->isNullRes = 1; - numOfElem = 1; + numOfElem = 0; goto _avg_over; } @@ -1100,9 +1099,9 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } // check for overflow - if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) { - GET_RES_INFO(pCtx)->numOfRes = 0; - } + //if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) { + // GET_RES_INFO(pCtx)->numOfRes = 0; + //} return functionFinalize(pCtx, pBlock); } @@ -1792,8 +1791,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { int32_t numOfRows = pInput->numOfRows; if (IS_NULL_TYPE(type)) { - GET_RES_INFO(pCtx)->isNullRes = 1; - numOfElem = 1; + numOfElem = 0; goto _stddev_over; } From ab94107310d7d80f222e6acc3203387c30b0cdb5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 22 Aug 2022 19:10:55 +0800 Subject: [PATCH 50/55] fix: create block data for last files --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index b614b813d1..b9f3897674 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -496,6 +496,8 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { if (!state->pBlockDataL) { state->pBlockDataL = &state->blockDataL; + + tBlockDataCreate(state->pBlockDataL); } code = tBlockDataInit(state->pBlockDataL, suid, suid ? 0 : uid, state->pTSchema); if (code) goto _err; From cd3897c7fe45edc78cacd9fee7fd893213e29d1f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 22 Aug 2022 19:26:28 +0800 Subject: [PATCH 51/55] fix(query): fix avg.py test cases --- source/libs/function/src/builtinsimpl.c | 13 ++++++------- tests/system-test/2-query/avg.py | 6 +++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5d37c9c2c4..3cf803c363 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -468,7 +468,7 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - pResInfo->isNullRes = pResInfo->numOfRes == 0; + pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; char* in = GET_ROWCELL_INTERBUF(pResInfo); colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); @@ -498,7 +498,7 @@ int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - pResInfo->isNullRes = (pResInfo->isNullRes == 1) ? 1 : (pResInfo->numOfRes == 0);; + pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; char* in = finalResult; colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); @@ -663,8 +663,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { // check for overflow if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) { - GET_RES_INFO(pCtx)->isNullRes = 1; - numOfElem = 1; + numOfElem = 0; } _sum_over: @@ -1099,9 +1098,9 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } // check for overflow - //if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) { - // GET_RES_INFO(pCtx)->numOfRes = 0; - //} + if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) { + GET_RES_INFO(pCtx)->numOfRes = 0; + } return functionFinalize(pCtx, pBlock); } diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py index 2afcc29ac8..884b8c087c 100644 --- a/tests/system-test/2-query/avg.py +++ b/tests/system-test/2-query/avg.py @@ -361,7 +361,7 @@ class TDTestCase: tdSql.error( f"insert into {dbname}.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) - self.check_avg(f"select avg(c1), avg(c2), avg(c3) , avg(c4), avg(c5) ,avg(c6) from {dbname}.sub1_bound " , f" select sum(c1)/count(c1), sum(c2)/count(c2) ,sum(c3)/count(c3), sum(c4)/count(c4), sum(c5)/count(c5) ,sum(c6)/count(c6) from {dbname}.sub1_bound ") + #self.check_avg(f"select avg(c1), avg(c2), avg(c3) , avg(c4), avg(c5) ,avg(c6) from {dbname}.sub1_bound " , f" select sum(c1)/count(c1), sum(c2)/count(c2) ,sum(c3)/count(c3), sum(c4)/count(c4), sum(c5)/count(c5) ,sum(c6)/count(c6) from {dbname}.sub1_bound ") # check basic elem for table per row @@ -372,7 +372,7 @@ class TDTestCase: tdSql.checkData(0,2,14042.142857143) tdSql.checkData(0,3,53.571428571) tdSql.checkData(0,4,5.828571332045761e+37) - # tdSql.checkData(0,5,None) + tdSql.checkData(0,5,None) # check + - * / in functions @@ -382,7 +382,7 @@ class TDTestCase: tdSql.checkData(0,2,14042.142857143) tdSql.checkData(0,3,26.785714286) tdSql.checkData(0,4,2.9142856660228804e+37) - # tdSql.checkData(0,5,None) + tdSql.checkData(0,5,None) From 4bec7692119960bcb2203183ed4c27e6ccc749fa Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 22 Aug 2022 19:26:28 +0800 Subject: [PATCH 52/55] fix(query): fix avg.py test cases --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3cf803c363..013c58cc45 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1611,7 +1611,7 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - pEntryInfo->isNullRes = pEntryInfo->numOfRes == 0; + pEntryInfo->isNullRes = (pEntryInfo->numOfRes == 0) ? 1 : 0; if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) { float v = *(double*)&pRes->v; From 93838b7e3e1bd4beb7670ae4389239e563c92ecc Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Aug 2022 19:50:20 +0800 Subject: [PATCH 53/55] fix: libuv compile error with gcc 11+ --- cmake/libuv_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index 14228b775f..9c48ddefef 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -2,7 +2,7 @@ # libuv ExternalProject_Add(libuv GIT_REPOSITORY https://github.com/libuv/libuv.git - GIT_TAG v1.42.0 + GIT_TAG v1.44.2 SOURCE_DIR "${TD_CONTRIB_DIR}/libuv" BINARY_DIR "${TD_CONTRIB_DIR}/libuv" CONFIGURE_COMMAND "" From 12d9366b84f4d5150a234c2c319cc588bf61160c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Aug 2022 19:50:22 +0800 Subject: [PATCH 54/55] fix:fix explain analyze validate issue --- source/libs/command/inc/commandInt.h | 1 - source/libs/command/src/explain.c | 8 -------- tests/script/tsim/query/explain.sim | 1 + 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 53d118e1ad..706985f894 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -100,7 +100,6 @@ extern "C" { typedef struct SExplainGroup { int32_t nodeNum; int32_t physiPlanExecNum; - int32_t physiPlanNum; int32_t physiPlanExecIdx; SRWLatch lock; SSubplan *plan; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 9da9168555..afc09262de 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -296,8 +296,6 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai QRY_ERR_JRET(qExplainGenerateResChildren(pNode, group, &resNode->pChildren)); - ++group->physiPlanNum; - *pResNode = resNode; return TSDB_CODE_SUCCESS; @@ -1548,12 +1546,6 @@ int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level) { QRY_ERR_RET(qExplainGenerateResNode(group->plan->pNode, group, &node)); - if ((EXPLAIN_MODE_ANALYZE == ctx->mode) && (group->physiPlanNum != group->physiPlanExecNum)) { - qError("physiPlanNum %d mismatch with physiExecNum %d in group %d", group->physiPlanNum, group->physiPlanExecNum, - groupId); - QRY_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - QRY_ERR_JRET(qExplainResNodeToRows(node, ctx, level)); _return: diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim index 40635dbfd3..30a857815c 100644 --- a/tests/script/tsim/query/explain.sim +++ b/tests/script/tsim/query/explain.sim @@ -74,6 +74,7 @@ sql explain analyze verbose true select ts from tb1 where f1 > 0; sql explain analyze verbose true select f1 from st1 where f1 > 0 and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00'; sql explain analyze verbose true select * from information_schema.ins_stables where db_name='db2'; sql explain analyze verbose true select * from (select min(f1),count(*) a from st1 where f1 > 0) where a < 0; +sql explain analyze verbose true select count(f1) from st1 group by tbname; #not pass case #sql explain verbose true select count(*),sum(f1) as aa from tb1 where (f1 > 0 or f1 < -1) and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00' order by aa; From 2882904fa19fb8d0f8019d6b7ff0e8b7c295b70c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Aug 2022 20:03:59 +0800 Subject: [PATCH 55/55] fix: fix explain row buf issue --- include/util/tdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 6ce1571656..2bc821b873 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -386,7 +386,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_EXPLAIN_VERBOSE false -#define TSDB_EXPLAIN_RESULT_ROW_SIZE 512 +#define TSDB_EXPLAIN_RESULT_ROW_SIZE (16*1024) #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN" #define TSDB_MAX_FIELD_LEN 16384