From d2ae4a6e2666be1a41b53cc25bc7255fa7aaa69d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 5 Mar 2024 09:52:26 +0800 Subject: [PATCH 01/34] enh(s3/writing): enable writing to s3 always --- source/dnode/vnode/src/inc/vnd.h | 1 + source/dnode/vnode/src/vnd/vnodeSync.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 4036200d73..8121c8fdd7 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -133,6 +133,7 @@ int32_t vnodeAsyncCommit(SVnode* pVnode); bool vnodeShouldRollback(SVnode* pVnode); // vnodeSync.c +int32_t vnodeNodeId(SVnode* pVnode); int32_t vnodeSyncOpen(SVnode* pVnode, char* path, int32_t vnodeVersion); int32_t vnodeSyncStart(SVnode* pVnode); void vnodeSyncPreClose(SVnode* pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index f9f2ae6b21..ef2dfcee85 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -811,6 +811,11 @@ bool vnodeIsLeader(SVnode *pVnode) { return true; } +int32_t vnodeNodeId(SVnode *pVnode) { + SSyncCfg *syncCfg = &pVnode->config.syncCfg; + return syncCfg->nodeInfo[syncCfg->myIndex].nodeId; +} + int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { int code = 0; pSnap->lastApplyIndex = pVnode->state.committed; From 60ef86da6d61d411d9cdb182dc7561308cc834c5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 6 Mar 2024 14:35:44 +0800 Subject: [PATCH 02/34] vnd/clusterId: new interface to get cluster id for vnode --- source/dnode/vnode/src/inc/vnd.h | 1 + source/dnode/vnode/src/vnd/vnodeSync.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 8121c8fdd7..f9f4b603fd 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -133,6 +133,7 @@ int32_t vnodeAsyncCommit(SVnode* pVnode); bool vnodeShouldRollback(SVnode* pVnode); // vnodeSync.c +int64_t vnodeClusterId(SVnode* pVnode); int32_t vnodeNodeId(SVnode* pVnode); int32_t vnodeSyncOpen(SVnode* pVnode, char* path, int32_t vnodeVersion); int32_t vnodeSyncStart(SVnode* pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index ef2dfcee85..35b63c5792 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -811,6 +811,11 @@ bool vnodeIsLeader(SVnode *pVnode) { return true; } +int64_t vnodeClusterId(SVnode *pVnode) { + SSyncCfg *syncCfg = &pVnode->config.syncCfg; + return syncCfg->nodeInfo[syncCfg->myIndex].clusterId; +} + int32_t vnodeNodeId(SVnode *pVnode) { SSyncCfg *syncCfg = &pVnode->config.syncCfg; return syncCfg->nodeInfo[syncCfg->myIndex].nodeId; From 9a8d03f0ca34fefc0562a8c60ffa9d47a4d21c43 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 2 Apr 2024 14:03:02 +0800 Subject: [PATCH 03/34] adj last function --- source/libs/parser/src/parTranslater.c | 4 ++-- tests/script/tsim/parser/first_last_query.sim | 4 ++-- tests/script/tsim/parser/join_multitables.sim | 16 ++++++------- tests/script/tsim/parser/last_cache_query.sim | 4 ++-- tests/script/tsim/parser/lastrow2.sim | 6 ++--- tests/script/tsim/query/bi_star_table.sim | 24 +++++++++---------- tests/script/tsim/query/cache_last.sim | 5 ++-- .../system-test/2-query/last_and_last_row.py | 4 ++-- 8 files changed, 34 insertions(+), 33 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2110013310..1ea0ec8574 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3368,9 +3368,9 @@ static int32_t createMultiResFuncsParas(STranslateContext* pCxt, SNodeList* pSrc SNode* pPara = NULL; FOREACH(pPara, pSrcParas) { if (nodesIsStar(pPara)) { - code = createAllColumns(pCxt, true, &pExprs); + code = createAllColumns(pCxt, false, &pExprs); } else if (nodesIsTableStar(pPara)) { - code = createTableAllCols(pCxt, (SColumnNode*)pPara, true, &pExprs); + code = createTableAllCols(pCxt, (SColumnNode*)pPara, false, &pExprs); } else { code = nodesListMakeStrictAppend(&pExprs, nodesCloneNode(pPara)); } diff --git a/tests/script/tsim/parser/first_last_query.sim b/tests/script/tsim/parser/first_last_query.sim index 533f59f4b2..8d02f817e3 100644 --- a/tests/script/tsim/parser/first_last_query.sim +++ b/tests/script/tsim/parser/first_last_query.sim @@ -298,7 +298,7 @@ if $data01 != 112 then return -1 endi -if $data02 != @tm0@ then +if $data03 != @tm0@ then return -1 endi @@ -310,7 +310,7 @@ if $data11 != 421 then return -1 endi -if $data12 != @tm1@ then +if $data13 != @tm1@ then return -1 endi diff --git a/tests/script/tsim/parser/join_multitables.sim b/tests/script/tsim/parser/join_multitables.sim index d2f8ea4a88..59e41bc9d9 100644 --- a/tests/script/tsim/parser/join_multitables.sim +++ b/tests/script/tsim/parser/join_multitables.sim @@ -805,16 +805,16 @@ endi if $data04 != 01 then return -1 endi -if $data05 != @21-03-01 01:00:00.000@ then +if $data[0][10] != @21-03-01 01:00:00.000@ then return -1 endi -if $data06 != 9911 then +if $data[0][11] != 9911 then return -1 endi -if $data07 != 9911.000000000 then +if $data[0][12] != 9911.000000000 then return -1 endi -if $data08 != 11 then +if $data[0][13] != 11 then return -1 endi @@ -837,16 +837,16 @@ endi if $data04 != 05 then return -1 endi -if $data05 != @21-03-01 05:00:00.000@ then +if $data[0][10] != @21-03-01 05:00:00.000@ then return -1 endi -if $data06 != 9915 then +if $data[0][11] != 9915 then return -1 endi -if $data07 != 9915.000000000 then +if $data[0][12] != 9915.000000000 then return -1 endi -if $data08 != 15 then +if $data[0][13] != 15 then return -1 endi diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index 30196e0b62..7bf6a51731 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -243,11 +243,11 @@ endi if $data04 != @70-01-01 07:59:57.000@ then return -1 endi -if $data05 != @21-05-12 10:10:12.000@ then +if $data06 != @21-05-12 10:10:12.000@ then print $data00 return -1 endi -if $data06 != @70-01-01 07:59:57.000@ then +if $data07 != @70-01-01 07:59:57.000@ then return -1 endi diff --git a/tests/script/tsim/parser/lastrow2.sim b/tests/script/tsim/parser/lastrow2.sim index 33267e3cfd..278de7ab49 100644 --- a/tests/script/tsim/parser/lastrow2.sim +++ b/tests/script/tsim/parser/lastrow2.sim @@ -54,13 +54,13 @@ sql select last_row(*), ts, 'abc', 123.981, tbname from m1 if $rows != 1 then return -1 endi -if $data02 != @19-01-01 01:01:01.000@ then +if $data03 != @19-01-01 01:01:01.000@ then return -1 endi -if $data03 != @abc@ then +if $data04 != @abc@ then return -1 endi -if $data04 != 123.981000000 then +if $data05 != 123.981000000 then print expect 123.981000000, actual: $data04 return -1 endi diff --git a/tests/script/tsim/query/bi_star_table.sim b/tests/script/tsim/query/bi_star_table.sim index 6bd6938678..1d71d6a68e 100644 --- a/tests/script/tsim/query/bi_star_table.sim +++ b/tests/script/tsim/query/bi_star_table.sim @@ -33,29 +33,29 @@ if $data06 != tba1 then endi sql select last(*) from db1.sta; -if $cols != 4 then +if $cols != 7 then return -1 endi -if $data03 != tba2 then +if $data06 != tba2 then return -1 endi sql select last_row(*) from db1.sta; -if $cols != 4 then +if $cols != 7 then return -1 endi -if $data03 != tba2 then +if $data06 != tba2 then return -1 endi sql select first(*) from db1.sta; -if $cols != 4 then +if $cols != 7 then return -1 endi -if $data03 != tba1 then +if $data06 != tba1 then return -1 endi @@ -71,29 +71,29 @@ if $data06 != tba1 then endi sql select last(b.*) from db1.sta b; -if $cols != 4 then +if $cols != 7 then return -1 endi -if $data03 != tba2 then +if $data06 != tba2 then return -1 endi sql select last_row(b.*) from db1.sta b; -if $cols != 4 then +if $cols != 7 then return -1 endi -if $data03 != tba2 then +if $data06 != tba2 then return -1 endi sql select first(b.*) from db1.sta b; -if $cols != 4 then +if $cols != 7 then return -1 endi -if $data03 != tba1 then +if $data06 != tba1 then return -1 endi diff --git a/tests/script/tsim/query/cache_last.sim b/tests/script/tsim/query/cache_last.sim index 65eb46de69..f936f822a5 100644 --- a/tests/script/tsim/query/cache_last.sim +++ b/tests/script/tsim/query/cache_last.sim @@ -35,11 +35,12 @@ if $data03 != b then return -1 endi sql explain select count(*), last(*) from sta; -if $data00 != @-> Merge (columns=4 width=226 input_order=unknown output_order=unknown mode=column)@ then +if $data00 != @-> Merge (columns=5 width=230 input_order=unknown output_order=unknown mode=column)@ then + print $data00 return -1 endi sql explain select first(f1), last(*) from sta; -if $data00 != @-> Merge (columns=4 width=226 input_order=unknown output_order=unknown mode=column)@ then +if $data00 != @-> Merge (columns=5 width=230 input_order=unknown output_order=unknown mode=column)@ then return -1 endi sql select first(f1), last(*) from sta; diff --git a/tests/system-test/2-query/last_and_last_row.py b/tests/system-test/2-query/last_and_last_row.py index b04b3a75f3..ce25c3dd99 100644 --- a/tests/system-test/2-query/last_and_last_row.py +++ b/tests/system-test/2-query/last_and_last_row.py @@ -152,8 +152,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, last_ts4) tdSql.checkData(0, 1, 5 * maxRange - 1) - tdSql.checkData(0, 2, last_ts4) - tdSql.checkData(0, 3, 4 * maxRange + 1) + tdSql.checkData(0, 3, last_ts4) + tdSql.checkData(0, 4, 4 * maxRange + 1) explain_res = self.explain_sql(sql) self.check_explain_res_no_row("Last Row Scan", explain_res, sql) From 71ff0bcb843601ab6df73d97330925697197c1a5 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 2 Apr 2024 17:24:47 +0800 Subject: [PATCH 04/34] adj ci --- tests/script/tsim/parser/last_both_query.sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/parser/last_both_query.sim b/tests/script/tsim/parser/last_both_query.sim index 5f86412199..1cfb2a316f 100644 --- a/tests/script/tsim/parser/last_both_query.sim +++ b/tests/script/tsim/parser/last_both_query.sim @@ -243,11 +243,11 @@ endi if $data04 != @70-01-01 07:59:57.000@ then return -1 endi -if $data05 != @21-05-12 10:10:12.000@ then +if $data06 != @21-05-12 10:10:12.000@ then print $data00 return -1 endi -if $data06 != @70-01-01 07:59:57.000@ then +if $data07 != @70-01-01 07:59:57.000@ then return -1 endi From e0a5b2dd9df895a25053cf64ea04e018c8c7e07c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 8 Apr 2024 08:55:13 +0800 Subject: [PATCH 05/34] split last tag --- source/libs/planner/src/planOptimizer.c | 12 +++++++++--- tests/system-test/2-query/last_and_last_row.py | 12 ++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 40e1d12a13..d322174fd0 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2641,7 +2641,8 @@ static bool isNeedSplitCacheLastFunc(SFunctionNode* pFunc, SScanLogicNode* pScan int32_t funcType = pFunc->funcType; if ((FUNCTION_TYPE_LAST_ROW != funcType || (FUNCTION_TYPE_LAST_ROW == funcType && TSDB_CACHE_MODEL_LAST_VALUE == pScan->cacheLastMode)) && (FUNCTION_TYPE_LAST != funcType || (FUNCTION_TYPE_LAST == funcType && (TSDB_CACHE_MODEL_LAST_ROW == pScan->cacheLastMode || - QUERY_NODE_OPERATOR == nodeType(nodesListGetNode(pFunc->pParameterList, 0)) || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))))) && + QUERY_NODE_OPERATOR == nodeType(nodesListGetNode(pFunc->pParameterList, 0)) || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0)) || + COLUMN_TYPE_COLUMN != ((SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0))->colType))) && FUNCTION_TYPE_SELECT_VALUE != funcType && FUNCTION_TYPE_GROUP_KEY != funcType) { return true; } @@ -2661,8 +2662,9 @@ static bool lastRowScanOptCheckFuncList(SLogicNode* pNode, int8_t cacheLastModel if (FUNCTION_TYPE_LAST == pAggFunc->funcType) { if (QUERY_NODE_COLUMN == nodeType(pParam)) { SColumnNode* pCol = (SColumnNode*)pParam; - if (pCol->colType != COLUMN_TYPE_COLUMN) { - return false; + if (pCol->colType != COLUMN_TYPE_COLUMN && TSDB_CACHE_MODEL_LAST_ROW != cacheLastModel) { + needSplitFuncCount++; + *hasOtherFunc = true; } if (lastColId != pCol->colId) { lastColId = pCol->colId; @@ -3070,6 +3072,10 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg, if (TSDB_CODE_SUCCESS != code) { return code; } + code = nodesCollectColumnsFromNode((SNode*)list, NULL, COLLECT_COL_TYPE_TAG, &pScan->pScanPseudoCols); + if (TSDB_CODE_SUCCESS != code) { + return code; + } nodesFree(list); bool found = false; FOREACH(pNode, pScan->pScanCols) { diff --git a/tests/system-test/2-query/last_and_last_row.py b/tests/system-test/2-query/last_and_last_row.py index ce25c3dd99..cd572b05cb 100644 --- a/tests/system-test/2-query/last_and_last_row.py +++ b/tests/system-test/2-query/last_and_last_row.py @@ -302,8 +302,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, last_ts4) tdSql.checkData(0, 1, 5 * maxRange - 1) - tdSql.checkData(0, 2, last_ts4) - tdSql.checkData(0, 3, 4 * maxRange + 1) + tdSql.checkData(0, 3, last_ts4) + tdSql.checkData(0, 4, 4 * maxRange + 1) explain_res = self.explain_sql(sql) self.check_explain_res_has_row("Last Row Scan", explain_res, sql) @@ -453,8 +453,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, last_ts4) tdSql.checkData(0, 1, 5 * maxRange - 1) - tdSql.checkData(0, 2, last_ts4) - tdSql.checkData(0, 3, 4 * maxRange + 1) + tdSql.checkData(0, 3, last_ts4) + tdSql.checkData(0, 4, 4 * maxRange + 1) explain_res = self.explain_sql(sql) self.check_explain_res_has_row("Last Row Scan", explain_res, sql) @@ -587,7 +587,7 @@ class TDTestCase: explain_res = self.explain_sql(sql) self.check_explain_res_has_row("Last Row Scan", explain_res, sql) - self.check_explain_res_no_row("Table Scan", explain_res, sql) + self.check_explain_res_has_row("Table Scan", explain_res, sql) sql = f'select last_row(ts), last(ts), last_row(id), last(id) from last_test_both_model.st;' tdSql.query(sql) @@ -607,7 +607,7 @@ class TDTestCase: tdSql.checkData(0, 0, last_ts4) tdSql.checkData(0, 1, 5 * maxRange - 1) #tdSql.checkData(0, 2, last_ts4) - tdSql.checkData(0, 3, 4 * maxRange + 1) + tdSql.checkData(0, 4, 4 * maxRange + 1) explain_res = self.explain_sql(sql) self.check_explain_res_has_row("Last Row Scan", explain_res, sql) From ca1262c5c5c3fe5bad306fc7e365d2b63ad81fce Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 8 Apr 2024 10:31:26 +0800 Subject: [PATCH 06/34] cos/multi-write: parser part --- source/libs/command/src/command.c | 4 +- source/libs/nodes/src/nodesCodeFuncs.c | 84 ++-- source/libs/nodes/src/nodesUtilFuncs.c | 5 + source/libs/parser/inc/parAst.h | 78 ++-- source/libs/parser/inc/sql.y | 8 + source/libs/parser/src/parAstCreater.c | 46 ++- source/libs/parser/src/parTokenizer.c | 4 + source/libs/parser/src/parTranslater.c | 515 ++++++++++++++----------- 8 files changed, 442 insertions(+), 302 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index c45a8931d2..f96973ffb9 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -307,12 +307,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm " "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " - "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d", + "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " KEEP_TIME_OFFSET %d S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d", dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize, - pCfg->keepTimeOffset); + pCfg->keepTimeOffset, pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact); if (retentions) { len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 689886c366..773a49b3d5 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -109,6 +109,8 @@ const char* nodesNodeName(ENodeType type) { return "FlushDatabaseStmt"; case QUERY_NODE_TRIM_DATABASE_STMT: return "TrimDatabaseStmt"; + case QUERY_NODE_S3MIGRATE_DATABASE_STMT: + return "S3MigrateDatabaseStmt"; case QUERY_NODE_CREATE_TABLE_STMT: return "CreateTableStmt"; case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: @@ -266,7 +268,7 @@ const char* nodesNodeName(ENodeType type) { case QUERY_NODE_SHOW_COMPACTS_STMT: return "ShowCompactsStmt"; case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT: - return "ShowCompactDetailsStmt"; + return "ShowCompactDetailsStmt"; case QUERY_NODE_SHOW_GRANTS_FULL_STMT: return "ShowGrantsFullStmt"; case QUERY_NODE_SHOW_GRANTS_LOGS_STMT: @@ -807,7 +809,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { static const char* jkProjectLogicPlanProjections = "Projections"; static const char* jkProjectLogicPlanIgnoreGroupId = "IgnoreGroupId"; -static const char* jkProjectLogicPlanInputIgnoreGroup= "InputIgnoreGroup"; +static const char* jkProjectLogicPlanInputIgnoreGroup = "InputIgnoreGroup"; static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj; @@ -1323,7 +1325,6 @@ static int32_t jsonToLogicDynQueryCtrlNode(const SJson* pJson, void* pObj) { return code; } - static const char* jkSubplanIdQueryId = "QueryId"; static const char* jkSubplanIdGroupId = "GroupId"; static const char* jkSubplanIdSubplanId = "SubplanId"; @@ -1830,8 +1831,6 @@ static int32_t jsonToFuncType(const SJson* pJson, void* pObj) { return code; } - - static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) { const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj; @@ -1845,7 +1844,7 @@ static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkLastRowScanPhysiPlanTargets, pNode->pTargets); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code) { code = tjsonAddTArray(pJson, jkLastRowScanPhysiPlanFuncTypes, funcTypeToJson, pNode->pFuncTypes); } @@ -2232,7 +2231,6 @@ static int32_t physiHashJoinNodeToJson(const void* pObj, SJson* pJson) { return code; } - static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) { SHashJoinPhysiNode* pNode = (SHashJoinPhysiNode*)pObj; @@ -2267,7 +2265,6 @@ static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) { return code; } - static const char* jkAggPhysiPlanExprs = "Exprs"; static const char* jkAggPhysiPlanGroupKeys = "GroupKeys"; static const char* jkAggPhysiPlanAggFuncs = "AggFuncs"; @@ -2481,7 +2478,7 @@ static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkSortPhysiPlanCalcGroupIds, &pNode->calcGroupId); } if (TSDB_CODE_SUCCESS == code) { - code= tjsonGetBoolValue(pJson, jkSortPhysiPlanExcludePKCol, &pNode->excludePkCol); + code = tjsonGetBoolValue(pJson, jkSortPhysiPlanExcludePKCol, &pNode->excludePkCol); } return code; @@ -3157,23 +3154,22 @@ static const char* jkGroupCachePhysiPlanGroupByUid = "GroupByUid"; static const char* jkGroupCachePhysiPlanGlobalGroup = "GlobalGroup"; static const char* jkGroupCachePhysiPlanBatchFetch = "BatchFetch"; - static int32_t physiGroupCacheNodeToJson(const void* pObj, SJson* pJson) { const SGroupCachePhysiNode* pNode = (const SGroupCachePhysiNode*)pObj; int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkGroupCachePhysiPlanGrpColsMayBeNull, pNode->grpColsMayBeNull); - } + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkGroupCachePhysiPlanGroupByUid, pNode->grpByUid); - } + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkGroupCachePhysiPlanGlobalGroup, pNode->globalGrp); - } + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkGroupCachePhysiPlanBatchFetch, pNode->batchFetch); - } + } if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkGroupCachePhysiPlanGroupCols, pNode->pGroupCols); } @@ -3291,8 +3287,6 @@ static int32_t jsonToPhysiDynQueryCtrlNode(const SJson* pJson, void* pObj) { return code; } - - static const char* jkQueryNodeAddrId = "Id"; static const char* jkQueryNodeAddrInUse = "InUse"; static const char* jkQueryNodeAddrNumOfEps = "NumOfEps"; @@ -4788,6 +4782,10 @@ static const char* jkDatabaseOptionsNumOfVgroups = "NumOfVgroups"; static const char* jkDatabaseOptionsSingleStable = "SingleStable"; static const char* jkDatabaseOptionsRetentions = "Retentions"; static const char* jkDatabaseOptionsSchemaless = "Schemaless"; +static const char* jkDatabaseOptionsS3ChunkSize = "S3ChunkSize"; +static const char* jkDatabaseOptionsS3KeepLocalNode = "S3KeepLocalNode"; +static const char* jkDatabaseOptionsS3KeepLocal = "S3KeepLocal"; +static const char* jkDatabaseOptionsS3Compact = "S3Compact"; static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) { const SDatabaseOptions* pNode = (const SDatabaseOptions*)pObj; @@ -4847,6 +4845,18 @@ static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsSchemaless, pNode->schemaless); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3ChunkSize, pNode->s3ChunkSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDatabaseOptionsS3KeepLocalNode, nodeToJson, pNode->s3KeepLocalStr); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3KeepLocal, pNode->s3KeepLocal); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsS3Compact, pNode->s3Compact); + } return code; } @@ -4909,6 +4919,18 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsSchemaless, &pNode->schemaless); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkDatabaseOptionsS3ChunkSize, &pNode->s3ChunkSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDatabaseOptionsS3KeepLocalNode, (SNode**)&pNode->s3KeepLocalStr); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkDatabaseOptionsS3KeepLocal, &pNode->s3KeepLocal); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsS3Compact, &pNode->s3Compact); + } return code; } @@ -5497,6 +5519,24 @@ static int32_t jsonToTrimDatabaseStmt(const SJson* pJson, void* pObj) { return code; } +static const char* jkS3MigrateDatabaseStmtDbName = "DbName"; + +static int32_t s3migrateDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const SS3MigrateDatabaseStmt* pNode = (const SS3MigrateDatabaseStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkS3MigrateDatabaseStmtDbName, pNode->dbName); + + return code; +} + +static int32_t jsonToS3MigrateDatabaseStmt(const SJson* pJson, void* pObj) { + SS3MigrateDatabaseStmt* pNode = (SS3MigrateDatabaseStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkS3MigrateDatabaseStmtDbName, pNode->dbName); + + return code; +} + static const char* jkCreateTableStmtDbName = "DbName"; static const char* jkCreateTableStmtTableName = "TableName"; static const char* jkCreateTableStmtIgnoreExists = "IgnoreExists"; @@ -6166,10 +6206,6 @@ static int32_t jsonToRestoreVnodeStmt(const SJson* pJson, void* pObj) { return jsonToRestoreComponentNodeStmt(pJson, pObj); } - - - - static const char* jkCreateTopicStmtTopicName = "TopicName"; static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName"; static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists"; @@ -6821,7 +6857,6 @@ static int32_t showCreateViewStmtToJson(const void* pObj, SJson* pJson) { return code; } - static int32_t jsonToShowCreateViewStmt(const SJson* pJson, void* pObj) { SShowCreateViewStmt* pNode = (SShowCreateViewStmt*)pObj; @@ -6833,7 +6868,6 @@ static int32_t jsonToShowCreateViewStmt(const SJson* pJson, void* pObj) { return code; } - static const char* jkShowTableDistributedStmtDbName = "DbName"; static const char* jkShowTableDistributedStmtTableName = "TableName"; @@ -7079,6 +7113,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return alterDatabaseStmtToJson(pObj, pJson); case QUERY_NODE_TRIM_DATABASE_STMT: return trimDatabaseStmtToJson(pObj, pJson); + case QUERY_NODE_S3MIGRATE_DATABASE_STMT: + return s3migrateDatabaseStmtToJson(pObj, pJson); case QUERY_NODE_CREATE_TABLE_STMT: return createTableStmtToJson(pObj, pJson); case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: @@ -7416,6 +7452,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToAlterDatabaseStmt(pJson, pObj); case QUERY_NODE_TRIM_DATABASE_STMT: return jsonToTrimDatabaseStmt(pJson, pObj); + case QUERY_NODE_S3MIGRATE_DATABASE_STMT: + return jsonToS3MigrateDatabaseStmt(pJson, pObj); case QUERY_NODE_CREATE_TABLE_STMT: return jsonToCreateTableStmt(pJson, pObj); case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: @@ -7566,7 +7604,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToRestoreQnodeStmt(pJson, pObj); case QUERY_NODE_RESTORE_MNODE_STMT: return jsonToRestoreMnodeStmt(pJson, pObj); - case QUERY_NODE_RESTORE_VNODE_STMT: + case QUERY_NODE_RESTORE_VNODE_STMT: return jsonToRestoreVnodeStmt(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SCAN: return jsonToLogicScanNode(pJson, pObj); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1bdcda3ddf..e2dd88ead5 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -324,6 +324,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SFlushDatabaseStmt)); case QUERY_NODE_TRIM_DATABASE_STMT: return makeNode(type, sizeof(STrimDatabaseStmt)); + case QUERY_NODE_S3MIGRATE_DATABASE_STMT: + return makeNode(type, sizeof(SS3MigrateDatabaseStmt)); case QUERY_NODE_CREATE_TABLE_STMT: return makeNode(type, sizeof(SCreateTableStmt)); case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: @@ -802,6 +804,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_DATABASE_OPTIONS: { SDatabaseOptions* pOptions = (SDatabaseOptions*)pNode; nodesDestroyNode((SNode*)pOptions->pDaysPerFile); + nodesDestroyNode((SNode*)pOptions->s3KeepLocalStr); nodesDestroyList(pOptions->pKeep); nodesDestroyList(pOptions->pRetentions); break; @@ -939,6 +942,8 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_FLUSH_DATABASE_STMT: // no pointer field case QUERY_NODE_TRIM_DATABASE_STMT: // no pointer field break; + case QUERY_NODE_S3MIGRATE_DATABASE_STMT: // no pointer field + break; case QUERY_NODE_CREATE_TABLE_STMT: { SCreateTableStmt* pStmt = (SCreateTableStmt*)pNode; nodesDestroyList(pStmt->pCols); diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index f13f9f1b96..7afc8539e9 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -64,6 +64,9 @@ typedef enum EDatabaseOptionType { DB_OPTION_STT_TRIGGER, DB_OPTION_TABLE_PREFIX, DB_OPTION_TABLE_SUFFIX, + DB_OPTION_S3_CHUNKSIZE, + DB_OPTION_S3_KEEPLOCAL, + DB_OPTION_S3_COMPACT, DB_OPTION_KEEP_TIME_OFFSET } EDatabaseOptionType; @@ -90,7 +93,7 @@ typedef struct STokenPair { typedef struct SShowTablesOption { EShowKind kind; - SToken dbName; + SToken dbName; } SShowTablesOption; extern SToken nil_token; @@ -106,40 +109,40 @@ SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode); SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode); SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); -SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName); -SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral); +SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName); +SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral); SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral); -SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral); -SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); -SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt); -SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); -SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias); -SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); -SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); -SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); -SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); -SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); -SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt); -SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); -SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2); -SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias); -SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); -SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); -SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName); -SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); -SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); -SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); -SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr); -SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond); -SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken); -SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, - SNode* pFill); -SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); -SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); -SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd); -SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint); -SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen); -SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse); +SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral); +SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); +SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt); +SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias); +SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); +SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); +SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); +SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt); +SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); +SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2); +SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias); +SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); +SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); +SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName); +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); +SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr); +SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond); +SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken); +SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, + SNode* pFill); +SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); +SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); +SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd); +SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint); +SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen); +SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse); SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); @@ -152,7 +155,8 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit); SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange); SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery); SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill); -SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint); +SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, + SNodeList* pHint); SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); @@ -168,6 +172,7 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STo SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions); SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed); +SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd); SNode* createDefaultTableOptions(SAstCreateContext* pCxt); SNode* createAlterTableOptions(SAstCreateContext* pCxt); @@ -194,7 +199,8 @@ SNode* setShowKind(SAstCreateContext* pCxt, SNode* pStmt, EShowKind showKind); SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type); SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbName, EOperatorType tableCondType); -SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, EOperatorType tableCondType); +SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, + EOperatorType tableCondType); SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pDbName, ENodeType type); SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 4c59f3abd7..d555c75d6e 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -208,6 +208,7 @@ cmd ::= USE db_name(A). cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); } cmd ::= FLUSH DATABASE db_name(A). { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); } cmd ::= TRIM DATABASE db_name(A) speed_opt(B). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); } +cmd ::= S3MIGRATE DATABASE db_name(A). { pCxt->pRootNode = createS3MigrateDatabaseStmt(pCxt, &A); } cmd ::= COMPACT DATABASE db_name(A) start_opt(B) end_opt(C). { pCxt->pRootNode = createCompactStmt(pCxt, &A, B, C); } %type not_exists_opt { bool } @@ -260,6 +261,10 @@ db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C). db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); } db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); } db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); } +db_options(A) ::= db_options(B) S3_CHUNKSIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_CHUNKSIZE, &C); } +db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); } +db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); } +db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_COMPACT, &C); } db_options(A) ::= db_options(B) KEEP_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP_TIME_OFFSET, &C); } alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); } @@ -291,6 +296,9 @@ alter_db_option(A) ::= WAL_RETENTION_SIZE NK_MINUS(B) NK_INTEGER(C). t.n = (C.z + C.n) - B.z; A.type = DB_OPTION_WAL_RETENTION_SIZE; A.val = t; } +alter_db_option(A) ::= S3_KEEPLOCAL NK_INTEGER(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; } +alter_db_option(A) ::= S3_KEEPLOCAL NK_VARIABLE(B). { A.type = DB_OPTION_S3_KEEPLOCAL; A.val = B; } +alter_db_option(A) ::= S3_COMPACT NK_INTEGER(B). { A.type = DB_OPTION_S3_COMPACT, A.val = B; } alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_KEEP_TIME_OFFSET; A.val = B; } %type integer_list { SNodeList* } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 1d6c5e800e..b4f15f6c58 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -228,7 +228,6 @@ static bool checkViewName(SAstCreateContext* pCxt, SToken* pViewName) { return true; } - static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) { trimEscape(pStreamName); if (pStreamName->n >= TSDB_STREAM_NAME_LEN) { @@ -288,7 +287,7 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { } else if (pRawExpr->isPseudoColumn) { // all pseudo column are translate to function with same name strcpy(pExpr->userAlias, ((SFunctionNode*)pExpr)->functionName); - strcpy(pExpr->aliasName, ((SFunctionNode*)pExpr)->functionName); + strcpy(pExpr->aliasName, ((SFunctionNode*)pExpr)->functionName); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); @@ -833,7 +832,6 @@ SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewNam return (SNode*)pView; } - SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { CHECK_PARSER_STATUS(pCxt); SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); @@ -1104,11 +1102,11 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr return select; } -SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags) { +SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags) { if (pStmt && QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { if (pCxt->pQueryCxt->biMode) { ((SSelectStmt*)pStmt)->tagScan = true; - } else { + } else { ((SSelectStmt*)pStmt)->tagScan = bSelectTags; } } @@ -1178,6 +1176,9 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { pOptions->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; pOptions->tablePrefix = TSDB_DEFAULT_HASH_PREFIX; pOptions->tableSuffix = TSDB_DEFAULT_HASH_SUFFIX; + pOptions->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + pOptions->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pOptions->s3Compact = TSDB_DEFAULT_S3_COMPACT; return (SNode*)pOptions; } @@ -1213,6 +1214,9 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { pOptions->sstTrigger = -1; pOptions->tablePrefix = -1; pOptions->tableSuffix = -1; + pOptions->s3ChunkSize = -1; + pOptions->s3KeepLocal = -1; + pOptions->s3Compact = -1; return (SNode*)pOptions; } @@ -1327,6 +1331,21 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED nodesDestroyNode((SNode*)pNode); break; } + case DB_OPTION_S3_CHUNKSIZE: + pDbOptions->s3ChunkSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + break; + case DB_OPTION_S3_KEEPLOCAL: { + SToken* pToken = pVal; + if (TK_NK_INTEGER == pToken->type) { + pDbOptions->s3KeepLocal = taosStr2Int32(pToken->z, NULL, 10) * 1440; + } else { + pDbOptions->s3KeepLocalStr = (SValueNode*)createDurationValueNode(pCxt, pToken); + } + break; + } + case DB_OPTION_S3_COMPACT: + pDbOptions->s3Compact = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + break; case DB_OPTION_KEEP_TIME_OFFSET: { pDbOptions->keepTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; @@ -1413,6 +1432,17 @@ SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t return (SNode*)pStmt; } +SNode* createS3MigrateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { + CHECK_PARSER_STATUS(pCxt); + if (!checkDbName(pCxt, pDbName, false)) { + return NULL; + } + SS3MigrateDatabaseStmt* pStmt = (SS3MigrateDatabaseStmt*)nodesMakeNode(QUERY_NODE_S3MIGRATE_DATABASE_STMT); + CHECK_OUT_OF_MEM(pStmt); + COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); + return (SNode*)pStmt; +} + SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pStart, SNode* pEnd) { CHECK_PARSER_STATUS(pCxt); if (!checkDbName(pCxt, pDbName, false)) { @@ -1697,7 +1727,7 @@ SNode* createShowCompactsStmt(SAstCreateContext* pCxt, ENodeType type) { SNode* setShowKind(SAstCreateContext* pCxt, SNode* pStmt, EShowKind showKind) { if (pStmt == NULL) { return NULL; - } + } SShowStmt* pShow = (SShowStmt*)pStmt; pShow->showKind = showKind; return pStmt; @@ -1719,7 +1749,8 @@ SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pD return (SNode*)pStmt; } -SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, EOperatorType tableCondType) { +SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, + EOperatorType tableCondType) { CHECK_PARSER_STATUS(pCxt); SNode* pDbName = NULL; if (option.dbName.type == TK_NK_NIL) { @@ -1795,7 +1826,6 @@ SNode* createShowCreateViewStmt(SAstCreateContext* pCxt, ENodeType type, SNode* return (SNode*)pStmt; } - SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable) { CHECK_PARSER_STATUS(pCxt); SShowTableDistributedStmt* pStmt = (SShowTableDistributedStmt*)nodesMakeNode(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index f1013d6157..0eca541d19 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -302,6 +302,10 @@ static SKeyword keywordTable[] = { {"_WSTART", TK_WSTART}, {"ALIVE", TK_ALIVE}, {"VARBINARY", TK_VARBINARY}, + {"S3_CHUNKSIZE", TK_S3_CHUNKSIZE}, + {"S3_KEEPLOCAL", TK_S3_KEEPLOCAL}, + {"S3_COMPACT", TK_S3_COMPACT}, + {"S3MIGRATE", TK_S3MIGRATE}, {"KEEP_TIME_OFFSET", TK_KEEP_TIME_OFFSET}, }; // clang-format on diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index cc522c9b5f..a5e2694fe2 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -14,8 +14,8 @@ */ #include "parTranslater.h" -#include "tdatablock.h" #include "parInt.h" +#include "tdatablock.h" #include "catalog.h" #include "cmdnodes.h" @@ -374,13 +374,14 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pTable) { } #ifdef BUILD_NO_CALL -static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) { +static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, + STableMeta** pMeta) { #ifndef TD_ENTERPRISE return TSDB_CODE_PAR_TABLE_NOT_EXIST; #endif int32_t code = TSDB_CODE_SUCCESS; - + if (pParCxt->async) { code = getViewMetaFromCache(pMetaCache, pName, pMeta); } else { @@ -390,7 +391,7 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac .mgmtEps = pParCxt->mgmtEpSet}; code = catalogGetViewMeta(pParCxt->pCatalog, &conn, pName, pMeta); } - + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { parserError("0x%" PRIx64 " catalogGetViewMeta error, code:%s, dbName:%s, viewName:%s", pParCxt->requestId, tstrerror(code), pName->dbname, pName->tname); @@ -399,9 +400,10 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac } #endif -int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) { +int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, + bool couldBeView) { int32_t code = TSDB_CODE_SUCCESS; - + if (pParCxt->async) { code = getTableMetaFromCache(pMetaCache, pName, pMeta); #ifdef TD_ENTERPRISE @@ -420,7 +422,7 @@ int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, c .mgmtEps = pParCxt->mgmtEpSet}; code = catalogGetTableMeta(pParCxt->pCatalog, &conn, pName, pMeta); } - + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pParCxt->requestId, tstrerror(code), pName->dbname, pName->tname); @@ -950,7 +952,7 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum assNode.pPlace = (SNode**)pColRef; assNode.pAssociationNode = (SNode*)*pColRef; taosArrayPush(pExpr->pAssociation, &assNode); - + strcpy(pCol->tableAlias, pTable->table.tableAlias); pCol->colId = isPrimaryKey(pTable, (SNode*)pExpr) ? PRIMARYKEY_TIMESTAMP_COL_ID : 0; strcpy(pCol->colName, pExpr->aliasName); @@ -1116,7 +1118,7 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p SExprNode* pExpr = (SExprNode*)pNode; if (0 == strcmp((*pCol)->colName, pExpr->userAlias)) { if (true == *pFound) { - if(nodesEqualNode(pFoundNode, pNode)) { + if (nodesEqualNode(pFoundNode, pNode)) { continue; } pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ORDERBY_AMBIGUOUS, (*pCol)->colName); @@ -1167,9 +1169,8 @@ static SNode* biMakeTbnameProjectAstNode(char* funcName, char* tableAlias) { if (valNode != NULL) { nodesListMakeAppend(&tbNameFunc->pParameterList, (SNode*)valNode); } - snprintf(tbNameFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), - (tableAlias)? "%s.tbname" : "%stbname", - (tableAlias)? tableAlias : ""); + snprintf(tbNameFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), (tableAlias) ? "%s.tbname" : "%stbname", + (tableAlias) ? tableAlias : ""); strncpy(tbNameFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); if (funcName == NULL) { @@ -1180,27 +1181,26 @@ static SNode* biMakeTbnameProjectAstNode(char* funcName, char* tableAlias) { nodesListMakeAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc); if (tsKeepColumnName) { - snprintf(multiResFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), - (tableAlias)? "%s.tbname" : "%stbname", - (tableAlias)? tableAlias : ""); + snprintf(multiResFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), + (tableAlias) ? "%s.tbname" : "%stbname", (tableAlias) ? tableAlias : ""); strcpy(multiResFunc->node.aliasName, tbNameFunc->functionName); } else { - snprintf(multiResFunc->node.userAlias, sizeof(multiResFunc->node.userAlias), - tableAlias? "%s(%s.tbname)" : "%s(%stbname)", funcName, - tableAlias? tableAlias: ""); - biMakeAliasNameInMD5(multiResFunc->node.userAlias, strlen(multiResFunc->node.userAlias), multiResFunc->node.aliasName); + snprintf(multiResFunc->node.userAlias, sizeof(multiResFunc->node.userAlias), + tableAlias ? "%s(%s.tbname)" : "%s(%stbname)", funcName, tableAlias ? tableAlias : ""); + biMakeAliasNameInMD5(multiResFunc->node.userAlias, strlen(multiResFunc->node.userAlias), + multiResFunc->node.aliasName); } return (SNode*)multiResFunc; } } -static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt* pSelect, SNode* pNode, SListCell* pSelectListCell) { +static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt* pSelect, SNode* pNode, + SListCell* pSelectListCell) { SNodeList* pTbnameNodeList = nodesMakeList(); SFunctionNode* pFunc = (SFunctionNode*)pNode; - if (strcasecmp(pFunc->functionName, "last") == 0 || - strcasecmp(pFunc->functionName, "last_row") == 0 || + if (strcasecmp(pFunc->functionName, "last") == 0 || strcasecmp(pFunc->functionName, "last_row") == 0 || strcasecmp(pFunc->functionName, "first") == 0) { SNodeList* pParams = pFunc->pParameterList; SNode* pPara = NULL; @@ -1220,7 +1220,7 @@ static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList); } } else if (nodesIsTableStar(pPara)) { - char* pTableAlias = ((SColumnNode*)pPara)->tableAlias; + char* pTableAlias = ((SColumnNode*)pPara)->tableAlias; STableNode* pTable = NULL; int32_t code = findTable(pCxt, pTableAlias, &pTable); if (TSDB_CODE_SUCCESS == code && nodeType(pTable) == QUERY_NODE_REAL_TABLE && @@ -1241,16 +1241,15 @@ static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt // after translate from // before translate select list int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { - SNode* pNode = NULL; + SNode* pNode = NULL; SNodeList* pTbnameNodeList = nodesMakeList(); WHERE_EACH(pNode, pSelect->pProjectionList) { if (nodesIsStar(pNode)) { SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); - size_t n = taosArrayGetSize(pTables); + size_t n = taosArrayGetSize(pTables); for (int32_t i = 0; i < n; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); - if (nodeType(pTable) == QUERY_NODE_REAL_TABLE && - ((SRealTableNode*)pTable)->pMeta != NULL && + if (nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta != NULL && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { SNode* pTbnameNode = biMakeTbnameProjectAstNode(NULL, NULL); nodesListAppend(pTbnameNodeList, pTbnameNode); @@ -1260,13 +1259,11 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList); } } else if (nodesIsTableStar(pNode)) { - char* pTableAlias = ((SColumnNode*)pNode)->tableAlias; + char* pTableAlias = ((SColumnNode*)pNode)->tableAlias; STableNode* pTable = NULL; int32_t code = findTable(pCxt, pTableAlias, &pTable); - if (TSDB_CODE_SUCCESS == code && - nodeType(pTable) == QUERY_NODE_REAL_TABLE && - ((SRealTableNode*)pTable)->pMeta != NULL && - ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { + if (TSDB_CODE_SUCCESS == code && nodeType(pTable) == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pTable)->pMeta != NULL && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { SNode* pTbnameNode = biMakeTbnameProjectAstNode(NULL, pTableAlias); nodesListAppend(pTbnameNodeList, pTbnameNode); } @@ -1276,7 +1273,7 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { } else if (nodeType(pNode) == QUERY_NODE_FUNCTION) { biRewriteSelectFuncParamStar(pCxt, pSelect, pNode, cell); } - WHERE_NEXT; + WHERE_NEXT; } return TSDB_CODE_SUCCESS; @@ -1284,11 +1281,11 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) { SColumnNode* pCol = (SColumnNode*)(*ppNode); - if ((strcasecmp(pCol->colName, "tbname") == 0) && - ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable && - QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { + if ((strcasecmp(pCol->colName, "tbname") == 0) && ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable && + QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { SFunctionNode* tbnameFuncNode = NULL; - tbnameFuncNode = (SFunctionNode*)biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0]!='\0') ? pCol->tableAlias : NULL); + tbnameFuncNode = + (SFunctionNode*)biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0] != '\0') ? pCol->tableAlias : NULL); tbnameFuncNode->node.resType = pCol->node.resType; strcpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName); strcpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias); @@ -1303,24 +1300,26 @@ bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) { int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) { if (pStmt->pTags) { - SNode* pNode = NULL; - FOREACH(pNode, pStmt->pTags) { - SColumnDefNode* pTag = (SColumnDefNode*)pNode; - if (strcasecmp(pTag->colName, "tbname") == 0) { - int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, "tbname can not used for tags in BI mode"); - return code; - } + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pTags) { + SColumnDefNode* pTag = (SColumnDefNode*)pNode; + if (strcasecmp(pTag->colName, "tbname") == 0) { + int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, + "tbname can not used for tags in BI mode"); + return code; } + } } if (pStmt->pCols) { - SNode* pNode = NULL; - FOREACH(pNode, pStmt->pCols) { - SColumnDefNode* pCol = (SColumnDefNode*)pNode; - if (strcasecmp(pCol->colName, "tbname") == 0) { - int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "tbname can not used for columns in BI mode"); - return code; - } + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pCols) { + SColumnDefNode* pCol = (SColumnDefNode*)pNode; + if (strcasecmp(pCol->colName, "tbname") == 0) { + int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, + "tbname can not used for columns in BI mode"); + return code; } + } } return TSDB_CODE_SUCCESS; } @@ -1358,9 +1357,9 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { res = translateColumnWithoutPrefix(pCxt, pCol); } } - if(SQL_CLAUSE_ORDER_BY == pCxt->currClause && !(*pCol)->node.asParam - && res != DEAL_RES_CONTINUE && res != DEAL_RES_END) { - res = translateColumnUseAlias(pCxt, pCol, &found); + if (SQL_CLAUSE_ORDER_BY == pCxt->currClause && !(*pCol)->node.asParam && res != DEAL_RES_CONTINUE && + res != DEAL_RES_END) { + res = translateColumnUseAlias(pCxt, pCol, &found); } } return res; @@ -1940,8 +1939,8 @@ static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; - SNode* pNode = NULL; - bool bFound = false; + SNode* pNode = NULL; + bool bFound = false; FOREACH(pNode, pSelect->pProjectionList) { if (nodeType(pNode) == QUERY_NODE_FUNCTION && strcasecmp(((SFunctionNode*)pNode)->functionName, "interp") == 0) { bFound = true; @@ -1955,7 +1954,7 @@ static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SNode** return code; } translateColumn(pCxt, (SColumnNode**)ppNode); - return pCxt->errCode; + return pCxt->errCode; } return TSDB_CODE_SUCCESS; } @@ -2247,7 +2246,7 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode return TSDB_CODE_OUT_OF_MEMORY; } SExprNode* pOldExpr = (SExprNode*)(*ppNode); - //rewrite a.tbname == tbname(a) + // rewrite a.tbname == tbname(a) if (nodeType(*ppNode) == QUERY_NODE_FUNCTION && ((SFunctionNode*)(*ppNode))->funcType == FUNCTION_TYPE_TBNAME) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (0 != LIST_LENGTH(pFunc->pParameterList)) { @@ -2260,7 +2259,7 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode nodesDestroyNode(*ppNode); *ppNode = (SNode*)pCol; - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } } pCol->node.resType = pOldExpr->resType; @@ -2275,23 +2274,23 @@ static int32_t replacePsedudoColumnFuncWithColumn(STranslateContext* pCxt, SNode } static int32_t rewriteToColumnAndRetranslate(STranslateContext* pCxt, SNode** ppNode, int32_t errCode) { - int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - translateColumn(pCxt, (SColumnNode**)ppNode); - if (pCxt->errCode != TSDB_CODE_SUCCESS) { - return generateSyntaxErrMsg(&pCxt->msgBuf, errCode); - } else { - return TSDB_CODE_SUCCESS; - } + int32_t code = replacePsedudoColumnFuncWithColumn(pCxt, ppNode); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + translateColumn(pCxt, (SColumnNode**)ppNode); + if (pCxt->errCode != TSDB_CODE_SUCCESS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, errCode); + } else { + return TSDB_CODE_SUCCESS; + } } static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; - } + } if (!isSelectStmt(pCxt->pCurrStmt)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); } @@ -2308,7 +2307,7 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SNode** static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRewriteToColumn) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); - if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { + if (!fmIsScanPseudoColumnFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } if (0 == LIST_LENGTH(pFunc->pParameterList)) { @@ -2325,7 +2324,7 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** pp pCxt->errCode = findTable(pCxt, pVal->literal, &pTable); if (TSDB_CODE_SUCCESS != pCxt->errCode || (NULL == pTable)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); - } + } if (nodeType(pTable) != QUERY_NODE_REAL_TABLE) { *pRewriteToColumn = true; return rewriteToColumnAndRetranslate(pCxt, ppNode, TSDB_CODE_PAR_INVALID_TBNAME); @@ -2336,7 +2335,7 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SNode** pp static int32_t translateNormalFunction(STranslateContext* pCxt, SNode** ppNode) { SFunctionNode* pFunc = (SFunctionNode*)(*ppNode); - int32_t code = translateAggFunc(pCxt, pFunc); + int32_t code = translateAggFunc(pCxt, pFunc); if (TSDB_CODE_SUCCESS == code) { bool bRewriteToColumn = false; code = translateScanPseudoColumnFunc(pCxt, ppNode, &bRewriteToColumn); @@ -2783,18 +2782,17 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { } static int32_t checkIsEmptyResult(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (pSelect->timeRange.skey > pSelect->timeRange.ekey - && !pSelect->hasCountFunc) { + if (pSelect->timeRange.skey > pSelect->timeRange.ekey && !pSelect->hasCountFunc) { pSelect->isEmptyResult = true; } - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } static int32_t resetSelectFuncNumWithoutDup(SSelectStmt* pSelect) { if (pSelect->selectFuncNum <= 1) return TSDB_CODE_SUCCESS; pSelect->selectFuncNum = 0; SNodeList* pNodeList = nodesMakeList(); - int32_t code = nodesCollectSelectFuncs(pSelect, SQL_CLAUSE_FROM, NULL, fmIsSelectFunc, pNodeList); + int32_t code = nodesCollectSelectFuncs(pSelect, SQL_CLAUSE_FROM, NULL, fmIsSelectFunc, pNodeList); if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pNodeList); return code; @@ -2820,8 +2818,8 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (!pSelect->isDistinct) { nodesRewriteExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } - if (((!cxt.existCol && 0 < pSelect->selectFuncNum) || (cxt.existCol && 1 == pSelect->selectFuncNum) ) - && !pSelect->hasOtherVectorFunc) { + if (((!cxt.existCol && 0 < pSelect->selectFuncNum) || (cxt.existCol && 1 == pSelect->selectFuncNum)) && + !pSelect->hasOtherVectorFunc) { return rewriteColsToSelectValFunc(pCxt, pSelect); } if (cxt.existCol) { @@ -3871,57 +3869,57 @@ static void convertVarDuration(SValueNode* pOffset, uint8_t precision) { pOffset->unit = units[precision]; } -static const int64_t tsdbMaxKeepMS = (int64_t)60 * 1000 * TSDB_MAX_KEEP; -static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { - uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; +static const int64_t tsdbMaxKeepMS = (int64_t)60 * 1000 * TSDB_MAX_KEEP; +static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { + uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; - SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit); - if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, - getPrecisionStr(precision)); + SValueNode* pInter = (SValueNode*)pInterval->pInterval; + bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit); + if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, + getPrecisionStr(precision)); } else if (pInter->datum.i / getPrecisionMultiple(precision) > tsdbMaxKeepMS) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, 1000, "years"); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_BIG, 1000, "years"); } - if (NULL != pInterval->pOffset) { - SValueNode* pOffset = (SValueNode*)pInterval->pOffset; - if (pOffset->datum.i <= 0) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_NEGATIVE); + if (NULL != pInterval->pOffset) { + SValueNode* pOffset = (SValueNode*)pInterval->pOffset; + if (pOffset->datum.i <= 0) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_NEGATIVE); } - if (pInter->unit == 'n' && pOffset->unit == 'y') { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); + if (pInter->unit == 'n' && pOffset->unit == 'y') { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } - bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter; - if ((fixed && pOffset->datum.i >= pInter->datum.i) || + bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter; + if ((fixed && pOffset->datum.i >= pInter->datum.i) || (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG); } - if (pOffset->unit == 'n' || pOffset->unit == 'y') { - convertVarDuration(pOffset, precision); + if (pOffset->unit == 'n' || pOffset->unit == 'y') { + convertVarDuration(pOffset, precision); } } - if (NULL != pInterval->pSliding) { - const static int32_t INTERVAL_SLIDING_FACTOR = 100; + if (NULL != pInterval->pSliding) { + const static int32_t INTERVAL_SLIDING_FACTOR = 100; - SValueNode* pSliding = (SValueNode*)pInterval->pSliding; - if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); + SValueNode* pSliding = (SValueNode*)pInterval->pSliding; + if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } - if ((pSliding->datum.i < + if ((pSliding->datum.i < convertTimeFromPrecisionToUnit(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, pSliding->unit)) || (pInter->datum.i / pSliding->datum.i > INTERVAL_SLIDING_FACTOR)) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL); } - if (pSliding->datum.i > pInter->datum.i) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG); + if (pSliding->datum.i > pInter->datum.i) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG); } } - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } static int32_t translateIntervalWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -4048,11 +4046,11 @@ static EDealRes collectWindowsPseudocolumns(SNode* pNode, void* pContext) { } static int32_t checkWindowsConditonValid(SNode* pNode) { - int32_t code = TSDB_CODE_SUCCESS; - if(QUERY_NODE_EVENT_WINDOW != nodeType(pNode)) return code; + int32_t code = TSDB_CODE_SUCCESS; + if (QUERY_NODE_EVENT_WINDOW != nodeType(pNode)) return code; SEventWindowNode* pEventWindowNode = (SEventWindowNode*)pNode; - SNodeList* pCols = nodesMakeList(); + SNodeList* pCols = nodesMakeList(); if (NULL == pCols) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -4262,14 +4260,15 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec typedef struct SEqCondTbNameTableInfo { SRealTableNode* pRealTable; - SArray* aTbnames; + SArray* aTbnames; } SEqCondTbNameTableInfo; //[tableAlias.]tbname = tbNamVal -static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray** ppTabNames) { +static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, + SArray** ppTabNames) { if (pOperator->opType != OP_TYPE_EQUAL) return false; SFunctionNode* pTbnameFunc = NULL; - SValueNode* pValueNode = NULL; + SValueNode* pValueNode = NULL; if (nodeType(pOperator->pLeft) == QUERY_NODE_FUNCTION && ((SFunctionNode*)(pOperator->pLeft))->funcType == FUNCTION_TYPE_TBNAME && nodeType(pOperator->pRight) == QUERY_NODE_VALUE) { @@ -4300,7 +4299,8 @@ static bool isOperatorEqTbnameCond(STranslateContext* pCxt, SOperatorNode* pOper } //[tableAlias.]tbname in (value1, value2, ...) -static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, SArray** ppTbNames) { +static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOperator, char** ppTableAlias, + SArray** ppTbNames) { if (pOperator->opType != OP_TYPE_IN) return false; if (nodeType(pOperator->pLeft) != QUERY_NODE_FUNCTION || ((SFunctionNode*)(pOperator->pLeft))->funcType != FUNCTION_TYPE_TBNAME || @@ -4321,8 +4321,8 @@ static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOper } *ppTbNames = taosArrayInit(1, sizeof(void*)); SNodeListNode* pValueListNode = (SNodeListNode*)pOperator->pRight; - SNodeList* pValueNodeList = pValueListNode->pNodeList; - SNode* pValNode = NULL; + SNodeList* pValueNodeList = pValueListNode->pNodeList; + SNode* pValNode = NULL; FOREACH(pValNode, pValueNodeList) { if (nodeType(pValNode) != QUERY_NODE_VALUE) { return false; @@ -4330,13 +4330,12 @@ static bool isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pOper taosArrayPush(*ppTbNames, &((SValueNode*)pValNode)->literal); } return true; - } static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo) { int32_t code = TSDB_CODE_SUCCESS; - char* pTableAlias = NULL; - char* pTbNameVal = NULL; + char* pTableAlias = NULL; + char* pTbNameVal = NULL; if (isOperatorEqTbnameCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames) || isOperatorTbnameInCond(pCxt, (SOperatorNode*)pWhere, &pTableAlias, &pInfo->aTbnames)) { STableNode* pTable; @@ -4345,8 +4344,8 @@ static bool findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWher } else { code = findTable(pCxt, pTableAlias, &pTable); } - if (code == TSDB_CODE_SUCCESS && nodeType(pTable) == QUERY_NODE_REAL_TABLE && - ((SRealTableNode*)pTable)->pMeta && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { + if (code == TSDB_CODE_SUCCESS && nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta && + ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { pInfo->pRealTable = (SRealTableNode*)pTable; return true; } @@ -4371,17 +4370,17 @@ static void findEqualCondTbnameInLogicCondAnd(STranslateContext* pCxt, SNode* pW FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; - bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); + bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); if (bIsEqTbnameCond) { if (!isTableExistInTableTbnames(aTableTbnames, info.pRealTable)) { - //TODO: intersect tbNames of same table? speed + // TODO: intersect tbNames of same table? speed taosArrayPush(aTableTbnames, &info); } else { taosArrayDestroy(info.aTbnames); } } } - //TODO: logic cond + // TODO: logic cond } } @@ -4406,10 +4405,10 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh bool bAllTbName = true; SNode* pTmpNode = NULL; FOREACH(pTmpNode, ((SLogicConditionNode*)pWhere)->pParameterList) { - //TODO: logic cond + // TODO: logic cond if (nodeType(pTmpNode) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; - bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); + bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pTmpNode, &info); if (!bIsEqTbnameCond) { bAllTbName = false; break; @@ -4432,10 +4431,10 @@ static void findEqualCondTbnameInLogicCondOr(STranslateContext* pCxt, SNode* pWh } static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArray* aTableTbnames) { - //TODO: optimize nested and/or condition. now only the fist level is processed. + // TODO: optimize nested and/or condition. now only the fist level is processed. if (nodeType(pWhere) == QUERY_NODE_OPERATOR) { SEqCondTbNameTableInfo info = {0}; - bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info); + bool bIsEqTbnameCond = findEqCondTbNameInOperatorNode(pCxt, pWhere, &info); if (bIsEqTbnameCond) { taosArrayPush(aTableTbnames, &info); } @@ -4449,7 +4448,8 @@ static int32_t findEqualCondTbname(STranslateContext* pCxt, SNode* pWhere, SArra return TSDB_CODE_SUCCESS; } -static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo, SVgroupsInfo* vgsInfo) { +static int32_t findVgroupsFromEqualTbname(STranslateContext* pCxt, SEqCondTbNameTableInfo* pInfo, + SVgroupsInfo* vgsInfo) { int32_t nVgroups = 0; int32_t nTbls = taosArrayGetSize(pInfo->aTbnames); @@ -4491,17 +4491,17 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt* int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < taosArrayGetSize(aTables); ++i) { SEqCondTbNameTableInfo* pInfo = taosArrayGet(aTables, i); - int32_t nTbls = taosArrayGetSize(pInfo->aTbnames); + int32_t nTbls = taosArrayGetSize(pInfo->aTbnames); SVgroupsInfo* vgsInfo = taosMemoryMalloc(sizeof(SVgroupsInfo) + nTbls * sizeof(SVgroupInfo)); - int32_t nVgroups = 0; + int32_t nVgroups = 0; findVgroupsFromEqualTbname(pCxt, pInfo, vgsInfo); if (vgsInfo->numOfVgroups != 0) { taosMemoryFree(pInfo->pRealTable->pVgroupList); pInfo->pRealTable->pVgroupList = vgsInfo; } else { taosMemoryFree(vgsInfo); - } + } } return TSDB_CODE_SUCCESS; } @@ -5019,7 +5019,6 @@ static int32_t translateInsertTable(STranslateContext* pCxt, SNode** pTable) { if (TSDB_CODE_SUCCESS == code && TSDB_CHILD_TABLE != ((SRealTableNode*)*pTable)->pMeta->tableType && TSDB_NORMAL_TABLE != ((SRealTableNode*)*pTable)->pMeta->tableType) { code = buildInvalidOperationMsg(&pCxt->msgBuf, "insert data into super table is not supported"); - } return code; } @@ -5115,6 +5114,9 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS pReq->hashSuffix = pStmt->pOptions->tableSuffix; pReq->tsdbPageSize = pStmt->pOptions->tsdbPageSize; pReq->keepTimeOffset = pStmt->pOptions->keepTimeOffset; + pReq->s3ChunkSize = pStmt->pOptions->s3ChunkSize; + pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal; + pReq->s3Compact = pStmt->pOptions->s3Compact; pReq->ignoreExist = pStmt->ignoreExists; return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); } @@ -5139,6 +5141,22 @@ static int32_t checkTableRangeOption(STranslateContext* pCxt, const char* pName, return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal); } +static int32_t checkDbS3KeepLocalOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { + if (NULL != pOptions->s3KeepLocalStr) { + if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->s3KeepLocalStr)) { + return pCxt->errCode; + } + if (TIME_UNIT_MINUTE != pOptions->s3KeepLocalStr->unit && TIME_UNIT_HOUR != pOptions->s3KeepLocalStr->unit && + TIME_UNIT_DAY != pOptions->s3KeepLocalStr->unit) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option s3_keeplocal unit: %c, only %c, %c, %c allowed", + pOptions->s3KeepLocalStr->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY); + } + pOptions->s3KeepLocal = getBigintFromValueNode(pOptions->s3KeepLocalStr); + } + return checkDbRangeOption(pCxt, "s3KeepLocal", pOptions->s3KeepLocal, TSDB_MIN_S3_KEEP_LOCAL, TSDB_MAX_S3_KEEP_LOCAL); +} + static int32_t checkDbDaysOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { if (NULL != pOptions->pDaysPerFile) { if (DEAL_RES_ERROR == translateValue(pCxt, pOptions->pDaysPerFile)) { @@ -5353,7 +5371,7 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete "Invalid option retentions(freq/keep): %s should larger than %s", pKeep->literal, pFreq->literal); } - + if (NULL != pPrevFreq && pPrevFreq->datum.i >= pFreq->datum.i) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option retentions(freq): %s should larger than %s", pFreq->literal, @@ -5401,6 +5419,7 @@ static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbP static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t daysPerFile = pOptions->daysPerFile; + int32_t s3KeepLocal = pOptions->s3KeepLocal; int64_t daysToKeep0 = pOptions->keep[0]; if (-1 == daysPerFile && -1 == daysToKeep0) { return TSDB_CODE_SUCCESS; @@ -5417,6 +5436,10 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid duration value, should be keep2 >= keep1 >= keep0 >= 3 * duration"); } + if (s3KeepLocal > 0 && daysPerFile > s3KeepLocal / 3) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid parameters, should be s3_keeplocal >= 3 * duration"); + } return TSDB_CODE_SUCCESS; } @@ -5511,9 +5534,19 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName if (TSDB_CODE_SUCCESS == code) { code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix); } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbS3KeepLocalOption(pCxt, pOptions); + } if (TSDB_CODE_SUCCESS == code) { code = checkOptionsDependency(pCxt, pDbName, pOptions); } + if (TSDB_CODE_SUCCESS == code) { + code = + checkDbRangeOption(pCxt, "s3_chunksize", pOptions->s3ChunkSize, TSDB_MIN_S3_CHUNK_SIZE, TSDB_MAX_S3_CHUNK_SIZE); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT); + } return code; } @@ -5526,18 +5559,18 @@ static int32_t checkCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* } #define FILL_CMD_SQL(sql, sqlLen, pCmdReq, CMD_TYPE, genericCmd) \ - CMD_TYPE* pCmdReq = genericCmd; \ - char* cmdSql = taosMemoryMalloc(sqlLen); \ - if (cmdSql == NULL) { \ - return TSDB_CODE_OUT_OF_MEMORY; \ - } \ - memcpy(cmdSql, sql, sqlLen); \ - pCmdReq->sqlLen = sqlLen; \ - pCmdReq->sql = cmdSql; \ + CMD_TYPE* pCmdReq = genericCmd; \ + char* cmdSql = taosMemoryMalloc(sqlLen); \ + if (cmdSql == NULL) { \ + return TSDB_CODE_OUT_OF_MEMORY; \ + } \ + memcpy(cmdSql, sql, sqlLen); \ + pCmdReq->sqlLen = sqlLen; \ + pCmdReq->sql = cmdSql; static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq) { const char* sql = pCxt->pParseCxt->pSql; - size_t sqlLen = pCxt->pParseCxt->sqlLen; + size_t sqlLen = pCxt->pParseCxt->sqlLen; switch (msgType) { case TDMT_MND_CREATE_DB: { @@ -5647,7 +5680,6 @@ static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq) default: { break; } - } return TSDB_CODE_SUCCESS; @@ -5720,6 +5752,8 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, pReq->minRows = pStmt->pOptions->minRowsPerBlock; pReq->walRetentionPeriod = pStmt->pOptions->walRetentionPeriod; pReq->walRetentionSize = pStmt->pOptions->walRetentionSize; + pReq->s3KeepLocal = pStmt->pOptions->s3KeepLocal; + pReq->s3Compact = pStmt->pOptions->s3Compact; return; } @@ -5745,6 +5779,14 @@ static int32_t translateTrimDatabase(STranslateContext* pCxt, STrimDatabaseStmt* return buildCmdMsg(pCxt, TDMT_MND_TRIM_DB, (FSerializeFunc)tSerializeSTrimDbReq, &req); } +static int32_t translateS3MigrateDatabase(STranslateContext* pCxt, SS3MigrateDatabaseStmt* pStmt) { + SS3MigrateDbReq req = {0}; + SName name = {0}; + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + tNameGetFullDbName(&name, req.db); + return buildCmdMsg(pCxt, TDMT_MND_S3MIGRATE_DB, (FSerializeFunc)tSerializeSS3MigrateDbReq, &req); +} + static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) { *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); SNode* pNode; @@ -5788,7 +5830,7 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS } static bool validRollupFunc(const char* pFunc) { - static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"}; + static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"}; static const int32_t numOfRollupFuncs = (sizeof(rollupFuncs) / sizeof(char*)); for (int i = 0; i < numOfRollupFuncs; ++i) { if (0 == strcmp(rollupFuncs[i], pFunc)) { @@ -6078,7 +6120,6 @@ static int32_t checkTableDeleteMarkOption(STranslateContext* pCxt, STableOptions return code; } - static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) { if (NULL != strchr(pStmt->tableName, '.')) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, @@ -7168,8 +7209,8 @@ static int16_t getCreateComponentNodeMsgType(ENodeType type) { static int32_t translateCreateComponentNode(STranslateContext* pCxt, SCreateComponentNodeStmt* pStmt) { SMCreateQnodeReq createReq = {.dnodeId = pStmt->dnodeId}; - int32_t code = buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)), - (FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &createReq); + int32_t code = buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)), + (FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &createReq); tFreeSMCreateQnodeReq(&createReq); return code; } @@ -7192,8 +7233,8 @@ static int16_t getDropComponentNodeMsgType(ENodeType type) { static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponentNodeStmt* pStmt) { SDDropQnodeReq dropReq = {.dnodeId = pStmt->dnodeId}; - int32_t code = buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)), - (FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &dropReq); + int32_t code = buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)), + (FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &dropReq); tFreeSDDropQnodeReq(&dropReq); return code; } @@ -7318,7 +7359,8 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt .mgmtEps = pParCxt->mgmtEpSet}; SName name; STableMeta* pMeta = NULL; - int32_t code = getTargetMeta(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta, false); + int32_t code = + getTargetMeta(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta, false); if (code) { taosMemoryFree(pMeta); return code; @@ -7426,7 +7468,7 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) #ifdef TD_ENTERPRISE if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { int32_t origCode = code; - SName name; + SName name; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); SViewMeta* pMeta = NULL; code = getViewMetaFromMetaCache(pCxt, &name, &pMeta); @@ -7434,9 +7476,10 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) code = origCode; } else { SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA}; - char dbFName[TSDB_DB_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(&name, dbFName); - code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, name.dbname, pMeta->querySql, false, pMeta->user, &res); + code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, name.dbname, pMeta->querySql, false, + pMeta->user, &res); if (TSDB_CODE_SUCCESS == code) { code = collectUseTable(&name, pCxt->pTargetTables); } @@ -7449,12 +7492,13 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) viewMeta.numOfCols = res.schemaRes.numOfCols; viewMeta.pSchema = res.schemaRes.pSchema; code = buildTableMetaFromViewMeta(&pStmt->pMeta, &viewMeta); - parserDebug("rebuild view meta, view:%s.%s, numOfCols:%d, code:0x%x", dbFName, pStmt->tableName, viewMeta.numOfCols, code); + parserDebug("rebuild view meta, view:%s.%s, numOfCols:%d, code:0x%x", dbFName, pStmt->tableName, + viewMeta.numOfCols, code); } taosMemoryFree(res.schemaRes.pSchema); } } -#endif +#endif return code; } @@ -7539,12 +7583,12 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt #ifdef TD_ENTERPRISE SRealTableNode* pRealTable = (SRealTableNode*)((SSelectStmt*)pStmt->pQuery)->pFromTable; - SName name; - STableMeta* pMeta = NULL; - int8_t tableType = 0; - int32_t code = getTargetMeta( - pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), - &pMeta, true); + SName name; + STableMeta* pMeta = NULL; + int8_t tableType = 0; + int32_t code = + getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), + &pMeta, true); if (NULL != pMeta) { tableType = pMeta->tableType; taosMemoryFree(pMeta); @@ -7555,8 +7599,8 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt if (TSDB_VIEW_TABLE == tableType) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } -#endif - +#endif + return TSDB_CODE_SUCCESS; } @@ -7780,10 +7824,10 @@ static int32_t subtableExprHasColumnOrPseudoColumn(SNode* pNode) { static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; - if ( (SRealTableNode*)pSelect->pFromTable && ((SRealTableNode*)pSelect->pFromTable)->pMeta - && TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType - && !hasPartitionByTbname(pSelect->pPartitionByList) - && pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) { + if ((SRealTableNode*)pSelect->pFromTable && ((SRealTableNode*)pSelect->pFromTable)->pMeta && + TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType && + !hasPartitionByTbname(pSelect->pPartitionByList) && pSelect->pWindow != NULL && + pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Event window for stream on super table must patitioned by table name"); } @@ -7808,27 +7852,27 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm } if (pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_COUNT_WINDOW) { - if ( (SRealTableNode*)pSelect->pFromTable && ((SRealTableNode*)pSelect->pFromTable)->pMeta - && TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType - && !hasPartitionByTbname(pSelect->pPartitionByList) ) { + if ((SRealTableNode*)pSelect->pFromTable && ((SRealTableNode*)pSelect->pFromTable)->pMeta && + TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType && + !hasPartitionByTbname(pSelect->pPartitionByList)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Count window for stream on super table must patitioned by table name"); + "Count window for stream on super table must patitioned by table name"); } int64_t watermark = 0; if (pStmt->pOptions->pWatermark) { translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark); - watermark =((SValueNode*)pStmt->pOptions->pWatermark)->datum.i; + watermark = ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i; } if (watermark <= 0) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Watermark of Count window must exceed 0."); + "Watermark of Count window must exceed 0."); } if (pStmt->pOptions->ignoreExpired != 1) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Ignore expired data of Count window must be 1."); - } + "Ignore expired data of Count window must be 1."); + } SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow; if (pCountWin->windowCount <= 1) { @@ -7850,7 +7894,6 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Size of Count window must less than 2147483647(INT32_MAX)."); } - } return TSDB_CODE_SUCCESS; @@ -8240,7 +8283,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta } snprintf(pFunc1->node.aliasName, sizeof(pFunc1->node.aliasName), "%s.%p", pFunc1->functionName, pFunc1); - code = nodesListStrictAppend(pProjectionList, (SNode*) pFunc1); + code = nodesListStrictAppend(pProjectionList, (SNode*)pFunc1); if (code) { nodesDestroyList(pProjectionList); return code; @@ -8253,7 +8296,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta } snprintf(pFunc2->node.aliasName, sizeof(pFunc2->node.aliasName), "%s.%p", pFunc2->functionName, pFunc2); - code = nodesListStrictAppend(pProjectionList, (SNode*) pFunc2); + code = nodesListStrictAppend(pProjectionList, (SNode*)pFunc2); if (code) { nodesDestroyList(pProjectionList); return code; @@ -8476,7 +8519,7 @@ static int32_t createStreamReqVersionInfo(SSDataBlock* pBlock, SArray** pArray, taosArrayPush(*pArray, &v); } } else { - int32_t precision = (pInterval->interval > 0)? pInterval->precision:TSDB_TIME_PRECISION_MILLI; + int32_t precision = (pInterval->interval > 0) ? pInterval->precision : TSDB_TIME_PRECISION_MILLI; *lastTs = taosGetTimestamp(precision); } @@ -8500,7 +8543,8 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDa if (TSDB_CODE_SUCCESS == code) { if (interval.interval > 0) { - pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit, interval.precision); + pStmt->pReq->lastTs = taosTimeAdd(taosTimeTruncate(lastTs, &interval), interval.interval, interval.intervalUnit, + interval.precision); } else { pStmt->pReq->lastTs = lastTs + 1; // start key of the next time window } @@ -8555,32 +8599,34 @@ static int32_t validateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStm return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type"); } -/* - STableMeta* pMetaCache = NULL; - int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->viewName, &pMetaCache); - if (TSDB_CODE_SUCCESS == code) { - taosMemoryFreeClear(pMetaCache); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with table"); - } -*/ + /* + STableMeta* pMetaCache = NULL; + int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->viewName, &pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + taosMemoryFreeClear(pMetaCache); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with + table"); + } + */ return TSDB_CODE_SUCCESS; } static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStmt) { #ifndef TD_ENTERPRISE - return TSDB_CODE_OPS_NOT_SUPPORT; + return TSDB_CODE_OPS_NOT_SUPPORT; #endif SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA}; - SName name; - char dbFName[TSDB_DB_FNAME_LEN]; + SName name; + char dbFName[TSDB_DB_FNAME_LEN]; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); tNameGetFullDbName(&name, dbFName); int32_t code = validateCreateView(pCxt, pStmt); - if (TSDB_CODE_SUCCESS == code) { - code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, pStmt->dbName, pStmt->pQuerySql, false, NULL, &res); + if (TSDB_CODE_SUCCESS == code) { + code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, pStmt->dbName, pStmt->pQuerySql, false, NULL, + &res); } if (TSDB_CODE_SUCCESS == code) { code = collectUseTable(&name, pCxt->pTargetTables); @@ -8591,7 +8637,8 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt pStmt->createReq.pSchema = res.schemaRes.pSchema; strncpy(pStmt->createReq.name, pStmt->viewName, sizeof(pStmt->createReq.name) - 1); tstrncpy(pStmt->createReq.dbFName, dbFName, sizeof(pStmt->createReq.dbFName)); - snprintf(pStmt->createReq.fullname, sizeof(pStmt->createReq.fullname) - 1, "%s.%s", pStmt->createReq.dbFName, pStmt->viewName); + snprintf(pStmt->createReq.fullname, sizeof(pStmt->createReq.fullname) - 1, "%s.%s", pStmt->createReq.dbFName, + pStmt->viewName); TSWAP(pStmt->createReq.querySql, pStmt->pQuerySql); pStmt->createReq.orReplace = pStmt->orReplace; pStmt->createReq.sql = tstrdup(pCxt->pParseCxt->pSql); @@ -8602,19 +8649,18 @@ static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pSt if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_VIEW, (FSerializeFunc)tSerializeSCMCreateViewReq, &pStmt->createReq); } - + tFreeSCMCreateViewReq(&pStmt->createReq); return code; } - static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) { #ifndef TD_ENTERPRISE return TSDB_CODE_OPS_NOT_SUPPORT; #endif - SCMDropViewReq dropReq = {0}; - SName name; + SCMDropViewReq dropReq = {0}; + SName name; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameGetFullDbName(&name, dropReq.dbFName); strncpy(dropReq.name, pStmt->viewName, sizeof(dropReq.name) - 1); @@ -8630,11 +8676,10 @@ static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) if (TSDB_CODE_SUCCESS != code) { return code; } - + return buildCmdMsg(pCxt, TDMT_MND_DROP_VIEW, (FSerializeFunc)tSerializeSCMDropViewReq, &dropReq); } - static int32_t readFromFile(char* pName, int32_t* len, char** buf) { int64_t filesize = 0; if (taosStatFile(pName, &filesize, NULL, NULL) < 0) { @@ -8735,7 +8780,7 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, if (TSDB_CODE_SUCCESS == code) { SName name; code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name), - &(pTable->pMeta), false); + &(pTable->pMeta), false); if (code) { nodesDestroyNode((SNode*)pTable); return code; @@ -8769,15 +8814,16 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, } static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { - int32_t code = 0; + int32_t code = 0; SAlterUserReq req = {0}; req.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES; req.privileges = pStmt->privileges; #ifdef TD_ENTERPRISE if (0 != pStmt->tabName[0]) { - SName name; + SName name; STableMeta* pTableMeta = NULL; - code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); + code = + getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); @@ -8803,16 +8849,17 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { } static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { - int32_t code = 0; + int32_t code = 0; SAlterUserReq req = {0}; req.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES; req.privileges = pStmt->privileges; #ifdef TD_ENTERPRISE if (0 != pStmt->tabName[0]) { - SName name; + SName name; STableMeta* pTableMeta = NULL; - code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); + code = + getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true); if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); @@ -8823,7 +8870,7 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { taosMemoryFree(pTableMeta); } #endif - + strcpy(req.user, pStmt->userName); sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); sprintf(req.tabName, "%s", pStmt->tabName); @@ -8842,7 +8889,8 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) { SBalanceVgroupLeaderReq req = {0}; req.vgId = pStmt->vgId; - int32_t code = buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req); + int32_t code = + buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req); tFreeSBalanceVgroupLeaderReq(&req); return code; } @@ -8932,7 +8980,7 @@ static int32_t translateShowCreateTable(STranslateContext* pCxt, SShowCreateTabl static int32_t translateShowCreateView(STranslateContext* pCxt, SShowCreateViewStmt* pStmt) { #ifndef TD_ENTERPRISE return TSDB_CODE_OPS_NOT_SUPPORT; -#else +#else SName name; toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name); return getViewMetaFromMetaCache(pCxt, &name, (SViewMeta**)&pStmt->pViewMeta); @@ -8966,6 +9014,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_TRIM_DATABASE_STMT: code = translateTrimDatabase(pCxt, (STrimDatabaseStmt*)pNode); break; + case QUERY_NODE_S3MIGRATE_DATABASE_STMT: + code = translateS3MigrateDatabase(pCxt, (SS3MigrateDatabaseStmt*)pNode); + break; case QUERY_NODE_CREATE_TABLE_STMT: code = translateCreateSuperTable(pCxt, (SCreateTableStmt*)pNode); break; @@ -9275,7 +9326,6 @@ static int32_t extractShowCreateViewResultSchema(int32_t* numOfCols, SSchema** p return TSDB_CODE_SUCCESS; } - static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) { *numOfCols = 3; *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); @@ -9347,7 +9397,7 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS case QUERY_NODE_SHOW_VARIABLES_STMT: return extractShowVariablesResultSchema(numOfCols, pSchema); case QUERY_NODE_COMPACT_DATABASE_STMT: - return extractCompactDbResultSchema(numOfCols, pSchema); + return extractCompactDbResultSchema(numOfCols, pSchema); default: break; } @@ -9466,17 +9516,16 @@ static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SN static const char* getTbNameColName(ENodeType type) { const char* colName; - switch (type) - { - case QUERY_NODE_SHOW_VIEWS_STMT: - colName = "view_name"; - break; - case QUERY_NODE_SHOW_STABLES_STMT: - colName = "stable_name"; - break; - default: - colName = "table_name"; - break; + switch (type) { + case QUERY_NODE_SHOW_VIEWS_STMT: + colName = "view_name"; + break; + case QUERY_NODE_SHOW_STABLES_STMT: + colName = "stable_name"; + break; + default: + colName = "table_name"; + break; } return colName; } @@ -9510,14 +9559,14 @@ static int32_t insertCondIntoSelectStmt(SSelectStmt* pSelect, SNode* pCond) { SNodeList* pLogicCondList2 = NULL; if (nodeType(pSelect->pWhere) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pSelect->pWhere)->condType == LOGIC_COND_TYPE_AND) { - pLogicCondListWhere = ((SLogicConditionNode*)pSelect->pWhere)->pParameterList; + pLogicCondListWhere = ((SLogicConditionNode*)pSelect->pWhere)->pParameterList; } else { nodesListMakeAppend(&pLogicCondListWhere, pSelect->pWhere); } if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pCond)->condType == LOGIC_COND_TYPE_AND) { - pLogicCondList2 = ((SLogicConditionNode*)pCond)->pParameterList; + pLogicCondList2 = ((SLogicConditionNode*)pCond)->pParameterList; } else { nodesListMakeAppend(&pLogicCondList2, pCond); } @@ -10844,8 +10893,8 @@ static int32_t rewriteFlushDatabase(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteShowCompacts(STranslateContext* pCxt, SQuery* pQuery) { SShowCompactsStmt* pShow = (SShowCompactsStmt*)(pQuery->pRoot); - SSelectStmt* pStmt = NULL; - int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACTS_STMT, &pStmt); + SSelectStmt* pStmt = NULL; + int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACTS_STMT, &pStmt); if (TSDB_CODE_SUCCESS == code) { pCxt->showRewrite = true; pQuery->showRewrite = true; @@ -10857,8 +10906,8 @@ static int32_t rewriteShowCompacts(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteShowCompactDetailsStmt(STranslateContext* pCxt, SQuery* pQuery) { SShowCompactDetailsStmt* pShow = (SShowCompactDetailsStmt*)(pQuery->pRoot); - SSelectStmt* pStmt = NULL; - int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACT_DETAILS_STMT, &pStmt); + SSelectStmt* pStmt = NULL; + int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_COMPACT_DETAILS_STMT, &pStmt); if (TSDB_CODE_SUCCESS == code) { if (NULL != pShow->pCompactId) { code = createOperatorNode(OP_TYPE_EQUAL, "compact_id", pShow->pCompactId, &pStmt->pWhere); @@ -10870,7 +10919,7 @@ static int32_t rewriteShowCompactDetailsStmt(STranslateContext* pCxt, SQuery* pQ nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pStmt; } - return code; + return code; } static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { From d5e9169769389eac3c736139158dffdc8554fab4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 8 Apr 2024 10:31:49 +0800 Subject: [PATCH 07/34] cos/multi-write: dnode part including mnode/vnode/tsdb --- source/dnode/mgmt/CMakeLists.txt | 8 + source/dnode/mgmt/exe/dmMain.c | 37 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 40 +- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 5 +- source/dnode/mgmt/node_mgmt/CMakeLists.txt | 10 +- source/dnode/mnode/impl/inc/mndDef.h | 3 + source/dnode/mnode/impl/src/mndDb.c | 170 +++++- source/dnode/mnode/impl/src/mndMain.c | 18 +- source/dnode/mnode/impl/src/mndStb.c | 66 +- source/dnode/mnode/impl/src/mndVgroup.c | 13 +- source/dnode/vnode/inc/vnode.h | 19 +- source/dnode/vnode/src/inc/tsdb.h | 3 +- source/dnode/vnode/src/inc/vnodeInt.h | 25 +- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 39 -- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 11 +- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 15 +- source/dnode/vnode/src/tsdb/tsdbDataFileRW.h | 1 + source/dnode/vnode/src/tsdb/tsdbDef.h | 2 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 96 ++- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 6 +- source/dnode/vnode/src/tsdb/tsdbFSetRW.c | 1 + source/dnode/vnode/src/tsdb/tsdbFSetRW.h | 1 + source/dnode/vnode/src/tsdb/tsdbFile2.c | 145 ++++- source/dnode/vnode/src/tsdb/tsdbFile2.h | 4 +- source/dnode/vnode/src/tsdb/tsdbMerge.c | 7 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 219 +++++-- source/dnode/vnode/src/tsdb/tsdbRetention.c | 569 ++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 6 +- source/dnode/vnode/src/tsdb/tsdbUpgrade.c | 6 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 4 +- source/dnode/vnode/src/vnd/vnodeCfg.c | 24 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 39 +- source/dnode/vnode/src/vnd/vnodeRetention.c | 10 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 82 ++- 35 files changed, 1255 insertions(+), 451 deletions(-) diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 762b8fd529..e6be56cfb7 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -14,6 +14,14 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" ) +IF (TD_ENTERPRISE) + IF(${BUILD_WITH_S3}) + add_definitions(-DUSE_S3) + ELSEIF(${BUILD_WITH_COS}) + add_definitions(-DUSE_COS) + ENDIF() +ENDIF() + IF (TD_LINUX_64 AND JEMALLOC_ENABLED) ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index f9456f1729..4f8858f3c2 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -27,15 +27,15 @@ #include "cus_name.h" #else #ifndef CUS_NAME - #define CUS_NAME "TDengine" +#define CUS_NAME "TDengine" #endif #ifndef CUS_PROMPT - #define CUS_PROMPT "taos" +#define CUS_PROMPT "taos" #endif #ifndef CUS_EMAIL - #define CUS_EMAIL "" +#define CUS_EMAIL "" #endif #endif // clang-format off @@ -58,6 +58,7 @@ static struct { bool dumpSdb; bool generateGrant; bool memDbg; + bool checkS3; bool printAuth; bool printVersion; bool printHelp; @@ -169,7 +170,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { return -1; } } else if (strcmp(argv[i], "-a") == 0) { - if(i < argc - 1) { + if (i < argc - 1) { if (strlen(argv[++i]) >= PATH_MAX) { printf("apollo url overflow"); return -1; @@ -182,7 +183,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } else if (strcmp(argv[i], "-s") == 0) { global.dumpSdb = true; } else if (strcmp(argv[i], "-E") == 0) { - if(i < argc - 1) { + if (i < argc - 1) { if (strlen(argv[++i]) >= PATH_MAX) { printf("env file path overflow"); return -1; @@ -207,6 +208,8 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { cmdEnvIndex++; } else if (strcmp(argv[i], "-dm") == 0) { global.memDbg = true; + } else if (strcmp(argv[i], "--checks3") == 0) { + global.checkS3 = true; } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0) { global.printHelp = true; @@ -267,8 +270,21 @@ static void dmDumpCfg() { cfgDumpCfg(pCfg, 0, true); } +static int32_t dmCheckS3() { + int32_t code = 0; + SConfig *pCfg = taosGetCfg(); + cfgDumpCfgS3(pCfg, 0, true); +#if defined(USE_S3) + extern int32_t s3CheckCfg(); + + code = s3CheckCfg(); +#endif + return code; +} + static int32_t dmInitLog() { - return taosCreateLog(CUS_PROMPT"dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); + return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, + 0); } static void taosCleanupArgs() { @@ -355,6 +371,15 @@ int mainWindows(int argc, char **argv) { return -1; } + if (global.checkS3) { + int32_t code = dmCheckS3(); + taosCleanupCfg(); + taosCloseLog(); + taosCleanupArgs(); + taosConvDestroy(); + return code; + } + if (global.dumpConfig) { dmDumpCfg(); taosCleanupCfg(); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index a3a5f1d971..3d3b7df3be 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -141,6 +141,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TRIM_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_S3MIGRATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -208,6 +209,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index bfac0bab9d..3b64dfcfd1 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -53,7 +53,7 @@ void vmGetVnodeLoadsLite(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { SVnodeObj **ppVnode = pIter; if (ppVnode == NULL || *ppVnode == NULL) continue; - SVnodeObj *pVnode = *ppVnode; + SVnodeObj *pVnode = *ppVnode; if (!pVnode->failed) { SVnodeLoadLite vload = {0}; if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) { @@ -157,6 +157,10 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->hashSuffix = pCreate->hashSuffix; pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; + pCfg->s3ChunkSize = pCreate->s3ChunkSize; + pCfg->s3KeepLocal = pCreate->s3KeepLocal; + pCfg->s3Compact = pCreate->s3Compact; + pCfg->standby = 0; pCfg->syncCfg.replicaNum = 0; pCfg->syncCfg.totalReplicaNum = 0; @@ -236,17 +240,18 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo( "vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d " "szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64 - ", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d" + ", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d s3ChunkSize:%d s3KeepLocal:%d s3Compact:%d tsma:%d " + "precision:%d compression:%d minRows:%d maxRows:%d" ", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64 ", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d " "learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d", req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, - req.keepTimeOffset, req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod, - req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod, - req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica, - req.learnerSelfIndex, req.strict, req.changeVersion); + req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression, + req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize, + req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, + req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion); for (int32_t i = 0; i < req.replica; ++i) { dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port, @@ -345,7 +350,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { _OVER: if (code != 0) { vnodeClose(pImpl); - vnodeDestroy(0, path, pMgmt->pTfs); + vnodeDestroy(0, path, pMgmt->pTfs, 0); } else { dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", req.vgId, TMSG_INFO(pMsg->msgType)); @@ -356,7 +361,7 @@ _OVER: return code; } -//alter replica doesn't use this, but restore dnode still use this +// alter replica doesn't use this, but restore dnode still use this int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SAlterVnodeTypeReq req = {0}; if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &req) != 0) { @@ -398,8 +403,8 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pMgmt->name); int32_t vgId = req.vgId; - dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d changeVersion:%d", - vgId, req.replica, req.selfIndex, req.strict, req.changeVersion); + dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d changeVersion:%d", vgId, req.replica, + req.selfIndex, req.strict, req.changeVersion); for (int32_t i = 0; i < req.replica; ++i) { SReplica *pReplica = &req.replicas[i]; dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id); @@ -482,12 +487,12 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - if(req.learnerReplicas == 0){ + if (req.learnerReplicas == 0) { req.learnerSelfIndex = -1; } - dInfo("vgId:%d, vnode management handle msgType:%s, start to process check-learner-catchup-request", - req.vgId, TMSG_INFO(pMsg->msgType)); + dInfo("vgId:%d, vnode management handle msgType:%s, start to process check-learner-catchup-request", req.vgId, + TMSG_INFO(pMsg->msgType)); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); if (pVnode == NULL) { @@ -499,7 +504,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { ESyncRole role = vnodeGetRole(pVnode->pImpl); dInfo("vgId:%d, checking node role:%d", req.vgId, role); - if(role == TAOS_SYNC_ROLE_VOTER){ + if (role == TAOS_SYNC_ROLE_VOTER) { dError("vgId:%d, failed to alter vnode type since node already is role:%d", req.vgId, role); terrno = TSDB_CODE_VND_ALREADY_IS_VOTER; vmReleaseVnode(pMgmt, pVnode); @@ -507,7 +512,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } dInfo("vgId:%d, checking node catch up", req.vgId); - if(vnodeIsCatchUp(pVnode->pImpl) != 1){ + if (vnodeIsCatchUp(pVnode->pImpl) != 1) { terrno = TSDB_CODE_VND_NOT_CATCH_UP; vmReleaseVnode(pMgmt, pVnode); return -1; @@ -517,8 +522,8 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { vmReleaseVnode(pMgmt, pVnode); - dInfo("vgId:%d, vnode management handle msgType:%s, end to process check-learner-catchup-request", - req.vgId, TMSG_INFO(pMsg->msgType)); + dInfo("vgId:%d, vnode management handle msgType:%s, end to process check-learner-catchup-request", req.vgId, + TMSG_INFO(pMsg->msgType)); return 0; } @@ -848,6 +853,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 3dfc2bd96f..296372a955 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -15,9 +15,9 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "libs/function/tudf.h" #include "tfs.h" #include "vnd.h" -#include "libs/function/tudf.h" int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { int32_t diskId = -1; @@ -234,6 +234,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) dInfo("vgId:%d, commit data finished", pVnode->vgId); } + int32_t nodeId = vnodeNodeId(pVnode->pImpl); vnodeClose(pVnode->pImpl); pVnode->pImpl = NULL; @@ -250,7 +251,7 @@ _closed: if (pVnode->dropped) { dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped); snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId); - vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs); + vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs, nodeId); } vmFreeVnodeObj(&pVnode); diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt index 0cdc68345a..82b9384d66 100644 --- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -4,15 +4,13 @@ target_link_libraries( dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw ) -IF (TD_STORAGE) - +IF (TD_ENTERPRISE) IF(${BUILD_WITH_S3}) - add_definitions(-DUSE_S3) + add_definitions(-DUSE_S3) ELSEIF(${BUILD_WITH_COS}) - add_definitions(-DUSE_COS) + add_definitions(-DUSE_COS) ENDIF() - -ENDIF () +ENDIF() IF (DEFINED GRANT_CFG_INCLUDE_DIR) add_definitions(-DGRANTS_CFG) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 4a092057ce..a26dbe890b 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -343,6 +343,9 @@ typedef struct { int32_t walRollPeriod; int64_t walRetentionSize; int64_t walSegmentSize; + int32_t s3ChunkSize; + int32_t s3KeepLocal; + int8_t s3Compact; } SDbCfg; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 37c2d19bd4..8430b5c6b9 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDb.h" +#include "audit.h" #include "mndCluster.h" #include "mndDnode.h" #include "mndIndex.h" @@ -30,12 +31,11 @@ #include "mndVgroup.h" #include "mndView.h" #include "systable.h" -#include "tjson.h" #include "thttp.h" -#include "audit.h" +#include "tjson.h" #define DB_VER_NUMBER 1 -#define DB_RESERVE_SIZE 42 +#define DB_RESERVE_SIZE 33 static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); @@ -43,14 +43,15 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew); static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw); -static int32_t mndProcessCreateDbReq(SRpcMsg *pReq); -static int32_t mndProcessAlterDbReq(SRpcMsg *pReq); -static int32_t mndProcessDropDbReq(SRpcMsg *pReq); -static int32_t mndProcessUseDbReq(SRpcMsg *pReq); -static int32_t mndProcessTrimDbReq(SRpcMsg *pReq); -static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); -static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); -static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq); +static int32_t mndProcessCreateDbReq(SRpcMsg *pReq); +static int32_t mndProcessAlterDbReq(SRpcMsg *pReq); +static int32_t mndProcessDropDbReq(SRpcMsg *pReq); +static int32_t mndProcessUseDbReq(SRpcMsg *pReq); +static int32_t mndProcessTrimDbReq(SRpcMsg *pReq); +static int32_t mndProcessS3MigrateDbReq(SRpcMsg *pReq); +static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); +static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); +static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq); #ifndef TD_ENTERPRISE int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { return TSDB_CODE_OPS_NOT_SUPPORT; } @@ -75,6 +76,7 @@ int32_t mndInitDb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB, mndProcessTrimDbReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); + mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB, mndProcessS3MigrateDbReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); @@ -139,6 +141,9 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER) SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.keepTimeOffset, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3ChunkSize, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3KeepLocal, _OVER) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -230,6 +235,9 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER) SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.keepTimeOffset, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3ChunkSize, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3KeepLocal, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) taosInitRWLatch(&pDb->lock); @@ -319,6 +327,9 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.minRows = pNew->cfg.minRows; pOld->cfg.maxRows = pNew->cfg.maxRows; pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize; + pOld->cfg.s3ChunkSize = pNew->cfg.s3ChunkSize; + pOld->cfg.s3KeepLocal = pNew->cfg.s3KeepLocal; + pOld->cfg.s3Compact = pNew->cfg.s3Compact; pOld->compactStartTime = pNew->compactStartTime; taosWUnLockLatch(&pOld->lock); return 0; @@ -414,6 +425,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1; if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1; + if (pCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1; + if (pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1; + if (pCfg->s3Compact < TSDB_MIN_S3_COMPACT || pCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1; + terrno = 0; return terrno; } @@ -448,6 +463,9 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pCfg) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; return -1; } + if (pCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1; + if (pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1; + if (pCfg->s3Compact < TSDB_MIN_S3_COMPACT || pCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1; terrno = 0; return terrno; @@ -484,6 +502,9 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; + if (pCfg->s3ChunkSize < 0) pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT; } static int32_t mndSetCreateDbPrepareAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { @@ -628,6 +649,9 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .sstTrigger = pCreate->sstTrigger, .hashPrefix = pCreate->hashPrefix, .hashSuffix = pCreate->hashSuffix, + .s3ChunkSize = pCreate->s3ChunkSize, + .s3KeepLocal = pCreate->s3KeepLocal, + .s3Compact = pCreate->s3Compact, .tsdbPageSize = pCreate->tsdbPageSize, }; @@ -698,17 +722,17 @@ _OVER: return code; } -static void mndBuildAuditDetailInt32(char* detail, char* tmp, char* format, int32_t para){ - if(para > 0){ - if(strlen(detail) > 0) strcat(detail, ", "); +static void mndBuildAuditDetailInt32(char *detail, char *tmp, char *format, int32_t para) { + if (para > 0) { + if (strlen(detail) > 0) strcat(detail, ", "); sprintf(tmp, format, para); strcat(detail, tmp); } } -static void mndBuildAuditDetailInt64(char* detail, char* tmp, char* format, int64_t para){ - if(para > 0){ - if(strlen(detail) > 0) strcat(detail, ", "); +static void mndBuildAuditDetailInt64(char *detail, char *tmp, char *format, int64_t para) { + if (para > 0) { + if (strlen(detail) > 0) strcat(detail, ", "); sprintf(tmp, format, para); strcat(detail, tmp); } @@ -893,6 +917,18 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = 0; } + if (pAlter->s3KeepLocal > TSDB_MIN_S3_KEEP_LOCAL && pAlter->s3KeepLocal != pDb->cfg.s3KeepLocal) { + pDb->cfg.s3KeepLocal = pAlter->s3KeepLocal; + pDb->vgVersion++; + terrno = 0; + } + + if (pAlter->s3Compact > TSDB_MIN_S3_COMPACT && pAlter->s3Compact != pDb->cfg.s3Compact) { + pDb->cfg.s3Compact = pAlter->s3Compact; + pDb->vgVersion++; + terrno = 0; + } + return terrno; } @@ -1081,6 +1117,9 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); cfgRsp->schemaless = pDb->cfg.schemaless; cfgRsp->sstTrigger = pDb->cfg.sstTrigger; + cfgRsp->s3ChunkSize = pDb->cfg.s3ChunkSize; + cfgRsp->s3KeepLocal = pDb->cfg.s3KeepLocal; + cfgRsp->s3Compact = pDb->cfg.s3Compact; } static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { @@ -1277,7 +1316,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER; #ifdef TD_ENTERPRISE if (mndDropViewByDb(pMnode, pTrans, pDb) != 0) goto _OVER; -#endif +#endif if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropIdxsByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; @@ -1562,21 +1601,19 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pDbCacheInfo->vgVersion >= pDb->vgVersion && - pDbCacheInfo->cfgVersion >= pDb->cfgVersion && - numOfTable == pDbCacheInfo->numOfTable && - pDbCacheInfo->stateTs == pDb->stateTs) { + if (pDbCacheInfo->vgVersion >= pDb->vgVersion && pDbCacheInfo->cfgVersion >= pDb->cfgVersion && + numOfTable == pDbCacheInfo->numOfTable && pDbCacheInfo->stateTs == pDb->stateTs) { mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d, not changed vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", - pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, pDbCacheInfo->numOfTable, - pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); + pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, + pDbCacheInfo->numOfTable, pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); mndReleaseDb(pMnode, pDb); continue; } else { mInfo("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d, changed to vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", - pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, pDbCacheInfo->numOfTable, - pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); + pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, + pDbCacheInfo->numOfTable, pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); } if (pDbCacheInfo->cfgVersion < pDb->cfgVersion) { @@ -1584,8 +1621,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, mndDumpDbCfgInfo(rsp.cfgRsp, pDb); } - if (pDbCacheInfo->vgVersion < pDb->vgVersion || - numOfTable != pDbCacheInfo->numOfTable || + if (pDbCacheInfo->vgVersion < pDb->vgVersion || numOfTable != pDbCacheInfo->numOfTable || pDbCacheInfo->stateTs != pDb->stateTs) { rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp)); rsp.useDbRsp->pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); @@ -1695,6 +1731,77 @@ _OVER: return code; } +static int32_t mndS3MigrateDb(SMnode *pMnode, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + SVS3MigrateDbReq s3migrateReq = {.timestamp = taosGetTimestampSec()}; + int32_t reqLen = tSerializeSVS3MigrateDbReq(NULL, 0, &s3migrateReq); + int32_t contLen = reqLen + sizeof(SMsgHead); + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (pVgroup->dbUid != pDb->uid) continue; + + SMsgHead *pHead = rpcMallocCont(contLen); + if (pHead == NULL) { + sdbCancelFetch(pSdb, pVgroup); + sdbRelease(pSdb, pVgroup); + continue; + } + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), contLen, &s3migrateReq); + + SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen}; + SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); + int32_t code = tmsgSendReq(&epSet, &rpcMsg); + if (code != 0) { + mError("vgId:%d, failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code); + } else { + mInfo("vgId:%d, send vnode-s3migrate request to vnode, time:%d", pVgroup->vgId, s3migrateReq.timestamp); + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndProcessS3MigrateDbReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SDbObj *pDb = NULL; + SS3MigrateDbReq s3migrateReq = {0}; + + if (tDeserializeSS3MigrateDbReq(pReq->pCont, pReq->contLen, &s3migrateReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mInfo("db:%s, start to s3migrate", s3migrateReq.db); + + pDb = mndAcquireDb(pMnode, s3migrateReq.db); + if (pDb == NULL) { + goto _OVER; + } + + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_TRIM_DB, pDb) != 0) { + goto _OVER; + } + + code = mndS3MigrateDb(pMnode, pDb); + +_OVER: + if (code != 0) { + mError("db:%s, failed to process s3migrate db req since %s", s3migrateReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + return code; +} + const char *mndGetDbStr(const char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); if (pos != NULL) ++pos; @@ -1991,6 +2098,13 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.keepTimeOffset, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3ChunkSize, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3KeepLocal, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3Compact, false); } taosMemoryFree(buf); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 69a0bd477d..0bb1b63028 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -133,6 +133,14 @@ static void mndPullupTtl(SMnode *pMnode) { } static void mndPullupTrimDb(SMnode *pMnode) { + mTrace("pullup s3migrate"); + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_S3MIGRATE_DB_TIMER, .pCont = pReq, .contLen = contLen}; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); +} + +static void mndPullupS3MigrateDb(SMnode *pMnode) { mTrace("pullup trim"); int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); @@ -302,6 +310,7 @@ static int32_t minCronTime() { int32_t min = INT32_MAX; min = TMIN(min, tsTtlPushIntervalSec); min = TMIN(min, tsTrimVDbIntervalSec); + min = TMIN(min, tsS3MigrateIntervalSec); min = TMIN(min, tsTransPullupInterval); min = TMIN(min, tsCompactPullupInterval); min = TMIN(min, tsMqRebalanceInterval); @@ -325,6 +334,10 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { mndPullupTrimDb(pMnode); } + if (tsS3MigrateEnabled && sec % tsS3MigrateIntervalSec == 0) { + mndPullupS3MigrateDb(pMnode); + } + if (sec % tsTransPullupInterval == 0) { mndPullupTrans(pMnode); } @@ -768,7 +781,8 @@ _OVER: pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER || pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER || pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE || - pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT) { + pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT || + pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER) { mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, pMnode->stopped, state.restored, syncStr(state.state)); return -1; @@ -912,7 +926,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pObj->id == pMnode->selfDnodeId) { pClusterInfo->first_ep_dnode_id = pObj->id; tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); - //pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; + // pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode); // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 7ee1b36916..5bc855207f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -42,6 +42,8 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); static int32_t mndProcessTtlTimer(SRpcMsg *pReq); static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq); +static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq); +static int32_t mndProcessS3MigrateDbRsp(SRpcMsg *pReq); static int32_t mndProcessCreateStbReq(SRpcMsg *pReq); static int32_t mndProcessAlterStbReq(SRpcMsg *pReq); static int32_t mndProcessDropStbReq(SRpcMsg *pReq); @@ -82,6 +84,8 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer); mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB_TIMER, mndProcessTrimDbTimer); + mndSetMsgHandle(pMnode, TDMT_VND_S3MIGRATE_RSP, mndProcessS3MigrateDbRsp); + mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB_TIMER, mndProcessS3MigrateDbTimer); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq); // mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq); @@ -762,9 +766,9 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN); pDst->createdTime = taosGetTimestampMs(); pDst->updateTime = pDst->createdTime; - pDst->uid = - (pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX) - ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + pDst->uid = (pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX) + ? pCreate->suid + : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); pDst->dbUid = pDb->uid; pDst->tagVer = 1; pDst->colVer = 1; @@ -983,6 +987,43 @@ static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq) { return 0; } +static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + SVS3MigrateDbReq s3migrateReq = {.timestamp = taosGetTimestampSec()}; + int32_t reqLen = tSerializeSVS3MigrateDbReq(NULL, 0, &s3migrateReq); + int32_t contLen = reqLen + sizeof(SMsgHead); + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + SMsgHead *pHead = rpcMallocCont(contLen); + if (pHead == NULL) { + sdbCancelFetch(pSdb, pVgroup); + sdbRelease(pSdb, pVgroup); + continue; + } + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &s3migrateReq); + + SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen}; + SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); + int32_t code = tmsgSendReq(&epSet, &rpcMsg); + if (code != 0) { + mError("vgId:%d, timer failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code); + } else { + mInfo("vgId:%d, timer send vnode-s3migrate request to vnode, time:%d", pVgroup->vgId, s3migrateReq.timestamp); + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) { for (int32_t tag = 0; tag < pStb->numOfTags; tag++) { if (strcmp(pStb->pTags[tag].name, tagName) == 0) { @@ -1131,7 +1172,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { } } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { goto _OVER; - } else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) && (createReq.tagVer != 1 || createReq.colVer != 1)) { + } else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) && + (createReq.tagVer != 1 || createReq.colVer != 1)) { mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name); code = 0; goto _OVER; @@ -1182,14 +1224,13 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { SName name = {0}; tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - if(createReq.sql == NULL && createReq.sqlLen == 0){ + if (createReq.sql == NULL && createReq.sqlLen == 0) { char detail[1000] = {0}; sprintf(detail, "dbname:%s, stable name:%s", name.dbname, name.tname); auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, detail, strlen(detail)); - } - else{ + } else { auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, createReq.sql, createReq.sqlLen); } _OVER: @@ -1571,7 +1612,7 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj for (int32_t i = 0; i < pOld->numOfTags; ++i) { nLen += (pOld->pTags[i].colId == colId) ? pField->bytes : pOld->pTags[i].bytes; } - + if (nLen > TSDB_MAX_TAGS_LEN) { terrno = TSDB_CODE_PAR_INVALID_TAGS_LENGTH; return -1; @@ -1934,7 +1975,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, return 0; } -static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion* pStbVer, bool* schema, bool* sma) { +static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion *pStbVer, bool *schema, bool *sma) { char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", pStbVer->dbFName, pStbVer->stbName); @@ -1964,7 +2005,7 @@ static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion* pStbVer, bo } else { *schema = false; } - + if (pStbVer->smaVer && pStbVer->smaVer != pStb->smaVer) { *sma = true; } else { @@ -2535,6 +2576,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, static int32_t mndProcessDropTtltbRsp(SRpcMsg *pRsp) { return 0; } static int32_t mndProcessTrimDbRsp(SRpcMsg *pRsp) { return 0; } +static int32_t mndProcessS3MigrateDbRsp(SRpcMsg *pRsp) { return 0; } static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; @@ -2748,8 +2790,8 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t pStbVersion->tversion = ntohl(pStbVersion->tversion); pStbVersion->smaVer = ntohl(pStbVersion->smaVer); - bool schema = false; - bool sma = false; + bool schema = false; + bool sma = false; int32_t code = mndValidateStbVersion(pMnode, pStbVersion, &schema, &sma); if (TSDB_CODE_SUCCESS != code) { STableMetaRsp metaRsp = {0}; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index b0290191bc..612db94bb6 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -31,10 +31,10 @@ #define VGROUP_VER_NUMBER 1 #define VGROUP_RESERVE_SIZE 64 -static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); -static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); -static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); -static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw); +static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); +static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw); static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); @@ -275,6 +275,9 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.daysToKeep1 = pDb->cfg.daysToKeep1; createReq.daysToKeep2 = pDb->cfg.daysToKeep2; createReq.keepTimeOffset = pDb->cfg.keepTimeOffset; + createReq.s3ChunkSize = pDb->cfg.s3ChunkSize; + createReq.s3KeepLocal = pDb->cfg.s3KeepLocal; + createReq.s3Compact = pDb->cfg.s3Compact; createReq.minRows = pDb->cfg.minRows; createReq.maxRows = pDb->cfg.maxRows; createReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod; @@ -398,6 +401,8 @@ static void *mndBuildAlterVnodeConfigReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pV alterReq.minRows = pDb->cfg.minRows; alterReq.walRetentionPeriod = pDb->cfg.walRetentionPeriod; alterReq.walRetentionSize = pDb->cfg.walRetentionSize; + alterReq.s3KeepLocal = pDb->cfg.s3KeepLocal; + alterReq.s3Compact = pDb->cfg.s3Compact; mInfo("vgId:%d, build alter vnode config req", pVgroup->vgId); int32_t contLen = tSerializeSAlterVnodeConfigReq(NULL, 0, &alterReq); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 294e75602e..7c033ac0f3 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -57,7 +57,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod int32_t diskPrimary, STfs *pTfs); int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, int32_t diskPrimary, STfs *pTfs); -void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs); +void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId); SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force); void vnodePreClose(SVnode *pVnode); void vnodePostClose(SVnode *pVnode); @@ -69,7 +69,7 @@ int32_t vnodeBegin(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode); void vnodeStop(SVnode *pVnode); int64_t vnodeGetSyncHandle(SVnode *pVnode); -int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); +int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList); @@ -134,8 +134,8 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); -bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type); -int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type); +bool metaTbInFilterCache(SMeta *pMeta, const void *key, int8_t type); +int32_t metaPutTbToFilterCache(SMeta *pMeta, const void *key, int8_t type); int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type); int32_t metaInitTbFilterCache(SMeta *pMeta); @@ -172,13 +172,13 @@ void *tsdbGetIvtIdx2(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); -void tsdbSetFilesetDelimited(STsdbReader* pReader); -void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param); +void tsdbSetFilesetDelimited(STsdbReader *pReader); +void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn notifyFn, void *param); int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr, - SArray* pFuncTypeList); + SArray *pFuncTypeList); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUids); void *tsdbCacherowsReaderClose(void *pReader); @@ -234,7 +234,7 @@ int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, i bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); -int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, bool* fhFinished); +int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); @@ -310,6 +310,9 @@ struct SVnodeCfg { int16_t hashPrefix; int16_t hashSuffix; int32_t tsdbPageSize; + int32_t s3ChunkSize; + int32_t s3KeepLocal; + int8_t s3Compact; }; #define TABLE_ROLLUP_ON ((int8_t)0x1) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index cac3be9ee3..a598e31fcc 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -279,7 +279,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); void tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); -int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_t* suid); +int32_t tsdbGetTableSchema(SMeta *pMeta, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbMerge.c ============================================================================================== typedef struct { @@ -639,6 +639,7 @@ typedef struct { STsdb *pTsdb; const char *objName; uint8_t s3File; + int32_t lcn; int32_t fid; int64_t cid; int64_t blkno; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index e610161544..35d32948e0 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -108,13 +108,13 @@ typedef struct SQueryNode SQueryNode; #define VNODE_METRIC_SQL_COUNT "taos_sql_req:count" -#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" +#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" #define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" -#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id" -#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep" -#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id" -#define VNODE_METRIC_TAG_NAME_USERNAME "username" -#define VNODE_METRIC_TAG_NAME_RESULT "result" +#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id" +#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep" +#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id" +#define VNODE_METRIC_TAG_NAME_USERNAME "username" +#define VNODE_METRIC_TAG_NAME_RESULT "result" #define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows" //#define VNODE_METRIC_TAG_VALUE_INSERT "insert" @@ -239,6 +239,7 @@ int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t col_type); int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync); +int32_t tsdbS3Migrate(STsdb* tsdb, int64_t now, int32_t sync); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); @@ -461,12 +462,12 @@ typedef struct SVCommitSched { int64_t maxWaitMs; } SVCommitSched; -typedef struct SVMonitorObj{ - char strClusterId[TSDB_CLUSTER_ID_LEN]; - char strDnodeId[TSDB_NODE_ID_LEN]; - char strVgId[TSDB_VGROUP_ID_LEN]; - taos_counter_t *insertCounter; -}SVMonitorObj; +typedef struct SVMonitorObj { + char strClusterId[TSDB_CLUSTER_ID_LEN]; + char strDnodeId[TSDB_NODE_ID_LEN]; + char strVgId[TSDB_VGROUP_ID_LEN]; + taos_counter_t* insertCounter; +} SVMonitorObj; struct SVnode { char* path; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index dc76aa61b2..34476c5cfd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -46,7 +46,6 @@ typedef struct { STFileSet *fset; TABLEID tbid[1]; bool hasTSData; - bool skipTsRow; } ctx[1]; // reader @@ -128,21 +127,8 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { continue; } } - /* - extern int8_t tsS3Enabled; - int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs); - committer->ctx->skipTsRow = false; - if (tsS3Enabled && nlevel > 1 && committer->ctx->did.level == nlevel - 1) { - committer->ctx->skipTsRow = true; - } - */ int64_t ts = TSDBROW_TS(&row->row); - - if (committer->ctx->skipTsRow && ts <= committer->ctx->maxKey) { - ts = committer->ctx->maxKey + 1; - } - if (ts > committer->ctx->maxKey) { committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); @@ -403,31 +389,6 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { // reset nextKey committer->ctx->nextKey = TSKEY_MAX; - committer->ctx->skipTsRow = false; - - extern int8_t tsS3Enabled; - extern int32_t tsS3UploadDelaySec; - long s3Size(const char *object_name); - int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs); - if (tsS3Enabled && nlevel > 1 && committer->ctx->fset) { - STFileObj *fobj = committer->ctx->fset->farr[TSDB_FTYPE_DATA]; - if (fobj && fobj->f->did.level == nlevel - 1) { - // if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay - const char *object_name = taosDirEntryBaseName((char *)fobj->fname); - - if (taosCheckExistFile(fobj->fname)) { - int32_t mtime = 0; - taosStatFile(fobj->fname, NULL, &mtime, NULL); - if (mtime < committer->ctx->now - tsS3UploadDelaySec) { - committer->ctx->skipTsRow = true; - } - } else /*if (s3Size(object_name) > 0) */ { - committer->ctx->skipTsRow = true; - } - } - // new fset can be written with ts data - } - _exit: if (code) { TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index d2e3cb08e5..f8cf591777 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -29,15 +29,16 @@ int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderCon reader[0]->config[0] = config[0]; + int32_t lcn = config->file.lcn; if (fname) { if (fname) { - code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd); + code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn); TSDB_CHECK_CODE(code, lino, _exit); } } else { char fname1[TSDB_FILENAME_LEN]; tsdbTFileName(config->tsdb, &config->file, fname1); - code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd); + code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn); TSDB_CHECK_CODE(code, lino, _exit); } @@ -113,8 +114,8 @@ static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) { static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; } static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) { - int32_t code = 0; - int32_t lino = 0; + int32_t code = 0; + int32_t lino = 0; ASSERT(writer->ctx->offset <= writer->file.size); ASSERT(writer->config->fid == writer->file.fid); @@ -151,7 +152,7 @@ static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) { } tsdbTFileName(writer->config->tsdb, &writer->file, fname); - code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd); + code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, writer->file.lcn); TSDB_CHECK_CODE(code, lino, _exit); _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index e1625c9ddb..3f35bcb166 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -97,7 +97,8 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig if (fname) { for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { if (fname[i]) { - code = tsdbOpenFile(fname[i], config->tsdb, TD_FILE_READ, &reader[0]->fd[i]); + int32_t lcn = config->files[i].file.lcn; + code = tsdbOpenFile(fname[i], config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -106,7 +107,8 @@ int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig if (config->files[i].exist) { char fname1[TSDB_FILENAME_LEN]; tsdbTFileName(config->tsdb, &config->files[i].file, fname1); - code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd[i]); + int32_t lcn = config->files[i].file.lcn; + code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd[i], lcn); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -303,7 +305,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe } int64_t szHint = 0; - if (bData->nColData > 3) { + if (bData->nColData > 2) { int64_t offset = 0; SBlockCol bc = {.cid = 0}; SBlockCol *blockCol = &bc; @@ -642,6 +644,7 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { .fid = writer->config->fid, .cid = writer->config->cid, .size = 0, + .lcn = writer->config->lcn == -1 ? 0 : -1, .minVer = VERSION_MAX, .maxVer = VERSION_MIN, }; @@ -1620,8 +1623,9 @@ static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) { flag |= (TD_FILE_CREATE | TD_FILE_TRUNC); } + int32_t lcn = writer->files[ftype].lcn; tsdbTFileName(writer->config->tsdb, &writer->files[ftype], fname); - code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype]); + code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn); TSDB_CHECK_CODE(code, lino, _exit); if (writer->files[ftype].size == 0) { @@ -1789,8 +1793,9 @@ static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) { int32_t flag = (TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + int32_t lcn = writer->files[ftype].lcn; tsdbTFileName(writer->config->tsdb, writer->files + ftype, fname); - code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype]); + code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd[ftype], lcn); TSDB_CHECK_CODE(code, lino, _exit); uint8_t hdr[TSDB_FHDR_SIZE] = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h index c4aed6e787..c6d754b014 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h @@ -79,6 +79,7 @@ typedef struct SDataFileWriterConfig { int64_t cid; SDiskID did; int64_t compactVersion; + int32_t lcn; struct { bool exist; STFile file; diff --git a/source/dnode/vnode/src/tsdb/tsdbDef.h b/source/dnode/vnode/src/tsdb/tsdbDef.h index 0f512e1306..6aa6e32804 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDef.h +++ b/source/dnode/vnode/src/tsdb/tsdbDef.h @@ -31,7 +31,7 @@ typedef struct SFDataPtr { int64_t size; } SFDataPtr; -extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD); +extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn); extern void tsdbCloseFile(STsdbFD **ppFD); extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size); extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 1d2fd60df2..6195c37ab2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -20,7 +20,7 @@ #define BLOCK_COMMIT_FACTOR 3 -extern void remove_file(const char *fname, bool last_level); +extern void remove_file(const char *fname); #define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT #define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1) @@ -355,17 +355,32 @@ static int32_t tsdbFSDoScanAndFixFile(STFileSystem *fs, const STFileObj *fobj) { // check file existence if (!taosCheckExistFile(fobj->fname)) { - if (tsS3Enabled) { - const char *object_name = taosDirEntryBaseName((char *)fobj->fname); - long s3_size = s3Size(object_name); - if (s3_size > 0) { - return 0; + bool found = false; + + if (tsS3Enabled && fobj->f->lcn > 1) { + char fname1[TSDB_FILENAME_LEN]; + tsdbTFileLastChunkName(fs->tsdb, fobj->f, fname1); + if (!taosCheckExistFile(fname1)) { + code = TSDB_CODE_FILE_CORRUPTED; + tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fname1); + return code; } + + found = true; + /* + const char *object_name = taosDirEntryBaseName((char *)fobj->fname); + long s3_size = s3Size(object_name); + if (s3_size > 0) { + return 0; + } + */ } - code = TSDB_CODE_FILE_CORRUPTED; - tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fobj->fname); - return code; + if (!found) { + code = TSDB_CODE_FILE_CORRUPTED; + tsdbError("vgId:%d %s failed since file:%s does not exist", TD_VID(fs->tsdb->pVnode), __func__, fobj->fname); + return code; + } } { // TODO: check file size @@ -530,9 +545,9 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { if (taosIsDir(file->aname)) continue; if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL && - strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3)) { - int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs); - remove_file(file->aname, nlevel > 1 && file->did.level == nlevel - 1); + strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3) && + strncmp(file->aname + strlen(file->aname) - 5, ".data", 5)) { + remove_file(file->aname); } } @@ -900,57 +915,28 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { continue; } - bool skipMerge = false; + // bool skipMerge = false; int32_t numFile = TARRAY2_SIZE(lvl->fobjArr); if (numFile >= sttTrigger && (!fset->mergeScheduled)) { - // launch merge - { - extern int8_t tsS3Enabled; - extern int32_t tsS3UploadDelaySec; - long s3Size(const char *object_name); - int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs); - if (tsS3Enabled && nlevel > 1) { - STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA]; - if (fobj && fobj->f->did.level == nlevel - 1) { - // if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay - const char *object_name = taosDirEntryBaseName((char *)fobj->fname); + code = tsdbTFileSetOpenChannel(fset); + TSDB_CHECK_CODE(code, lino, _exit); - if (taosCheckExistFile(fobj->fname)) { - int32_t now = taosGetTimestampSec(); - int32_t mtime = 0; - taosStatFile(fobj->fname, NULL, &mtime, NULL); - if (mtime < now - tsS3UploadDelaySec) { - skipMerge = true; - } - } else /* if (s3Size(object_name) > 0) */ { - skipMerge = true; - } - } - // new fset can be written with ts data - } + SMergeArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } - if (!skipMerge) { - code = tsdbTFileSetOpenChannel(fset); - TSDB_CHECK_CODE(code, lino, _exit); + arg->tsdb = fs->tsdb; + arg->fid = fset->fid; - SMergeArg *arg = taosMemoryMalloc(sizeof(*arg)); - if (arg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - arg->tsdb = fs->tsdb; - arg->fid = fset->fid; - - code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, - arg, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - fset->mergeScheduled = true; - } + code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg, + NULL); + TSDB_CHECK_CODE(code, lino, _exit); + fset->mergeScheduled = true; } - if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR && !skipMerge) { + if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR) { tsdbFSSetBlockCommit(fset, true); } else { tsdbFSSetBlockCommit(fset, false); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index e088f54930..a2cdd9f381 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -386,7 +386,7 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f fobj2->f[0] = fobj1->f[0]; } } else { - tsdbTFileObjRemove(fobj2); + tsdbTFileObjRemoveUpdateLC(fobj2); code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]); if (code) return code; } @@ -585,7 +585,7 @@ int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr) { return 0; } -int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray** ppArr) { +int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr) { if (ppArr && ppArr[0]) { TARRAY2_DESTROY(ppArr[0], tsdbTFileSetRangeClear); taosMemoryFree(ppArr[0]); @@ -663,4 +663,4 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) { int32_t tsdbTFileSetOpenChannel(STFileSet *fset) { if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0; return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel); -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c index e6b3cf8f54..6eec43baff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -149,6 +149,7 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) { .skmTb = writer[0]->skmTb, .skmRow = writer[0]->skmRow, .bufArr = writer[0]->bufArr, + .lcn = config->lcn, }; for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) { dataWriterConfig.files[ftype].exist = config->files[ftype].exist; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRW.h index 0a8049cded..640e9db5cc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.h @@ -37,6 +37,7 @@ typedef struct { int64_t cid; SDiskID did; int32_t level; + int32_t lcn; struct { bool exist; STFile file; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 8cd9304188..792d94ce73 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -15,6 +15,7 @@ #include "tsdbFile2.h" #include "cos.h" +#include "vnd.h" // to_json static int32_t head_to_json(const STFile *file, cJSON *json); @@ -42,24 +43,9 @@ static const struct { [TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json}, }; -void remove_file(const char *fname, bool last_level) { - int32_t code = taosRemoveFile(fname); - if (code) { - if (tsS3Enabled && last_level) { - const char *object_name = taosDirEntryBaseName((char *)fname); - long s3_size = tsS3Enabled ? s3Size(object_name) : 0; - if (!strncmp(fname + strlen(fname) - 5, ".data", 5) && s3_size > 0) { - s3DeleteObjects(&object_name, 1); - tsdbInfo("file:%s is removed from s3", fname); - } else { - tsdbError("file:%s remove failed", fname); - } - } else { - tsdbError("file:%s remove failed", fname); - } - } else { - tsdbInfo("file:%s is removed", fname); - } +void remove_file(const char *fname) { + taosRemoveFile(fname); + tsdbInfo("file:%s is removed", fname); } static int32_t tfile_to_json(const STFile *file, cJSON *json) { @@ -73,6 +59,11 @@ static int32_t tfile_to_json(const STFile *file, cJSON *json) { return TSDB_CODE_OUT_OF_MEMORY; } + /* lcn - last chunk number */ + if (cJSON_AddNumberToObject(json, "lcn", file->lcn) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + /* fid */ if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -121,6 +112,14 @@ static int32_t tfile_from_json(const cJSON *json, STFile *file) { return TSDB_CODE_FILE_CORRUPTED; } + /* lcn */ + item = cJSON_GetObjectItem(json, "lcn"); + if (cJSON_IsNumber(item)) { + file->lcn = item->valuedouble; + } else { + // return TSDB_CODE_FILE_CORRUPTED; + } + /* fid */ item = cJSON_GetObjectItem(json, "fid"); if (cJSON_IsNumber(item)) { @@ -236,7 +235,8 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { fobj[0]->state = TSDB_FSTATE_LIVE; fobj[0]->ref = 1; tsdbTFileName(pTsdb, f, fobj[0]->fname); - fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs); + // fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs); + fobj[0]->nlevel = vnodeNodeId(pTsdb->pVnode); return 0; } @@ -258,7 +258,7 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) { tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); if (nRef == 0) { if (fobj->state == TSDB_FSTATE_DEAD) { - remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1); + remove_file(fobj->fname); } taosMemoryFree(fobj); } @@ -266,6 +266,61 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) { return 0; } +static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) { + if (fobj->f->type != TSDB_FTYPE_DATA) { + remove_file(fobj->fname); + return; + } + + if (!remove_all) { + if (fobj->f->lcn < 1) { + remove_file(fobj->fname); + return; + } else { + // remove local last chunk file + char lc_path[TSDB_FILENAME_LEN]; + tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN); + + char *dot = strrchr(lc_path, '.'); + if (!dot) { + tsdbError("unexpected path: %s", lc_path); + return; + } + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn); + + remove_file(lc_path); + } + } else { + // delete by data file prefix + char lc_path[TSDB_FILENAME_LEN]; + tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN); + + char *object_name = taosDirEntryBaseName(lc_path); + int32_t node_id = fobj->nlevel; + char object_name_prefix[TSDB_FILENAME_LEN]; + snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name); + + char *dot = strrchr(object_name_prefix, '.'); + if (!dot) { + tsdbError("unexpected path: %s", object_name_prefix); + return; + } + *(dot + 1) = 0; + + s3DeleteObjectsByPrefix(object_name_prefix); + + // remove local last chunk file + dot = strrchr(lc_path, '.'); + if (!dot) { + tsdbError("unexpected path: %s", lc_path); + return; + } + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn); + + remove_file(lc_path); + } +} + int32_t tsdbTFileObjRemove(STFileObj *fobj) { taosThreadMutexLock(&fobj->mutex); ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0); @@ -274,7 +329,21 @@ int32_t tsdbTFileObjRemove(STFileObj *fobj) { taosThreadMutexUnlock(&fobj->mutex); tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); if (nRef == 0) { - remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1); + tsdbTFileObjRemoveLC(fobj, true); + taosMemoryFree(fobj); + } + return 0; +} + +int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) { + taosThreadMutexLock(&fobj->mutex); + ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0); + fobj->state = TSDB_FSTATE_DEAD; + int32_t nRef = --fobj->ref; + taosThreadMutexUnlock(&fobj->mutex); + tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); + if (nRef == 0) { + tsdbTFileObjRemoveLC(fobj, false); taosMemoryFree(fobj); } return 0; @@ -310,13 +379,45 @@ int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]) { return 0; } +int32_t tsdbTFileLastChunkName(STsdb *pTsdb, const STFile *f, char fname[]) { + SVnode *pVnode = pTsdb->pVnode; + STfs *pTfs = pVnode->pTfs; + + if (pTfs) { + snprintf(fname, // + TSDB_FILENAME_LEN, // + "%s%s%s%sv%df%dver%" PRId64 ".%d.%s", // + tfsGetDiskPath(pTfs, f->did), // + TD_DIRSEP, // + pTsdb->path, // + TD_DIRSEP, // + TD_VID(pVnode), // + f->fid, // + f->cid, // + f->lcn, // + g_tfile_info[f->type].suffix); + } else { + snprintf(fname, // + TSDB_FILENAME_LEN, // + "%s%sv%df%dver%" PRId64 ".%d.%s", // + pTsdb->path, // + TD_DIRSEP, // + TD_VID(pVnode), // + f->fid, // + f->cid, // + f->lcn, // + g_tfile_info[f->type].suffix); + } + return 0; +} + bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) { if (f1->type != f2->type) return false; if (f1->did.level != f2->did.level) return false; if (f1->did.id != f2->did.id) return false; if (f1->fid != f2->fid) return false; if (f1->cid != f2->cid) return false; - if (f1->s3flag != f2->s3flag) return false; + if (f1->lcn != f2->lcn) return false; return true; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.h b/source/dnode/vnode/src/tsdb/tsdbFile2.h index b94f7a9fd0..70e167c1eb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.h @@ -45,6 +45,7 @@ enum { int32_t tsdbTFileToJson(const STFile *f, cJSON *json); int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f); int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]); +int32_t tsdbTFileLastChunkName(STsdb *pTsdb, const STFile *f, char fname[]); bool tsdbIsSameTFile(const STFile *f1, const STFile *f2); bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2); @@ -53,12 +54,13 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj); int32_t tsdbTFileObjRef(STFileObj *fobj); int32_t tsdbTFileObjUnref(STFileObj *fobj); int32_t tsdbTFileObjRemove(STFileObj *fobj); +int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj); int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2); struct STFile { tsdb_ftype_t type; SDiskID did; // disk id - int32_t s3flag; + int32_t lcn; // last chunk number int32_t fid; // file id int64_t cid; // commit id int64_t size; diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 7f33f08794..87f48acaac 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -544,7 +544,7 @@ int32_t tsdbMerge(void *arg) { TSDB_CHECK_CODE(code, lino, _exit); if (merger->fset == NULL) return 0; - + /* bool skipMerge = false; { extern int8_t tsS3Enabled; @@ -565,7 +565,8 @@ int32_t tsdbMerge(void *arg) { if (mtime < now - tsS3UploadDelaySec) { skipMerge = true; } - } else /* if (s3Size(object_name) > 0) */ { + } else // if (s3Size(object_name) > 0) + { skipMerge = true; } } @@ -577,7 +578,7 @@ int32_t tsdbMerge(void *arg) { code = 0; goto _exit; } - + */ // do merge tsdbDebug("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid); code = tsdbDoMerge(merger); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index babf8c75fb..c33f9c48d1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -15,16 +15,48 @@ #include "cos.h" #include "tsdb.h" +#include "vnd.h" static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { int32_t code = 0; const char *path = pFD->path; int32_t szPage = pFD->szPage; int32_t flag = pFD->flag; + int64_t lc_size = 0; pFD->pFD = taosOpenFile(path, flag); if (pFD->pFD == NULL) { - int errsv = errno; + if (tsS3Enabled && pFD->lcn > 1 && !strncmp(path + strlen(path) - 5, ".data", 5)) { + char lc_path[TSDB_FILENAME_LEN]; + tstrncpy(lc_path, path, TSDB_FQDN_LEN); + + char *dot = strrchr(lc_path, '.'); + if (!dot) { + tsdbError("unexpected path: %s", lc_path); + code = TAOS_SYSTEM_ERROR(ENOENT); + goto _exit; + } + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", pFD->lcn); + + pFD->pFD = taosOpenFile(lc_path, flag); + if (pFD->pFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + // taosMemoryFree(pFD); + goto _exit; + } + if (taosStatFile(lc_path, &lc_size, NULL, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + // taosCloseFile(&pFD->pFD); + // taosMemoryFree(pFD); + goto _exit; + } + } else { + tsdbInfo("no file: %s", path); + code = TAOS_SYSTEM_ERROR(errno); + // taosMemoryFree(pFD); + goto _exit; + } + /* const char *object_name = taosDirEntryBaseName((char *)path); long s3_size = 0; if (tsS3Enabled) { @@ -43,7 +75,6 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { s3Get(object_name, path); pFD->pFD = taosOpenFile(path, flag); - if (pFD->pFD == NULL) { code = TAOS_SYSTEM_ERROR(ENOENT); // taosMemoryFree(pFD); @@ -57,12 +88,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { pFD->objName = object_name; // pFD->szFile = s3_size; #endif - } else { - tsdbInfo("no file: %s", path); - code = TAOS_SYSTEM_ERROR(errsv); - // taosMemoryFree(pFD); - goto _exit; - } + */ } pFD->pBuf = taosMemoryCalloc(1, szPage); @@ -73,26 +99,33 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { goto _exit; } + if (lc_size > 0) { + SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config; + int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; + + pFD->szFile = lc_size + chunksize * (pFD->lcn - 1); + } + // not check file size when reading data files. - if (flag != TD_FILE_READ && !pFD->s3File) { - if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) { + if (flag != TD_FILE_READ /* && !pFD->s3File*/) { + if (!lc_size && taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) { code = TAOS_SYSTEM_ERROR(errno); // taosMemoryFree(pFD->pBuf); // taosCloseFile(&pFD->pFD); // taosMemoryFree(pFD); goto _exit; } - - ASSERT(pFD->szFile % szPage == 0); - pFD->szFile = pFD->szFile / szPage; } + ASSERT(pFD->szFile % szPage == 0); + pFD->szFile = pFD->szFile / szPage; + _exit: return code; } // =============== PAGE-WISE FILE =============== -int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD) { +int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn) { int32_t code = 0; STsdbFD *pFD = NULL; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; @@ -111,6 +144,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF pFD->flag = flag; pFD->szPage = szPage; pFD->pgno = 0; + pFD->lcn = lcn; pFD->pTsdb = pTsdb; *ppFD = pFD; @@ -123,9 +157,9 @@ void tsdbCloseFile(STsdbFD **ppFD) { STsdbFD *pFD = *ppFD; if (pFD) { taosMemoryFree(pFD->pBuf); - if (!pFD->s3File) { - taosCloseFile(&pFD->pFD); - } + // if (!pFD->s3File) { + taosCloseFile(&pFD->pFD); + //} taosMemoryFree(pFD); *ppFD = NULL; } @@ -141,12 +175,18 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD) { } } - if (pFD->s3File) { - tsdbWarn("%s file: %s", __func__, pFD->path); - return code; - } if (pFD->pgno > 0) { - int64_t n = taosLSeekFile(pFD->pFD, PAGE_OFFSET(pFD->pgno, pFD->szPage), SEEK_SET); + int64_t offset = PAGE_OFFSET(pFD->pgno, pFD->szPage); + if (pFD->lcn > 1) { + SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config; + int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; + int64_t chunkoffset = chunksize * (pFD->lcn - 1); + + offset -= chunkoffset; + } + ASSERT(offset >= 0); + + int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _exit; @@ -182,7 +222,15 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { } int64_t offset = PAGE_OFFSET(pgno, pFD->szPage); + if (pFD->lcn > 1) { + SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config; + int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; + int64_t chunkoffset = chunksize * (pFD->lcn - 1); + offset -= chunkoffset; + } + ASSERT(offset >= 0); + /* if (pFD->s3File) { LRUHandle *handle = NULL; @@ -203,24 +251,25 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { tsdbCacheRelease(pFD->pTsdb->bCache, handle); } else { - // seek - int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } - - // read - n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } else if (n < pFD->szPage) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } + */ + // seek + int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; } + // read + n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } else if (n < pFD->szPage) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + //} + // check if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { code = TSDB_CODE_FILE_CORRUPTED; @@ -294,6 +343,74 @@ _exit: return code; } +static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { + int32_t code = 0; + SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config; + int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; + int64_t cOffset = offset % chunksize; + int64_t n = 0; + + char *object_name = taosDirEntryBaseName(pFD->path); + char object_name_prefix[TSDB_FILENAME_LEN]; + int32_t node_id = vnodeNodeId(pFD->pTsdb->pVnode); + snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name); + + char *dot = strrchr(object_name_prefix, '.'); + if (!dot) { + tsdbError("unexpected path: %s", object_name_prefix); + code = TAOS_SYSTEM_ERROR(ENOENT); + goto _exit; + } + + char *buf = taosMemoryCalloc(1, size); + + for (int32_t chunkno = offset / chunksize + 1; n < size; ++chunkno) { + int64_t nRead = TMIN(chunksize - cOffset, size - n); + + if (chunkno >= pFD->lcn) { + // read last chunk + int64_t ret = taosLSeekFile(pFD->pFD, chunksize * (chunkno - pFD->lcn) + cOffset, SEEK_SET); + if (ret < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(buf); + goto _exit; + } + + ret = taosReadFile(pFD->pFD, buf + n, nRead); + if (ret < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(buf); + goto _exit; + } else if (ret < nRead) { + code = TSDB_CODE_FILE_CORRUPTED; + taosMemoryFree(buf); + goto _exit; + } + } else { + uint8_t *pBlock = NULL; + + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno); + + code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(buf); + goto _exit; + } + + memcpy(buf + n, pBlock, nRead); + taosMemoryFree(pBlock); + } + + n += nRead; + cOffset = 0; + } + + *ppBlock = buf; + +_exit: + return code; +} + static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) { int32_t code = 0; int64_t n = 0; @@ -356,15 +473,22 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 } int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage; + /* code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock); if (code != TSDB_CODE_SUCCESS) { goto _exit; } - + */ + code = tsdbReadFileBlock(pFD, retrieve_offset, retrieve_size, 1, &pBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } // 3, Store Pages in Cache int nPage = pgnoEnd - pgno + 1; for (int i = 0; i < nPage; ++i) { - tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage); + if (pFD->szFile != pgno) { // DONOT cache last volatile page + tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage); + } if (szHint > 0 && n >= size) { ++pgno; @@ -404,7 +528,7 @@ int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, } } - if (pFD->s3File && tsS3BlockSize < 0) { + if (pFD->lcn > 1 /*pFD->s3File && tsS3BlockSize < 0*/) { return tsdbReadFileS3(pFD, offset, pBuf, size, szHint); } else { return tsdbReadFileImp(pFD, offset, pBuf, size); @@ -416,11 +540,12 @@ _exit: int32_t tsdbFsyncFile(STsdbFD *pFD) { int32_t code = 0; - + /* if (pFD->s3File) { tsdbWarn("%s file: %s", __func__, pFD->path); return code; } + */ code = tsdbWriteFilePage(pFD); if (code) goto _exit; @@ -452,23 +577,23 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS // head tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); - code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pHeadFD); + code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pHeadFD, 0); TSDB_CHECK_CODE(code, lino, _exit); // data tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); - code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pDataFD); + code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pDataFD, 0); TSDB_CHECK_CODE(code, lino, _exit); // sma tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); - code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pSmaFD); + code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->pSmaFD, 0); TSDB_CHECK_CODE(code, lino, _exit); // stt for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); - code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->aSttFD[iStt]); + code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pReader->aSttFD[iStt], 0); TSDB_CHECK_CODE(code, lino, _exit); } @@ -875,7 +1000,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb pDelFReader->fDel = *pFile; tsdbDelFileName(pTsdb, pFile, fname); - code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH); + code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH, 0); if (code) { taosMemoryFree(pDelFReader); goto _exit; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 9ecb2fd5ba..2e0b44f5f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -38,24 +38,42 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) { return TARRAY2_APPEND(rtner->fopArr, op); } -static int32_t tsdbRemoveFileObjectS3(SRTNer *rtner, const STFileObj *fobj) { - int32_t code = 0, lino = 0; +static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) { + int32_t code = 0; + int32_t lino = 0; + TdFilePtr fdFrom = NULL, fdTo = NULL; + char fname_from[TSDB_FILENAME_LEN]; + char fname_to[TSDB_FILENAME_LEN]; - STFileOp op = { - .optype = TSDB_FOP_REMOVE, - .fid = fobj->f->fid, - .of = fobj->f[0], - }; + tsdbTFileLastChunkName(rtner->tsdb, from->f, fname_from); + tsdbTFileLastChunkName(rtner->tsdb, to, fname_to); - code = TARRAY2_APPEND(rtner->fopArr, op); + fdFrom = taosOpenFile(fname_from, TD_FILE_READ); + if (fdFrom == NULL) code = terrno; TSDB_CHECK_CODE(code, lino, _exit); - const char *object_name = taosDirEntryBaseName((char *)fobj->fname); - s3DeleteObjects(&object_name, 1); + tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname_to, from->f->size); + + fdTo = taosOpenFile(fname_to, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (fdTo == NULL) code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + + SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config; + int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; + int64_t lc_size = tsdbLogicToFileSize(to->size, rtner->szPage) - chunksize * (to->lcn - 1); + int64_t n = taosFSendFile(fdTo, fdFrom, 0, lc_size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); _exit: if (code) { TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + if (fdFrom) taosCloseFile(&fdFrom); + if (fdTo) taosCloseFile(&fdTo); } return code; } @@ -97,38 +115,11 @@ _exit: return code; } -static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) { - int32_t code = 0; - int32_t lino = 0; - - char fname[TSDB_FILENAME_LEN]; - TdFilePtr fdFrom = NULL; - // TdFilePtr fdTo = NULL; - - tsdbTFileName(rtner->tsdb, to, fname); - - fdFrom = taosOpenFile(from->fname, TD_FILE_READ); - if (fdFrom == NULL) code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - - char *object_name = taosDirEntryBaseName(fname); - code = s3PutObjectFromFile2(from->fname, object_name, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - taosCloseFile(&fdFrom); - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); - taosCloseFile(&fdFrom); - } - return code; -} - static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) { int32_t code = 0; int32_t lino = 0; STFileOp op = {0}; + int32_t lcn = fobj->f->lcn; // remove old op = (STFileOp){ @@ -153,6 +144,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const .maxVer = fobj->f->maxVer, .cid = fobj->f->cid, .size = fobj->f->size, + .lcn = lcn, .stt[0] = { .level = fobj->f->stt[0].level, @@ -164,59 +156,14 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const TSDB_CHECK_CODE(code, lino, _exit); // do copy the file - code = tsdbDoCopyFile(rtner, fobj, &op.nf); - TSDB_CHECK_CODE(code, lino, _exit); -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + if (lcn < 1) { + code = tsdbDoCopyFile(rtner, fobj, &op.nf); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDoCopyFileLC(rtner, fobj, &op.nf); + TSDB_CHECK_CODE(code, lino, _exit); } - return code; -} - -static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) { - int32_t code = 0; - int32_t lino = 0; - STFileOp op = {0}; - - // remove old - op = (STFileOp){ - .optype = TSDB_FOP_REMOVE, - .fid = fobj->f->fid, - .of = fobj->f[0], - }; - - code = TARRAY2_APPEND(rtner->fopArr, op); - TSDB_CHECK_CODE(code, lino, _exit); - - // create new - op = (STFileOp){ - .optype = TSDB_FOP_CREATE, - .fid = fobj->f->fid, - .nf = - { - .type = fobj->f->type, - .did = did[0], - .fid = fobj->f->fid, - .minVer = fobj->f->minVer, - .maxVer = fobj->f->maxVer, - .cid = fobj->f->cid, - .size = fobj->f->size, - .stt[0] = - { - .level = fobj->f->stt[0].level, - }, - }, - }; - - op.nf.s3flag = true; - - code = TARRAY2_APPEND(rtner->fopArr, op); - TSDB_CHECK_CODE(code, lino, _exit); - - // do copy the file - code = tsdbCopyFileS3(rtner, fobj, &op.nf); - TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { @@ -327,38 +274,14 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = fset->farr[ftype], 1); ++ftype) { if (fobj == NULL) continue; - int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); - if (fobj->f->did.level == did.level) { - if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1 && - taosCheckExistFile(fobj->fname)) { - int32_t mtime = 0; - taosStatFile(fobj->fname, NULL, &mtime, NULL); - if (mtime < rtner->now - tsS3UploadDelaySec) { - tsdbInfo("file:%s size: %" PRId64 " do migrate s3", fobj->fname, fobj->f->size); - code = tsdbMigrateDataFileS3(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - + /* + code = tsdbCheckMigrateS3(rtner, fobj, ftype, &did); + TSDB_CHECK_CODE(code, lino, _exit); + */ continue; } - /* - if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) { - code = tsdbMigrateDataFileS3(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - if (tsS3Enabled) { - int64_t fsize = 0; - if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(terrno); - tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(rtner->tsdb->pVnode), - __func__, fobj->fname, tstrerror(code)); TSDB_CHECK_CODE(code, lino, _exit); - } - s3EvictCache(fobj->fname, fsize * 2); - } - */ if (fobj->f->did.level > did.level) { continue; } @@ -367,7 +290,6 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { code = tsdbDoMigrateFileObj(rtner, fobj, &did); TSDB_CHECK_CODE(code, lino, _exit); - //} } // stt @@ -471,3 +393,414 @@ int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) { return code; } + +static int32_t tsdbS3FidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int32_t s3KeepLocal, int64_t nowSec) { + int32_t localFid; + TSKEY key; + + if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) { + nowSec = nowSec * 1000; + } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) { + nowSec = nowSec * 1000000l; + } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) { + nowSec = nowSec * 1000000000l; + } else { + ASSERT(0); + } + + nowSec = nowSec - pKeepCfg->keepTimeOffset * tsTickPerHour[pKeepCfg->precision]; + + key = nowSec - s3KeepLocal * tsTickPerMin[pKeepCfg->precision]; + localFid = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision); + + if (fid >= localFid) { + return 0; + } else { + return 1; + } +} + +static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile *to) { + int32_t code = 0; + int32_t lino = 0; + + char fname[TSDB_FILENAME_LEN]; + TdFilePtr fdFrom = NULL; + // TdFilePtr fdTo = NULL; + + tsdbTFileName(rtner->tsdb, to, fname); + + fdFrom = taosOpenFile(from->fname, TD_FILE_READ); + if (fdFrom == NULL) code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + + char *object_name = taosDirEntryBaseName(fname); + code = s3PutObjectFromFile2(from->fname, object_name, 1); + TSDB_CHECK_CODE(code, lino, _exit); + + taosCloseFile(&fdFrom); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + taosCloseFile(&fdFrom); + } + return code; +} + +static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) { + int32_t code = 0; + int32_t lino = 0; + STFileOp op = {0}; + TdFilePtr fdFrom = NULL, fdTo = NULL; + int32_t lcn = fobj->f->lcn + (size - 1) / chunksize; + + // remove old + op = (STFileOp){ + .optype = TSDB_FOP_REMOVE, + .fid = fobj->f->fid, + .of = fobj->f[0], + }; + + code = TARRAY2_APPEND(rtner->fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + // create new + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = fobj->f->fid, + .nf = + { + .type = fobj->f->type, + .did = fobj->f->did, + .fid = fobj->f->fid, + .minVer = fobj->f->minVer, + .maxVer = fobj->f->maxVer, + .cid = fobj->f->cid, + .size = fobj->f->size, + .lcn = lcn, + .stt[0] = + { + .level = fobj->f->stt[0].level, + }, + }, + }; + + code = TARRAY2_APPEND(rtner->fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + char fname[TSDB_FILENAME_LEN]; + tsdbTFileName(rtner->tsdb, &op.nf, fname); + char *object_name = taosDirEntryBaseName(fname); + char object_name_prefix[TSDB_FILENAME_LEN]; + int32_t node_id = vnodeNodeId(rtner->tsdb->pVnode); + snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name); + + char *dot = strrchr(object_name_prefix, '.'); + if (!dot) { + tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); + return -1; + } + + char *dot2 = strchr(object_name, '.'); + if (!dot) { + tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); + return -1; + } + snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn); + + // do copy the file + for (int32_t cn = fobj->f->lcn; cn < lcn; ++cn) { + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn); + int64_t c_offset = chunksize * (cn - fobj->f->lcn); + + code = s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // copy last chunk + int64_t lc_offset = chunksize * (lcn - fobj->f->lcn); + int64_t lc_size = size - lc_offset; + + snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn); + + fdFrom = taosOpenFile(fname, TD_FILE_READ); + if (fdFrom == NULL) code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size); + + snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn); + fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (fdTo == NULL) code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + + int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + if (fdFrom) taosCloseFile(&fdFrom); + if (fdTo) taosCloseFile(&fdTo); + } + return code; +} + +static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64_t size, int64_t chunksize) { + int32_t code = 0; + int32_t lino = 0; + STFileOp op = {0}; + int32_t lcn = (size - 1) / chunksize + 1; + + // remove old + op = (STFileOp){ + .optype = TSDB_FOP_REMOVE, + .fid = fobj->f->fid, + .of = fobj->f[0], + }; + + code = TARRAY2_APPEND(rtner->fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + // create new + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = fobj->f->fid, + .nf = + { + .type = fobj->f->type, + .did = fobj->f->did, + .fid = fobj->f->fid, + .minVer = fobj->f->minVer, + .maxVer = fobj->f->maxVer, + .cid = fobj->f->cid, + .size = fobj->f->size, + .lcn = lcn, + .stt[0] = + { + .level = fobj->f->stt[0].level, + }, + }, + }; + + code = TARRAY2_APPEND(rtner->fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + char fname[TSDB_FILENAME_LEN]; + tsdbTFileName(rtner->tsdb, &op.nf, fname); + char *object_name = taosDirEntryBaseName(fname); + char object_name_prefix[TSDB_FILENAME_LEN]; + int32_t node_id = vnodeNodeId(rtner->tsdb->pVnode); + snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name); + + char *dot = strrchr(object_name_prefix, '.'); + if (!dot) { + tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); + return -1; + } + + // do copy the file + for (int32_t cn = 1; cn < lcn; ++cn) { + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn); + int64_t c_offset = chunksize * (cn - 1); + + code = s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // copy last chunk + TdFilePtr fdFrom = NULL, fdTo = NULL; + int64_t lc_offset = (int64_t)(lcn - 1) * chunksize; + int64_t lc_size = size - lc_offset; + + dot = strchr(object_name, '.'); + if (!dot) { + tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); + return -1; + } + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name), "%d.data", lcn); + + fdFrom = taosOpenFile(fobj->fname, TD_FILE_READ); + if (fdFrom == NULL) code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, fobj->f->size); + + fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (fdTo == NULL) code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + + int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); + } + return code; +} + +static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA]; + if (!fobj) return code; + + int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now); + if (expLevel < 0) return code; // expired + + SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config; + int32_t s3KeepLocal = pCfg->s3KeepLocal; + int32_t s3ExpLevel = tsdbS3FidLevel(fset->fid, &rtner->tsdb->keepCfg, s3KeepLocal, rtner->now); + if (s3ExpLevel < 1) return code; // keep on local storage + + int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; + int32_t lcn = fobj->f->lcn; + + if (lcn < 1 && taosCheckExistFile(fobj->fname)) { + int32_t mtime = 0; + int64_t size = 0; + taosStatFile(fobj->fname, &size, &mtime, NULL); + if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) { + if (pCfg->s3Compact && lcn < 0) { + extern int32_t tsdbAsyncCompact(STsdb * tsdb, const STimeWindow *tw, bool sync); + + STimeWindow win = {0}; + tsdbFidKeyRange(fset->fid, rtner->tsdb->keepCfg.days, rtner->tsdb->keepCfg.precision, &win.skey, &win.ekey); + + tsdbInfo("vgId:%d, compact begin lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn); + tsdbAsyncCompact(rtner->tsdb, &win, pCfg->sttTrigger == 1); + tsdbInfo("vgId:%d, compact end lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn); + return code; + } + + code = tsdbMigrateDataFileS3(rtner, fobj, size, chunksize); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + if (lcn <= 1) { + tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); + return code; + } + char fname1[TSDB_FILENAME_LEN]; + tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1); + + if (taosCheckExistFile(fname1)) { + int32_t mtime = 0; + int64_t size = 0; + taosStatFile(fname1, &size, &mtime, NULL); + if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) { + code = tsdbMigrateDataFileLCS3(rtner, fobj, size, chunksize); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + tsdbError("vgId:%d, file: %s not found, %s at line %d", TD_VID(rtner->tsdb->pVnode), fname1, __func__, lino); + return code; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDoS3MigrateAsync(void *arg) { + int32_t code = 0; + int32_t lino = 0; + SRTNer rtner[1] = {0}; + + code = tsdbDoRetentionBegin(arg, rtner); + TSDB_CHECK_CODE(code, lino, _exit); + + STFileSet *fset; + TARRAY2_FOREACH(rtner->fsetArr, fset) { + if (fset->fid != ((SRtnArg *)arg)->fid) continue; + + code = tsdbDoS3MigrateOnFileSet(rtner, fset); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDoRetentionEnd(rtner); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + if (TARRAY2_DATA(rtner->fopArr)) { + TARRAY2_DESTROY(rtner->fopArr, NULL); + } + TFileSetArray **fsetArr = &rtner->fsetArr; + if (fsetArr[0]) { + tsdbFSDestroyCopySnapshot(&rtner->fsetArr); + } + + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) { + int32_t code = 0; + + if (!tsS3Enabled) { + return code; + } + + taosThreadMutexLock(&tsdb->mutex); + + if (tsdb->bgTaskDisabled) { + taosThreadMutexUnlock(&tsdb->mutex); + return 0; + } + + STFileSet *fset; + TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { + code = tsdbTFileSetOpenChannel(fset); + if (code) { + taosThreadMutexUnlock(&tsdb->mutex); + return code; + } + + SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) { + taosThreadMutexUnlock(&tsdb->mutex); + return TSDB_CODE_OUT_OF_MEMORY; + } + + arg->tsdb = tsdb; + arg->now = now; + arg->fid = fset->fid; + + if (sync) { + code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync, + tsdbFreeRtnArg, arg, NULL); + } else { + code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync, + tsdbFreeRtnArg, arg, NULL); + } + if (code) { + tsdbFreeRtnArg(arg); + taosThreadMutexUnlock(&tsdb->mutex); + return code; + } + } + + taosThreadMutexUnlock(&tsdb->mutex); + + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index f26c6540df..b2a091fbb3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -47,12 +47,12 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con // open file if (fname) { - code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd); + code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0); TSDB_CHECK_CODE(code, lino, _exit); } else { char fname1[TSDB_FILENAME_LEN]; tsdbTFileName(config->tsdb, config->file, fname1); - code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd); + code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0); TSDB_CHECK_CODE(code, lino, _exit); } @@ -642,7 +642,7 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { char fname[TSDB_FILENAME_LEN]; tsdbTFileName(writer->config->tsdb, writer->file, fname); - code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd); + code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0); TSDB_CHECK_CODE(code, lino, _exit); uint8_t hdr[TSDB_FHDR_SIZE] = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c index 876c0df4a0..b8872a0401 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -80,7 +80,7 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader * char fname[TSDB_FILENAME_LEN]; tsdbTFileName(tsdb, &file, fname); - code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd); + code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0); TSDB_CHECK_CODE(code, lino, _exit); // convert @@ -258,7 +258,7 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade code = tsdbTFileObjInit(tsdb, &file, &fobj); TSDB_CHECK_CODE(code, lino, _exit1); - code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd); + code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0); TSDB_CHECK_CODE(code, lino, _exit1); for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) { @@ -413,7 +413,7 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f } char fname[TSDB_FILENAME_LEN] = {0}; - code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd); + code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd, 0); TSDB_CHECK_CODE(code, lino, _exit); uint8_t hdr[TSDB_FHDR_SIZE] = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 44621bf4e6..076db8b3ca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -642,8 +642,8 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { return &pIter->cv; } - if (pIter->iColData < pIter->pRow->pBlockData->nColData) { - tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData], pIter->pRow->iRow, &pIter->cv); + if (pIter->iColData <= pIter->pRow->pBlockData->nColData) { + tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData - 1], pIter->pRow->iRow, &pIter->cv); ++pIter->iColData; return &pIter->cv; } else { diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 07bfa6c719..53edc4da76 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -51,6 +51,9 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .hashEnd = 0, .hashMethod = 0, .sttTrigger = TSDB_DEFAULT_SST_TRIGGER, + .s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE, + .s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL, + .s3Compact = TSDB_DEFAULT_S3_COMPACT, .tsdbPageSize = TSDB_DEFAULT_PAGE_SIZE}; int vnodeCheckCfg(const SVnodeCfg *pCfg) { @@ -106,6 +109,9 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact) < 0) return -1; if (pCfg->tsdbCfg.retentions[0].keep > 0) { int32_t nRetention = 1; if (pCfg->tsdbCfg.retentions[1].freq > 0) { @@ -154,9 +160,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { SJson *nodeInfo = tjsonCreateArray(); if (nodeInfo == NULL) return -1; if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1; - vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", - pCfg->vgId, pCfg->syncCfg.replicaNum, - pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion); + vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pCfg->vgId, + pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion); for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) { SJson *info = tjsonCreateObject(); SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i]; @@ -317,6 +322,19 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024; } + tjsonGetNumberValue(pJson, "s3ChunkSize", pCfg->s3ChunkSize, code); + if (code < 0) { + pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + } + tjsonGetNumberValue(pJson, "s3KeepLocal", pCfg->s3KeepLocal, code); + if (code < 0) { + pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + } + tjsonGetNumberValue(pJson, "s3Compact", pCfg->s3Compact, code); + if (code < 0) { + pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT; + } + return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 63ca7251f5..463077590e 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -305,14 +305,14 @@ int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t s return dstVgId; } -void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs) { +void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId) { vInfo("path:%s is removed while destroy vnode", path); tfsRmdir(pTfs, path); - int32_t nlevel = tfsGetLevel(pTfs); - if (vgId > 0 && nlevel > 1 && tsS3Enabled) { + // int32_t nlevel = tfsGetLevel(pTfs); + if (nodeId > 0 && vgId > 0 /*&& nlevel > 1*/ && tsS3Enabled) { char vnode_prefix[TSDB_FILENAME_LEN]; - snprintf(vnode_prefix, TSDB_FILENAME_LEN, "v%df", vgId); + snprintf(vnode_prefix, TSDB_FILENAME_LEN, "%d/v%df", nodeId, vgId); s3DeleteObjectsByPrefix(vnode_prefix); } } @@ -480,30 +480,29 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC vnodeRollback(pVnode); } - snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%"PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId); - snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%"PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId); - snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%"PRId32, pVnode->config.vgId); + snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%" PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId); + snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%" PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId); + snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%" PRId32, pVnode->config.vgId); - if(tsEnableMonitor && pVnode->monitor.insertCounter == NULL){ + if (tsEnableMonitor && pVnode->monitor.insertCounter == NULL) { taos_counter_t *counter = NULL; counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT); - if(counter == NULL){ - int32_t label_count = 7; - const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID, - VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP, - VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME, - VNODE_METRIC_TAG_NAME_RESULT}; - counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", - label_count, sample_labels); - vInfo("vgId:%d, new metric:%p",TD_VID(pVnode), counter); - if(taos_collector_registry_register_metric(counter) == 1){ + if (counter == NULL) { + int32_t label_count = 7; + const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID, + VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP, + VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME, + VNODE_METRIC_TAG_NAME_RESULT}; + counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", label_count, sample_labels); + vInfo("vgId:%d, new metric:%p", TD_VID(pVnode), counter); + if (taos_collector_registry_register_metric(counter) == 1) { taos_counter_destroy(counter); counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT); - vInfo("vgId:%d, get metric from registry:%p",TD_VID(pVnode), counter); + vInfo("vgId:%d, get metric from registry:%p", TD_VID(pVnode), counter); } } pVnode->monitor.insertCounter = counter; - vInfo("vgId:%d, succeed to set metric:%p",TD_VID(pVnode), counter); + vInfo("vgId:%d, succeed to set metric:%p", TD_VID(pVnode), counter); } return pVnode; diff --git a/source/dnode/vnode/src/vnd/vnodeRetention.c b/source/dnode/vnode/src/vnd/vnodeRetention.c index c510c0fe92..5db20b8fc7 100644 --- a/source/dnode/vnode/src/vnd/vnodeRetention.c +++ b/source/dnode/vnode/src/vnd/vnodeRetention.c @@ -23,4 +23,12 @@ int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) { if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now); return code; -} \ No newline at end of file +} + +int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now) { + int32_t code = TSDB_CODE_SUCCESS; + + code = tsdbS3Migrate(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1); + + return code; +} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e32d4b70e0..2c2c64873b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -31,15 +31,16 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, SRpcMsg *pOriginRpc); -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, - SRpcMsg *pOriginalMsg); +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg); static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, - SRpcMsg *pOriginalMsg); +static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg); static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -540,6 +541,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg case TDMT_VND_TRIM: if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; + case TDMT_VND_S3MIGRATE: + if (vnodeProcessS3MigrateReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; + break; case TDMT_VND_CREATE_SMA: if (vnodeProcessCreateTSmaReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; @@ -846,6 +850,26 @@ _exit: return code; } +extern int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now); + +static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { + int32_t code = 0; + SVS3MigrateDbReq s3migrateReq = {0}; + + // decode + if (tDeserializeSVS3MigrateDbReq(pReq, len, &s3migrateReq) != 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + vInfo("vgId:%d, s3migrate vnode request will be processed, time:%d", pVnode->config.vgId, s3migrateReq.timestamp); + + code = vnodeDoS3Migrate(pVnode, s3migrateReq.timestamp); + +_exit: + return code; +} + static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropTtlTableReq ttlReq = {0}; if (tDeserializeSVDropTtlTableReq(pReq, len, &ttlReq) != 0) { @@ -1489,8 +1513,8 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { return code; } -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, - SRpcMsg *pOriginalMsg) { +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg) { int32_t code = 0; terrno = 0; @@ -1709,10 +1733,14 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); - if(tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0){ - const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId, - pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, - pOriginalMsg->info.conn.user, "Success"}; + if (tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0) { + const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, + pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, + tsLocalEp, + pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, + "Success"}; taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels); } @@ -1725,14 +1753,14 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len); - const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, - pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, pOriginalMsg->info.conn.user, "Success"}; taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); } else{ - const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, - pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, pOriginalMsg->info.conn.user, "Failed"}; taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); } @@ -1845,11 +1873,13 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe } vInfo("vgId:%d, start to alter vnode config, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 - " cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset:%d fsync:%d level:%d " + " cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset:%d s3KeepLocal:%d " + "s3Compact:%d fsync:%d level:%d " "walRetentionPeriod:%d walRetentionSize:%d", TD_VID(pVnode), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, - req.keepTimeOffset, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize); + req.keepTimeOffset, req.s3KeepLocal, req.s3Compact, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, + req.walRetentionSize); if (pVnode->config.cacheLastSize != req.cacheLastSize) { pVnode->config.cacheLastSize = req.cacheLastSize; @@ -1933,6 +1963,13 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pRe pVnode->config.tsdbCfg.minRows = req.minRows; } + if (req.s3KeepLocal != -1 && req.s3KeepLocal != pVnode->config.s3KeepLocal) { + pVnode->config.s3KeepLocal = req.s3KeepLocal; + } + if (req.s3Compact != -1 && req.s3Compact != pVnode->config.s3Compact) { + pVnode->config.s3Compact = req.s3Compact; + } + if (walChanged) { walAlter(pVnode->pWal, &pVnode->config.walCfg); } @@ -1992,8 +2029,8 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe return 0; } -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, - SRpcMsg *pOriginalMsg) { +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; SDeleteRes *pRes = &(SDeleteRes){0}; @@ -2038,14 +2075,14 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, in _err: /* if(code == TSDB_CODE_SUCCESS){ - const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, - pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, pOriginalMsg->info.conn.user, "Success"}; taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); } else{ - const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, - pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, pOriginalMsg->info.conn.user, "Failed"}; taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); } @@ -2123,4 +2160,5 @@ static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pR int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } +int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync); #endif From 44b9785853bdaca643672f4df592964ed211f69a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 8 Apr 2024 10:33:02 +0800 Subject: [PATCH 08/34] cos/multi-write: include headers part --- include/common/cos.h | 6 +- include/common/tglobal.h | 14 +- include/common/tmsg.h | 50 ++- include/common/tmsgdef.h | 3 + include/common/ttokendef.h | 572 +++++++++++++++++----------------- include/libs/nodes/cmdnodes.h | 9 + include/util/tconfig.h | 7 +- include/util/tdef.h | 26 +- source/common/CMakeLists.txt | 5 +- source/common/src/cos.c | 207 +++++++++++- source/common/src/systable.c | 5 +- source/common/src/tglobal.c | 37 ++- source/common/src/tmsg.c | 189 ++++++++--- source/util/src/tconfig.c | 110 ++++++- 14 files changed, 844 insertions(+), 396 deletions(-) diff --git a/include/common/cos.h b/include/common/cos.h index afeca3ca03..8e48533304 100644 --- a/include/common/cos.h +++ b/include/common/cos.h @@ -33,10 +33,12 @@ extern int32_t tsS3UploadDelaySec; int32_t s3Init(); void s3CleanUp(); +int32_t s3CheckCfg(); int32_t s3PutObjectFromFile(const char *file, const char *object); int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp); +int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size); void s3DeleteObjectsByPrefix(const char *prefix); -void s3DeleteObjects(const char *object_name[], int nobject); +int32_t s3DeleteObjects(const char *object_name[], int nobject); bool s3Exists(const char *object_name); bool s3Get(const char *object_name, const char *path); int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock); @@ -45,6 +47,8 @@ void s3EvictCache(const char *path, long object_size); long s3Size(const char *object_name); int32_t s3GetObjectToFile(const char *object_name, char *fileName); +#define S3_DATA_CHUNK_PAGES (256 * 1024 * 1024) + #ifdef __cplusplus } #endif diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 93f17fa887..0763321bba 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -111,9 +111,9 @@ extern int32_t tsMonitorIntervalForBasic; extern bool tsMonitorForceV2; // audit -extern bool tsEnableAudit; -extern bool tsEnableAuditCreateTable; -extern int32_t tsAuditInterval; +extern bool tsEnableAudit; +extern bool tsEnableAuditCreateTable; +extern int32_t tsAuditInterval; // telem extern bool tsEnableTelem; @@ -121,9 +121,9 @@ extern int32_t tsTelemInterval; extern char tsTelemServer[]; extern uint16_t tsTelemPort; extern bool tsEnableCrashReport; -extern char * tsTelemUri; -extern char * tsClientCrashReportUri; -extern char * tsSvrCrashReportUri; +extern char *tsTelemUri; +extern char *tsClientCrashReportUri; +extern char *tsSvrCrashReportUri; // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing @@ -209,6 +209,8 @@ extern int32_t tsTtlUnit; extern int32_t tsTtlPushIntervalSec; extern int32_t tsTtlBatchDropNum; extern int32_t tsTrimVDbIntervalSec; +extern int32_t tsS3MigrateIntervalSec; +extern bool tsS3MigrateEnabled; extern int32_t tsGrantHBInterval; extern int32_t tsUptimeInterval; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 958789178a..44b705a6cb 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -300,7 +300,8 @@ typedef enum ENodeType { QUERY_NODE_GRANT_STMT, QUERY_NODE_REVOKE_STMT, QUERY_NODE_ALTER_CLUSTER_STMT, - // placeholder for [153, 180] + QUERY_NODE_S3MIGRATE_DATABASE_STMT, + // placeholder for [154, 180] QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181, QUERY_NODE_SHOW_CREATE_DATABASE_STMT, QUERY_NODE_SHOW_CREATE_TABLE_STMT, @@ -584,7 +585,7 @@ typedef struct { // int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); // int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); // void tFreeSSubmitBlkRsp(void* param); -void tFreeSSubmitRsp(SSubmitRsp* pRsp); +void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_SMA_ON ((int8_t)0x1) #define COL_IDX_ON ((int8_t)0x2) @@ -1149,6 +1150,9 @@ typedef struct { int32_t sstTrigger; int16_t hashPrefix; int16_t hashSuffix; + int32_t s3ChunkSize; + int32_t s3KeepLocal; + int8_t s3Compact; int32_t tsdbPageSize; int32_t sqlLen; char* sql; @@ -1178,6 +1182,8 @@ typedef struct { int32_t minRows; int32_t walRetentionPeriod; int32_t walRetentionSize; + int32_t s3KeepLocal; + int8_t s3Compact; int32_t sqlLen; char* sql; } SAlterDbReq; @@ -1257,6 +1263,20 @@ typedef struct { int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); +typedef struct { + char db[TSDB_DB_FNAME_LEN]; +} SS3MigrateDbReq; + +int32_t tSerializeSS3MigrateDbReq(void* buf, int32_t bufLen, SS3MigrateDbReq* pReq); +int32_t tDeserializeSS3MigrateDbReq(void* buf, int32_t bufLen, SS3MigrateDbReq* pReq); + +typedef struct { + int32_t timestamp; +} SVS3MigrateDbReq; + +int32_t tSerializeSVS3MigrateDbReq(void* buf, int32_t bufLen, SVS3MigrateDbReq* pReq); +int32_t tDeserializeSVS3MigrateDbReq(void* buf, int32_t bufLen, SVS3MigrateDbReq* pReq); + typedef struct { int32_t timestampSec; int32_t ttlDropMaxCount; @@ -1293,6 +1313,9 @@ typedef struct { int8_t replications; int8_t strict; int8_t cacheLast; + int32_t s3ChunkSize; + int32_t s3KeepLocal; + int8_t s3Compact; int32_t tsdbPageSize; int32_t walRetentionPeriod; int32_t walRollPeriod; @@ -1582,13 +1605,13 @@ int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq); typedef struct { - int32_t contLen; - char* pCont; + int32_t contLen; + char* pCont; } SStatisReq; int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); -void tFreeSStatisReq(SStatisReq *pReq); +void tFreeSStatisReq(SStatisReq* pReq); typedef struct { int32_t dnodeId; @@ -1693,7 +1716,10 @@ typedef struct { int16_t hashPrefix; int16_t hashSuffix; int32_t tsdbPageSize; - int64_t reserved[8]; + int32_t s3ChunkSize; + int32_t s3KeepLocal; + int8_t s3Compact; + int64_t reserved[6]; int8_t learnerReplica; int8_t learnerSelfIndex; SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; @@ -1781,13 +1807,15 @@ typedef struct { int8_t walLevel; int8_t strict; int8_t cacheLast; - int64_t reserved[8]; + int64_t reserved[7]; // 1st modification int16_t sttTrigger; int32_t minRows; // 2nd modification int32_t walRetentionPeriod; int32_t walRetentionSize; + int32_t s3KeepLocal; + int8_t s3Compact; } SAlterVnodeConfigReq; int32_t tSerializeSAlterVnodeConfigReq(void* buf, int32_t bufLen, SAlterVnodeConfigReq* pReq); @@ -1946,7 +1974,7 @@ typedef struct { int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); // int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); -void tFreeSShowReq(SShowReq* pReq); +void tFreeSShowReq(SShowReq* pReq); typedef struct { int64_t showId; @@ -2735,7 +2763,7 @@ typedef struct { SVCreateTbReq* pReqs; SArray* pArray; }; - int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient + int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient } SVCreateTbBatchReq; int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq); @@ -2828,7 +2856,7 @@ typedef struct { int32_t newCommentLen; char* newComment; int64_t ctimeMs; // fill by vnode - int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient + int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient } SVAlterTbReq; int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); @@ -3932,7 +3960,7 @@ int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq); #define SUBMIT_REQ_FROM_FILE 0x4 #define TD_REQ_FROM_TAOX 0x8 -#define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility +#define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility typedef struct { int32_t flags; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 62392a9caa..4481f1f153 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -220,6 +220,8 @@ TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) @@ -272,6 +274,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_S3MIGRATE, "vnode-s3migrate", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 8f89857d33..fed96c3b2b 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -16,7 +16,6 @@ #ifndef _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_ - #define TK_OR 1 #define TK_AND 2 #define TK_UNION 3 @@ -85,288 +84,292 @@ #define TK_USE 66 #define TK_FLUSH 67 #define TK_TRIM 68 -#define TK_COMPACT 69 -#define TK_IF 70 -#define TK_NOT 71 -#define TK_EXISTS 72 -#define TK_BUFFER 73 -#define TK_CACHEMODEL 74 -#define TK_CACHESIZE 75 -#define TK_COMP 76 -#define TK_DURATION 77 -#define TK_NK_VARIABLE 78 -#define TK_MAXROWS 79 -#define TK_MINROWS 80 -#define TK_KEEP 81 -#define TK_PAGES 82 -#define TK_PAGESIZE 83 -#define TK_TSDB_PAGESIZE 84 -#define TK_PRECISION 85 -#define TK_REPLICA 86 -#define TK_VGROUPS 87 -#define TK_SINGLE_STABLE 88 -#define TK_RETENTIONS 89 -#define TK_SCHEMALESS 90 -#define TK_WAL_LEVEL 91 -#define TK_WAL_FSYNC_PERIOD 92 -#define TK_WAL_RETENTION_PERIOD 93 -#define TK_WAL_RETENTION_SIZE 94 -#define TK_WAL_ROLL_PERIOD 95 -#define TK_WAL_SEGMENT_SIZE 96 -#define TK_STT_TRIGGER 97 -#define TK_TABLE_PREFIX 98 -#define TK_TABLE_SUFFIX 99 -#define TK_KEEP_TIME_OFFSET 100 -#define TK_NK_COLON 101 -#define TK_BWLIMIT 102 -#define TK_START 103 -#define TK_TIMESTAMP 104 -#define TK_END 105 -#define TK_TABLE 106 -#define TK_NK_LP 107 -#define TK_NK_RP 108 -#define TK_STABLE 109 -#define TK_COLUMN 110 -#define TK_MODIFY 111 -#define TK_RENAME 112 -#define TK_TAG 113 -#define TK_SET 114 -#define TK_NK_EQ 115 -#define TK_USING 116 -#define TK_TAGS 117 -#define TK_BOOL 118 -#define TK_TINYINT 119 -#define TK_SMALLINT 120 -#define TK_INT 121 -#define TK_INTEGER 122 -#define TK_BIGINT 123 -#define TK_FLOAT 124 -#define TK_DOUBLE 125 -#define TK_BINARY 126 -#define TK_NCHAR 127 -#define TK_UNSIGNED 128 -#define TK_JSON 129 -#define TK_VARCHAR 130 -#define TK_MEDIUMBLOB 131 -#define TK_BLOB 132 -#define TK_VARBINARY 133 -#define TK_GEOMETRY 134 -#define TK_DECIMAL 135 -#define TK_COMMENT 136 -#define TK_MAX_DELAY 137 -#define TK_WATERMARK 138 -#define TK_ROLLUP 139 -#define TK_TTL 140 -#define TK_SMA 141 -#define TK_DELETE_MARK 142 -#define TK_FIRST 143 -#define TK_LAST 144 -#define TK_SHOW 145 -#define TK_PRIVILEGES 146 -#define TK_DATABASES 147 -#define TK_TABLES 148 -#define TK_STABLES 149 -#define TK_MNODES 150 -#define TK_QNODES 151 -#define TK_FUNCTIONS 152 -#define TK_INDEXES 153 -#define TK_ACCOUNTS 154 -#define TK_APPS 155 -#define TK_CONNECTIONS 156 -#define TK_LICENCES 157 -#define TK_GRANTS 158 -#define TK_FULL 159 -#define TK_LOGS 160 -#define TK_MACHINES 161 -#define TK_QUERIES 162 -#define TK_SCORES 163 -#define TK_TOPICS 164 -#define TK_VARIABLES 165 -#define TK_BNODES 166 -#define TK_SNODES 167 -#define TK_TRANSACTIONS 168 -#define TK_DISTRIBUTED 169 -#define TK_CONSUMERS 170 -#define TK_SUBSCRIPTIONS 171 -#define TK_VNODES 172 -#define TK_ALIVE 173 -#define TK_VIEWS 174 -#define TK_VIEW 175 -#define TK_COMPACTS 176 -#define TK_NORMAL 177 -#define TK_CHILD 178 -#define TK_LIKE 179 -#define TK_TBNAME 180 -#define TK_QTAGS 181 -#define TK_AS 182 -#define TK_SYSTEM 183 -#define TK_INDEX 184 -#define TK_FUNCTION 185 -#define TK_INTERVAL 186 -#define TK_COUNT 187 -#define TK_LAST_ROW 188 -#define TK_META 189 -#define TK_ONLY 190 -#define TK_TOPIC 191 -#define TK_CONSUMER 192 -#define TK_GROUP 193 -#define TK_DESC 194 -#define TK_DESCRIBE 195 -#define TK_RESET 196 -#define TK_QUERY 197 -#define TK_CACHE 198 -#define TK_EXPLAIN 199 -#define TK_ANALYZE 200 -#define TK_VERBOSE 201 -#define TK_NK_BOOL 202 -#define TK_RATIO 203 -#define TK_NK_FLOAT 204 -#define TK_OUTPUTTYPE 205 -#define TK_AGGREGATE 206 -#define TK_BUFSIZE 207 -#define TK_LANGUAGE 208 -#define TK_REPLACE 209 -#define TK_STREAM 210 -#define TK_INTO 211 -#define TK_PAUSE 212 -#define TK_RESUME 213 -#define TK_TRIGGER 214 -#define TK_AT_ONCE 215 -#define TK_WINDOW_CLOSE 216 -#define TK_IGNORE 217 -#define TK_EXPIRED 218 -#define TK_FILL_HISTORY 219 -#define TK_UPDATE 220 -#define TK_SUBTABLE 221 -#define TK_UNTREATED 222 -#define TK_KILL 223 -#define TK_CONNECTION 224 -#define TK_TRANSACTION 225 -#define TK_BALANCE 226 -#define TK_VGROUP 227 -#define TK_LEADER 228 -#define TK_MERGE 229 -#define TK_REDISTRIBUTE 230 -#define TK_SPLIT 231 -#define TK_DELETE 232 -#define TK_INSERT 233 -#define TK_NULL 234 -#define TK_NK_QUESTION 235 -#define TK_NK_ALIAS 236 -#define TK_NK_ARROW 237 -#define TK_ROWTS 238 -#define TK_QSTART 239 -#define TK_QEND 240 -#define TK_QDURATION 241 -#define TK_WSTART 242 -#define TK_WEND 243 -#define TK_WDURATION 244 -#define TK_IROWTS 245 -#define TK_ISFILLED 246 -#define TK_CAST 247 -#define TK_NOW 248 -#define TK_TODAY 249 -#define TK_TIMEZONE 250 -#define TK_CLIENT_VERSION 251 -#define TK_SERVER_VERSION 252 -#define TK_SERVER_STATUS 253 -#define TK_CURRENT_USER 254 -#define TK_CASE 255 -#define TK_WHEN 256 -#define TK_THEN 257 -#define TK_ELSE 258 -#define TK_BETWEEN 259 -#define TK_IS 260 -#define TK_NK_LT 261 -#define TK_NK_GT 262 -#define TK_NK_LE 263 -#define TK_NK_GE 264 -#define TK_NK_NE 265 -#define TK_MATCH 266 -#define TK_NMATCH 267 -#define TK_CONTAINS 268 -#define TK_IN 269 -#define TK_JOIN 270 -#define TK_INNER 271 -#define TK_SELECT 272 -#define TK_NK_HINT 273 -#define TK_DISTINCT 274 -#define TK_WHERE 275 -#define TK_PARTITION 276 -#define TK_BY 277 -#define TK_SESSION 278 -#define TK_STATE_WINDOW 279 -#define TK_EVENT_WINDOW 280 -#define TK_COUNT_WINDOW 281 -#define TK_SLIDING 282 -#define TK_FILL 283 -#define TK_VALUE 284 -#define TK_VALUE_F 285 -#define TK_NONE 286 -#define TK_PREV 287 -#define TK_NULL_F 288 -#define TK_LINEAR 289 -#define TK_NEXT 290 -#define TK_HAVING 291 -#define TK_RANGE 292 -#define TK_EVERY 293 -#define TK_ORDER 294 -#define TK_SLIMIT 295 -#define TK_SOFFSET 296 -#define TK_LIMIT 297 -#define TK_OFFSET 298 -#define TK_ASC 299 -#define TK_NULLS 300 -#define TK_ABORT 301 -#define TK_AFTER 302 -#define TK_ATTACH 303 -#define TK_BEFORE 304 -#define TK_BEGIN 305 -#define TK_BITAND 306 -#define TK_BITNOT 307 -#define TK_BITOR 308 -#define TK_BLOCKS 309 -#define TK_CHANGE 310 -#define TK_COMMA 311 -#define TK_CONCAT 312 -#define TK_CONFLICT 313 -#define TK_COPY 314 -#define TK_DEFERRED 315 -#define TK_DELIMITERS 316 -#define TK_DETACH 317 -#define TK_DIVIDE 318 -#define TK_DOT 319 -#define TK_EACH 320 -#define TK_FAIL 321 -#define TK_FILE 322 -#define TK_FOR 323 -#define TK_GLOB 324 -#define TK_ID 325 -#define TK_IMMEDIATE 326 -#define TK_IMPORT 327 -#define TK_INITIALLY 328 -#define TK_INSTEAD 329 -#define TK_ISNULL 330 -#define TK_KEY 331 -#define TK_MODULES 332 -#define TK_NK_BITNOT 333 -#define TK_NK_SEMI 334 -#define TK_NOTNULL 335 -#define TK_OF 336 -#define TK_PLUS 337 -#define TK_PRIVILEGE 338 -#define TK_RAISE 339 -#define TK_RESTRICT 340 -#define TK_ROW 341 -#define TK_SEMI 342 -#define TK_STAR 343 -#define TK_STATEMENT 344 -#define TK_STRICT 345 -#define TK_STRING 346 -#define TK_TIMES 347 -#define TK_VALUES 348 -#define TK_VARIABLE 349 -#define TK_WAL 350 +#define TK_S3MIGRATE 69 +#define TK_COMPACT 70 +#define TK_IF 71 +#define TK_NOT 72 +#define TK_EXISTS 73 +#define TK_BUFFER 74 +#define TK_CACHEMODEL 75 +#define TK_CACHESIZE 76 +#define TK_COMP 77 +#define TK_DURATION 78 +#define TK_NK_VARIABLE 79 +#define TK_MAXROWS 80 +#define TK_MINROWS 81 +#define TK_KEEP 82 +#define TK_PAGES 83 +#define TK_PAGESIZE 84 +#define TK_TSDB_PAGESIZE 85 +#define TK_PRECISION 86 +#define TK_REPLICA 87 +#define TK_VGROUPS 88 +#define TK_SINGLE_STABLE 89 +#define TK_RETENTIONS 90 +#define TK_SCHEMALESS 91 +#define TK_WAL_LEVEL 92 +#define TK_WAL_FSYNC_PERIOD 93 +#define TK_WAL_RETENTION_PERIOD 94 +#define TK_WAL_RETENTION_SIZE 95 +#define TK_WAL_ROLL_PERIOD 96 +#define TK_WAL_SEGMENT_SIZE 97 +#define TK_STT_TRIGGER 98 +#define TK_TABLE_PREFIX 99 +#define TK_TABLE_SUFFIX 100 +#define TK_S3_CHUNKSIZE 101 +#define TK_S3_KEEPLOCAL 102 +#define TK_S3_COMPACT 103 +#define TK_KEEP_TIME_OFFSET 104 +#define TK_NK_COLON 105 +#define TK_BWLIMIT 106 +#define TK_START 107 +#define TK_TIMESTAMP 108 +#define TK_END 109 +#define TK_TABLE 110 +#define TK_NK_LP 111 +#define TK_NK_RP 112 +#define TK_STABLE 113 +#define TK_COLUMN 114 +#define TK_MODIFY 115 +#define TK_RENAME 116 +#define TK_TAG 117 +#define TK_SET 118 +#define TK_NK_EQ 119 +#define TK_USING 120 +#define TK_TAGS 121 +#define TK_BOOL 122 +#define TK_TINYINT 123 +#define TK_SMALLINT 124 +#define TK_INT 125 +#define TK_INTEGER 126 +#define TK_BIGINT 127 +#define TK_FLOAT 128 +#define TK_DOUBLE 129 +#define TK_BINARY 130 +#define TK_NCHAR 131 +#define TK_UNSIGNED 132 +#define TK_JSON 133 +#define TK_VARCHAR 134 +#define TK_MEDIUMBLOB 135 +#define TK_BLOB 136 +#define TK_VARBINARY 137 +#define TK_GEOMETRY 138 +#define TK_DECIMAL 139 +#define TK_COMMENT 140 +#define TK_MAX_DELAY 141 +#define TK_WATERMARK 142 +#define TK_ROLLUP 143 +#define TK_TTL 144 +#define TK_SMA 145 +#define TK_DELETE_MARK 146 +#define TK_FIRST 147 +#define TK_LAST 148 +#define TK_SHOW 149 +#define TK_PRIVILEGES 150 +#define TK_DATABASES 151 +#define TK_TABLES 152 +#define TK_STABLES 153 +#define TK_MNODES 154 +#define TK_QNODES 155 +#define TK_FUNCTIONS 156 +#define TK_INDEXES 157 +#define TK_ACCOUNTS 158 +#define TK_APPS 159 +#define TK_CONNECTIONS 160 +#define TK_LICENCES 161 +#define TK_GRANTS 162 +#define TK_FULL 163 +#define TK_LOGS 164 +#define TK_MACHINES 165 +#define TK_QUERIES 166 +#define TK_SCORES 167 +#define TK_TOPICS 168 +#define TK_VARIABLES 169 +#define TK_BNODES 170 +#define TK_SNODES 171 +#define TK_TRANSACTIONS 172 +#define TK_DISTRIBUTED 173 +#define TK_CONSUMERS 174 +#define TK_SUBSCRIPTIONS 175 +#define TK_VNODES 176 +#define TK_ALIVE 177 +#define TK_VIEWS 178 +#define TK_VIEW 179 +#define TK_COMPACTS 180 +#define TK_NORMAL 181 +#define TK_CHILD 182 +#define TK_LIKE 183 +#define TK_TBNAME 184 +#define TK_QTAGS 185 +#define TK_AS 186 +#define TK_SYSTEM 187 +#define TK_INDEX 188 +#define TK_FUNCTION 189 +#define TK_INTERVAL 190 +#define TK_COUNT 191 +#define TK_LAST_ROW 192 +#define TK_META 193 +#define TK_ONLY 194 +#define TK_TOPIC 195 +#define TK_CONSUMER 196 +#define TK_GROUP 197 +#define TK_DESC 198 +#define TK_DESCRIBE 199 +#define TK_RESET 200 +#define TK_QUERY 201 +#define TK_CACHE 202 +#define TK_EXPLAIN 203 +#define TK_ANALYZE 204 +#define TK_VERBOSE 205 +#define TK_NK_BOOL 206 +#define TK_RATIO 207 +#define TK_NK_FLOAT 208 +#define TK_OUTPUTTYPE 209 +#define TK_AGGREGATE 210 +#define TK_BUFSIZE 211 +#define TK_LANGUAGE 212 +#define TK_REPLACE 213 +#define TK_STREAM 214 +#define TK_INTO 215 +#define TK_PAUSE 216 +#define TK_RESUME 217 +#define TK_TRIGGER 218 +#define TK_AT_ONCE 219 +#define TK_WINDOW_CLOSE 220 +#define TK_IGNORE 221 +#define TK_EXPIRED 222 +#define TK_FILL_HISTORY 223 +#define TK_UPDATE 224 +#define TK_SUBTABLE 225 +#define TK_UNTREATED 226 +#define TK_KILL 227 +#define TK_CONNECTION 228 +#define TK_TRANSACTION 229 +#define TK_BALANCE 230 +#define TK_VGROUP 231 +#define TK_LEADER 232 +#define TK_MERGE 233 +#define TK_REDISTRIBUTE 234 +#define TK_SPLIT 235 +#define TK_DELETE 236 +#define TK_INSERT 237 +#define TK_NULL 238 +#define TK_NK_QUESTION 239 +#define TK_NK_ALIAS 240 +#define TK_NK_ARROW 241 +#define TK_ROWTS 242 +#define TK_QSTART 243 +#define TK_QEND 244 +#define TK_QDURATION 245 +#define TK_WSTART 246 +#define TK_WEND 247 +#define TK_WDURATION 248 +#define TK_IROWTS 249 +#define TK_ISFILLED 250 +#define TK_CAST 251 +#define TK_NOW 252 +#define TK_TODAY 253 +#define TK_TIMEZONE 254 +#define TK_CLIENT_VERSION 255 +#define TK_SERVER_VERSION 256 +#define TK_SERVER_STATUS 257 +#define TK_CURRENT_USER 258 +#define TK_CASE 259 +#define TK_WHEN 260 +#define TK_THEN 261 +#define TK_ELSE 262 +#define TK_BETWEEN 263 +#define TK_IS 264 +#define TK_NK_LT 265 +#define TK_NK_GT 266 +#define TK_NK_LE 267 +#define TK_NK_GE 268 +#define TK_NK_NE 269 +#define TK_MATCH 270 +#define TK_NMATCH 271 +#define TK_CONTAINS 272 +#define TK_IN 273 +#define TK_JOIN 274 +#define TK_INNER 275 +#define TK_SELECT 276 +#define TK_NK_HINT 277 +#define TK_DISTINCT 278 +#define TK_WHERE 279 +#define TK_PARTITION 280 +#define TK_BY 281 +#define TK_SESSION 282 +#define TK_STATE_WINDOW 283 +#define TK_EVENT_WINDOW 284 +#define TK_COUNT_WINDOW 285 +#define TK_SLIDING 286 +#define TK_FILL 287 +#define TK_VALUE 288 +#define TK_VALUE_F 289 +#define TK_NONE 290 +#define TK_PREV 291 +#define TK_NULL_F 292 +#define TK_LINEAR 293 +#define TK_NEXT 294 +#define TK_HAVING 295 +#define TK_RANGE 296 +#define TK_EVERY 297 +#define TK_ORDER 298 +#define TK_SLIMIT 299 +#define TK_SOFFSET 300 +#define TK_LIMIT 301 +#define TK_OFFSET 302 +#define TK_ASC 303 +#define TK_NULLS 304 +#define TK_ABORT 305 +#define TK_AFTER 306 +#define TK_ATTACH 307 +#define TK_BEFORE 308 +#define TK_BEGIN 309 +#define TK_BITAND 310 +#define TK_BITNOT 311 +#define TK_BITOR 312 +#define TK_BLOCKS 313 +#define TK_CHANGE 314 +#define TK_COMMA 315 +#define TK_CONCAT 316 +#define TK_CONFLICT 317 +#define TK_COPY 318 +#define TK_DEFERRED 319 +#define TK_DELIMITERS 320 +#define TK_DETACH 321 +#define TK_DIVIDE 322 +#define TK_DOT 323 +#define TK_EACH 324 +#define TK_FAIL 325 +#define TK_FILE 326 +#define TK_FOR 327 +#define TK_GLOB 328 +#define TK_ID 329 +#define TK_IMMEDIATE 330 +#define TK_IMPORT 331 +#define TK_INITIALLY 332 +#define TK_INSTEAD 333 +#define TK_ISNULL 334 +#define TK_KEY 335 +#define TK_MODULES 336 +#define TK_NK_BITNOT 337 +#define TK_NK_SEMI 338 +#define TK_NOTNULL 339 +#define TK_OF 340 +#define TK_PLUS 341 +#define TK_PRIVILEGE 342 +#define TK_RAISE 343 +#define TK_RESTRICT 344 +#define TK_ROW 345 +#define TK_SEMI 346 +#define TK_STAR 347 +#define TK_STATEMENT 348 +#define TK_STRICT 349 +#define TK_STRING 350 +#define TK_TIMES 351 +#define TK_VALUES 352 +#define TK_VARIABLE 353 +#define TK_WAL 354 #define TK_NK_SPACE 600 @@ -379,8 +382,7 @@ #define TK_NO_BATCH_SCAN 607 #define TK_SORT_FOR_GROUP 608 #define TK_PARTITION_FIRST 609 -#define TK_PARA_TABLES_SORT 610 - +#define TK_PARA_TABLES_SORT 610 #define TK_NK_NIL 65535 diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 9a12d7b98f..675fd9a3e4 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -100,6 +100,10 @@ typedef struct SDatabaseOptions { int32_t sstTrigger; int32_t tablePrefix; int32_t tableSuffix; + int32_t s3ChunkSize; + int32_t s3KeepLocal; + SValueNode* s3KeepLocalStr; + int8_t s3Compact; } SDatabaseOptions; typedef struct SCreateDatabaseStmt { @@ -137,6 +141,11 @@ typedef struct STrimDatabaseStmt { int32_t maxSpeed; } STrimDatabaseStmt; +typedef struct SS3MigrateDatabaseStmt { + ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; +} SS3MigrateDatabaseStmt; + typedef struct SCompactDatabaseStmt { ENodeType type; char dbName[TSDB_DB_NAME_LEN]; diff --git a/include/util/tconfig.h b/include/util/tconfig.h index f2a9446700..45abe2ff83 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -51,11 +51,7 @@ typedef enum { CFG_DTYPE_TIMEZONE } ECfgDataType; -typedef enum { - CFG_SCOPE_SERVER, - CFG_SCOPE_CLIENT, - CFG_SCOPE_BOTH -} ECfgScopeType; +typedef enum { CFG_SCOPE_SERVER, CFG_SCOPE_CLIENT, CFG_SCOPE_BOTH } ECfgScopeType; typedef enum { CFG_DYN_NONE = 0, @@ -138,6 +134,7 @@ void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); +void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump); int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl); diff --git a/include/util/tdef.h b/include/util/tdef.h index e2d1beb5a5..0dfc88cc1d 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -197,8 +197,8 @@ typedef enum ELogicConditionType { #define TSDB_POINTER_PRINT_BYTES 18 // 0x1122334455667788 // ACCOUNT is a 32 bit positive integer // this is the length of its string representation, including the terminator zero -#define TSDB_ACCT_ID_LEN 11 -#define TSDB_NODE_ID_LEN 11 +#define TSDB_ACCT_ID_LEN 11 +#define TSDB_NODE_ID_LEN 11 #define TSDB_VGROUP_ID_LEN 11 #define TSDB_MAX_COLUMNS 4096 @@ -290,8 +290,8 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_CONFIG_LEN 128 #define TSDB_DNODE_VALUE_LEN 256 -#define TSDB_CLUSTER_VALUE_LEN 1000 -#define TSDB_GRANT_LOG_COL_LEN 15600 +#define TSDB_CLUSTER_VALUE_LEN 1000 +#define TSDB_GRANT_LOG_COL_LEN 15600 #define TSDB_ACTIVE_KEY_LEN 109 #define TSDB_CONN_ACTIVE_KEY_LEN 255 @@ -413,6 +413,16 @@ typedef enum ELogicConditionType { #define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_DEFAULT_HASH_SUFFIX 0 +#define TSDB_MIN_S3_CHUNK_SIZE (32 * 1024) +#define TSDB_MAX_S3_CHUNK_SIZE (1024 * 1024) +#define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024) +#define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute +#define TSDB_MAX_S3_KEEP_LOCAL (365000 * 1440) +#define TSDB_DEFAULT_S3_KEEP_LOCAL (30 * 1440) +#define TSDB_MIN_S3_COMPACT 0 +#define TSDB_MAX_S3_COMPACT 1 +#define TSDB_DEFAULT_S3_COMPACT 0 + #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 #define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600 #define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 3600 @@ -551,10 +561,10 @@ enum { #define VNODE_TIMEOUT_SEC 60 #define MNODE_TIMEOUT_SEC 60 -#define MONITOR_TABLENAME_LEN 200 -#define MONITOR_TAG_NAME_LEN 100 -#define MONITOR_TAG_VALUE_LEN 300 -#define MONITOR_METRIC_NAME_LEN 100 +#define MONITOR_TABLENAME_LEN 200 +#define MONITOR_TAG_NAME_LEN 100 +#define MONITOR_TAG_VALUE_LEN 300 +#define MONITOR_METRIC_NAME_LEN 100 #ifdef __cplusplus } #endif diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index d3df1345df..eb3dd95e95 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -16,14 +16,15 @@ ENDIF () IF (TD_STORAGE) ADD_DEFINITIONS(-D_STORAGE) TARGET_LINK_LIBRARIES(common PRIVATE storage) +ENDIF () +IF (TD_ENTERPRISE) IF(${BUILD_WITH_S3}) add_definitions(-DUSE_S3) ELSEIF(${BUILD_WITH_COS}) add_definitions(-DUSE_COS) ENDIF() - -ENDIF () +ENDIF() target_include_directories( common diff --git a/source/common/src/cos.c b/source/common/src/cos.c index fcc777ac99..95b216c285 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -53,6 +53,100 @@ int32_t s3Init() { return 0; /*s3Begin();*/ } void s3CleanUp() { /*s3End();*/ } +static int32_t s3ListBucket(char const *bucketname); + +int32_t s3CheckCfg() { + int32_t code = 0; + + code = s3Begin(); + if (code != 0) { + fprintf(stderr, "failed to initialize s3.\n"); + goto _exit; + } + + // test put + char testdata[17] = "0123456789abcdef"; + const char *objectname[] = {"s3test.txt"}; + char path[PATH_MAX] = {0}; + int ds_len = strlen(TD_DIRSEP); + int tmp_len = strlen(tsTempDir); + + snprintf(path, PATH_MAX, "%s", tsTempDir); + if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) { + snprintf(path + tmp_len, PATH_MAX, "%s", TD_DIRSEP); + snprintf(path + tmp_len + ds_len, PATH_MAX, "%s", objectname[0]); + } else { + snprintf(path + tmp_len, PATH_MAX, "%s", objectname[0]); + } + + TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (!fp) { + code = TAOS_SYSTEM_ERROR(errno); + fprintf(stderr, "failed to open test file: %s.\n", path); + // uError("ERROR: %s Failed to open %s", __func__, path); + goto _exit; + } + if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + fprintf(stderr, "failed to write test file: %s.\n", path); + goto _exit; + } + if (taosFsyncFile(fp) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + fprintf(stderr, "failed to fsync test file: %s.\n", path); + goto _exit; + } + taosCloseFile(&fp); + + fprintf(stderr, "\nstart to put object: %s, file: %s content: %s\n", objectname[0], path, testdata); + code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16); + if (code != 0) { + fprintf(stderr, "put object %s : failed.\n", objectname[0]); + goto _exit; + } + fprintf(stderr, "put object %s: success.\n\n", objectname[0]); + + // list buckets + fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName); + code = s3ListBucket(tsS3BucketName); + if (code != 0) { + fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName); + goto _exit; + } + fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName); + + // test range get + uint8_t *pBlock = NULL; + int c_offset = 10; + int c_len = 6; + + fprintf(stderr, "start to range get object %s offset: %d len: %d.\n", objectname[0], c_offset, c_len); + code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock); + if (code != 0) { + fprintf(stderr, "get object %s : failed.\n", objectname[0]); + goto _exit; + } + char buf[7] = {0}; + memcpy(buf, pBlock, c_len); + taosMemoryFree(pBlock); + fprintf(stderr, "object content: %s\n", buf); + fprintf(stderr, "get object %s: success.\n\n", objectname[0]); + + // delete test object + fprintf(stderr, "start to delete object: %s.\n", objectname[0]); + code = s3DeleteObjects(objectname, 1); + if (code != 0) { + fprintf(stderr, "delete object %s : failed.\n", objectname[0]); + goto _exit; + } + fprintf(stderr, "delete object %s: success.\n\n", objectname[0]); + + s3End(); + +_exit: + return code; +} + static int should_retry() { /* if (retriesG--) { @@ -85,7 +179,7 @@ typedef struct { } TS3GetData; typedef struct { - char err_msg[128]; + char err_msg[512]; S3Status status; uint64_t content_length; char *buf; @@ -137,6 +231,30 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro } } +static SArray *getListByPrefix(const char *prefix); +static void s3FreeObjectKey(void *pItem); + +static int32_t s3ListBucket(char const *bucketname) { + int32_t code = 0; + + SArray *objectArray = getListByPrefix("s3"); + if (objectArray == NULL) { + return -1; + } + + const char **object_name = TARRAY_DATA(objectArray); + int size = TARRAY_SIZE(objectArray); + + fprintf(stderr, "objects:\n"); + for (int i = 0; i < size; ++i) { + fprintf(stderr, "%s\n", object_name[i]); + } + + taosArrayDestroyEx(objectArray, s3FreeObjectKey); + + return code; +} + typedef struct growbuffer { // The total number of bytes, and the start byte int size; @@ -471,10 +589,11 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c } while (S3_status_is_retryable(data->status) && should_retry()); if (data->status != S3StatusOK) { - s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg); + // s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg); + s3PrintError(NULL, __LINE__, __func__, data->status, data->err_msg); code = TAOS_SYSTEM_ERROR(EIO); } else if (data->contentLength) { - uError("%s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data->contentLength); + uError("%s Failed to put remaining %llu bytes", __func__, (unsigned long long)data->contentLength); code = TAOS_SYSTEM_ERROR(EIO); } @@ -830,6 +949,66 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w return code; } +int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { + int32_t code = 0; + int32_t lmtime = 0; + const char *filename = 0; + uint64_t contentLength = 0; + const char *cacheControl = 0, *contentType = 0, *md5 = 0; + const char *contentDispositionFilename = 0, *contentEncoding = 0; + int64_t expires = -1; + S3CannedAcl cannedAcl = S3CannedAclPrivate; + int metaPropertiesCount = 0; + S3NameValue metaProperties[S3_MAX_METADATA_COUNT]; + char useServerSideEncryption = 0; + put_object_callback_data data = {0}; + + if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + uError("ERROR: %s Failed to stat file %s: ", __func__, file); + return code; + } + + contentLength = size; + + if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { + code = TAOS_SYSTEM_ERROR(errno); + uError("ERROR: %s Failed to open file %s: ", __func__, file); + return code; + } + if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) { + taosCloseFile(&data.infileFD); + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = + contentLength; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + + S3PutProperties putProperties = {contentType, md5, + cacheControl, contentDispositionFilename, + contentEncoding, expires, + cannedAcl, metaPropertiesCount, + metaProperties, useServerSideEncryption}; + + if (contentLength <= MULTIPART_CHUNK_SIZE) { + code = s3PutObjectFromFileSimple(&bucketContext, object_name, contentLength, &putProperties, &data); + } else { + code = s3PutObjectFromFileWithoutCp(&bucketContext, object_name, contentLength, &putProperties, &data); + } + + if (data.infileFD) { + taosCloseFile(&data.infileFD); + } else if (data.gb) { + growbuffer_destroy(data.gb); + } + + return code; +} + typedef struct list_bucket_callback_data { char err_msg[512]; S3Status status; @@ -888,7 +1067,7 @@ static SArray *getListByPrefix(const char *prefix) { const char *marker = 0, *delimiter = 0; int maxkeys = 0, allDetails = 0; - list_bucket_callback_data data; + list_bucket_callback_data data = {0}; data.objectArray = taosArrayInit(32, sizeof(void *)); if (!data.objectArray) { uError("%s: %s", __func__, "out of memoty"); @@ -918,14 +1097,17 @@ static SArray *getListByPrefix(const char *prefix) { return data.objectArray; } } else { - s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); + uError("failed to list with prefix %s: %s", prefix, S3_get_status_name(data.status)); + // s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); } taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); return NULL; } -void s3DeleteObjects(const char *object_name[], int nobject) { +int32_t s3DeleteObjects(const char *object_name[], int nobject) { + int32_t code = 0; + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; @@ -938,8 +1120,11 @@ void s3DeleteObjects(const char *object_name[], int nobject) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); + code = -1; } } + + return code; } void s3DeleteObjectsByPrefix(const char *prefix) { @@ -991,12 +1176,12 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, } while (S3_status_is_retryable(cbd.status) && should_retry()); if (cbd.status != S3StatusOK) { - uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg); return TAOS_SYSTEM_ERROR(EIO); } if (check && cbd.buf_pos != size) { - uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg); return TAOS_SYSTEM_ERROR(EIO); } @@ -1233,7 +1418,7 @@ void s3DeleteObjectsByPrefix(const char *prefix_str) { cos_pool_destroy(p); } -void s3DeleteObjects(const char *object_name[], int nobject) { +int32_t s3DeleteObjects(const char *object_name[], int nobject) { cos_pool_t *p = NULL; int is_cname = 0; cos_string_t bucket; @@ -1267,6 +1452,8 @@ void s3DeleteObjects(const char *object_name[], int nobject) { } else { cos_warn_log("delete objects failed\n"); } + + return 0; } bool s3Exists(const char *object_name) { @@ -1536,7 +1723,7 @@ void s3CleanUp() {} int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; } int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; } void s3DeleteObjectsByPrefix(const char *prefix) {} -void s3DeleteObjects(const char *object_name[], int nobject) {} +int32_t s3DeleteObjects(const char *object_name[], int nobject) { return 0; } bool s3Exists(const char *object_name) { return false; } bool s3Get(const char *object_name, const char *path) { return false; } int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 25cc5d7c79..e2dff7e95e 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -24,7 +24,7 @@ #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE) // clang-format off static const SSysDbTableSchema dnodesSchema[] = { @@ -107,6 +107,9 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "s3_keeplocal", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, }; static const SSysDbTableSchema userFuncSchema[] = { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 55df80ca44..157fe942de 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -259,7 +259,9 @@ float tsSinkDataRate = 2.0; int32_t tsStreamNodeCheckInterval = 16; int32_t tsTtlUnit = 86400; int32_t tsTtlPushIntervalSec = 10; -int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups +int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups +int32_t tsS3MigrateIntervalSec = 60 * 60; // interval of s3migrate db in all vgroups +bool tsS3MigrateEnabled = 1; int32_t tsGrantHBInterval = 60; int32_t tsUptimeInterval = 300; // seconds char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits @@ -285,7 +287,7 @@ char tsS3Hostname[TSDB_FQDN_LEN] = ""; int32_t tsS3BlockSize = -1; // number of tsdb pages (4096) int32_t tsS3BlockCacheSize = 16; // number of blocks int32_t tsS3PageCacheSize = 4096; // number of pages -int32_t tsS3UploadDelaySec = 60 * 60 * 24; +int32_t tsS3UploadDelaySec = 60; bool tsExperimental = true; @@ -345,7 +347,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { } if (tsS3BucketName[0] != '<') { #if defined(USE_COS) || defined(USE_S3) - if (tsDiskCfgNum > 1) tsS3Enabled = true; +#ifdef TD_ENTERPRISE + /*if (tsDiskCfgNum > 1) */ tsS3Enabled = true; +#endif tsS3StreamEnabled = true; #endif } @@ -539,7 +543,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; return 0; } @@ -680,8 +685,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, - (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), + CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) @@ -699,7 +704,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, + CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -742,6 +748,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -760,8 +770,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; + if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) @@ -792,6 +801,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + /* if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (tsS3BlockSize > -1 && tsS3BlockSize < 1024) { @@ -801,10 +811,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + */ if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, + if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; @@ -1241,8 +1252,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsResolveFQDNRetryTime = cfgGetItem(pCfg, "resolveFQDNRetryTime")->i32; tsMinDiskFreeSize = cfgGetItem(pCfg, "minDiskFreeSize")->i64; - tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32; - tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32; + // tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32; + // tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32; tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32; tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32; @@ -1516,6 +1527,8 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { {"ttlBatchDropNum", &tsTtlBatchDropNum}, {"ttlFlushThreshold", &tsTtlFlushThreshold}, {"ttlPushInterval", &tsTtlPushIntervalSec}, + {"s3MigrateIntervalSec", &tsS3MigrateIntervalSec}, + {"s3MigrateEnabled", &tsS3MigrateEnabled}, //{"s3BlockSize", &tsS3BlockSize}, {"s3BlockCacheSize", &tsS3BlockCacheSize}, {"s3PageCacheSize", &tsS3PageCacheSize}, diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index f8df2edd61..b6067c7266 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1471,9 +1471,7 @@ int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) { return 0; } -void tFreeSStatisReq(SStatisReq *pReq) { - taosMemoryFreeClear(pReq->pCont); -} +void tFreeSStatisReq(SStatisReq *pReq) { taosMemoryFreeClear(pReq->pCont); } // int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { // SEncoder encoder = {0}; @@ -1872,7 +1870,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) char *tb = taosHashIterate(pRsp->readTbs, NULL); while (tb != NULL) { size_t keyLen = 0; - void * key = taosHashGetKey(tb, &keyLen); + void *key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1887,7 +1885,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->writeTbs, NULL); while (tb != NULL) { size_t keyLen = 0; - void * key = taosHashGetKey(tb, &keyLen); + void *key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1902,7 +1900,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->alterTbs, NULL); while (tb != NULL) { size_t keyLen = 0; - void * key = taosHashGetKey(tb, &keyLen); + void *key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1917,7 +1915,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->readViews, NULL); while (tb != NULL) { size_t keyLen = 0; - void * key = taosHashGetKey(tb, &keyLen); + void *key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1932,7 +1930,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->writeViews, NULL); while (tb != NULL) { size_t keyLen = 0; - void * key = taosHashGetKey(tb, &keyLen); + void *key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1947,7 +1945,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->alterViews, NULL); while (tb != NULL) { size_t keyLen = 0; - void * key = taosHashGetKey(tb, &keyLen); + void *key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1962,7 +1960,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL); while (useDb != NULL) { size_t keyLen = 0; - void * key = taosHashGetKey(useDb, &keyLen); + void *key = taosHashGetKey(useDb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -3022,6 +3020,9 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { } if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1; + if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1; + if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1; ENCODESQL(); tEndEncode(&encoder); @@ -3091,6 +3092,15 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1; } + pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1; + } + DECODESQL(); tEndDecode(&decoder); @@ -3132,6 +3142,10 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1; if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1; + + if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1; + if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1; + ENCODESQL(); tEndEncode(&encoder); @@ -3181,6 +3195,13 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1; } + pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1; + } + DECODESQL(); tEndDecode(&decoder); @@ -3873,6 +3894,57 @@ int32_t tDeserializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) { return 0; } +int32_t tSerializeSS3MigrateDbReq(void *buf, int32_t bufLen, SS3MigrateDbReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSS3MigrateDbReq(void *buf, int32_t bufLen, SS3MigrateDbReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + +int32_t tSerializeSVS3MigrateDbReq(void *buf, int32_t bufLen, SVS3MigrateDbReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSVS3MigrateDbReq(void *buf, int32_t bufLen, SVS3MigrateDbReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3969,6 +4041,9 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) { if (tEncodeI8(encoder, pRsp->schemaless) < 0) return -1; if (tEncodeI16(encoder, pRsp->sstTrigger) < 0) return -1; if (tEncodeI32(encoder, pRsp->keepTimeOffset) < 0) return -1; + if (tEncodeI32(encoder, pRsp->s3ChunkSize) < 0) return -1; + if (tEncodeI32(encoder, pRsp->s3KeepLocal) < 0) return -1; + if (tEncodeI8(encoder, pRsp->s3Compact) < 0) return -1; return 0; } @@ -4042,6 +4117,15 @@ int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) { if (tDecodeI32(decoder, &pRsp->keepTimeOffset) < 0) return -1; } + pRsp->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + pRsp->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pRsp->s3Compact = TSDB_DEFAULT_S3_COMPACT; + if (!tDecodeIsEnd(decoder)) { + if (tDecodeI32(decoder, &pRsp->s3ChunkSize) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->s3KeepLocal) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->s3Compact) < 0) return -1; + } + return 0; } @@ -5065,7 +5149,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1; if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1; if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1; - for (int32_t i = 0; i < 8; ++i) { + for (int32_t i = 0; i < 6; ++i) { if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; } if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; @@ -5076,6 +5160,9 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR } if (tEncodeI32(&encoder, pReq->changeVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1; + if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1; + if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1; tEndEncode(&encoder); @@ -5151,7 +5238,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1; if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1; if (tDecodeI32(&decoder, &pReq->tsdbPageSize) < 0) return -1; - for (int32_t i = 0; i < 8; ++i) { + for (int32_t i = 0; i < 6; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } if (!tDecodeIsEnd(&decoder)) { @@ -5170,6 +5257,15 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1; } + pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1; + } + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -5425,7 +5521,7 @@ int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeCon if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; - for (int32_t i = 0; i < 8; ++i) { + for (int32_t i = 0; i < 7; ++i) { if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; } @@ -5436,6 +5532,10 @@ int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeCon if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1; if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1; + + if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1; + if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -5461,7 +5561,7 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; - for (int32_t i = 0; i < 8; ++i) { + for (int32_t i = 0; i < 7; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } @@ -5487,6 +5587,13 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1; } + pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1; + } + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -7008,27 +7115,27 @@ void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskSt // return 0; // } -//int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { - // SDecoder decoder = {0}; - // int32_t num = 0; - // tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { +// SDecoder decoder = {0}; +// int32_t num = 0; +// tDecoderInit(&decoder, buf, bufLen); - // if (tStartDecode(&decoder) < 0) return -1; - // if (tDecodeI32(&decoder, &num) < 0) return -1; - // if (num > 0) { - // pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp)); - // if (NULL == pRsp->rspList) return -1; - // for (int32_t i = 0; i < num; ++i) { - // SVCreateTbRsp rsp = {0}; - // if (tDecodeI32(&decoder, &rsp.code) < 0) return -1; - // if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1; - // } - // } else { - // pRsp->rspList = NULL; - // } - // tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &num) < 0) return -1; +// if (num > 0) { +// pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp)); +// if (NULL == pRsp->rspList) return -1; +// for (int32_t i = 0; i < num; ++i) { +// SVCreateTbRsp rsp = {0}; +// if (tDecodeI32(&decoder, &rsp.code) < 0) return -1; +// if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1; +// } +// } else { +// pRsp->rspList = NULL; +// } +// tEndDecode(&decoder); - // tDecoderClear(&decoder); +// tDecoderClear(&decoder); // return 0; //} @@ -7301,8 +7408,8 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS if (tEncodeI32(&encoder, taosArrayGetSize(pReq->pVgroupVerList)) < 0) return -1; - for(int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) { - SVgroupVer* p = taosArrayGet(pReq->pVgroupVerList, i); + for (int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) { + SVgroupVer *p = taosArrayGet(pReq->pVgroupVerList, i); if (tEncodeI32(&encoder, p->vgId) < 0) return -1; if (tEncodeI64(&encoder, p->ver) < 0) return -1; } @@ -8478,7 +8585,7 @@ int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) { for (int32_t i = 0; i < pRsp->blockNum; i++) { int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i); - void * data = taosArrayGetP(pRsp->blockData, i); + void *data = taosArrayGetP(pRsp->blockData, i); if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1; if (pRsp->withSchema) { SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i); @@ -8516,7 +8623,7 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { } for (int32_t i = 0; i < pRsp->blockNum; i++) { - void * data; + void *data; uint64_t bLen; if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1; taosArrayPush(pRsp->blockData, &data); @@ -8570,7 +8677,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { for (int32_t i = 0; i < pRsp->createTableNum; i++) { - void * createTableReq = taosArrayGetP(pRsp->createTableReq, i); + void *createTableReq = taosArrayGetP(pRsp->createTableReq, i); int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i); if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1; } @@ -8579,14 +8686,14 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { } int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { - if (tDecodeMqDataRspCommon(pDecoder, (SMqDataRsp*)pRsp) < 0) return -1; + if (tDecodeMqDataRspCommon(pDecoder, (SMqDataRsp *)pRsp) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *)); for (int32_t i = 0; i < pRsp->createTableNum; i++) { - void * pCreate = NULL; + void *pCreate = NULL; uint64_t len; if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1; int32_t l = (int32_t)len; @@ -8889,7 +8996,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) { taosArrayDestroy(pTbData->aCol); } else { int32_t nRow = TARRAY_SIZE(pTbData->aRowP); - SRow ** rows = (SRow **)TARRAY_DATA(pTbData->aRowP); + SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); for (int32_t i = 0; i < nRow; ++i) { tRowDestroy(rows[i]); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index ad3c766510..40cf4395d8 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -21,8 +21,8 @@ #include "tgrant.h" #include "tjson.h" #include "tlog.h" -#include "tutil.h" #include "tunit.h" +#include "tutil.h" #define CFG_NAME_PRINT_LEN 24 #define CFG_SRC_PRINT_LEN 12 @@ -310,19 +310,19 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool resetArray) { SConfigItem *pDebugFlagItem = cfgGetItem(pCfg, "debugFlag"); if (resetArray) { - // reset - if (pDebugFlagItem == NULL) return -1; + // reset + if (pDebugFlagItem == NULL) return -1; - // logflag names that should 'not' be set by 'debugFlag' + // logflag names that should 'not' be set by 'debugFlag' + if (pDebugFlagItem->array == NULL) { + pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar)); if (pDebugFlagItem->array == NULL) { - pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar)); - if (pDebugFlagItem->array == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - taosArrayClear(pDebugFlagItem->array); - return 0; + } + taosArrayClear(pDebugFlagItem->array); + return 0; } // update @@ -401,8 +401,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p case CFG_DTYPE_BOOL: { int32_t ival = (int32_t)atoi(pVal); if (ival != 0 && ival != 1) { - uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, - cfgDtypeStr(pItem->dtype), ival); + uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, cfgDtypeStr(pItem->dtype), ival); terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } @@ -670,6 +669,89 @@ void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p *pLen = len; } +void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump) { + if (dump) { + printf(" s3 config"); + printf("\n"); + printf("================================================================="); + printf("\n"); + } else { + uInfo(" s3 config"); + uInfo("================================================================="); + } + + char src[CFG_SRC_PRINT_LEN + 1] = {0}; + char name[CFG_NAME_PRINT_LEN + 1] = {0}; + + int32_t size = taosArrayGetSize(pCfg->array); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->array, i); + if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; + if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; + if (dump && strncmp(pItem->name, "s3", 2) != 0) continue; + tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); + for (int32_t j = 0; j < CFG_SRC_PRINT_LEN; ++j) { + if (src[j] == 0) src[j] = ' '; + } + + tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN); + for (int32_t j = 0; j < CFG_NAME_PRINT_LEN; ++j) { + if (name[j] == 0) name[j] = ' '; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + if (dump) { + printf("%s %s %u\n", src, name, pItem->bval); + } else { + uInfo("%s %s %u", src, name, pItem->bval); + } + + break; + case CFG_DTYPE_INT32: + if (dump) { + printf("%s %s %d\n", src, name, pItem->i32); + } else { + uInfo("%s %s %d", src, name, pItem->i32); + } + break; + case CFG_DTYPE_INT64: + if (dump) { + printf("%s %s %" PRId64 "\n", src, name, pItem->i64); + } else { + uInfo("%s %s %" PRId64, src, name, pItem->i64); + } + break; + case CFG_DTYPE_DOUBLE: + case CFG_DTYPE_FLOAT: + if (dump) { + printf("%s %s %.2f\n", src, name, pItem->fval); + } else { + uInfo("%s %s %.2f", src, name, pItem->fval); + } + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + case CFG_DTYPE_NONE: + if (dump) { + printf("%s %s %s\n", src, name, pItem->str); + } else { + uInfo("%s %s %s", src, name, pItem->str); + } + break; + } + } + + if (dump) { + printf("=================================================================\n"); + } else { + uInfo("================================================================="); + } +} + void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { if (dump) { printf(" global config"); @@ -717,7 +799,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { break; case CFG_DTYPE_INT64: if (dump) { - printf("%s %s %" PRId64"\n", src, name, pItem->i64); + printf("%s %s %" PRId64 "\n", src, name, pItem->i64); } else { uInfo("%s %s %" PRId64, src, name, pItem->i64); } From 26d66e8b323e2e20cd61b77d407f443cd443440a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 8 Apr 2024 10:33:40 +0800 Subject: [PATCH 09/34] multi-write: shell part --- tools/shell/src/shellAuto.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 847bbcf4be..f19a1f2f27 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -214,6 +214,7 @@ SWords shellCommands[] = { {"insert into using values(", 0, 0, NULL}, {"insert into file ", 0, 0, NULL}, {"trim database ", 0, 0, NULL}, + {"s3migrate database ", 0, 0, NULL}, {"use ", 0, 0, NULL}, {"quit", 0, 0, NULL}}; @@ -283,6 +284,9 @@ char* db_options[] = {"keep ", "wal_level ", "vgroups ", "single_stable ", + "s3_chunksize ", + "s3_keeplocal ", + "s3_compact ", "wal_retention_period ", "wal_roll_period ", "wal_retention_size ", @@ -290,6 +294,7 @@ char* db_options[] = {"keep ", char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "stt_trigger ", "wal_retention_period ", "wal_retention_size ", "cachesize ", + "s3_keeplocal ", "s3_compact ", "wal_fsync_period ", "buffer ", "pages " ,"wal_level "}; char* data_types[] = {"timestamp", "int", From 23d0037c03ab7fad54177a021907d785ba90b14c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 8 Apr 2024 11:02:28 +0800 Subject: [PATCH 10/34] cos/multi-write: fix var name from pCfg to pNewCfg --- source/dnode/mnode/impl/src/mndDb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b8d5f325e5..fde5cd0227 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -486,9 +486,9 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; return -1; } - if (pCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1; - if (pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1; - if (pCfg->s3Compact < TSDB_MIN_S3_COMPACT || pCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1; + if (pNewCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pNewCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1; + if (pNewCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pNewCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1; + if (pNewCfg->s3Compact < TSDB_MIN_S3_COMPACT || pNewCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1; terrno = 0; return terrno; From a491e250fba7435c2e587bd2f89ac4e97a522858 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 8 Apr 2024 13:24:47 +0800 Subject: [PATCH 11/34] cos/multi-write: fix compilation on mac --- source/common/src/cos.c | 1 + tests/script/tsim/db/alter_option.sim | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 95b216c285..990bfdcea3 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -1722,6 +1722,7 @@ int32_t s3Init() { return 0; } void s3CleanUp() {} int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; } int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; } +int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { return 0; } void s3DeleteObjectsByPrefix(const char *prefix) {} int32_t s3DeleteObjects(const char *object_name[], int nobject) { return 0; } bool s3Exists(const char *object_name) { return false; } diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 6c98d43794..749b956a9a 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -46,7 +46,7 @@ print ============= create database # | REPLICA value [1 | 3] # | WAL_LEVEL value [1 | 2] -sql create database db CACHEMODEL 'both' COMP 0 DURATION 240 WAL_FSYNC_PERIOD 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL_LEVEL 2 VGROUPS 6 SINGLE_STABLE 1 +sql create database db CACHEMODEL 'both' COMP 0 DURATION 240 WAL_FSYNC_PERIOD 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 S3_KEEPLOCAL 720 PRECISION 'ns' REPLICA 3 WAL_LEVEL 2 VGROUPS 6 SINGLE_STABLE 1 sql select * from information_schema.ins_databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 From 1dbb91bf6a40e215b3d3cd00e1f0b423f5cc9c1e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 8 Apr 2024 14:27:22 +0800 Subject: [PATCH 12/34] cos/multi-write: default s3_keeplocal to 3650 days --- include/util/tdef.h | 18 +++++++++--------- .../2-query/distribute_agg_apercentile.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 3d680de129..618a909f5d 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -421,7 +421,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024) #define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute #define TSDB_MAX_S3_KEEP_LOCAL (365000 * 1440) -#define TSDB_DEFAULT_S3_KEEP_LOCAL (30 * 1440) +#define TSDB_DEFAULT_S3_KEEP_LOCAL (3650 * 1440) #define TSDB_MIN_S3_COMPACT 0 #define TSDB_MAX_S3_COMPACT 1 #define TSDB_DEFAULT_S3_COMPACT 0 @@ -540,12 +540,12 @@ enum { SND_WORKER_TYPE__UNIQUE, }; -#define DEFAULT_HANDLE 0 -#define MNODE_HANDLE 1 -#define QNODE_HANDLE -1 -#define SNODE_HANDLE -2 -#define VNODE_HANDLE -3 -#define CLIENT_HANDLE -5 +#define DEFAULT_HANDLE 0 +#define MNODE_HANDLE 1 +#define QNODE_HANDLE -1 +#define SNODE_HANDLE -2 +#define VNODE_HANDLE -3 +#define CLIENT_HANDLE -5 #define TSDB_CONFIG_OPTION_LEN 32 #define TSDB_CONFIG_VALUE_LEN 64 @@ -565,8 +565,8 @@ enum { // sort page size by default #define DEFAULT_PAGESIZE 4096 -#define VNODE_TIMEOUT_SEC 60 -#define MNODE_TIMEOUT_SEC 60 +#define VNODE_TIMEOUT_SEC 60 +#define MNODE_TIMEOUT_SEC 60 #define MONITOR_TABLENAME_LEN 200 #define MONITOR_TAG_NAME_LEN 100 diff --git a/tests/system-test/2-query/distribute_agg_apercentile.py b/tests/system-test/2-query/distribute_agg_apercentile.py index 897580fbcc..23ca0b9fae 100644 --- a/tests/system-test/2-query/distribute_agg_apercentile.py +++ b/tests/system-test/2-query/distribute_agg_apercentile.py @@ -18,7 +18,7 @@ class TDTestCase: def prepare_datas_of_distribute(self, dbname="testdb"): # prepate datas for 20 tables distributed at different vgroups - tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") + tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 s3_keeplocal 3000 vgroups 5") tdSql.execute(f" use {dbname} ") tdSql.execute( f'''create table {dbname}.stb1 From ad2e7213026c75b4ebfe995726a55096e69f0421 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 9 Apr 2024 09:20:44 +0800 Subject: [PATCH 13/34] cos/multi-writing: suppress memalign uninitialized value --- source/common/src/systable.c | 6 +++--- tests/script/local.supp | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 66d4972903..94460c6dfa 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -117,9 +117,9 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, - {.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, - {.name = "s3_keeplocal", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, - {.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, + {.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, + {.name = "s3_keeplocal", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, + {.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, }; diff --git a/tests/script/local.supp b/tests/script/local.supp index c351846f2c..60a0dffb95 100644 --- a/tests/script/local.supp +++ b/tests/script/local.supp @@ -1655,3 +1655,25 @@ fun:start_thread fun:clone } + +{ + + Memcheck:Param + write(buf) + fun:__libc_write + fun:write + fun:uv__try_write + fun:uv__write + fun:uv_write2 + fun:uvStartSendRespImpl + fun:uvStartSendResp + fun:uvHandleResp + fun:uvWorkerAsyncCb + fun:uv__async_io + fun:uv__io_poll + fun:uv_run + fun:transWorkerThread + fun:start_thread + fun:clone +} + From 4d08390d565282f752f2e023a09d68b477dc59ec Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 9 Apr 2024 10:06:34 +0800 Subject: [PATCH 14/34] cos/multi-writing: fix chunsize's default value setting --- source/dnode/mnode/impl/src/mndDb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index fde5cd0227..9f2eb2e27d 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -525,7 +525,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; - if (pCfg->s3ChunkSize < 0) pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + if (pCfg->s3ChunkSize <= 0) pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT; if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; @@ -1257,7 +1257,7 @@ static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p if (pIter == NULL) break; if (pArbGroup->dbUid == pDb->uid) { - if (mndSetDropArbGroupPrepareLogs(pTrans,pArbGroup) != 0) { + if (mndSetDropArbGroupPrepareLogs(pTrans, pArbGroup) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pArbGroup); return -1; @@ -1285,7 +1285,7 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD if (pIter == NULL) break; if (pArbGroup->dbUid == pDb->uid) { - if (mndSetDropArbGroupCommitLogs(pTrans,pArbGroup) != 0) { + if (mndSetDropArbGroupCommitLogs(pTrans, pArbGroup) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pArbGroup); return -1; From fde386a7bdb3aba53b029cefd78e3d4cdfc4d03b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 9 Apr 2024 12:29:46 +0800 Subject: [PATCH 15/34] cos/multi-writing: suppress uninitialized memalign --- tests/script/local.supp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/script/local.supp b/tests/script/local.supp index 60a0dffb95..6ff52ebada 100644 --- a/tests/script/local.supp +++ b/tests/script/local.supp @@ -1655,6 +1655,25 @@ fun:start_thread fun:clone } +{ + + Memcheck:Param + write(buf) + fun:__libc_write + fun:write + fun:uv__try_write + fun:uv__write + fun:uv_write2 + fun:uvStartSendRespImpl + fun:uvStartSendResp + fun:uvHandleResp + fun:uvPrepareCb + fun:uv__run_prepare + fun:uv_run + fun:transWorkerThread + fun:start_thread + fun:clone +} { @@ -1676,4 +1695,3 @@ fun:start_thread fun:clone } - From 41bf5280f77f985f6588f5d57e8c309c3be73904 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 9 Apr 2024 12:30:20 +0800 Subject: [PATCH 16/34] cos/multi-writing: fix show create database format --- tests/develop-test/2-query/show_create_db.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index db9ba7b8fd..4ac3d3955c 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -42,17 +42,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") self.restartTaosd(1, dbname='scd') @@ -60,17 +60,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") tdSql.execute('drop database scd') From 0f6e87b3c931c7af219c5099f68679d90ae2798c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 9 Apr 2024 14:53:50 +0800 Subject: [PATCH 17/34] add tag for last row --- source/libs/planner/src/planOptimizer.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index d322174fd0..0cc6a5057d 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2943,6 +2943,13 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic } } } + FOREACH(pColNode, pScan->pScanPseudoCols) { + if (nodesEqualNode(pParamNode, pColNode)) { + if (funcType != FUNCTION_TYPE_LAST) { + nodesListMakeAppend(&pLastRowCols, nodesCloneNode(pColNode)); + } + } + } } } From 20efb35c6edcbf9a8066157bc0680472d5a625de Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 9 Apr 2024 19:09:20 +0800 Subject: [PATCH 18/34] add tag for split plan --- source/libs/planner/src/planOptimizer.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 0cc6a5057d..1eaa37cc03 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3104,6 +3104,10 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg, if (TSDB_CODE_SUCCESS != code) { return code; } + code = createColumnByRewriteExprs(pScan->pScanPseudoCols, &pScan->node.pTargets); + if (TSDB_CODE_SUCCESS != code) { + return code; + } OPTIMIZE_FLAG_CLEAR_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH); } From 3d733370f0a90b4c57ce267a3a2bc3e1cb4a73d1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 11 Apr 2024 10:48:54 +0800 Subject: [PATCH 19/34] mnd: fix mnode dump db info with arb --- source/dnode/mnode/impl/src/mndDb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9f2eb2e27d..9c512caeba 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -2212,6 +2212,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3KeepLocal, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3Compact, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false); } From f55ac240d5bc61bcc3d4c3e0390ba728e9be7234 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 11 Apr 2024 11:45:44 +0800 Subject: [PATCH 20/34] tbname supported --- source/libs/parser/src/parTranslater.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3a37d841aa..8910325912 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2464,6 +2464,20 @@ static int32_t translateFunctionImpl(STranslateContext* pCxt, SFunctionNode** pF static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc) { SNode* pParam = NULL; + if (strcmp((*pFunc)->functionName, "tbname") == 0 && (*pFunc)->pParameterList != NULL) { + pParam = nodesListGetNode((*pFunc)->pParameterList, 0); + if(pParam && nodeType(pParam) == QUERY_NODE_VALUE) { + if (pCxt && pCxt->pCurrStmt && pCxt->pCurrStmt->type == QUERY_NODE_SELECT_STMT && + ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable && + nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) == QUERY_NODE_REAL_TABLE) { + SRealTableNode* pRealTable = (SRealTableNode*)((SSelectStmt*)pCxt->pCurrStmt)->pFromTable; + if (strcmp(((SValueNode*)pParam)->literal, pRealTable->table.tableName) == 0) { + nodesClearList((*pFunc)->pParameterList); + (*pFunc)->pParameterList = NULL; + } + } + } + } FOREACH(pParam, (*pFunc)->pParameterList) { if (isMultiResFunc(pParam)) { pCxt->errCode = TSDB_CODE_FUNC_FUNTION_PARA_NUM; @@ -2822,7 +2836,9 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); } } - if (partionByTbname && QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { + if (partionByTbname && + ((QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) || + (QUERY_NODE_FUNCTION == nodeType(*pNode) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)*pNode)->funcType))) { return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { From 69a7db50b03fb949f5df2c1e99cb89cafcd59722 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 11 Apr 2024 15:40:36 +0800 Subject: [PATCH 21/34] mnd: fix s3_keeplocal's format --- source/common/src/systable.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 94460c6dfa..aca2e71b1f 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -118,7 +118,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "s3_keeplocal", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, + {.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, }; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9c512caeba..e564c7e5a2 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -2208,10 +2208,16 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3ChunkSize, false); + + char keeplocalVstr[128] = {0}; + len = sprintf(&keeplocalVstr[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.s3KeepLocal); + varDataSetLen(keeplocalVstr, len); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3KeepLocal, false); + colDataSetVal(pColInfo, rows, (const char *)keeplocalVstr, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3Compact, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false); } From 356f6685423e3ca3c26a9005e2ae305d03b9835d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 11 Apr 2024 16:26:39 +0800 Subject: [PATCH 22/34] mnd: fix s3_chunksize & s3_keeplocal if zero --- source/dnode/mnode/impl/src/mndDb.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e564c7e5a2..2eed36d3d7 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -245,6 +245,18 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) taosInitRWLatch(&pDb->lock); + if (pDb->cfg.s3ChunkSize == 0) { + pDb->cfg.s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; + + mInfo("db:%s, s3ChunkSize set from %d to default %d", pDb->name, pDb->cfg.s3ChunkSize, TSDB_DEFAULT_S3_CHUNK_SIZE); + } + + if (pDb->cfg.s3KeepLocal == 0) { + pDb->cfg.s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; + + mInfo("db:%s, s3KeepLocal set from %d to default %d", pDb->name, pDb->cfg.s3KeepLocal, TSDB_DEFAULT_S3_KEEP_LOCAL); + } + if (pDb->cfg.tsdbPageSize != TSDB_MIN_TSDB_PAGESIZE) { mInfo("db:%s, tsdbPageSize set from %d to default %d", pDb->name, pDb->cfg.tsdbPageSize, TSDB_DEFAULT_TSDB_PAGESIZE); From 4c7589148ad4ca5cd3e5456bfce8a23a1441f46b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 11 Apr 2024 16:35:10 +0800 Subject: [PATCH 23/34] mnd: fix show create database statement --- source/common/src/tmsg.c | 6 ++---- tests/develop-test/2-query/show_create_db.py | 12 ++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index b5f51e7797..1a619696ec 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4053,10 +4053,10 @@ int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) { if (tEncodeI8(encoder, pRsp->schemaless) < 0) return -1; if (tEncodeI16(encoder, pRsp->sstTrigger) < 0) return -1; if (tEncodeI32(encoder, pRsp->keepTimeOffset) < 0) return -1; + if (tEncodeI8(encoder, pRsp->withArbitrator) < 0) return -1; if (tEncodeI32(encoder, pRsp->s3ChunkSize) < 0) return -1; if (tEncodeI32(encoder, pRsp->s3KeepLocal) < 0) return -1; if (tEncodeI8(encoder, pRsp->s3Compact) < 0) return -1; - if (tEncodeI8(encoder, pRsp->withArbitrator) < 0) return -1; return 0; } @@ -6664,9 +6664,7 @@ int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) { return 0; } -void tDestroySMqHbRsp(SMqHbRsp *pRsp) { - taosArrayDestroy(pRsp->topicPrivileges); -} +void tDestroySMqHbRsp(SMqHbRsp *pRsp) { taosArrayDestroy(pRsp->topicPrivileges); } int32_t tSerializeSMqHbRsp(void *buf, int32_t bufLen, SMqHbRsp *pRsp) { SEncoder encoder = {0}; diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index 4ac3d3955c..4cd4582749 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -42,17 +42,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") self.restartTaosd(1, dbname='scd') @@ -60,17 +60,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 1073742848 S3_KEEPLOCAL 20531m S3_COMPACT 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.execute('drop database scd') From af97f9e0007938bd94db517331c6ed7496a25d4d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 11 Apr 2024 16:48:58 +0800 Subject: [PATCH 24/34] add config --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 8 ++++++- source/libs/parser/src/parTranslater.c | 4 ++-- tests/script/tsim/parser/first_last_query.sim | 4 ++-- tests/script/tsim/parser/join_multitables.sim | 16 ++++++------- tests/script/tsim/parser/last_both_query.sim | 4 ++-- tests/script/tsim/parser/last_cache_query.sim | 4 ++-- tests/script/tsim/parser/lastrow2.sim | 6 ++--- tests/script/tsim/query/bi_star_table.sim | 24 +++++++++---------- tests/script/tsim/query/cache_last.sim | 5 ++-- .../system-test/2-query/last_and_last_row.py | 16 ++++++------- 11 files changed, 49 insertions(+), 43 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 7a7f19c3af..e04488a68c 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -158,6 +158,7 @@ extern int32_t tsMetaCacheMaxSize; extern int32_t tsSlowLogThreshold; extern int32_t tsSlowLogScope; extern int32_t tsTimeSeriesThreshold; +extern bool tsMultiResultFunctionStarReturnTags; // client extern int32_t tsMinSlidingTime; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3381d52050..dae86f1a32 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -170,6 +170,7 @@ int32_t tsMetaCacheMaxSize = -1; // MB int32_t tsSlowLogThreshold = 3; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; int32_t tsTimeSeriesThreshold = 50; +bool tsMultiResultFunctionStarReturnTags = false; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -545,6 +546,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; return 0; } @@ -1103,6 +1106,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; tsExperimental = cfgGetItem(pCfg, "experimental")->bval; + + tsMultiResultFunctionStarReturnTags = cfgGetItem(pCfg, "multiResultFunctionStarReturnTags")->bval; return 0; } @@ -1742,7 +1747,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { {"shellActivityTimer", &tsShellActivityTimer}, {"slowLogThreshold", &tsSlowLogThreshold}, {"useAdapter", &tsUseAdapter}, - {"experimental", &tsExperimental}}; + {"experimental", &tsExperimental}, + {"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags} }; if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { taosCfgSetOption(options, tListLen(options), pItem, false); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1ea0ec8574..778c9b1590 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3368,9 +3368,9 @@ static int32_t createMultiResFuncsParas(STranslateContext* pCxt, SNodeList* pSrc SNode* pPara = NULL; FOREACH(pPara, pSrcParas) { if (nodesIsStar(pPara)) { - code = createAllColumns(pCxt, false, &pExprs); + code = createAllColumns(pCxt, !tsMultiResultFunctionStarReturnTags, &pExprs); } else if (nodesIsTableStar(pPara)) { - code = createTableAllCols(pCxt, (SColumnNode*)pPara, false, &pExprs); + code = createTableAllCols(pCxt, (SColumnNode*)pPara, !tsMultiResultFunctionStarReturnTags, &pExprs); } else { code = nodesListMakeStrictAppend(&pExprs, nodesCloneNode(pPara)); } diff --git a/tests/script/tsim/parser/first_last_query.sim b/tests/script/tsim/parser/first_last_query.sim index 8d02f817e3..533f59f4b2 100644 --- a/tests/script/tsim/parser/first_last_query.sim +++ b/tests/script/tsim/parser/first_last_query.sim @@ -298,7 +298,7 @@ if $data01 != 112 then return -1 endi -if $data03 != @tm0@ then +if $data02 != @tm0@ then return -1 endi @@ -310,7 +310,7 @@ if $data11 != 421 then return -1 endi -if $data13 != @tm1@ then +if $data12 != @tm1@ then return -1 endi diff --git a/tests/script/tsim/parser/join_multitables.sim b/tests/script/tsim/parser/join_multitables.sim index 59e41bc9d9..d2f8ea4a88 100644 --- a/tests/script/tsim/parser/join_multitables.sim +++ b/tests/script/tsim/parser/join_multitables.sim @@ -805,16 +805,16 @@ endi if $data04 != 01 then return -1 endi -if $data[0][10] != @21-03-01 01:00:00.000@ then +if $data05 != @21-03-01 01:00:00.000@ then return -1 endi -if $data[0][11] != 9911 then +if $data06 != 9911 then return -1 endi -if $data[0][12] != 9911.000000000 then +if $data07 != 9911.000000000 then return -1 endi -if $data[0][13] != 11 then +if $data08 != 11 then return -1 endi @@ -837,16 +837,16 @@ endi if $data04 != 05 then return -1 endi -if $data[0][10] != @21-03-01 05:00:00.000@ then +if $data05 != @21-03-01 05:00:00.000@ then return -1 endi -if $data[0][11] != 9915 then +if $data06 != 9915 then return -1 endi -if $data[0][12] != 9915.000000000 then +if $data07 != 9915.000000000 then return -1 endi -if $data[0][13] != 15 then +if $data08 != 15 then return -1 endi diff --git a/tests/script/tsim/parser/last_both_query.sim b/tests/script/tsim/parser/last_both_query.sim index 1cfb2a316f..5f86412199 100644 --- a/tests/script/tsim/parser/last_both_query.sim +++ b/tests/script/tsim/parser/last_both_query.sim @@ -243,11 +243,11 @@ endi if $data04 != @70-01-01 07:59:57.000@ then return -1 endi -if $data06 != @21-05-12 10:10:12.000@ then +if $data05 != @21-05-12 10:10:12.000@ then print $data00 return -1 endi -if $data07 != @70-01-01 07:59:57.000@ then +if $data06 != @70-01-01 07:59:57.000@ then return -1 endi diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index 7bf6a51731..30196e0b62 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -243,11 +243,11 @@ endi if $data04 != @70-01-01 07:59:57.000@ then return -1 endi -if $data06 != @21-05-12 10:10:12.000@ then +if $data05 != @21-05-12 10:10:12.000@ then print $data00 return -1 endi -if $data07 != @70-01-01 07:59:57.000@ then +if $data06 != @70-01-01 07:59:57.000@ then return -1 endi diff --git a/tests/script/tsim/parser/lastrow2.sim b/tests/script/tsim/parser/lastrow2.sim index 278de7ab49..33267e3cfd 100644 --- a/tests/script/tsim/parser/lastrow2.sim +++ b/tests/script/tsim/parser/lastrow2.sim @@ -54,13 +54,13 @@ sql select last_row(*), ts, 'abc', 123.981, tbname from m1 if $rows != 1 then return -1 endi -if $data03 != @19-01-01 01:01:01.000@ then +if $data02 != @19-01-01 01:01:01.000@ then return -1 endi -if $data04 != @abc@ then +if $data03 != @abc@ then return -1 endi -if $data05 != 123.981000000 then +if $data04 != 123.981000000 then print expect 123.981000000, actual: $data04 return -1 endi diff --git a/tests/script/tsim/query/bi_star_table.sim b/tests/script/tsim/query/bi_star_table.sim index 1d71d6a68e..6bd6938678 100644 --- a/tests/script/tsim/query/bi_star_table.sim +++ b/tests/script/tsim/query/bi_star_table.sim @@ -33,29 +33,29 @@ if $data06 != tba1 then endi sql select last(*) from db1.sta; -if $cols != 7 then +if $cols != 4 then return -1 endi -if $data06 != tba2 then +if $data03 != tba2 then return -1 endi sql select last_row(*) from db1.sta; -if $cols != 7 then +if $cols != 4 then return -1 endi -if $data06 != tba2 then +if $data03 != tba2 then return -1 endi sql select first(*) from db1.sta; -if $cols != 7 then +if $cols != 4 then return -1 endi -if $data06 != tba1 then +if $data03 != tba1 then return -1 endi @@ -71,29 +71,29 @@ if $data06 != tba1 then endi sql select last(b.*) from db1.sta b; -if $cols != 7 then +if $cols != 4 then return -1 endi -if $data06 != tba2 then +if $data03 != tba2 then return -1 endi sql select last_row(b.*) from db1.sta b; -if $cols != 7 then +if $cols != 4 then return -1 endi -if $data06 != tba2 then +if $data03 != tba2 then return -1 endi sql select first(b.*) from db1.sta b; -if $cols != 7 then +if $cols != 4 then return -1 endi -if $data06 != tba1 then +if $data03 != tba1 then return -1 endi diff --git a/tests/script/tsim/query/cache_last.sim b/tests/script/tsim/query/cache_last.sim index f936f822a5..65eb46de69 100644 --- a/tests/script/tsim/query/cache_last.sim +++ b/tests/script/tsim/query/cache_last.sim @@ -35,12 +35,11 @@ if $data03 != b then return -1 endi sql explain select count(*), last(*) from sta; -if $data00 != @-> Merge (columns=5 width=230 input_order=unknown output_order=unknown mode=column)@ then - print $data00 +if $data00 != @-> Merge (columns=4 width=226 input_order=unknown output_order=unknown mode=column)@ then return -1 endi sql explain select first(f1), last(*) from sta; -if $data00 != @-> Merge (columns=5 width=230 input_order=unknown output_order=unknown mode=column)@ then +if $data00 != @-> Merge (columns=4 width=226 input_order=unknown output_order=unknown mode=column)@ then return -1 endi sql select first(f1), last(*) from sta; diff --git a/tests/system-test/2-query/last_and_last_row.py b/tests/system-test/2-query/last_and_last_row.py index cd572b05cb..b04b3a75f3 100644 --- a/tests/system-test/2-query/last_and_last_row.py +++ b/tests/system-test/2-query/last_and_last_row.py @@ -152,8 +152,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, last_ts4) tdSql.checkData(0, 1, 5 * maxRange - 1) - tdSql.checkData(0, 3, last_ts4) - tdSql.checkData(0, 4, 4 * maxRange + 1) + tdSql.checkData(0, 2, last_ts4) + tdSql.checkData(0, 3, 4 * maxRange + 1) explain_res = self.explain_sql(sql) self.check_explain_res_no_row("Last Row Scan", explain_res, sql) @@ -302,8 +302,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, last_ts4) tdSql.checkData(0, 1, 5 * maxRange - 1) - tdSql.checkData(0, 3, last_ts4) - tdSql.checkData(0, 4, 4 * maxRange + 1) + tdSql.checkData(0, 2, last_ts4) + tdSql.checkData(0, 3, 4 * maxRange + 1) explain_res = self.explain_sql(sql) self.check_explain_res_has_row("Last Row Scan", explain_res, sql) @@ -453,8 +453,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, last_ts4) tdSql.checkData(0, 1, 5 * maxRange - 1) - tdSql.checkData(0, 3, last_ts4) - tdSql.checkData(0, 4, 4 * maxRange + 1) + tdSql.checkData(0, 2, last_ts4) + tdSql.checkData(0, 3, 4 * maxRange + 1) explain_res = self.explain_sql(sql) self.check_explain_res_has_row("Last Row Scan", explain_res, sql) @@ -587,7 +587,7 @@ class TDTestCase: explain_res = self.explain_sql(sql) self.check_explain_res_has_row("Last Row Scan", explain_res, sql) - self.check_explain_res_has_row("Table Scan", explain_res, sql) + self.check_explain_res_no_row("Table Scan", explain_res, sql) sql = f'select last_row(ts), last(ts), last_row(id), last(id) from last_test_both_model.st;' tdSql.query(sql) @@ -607,7 +607,7 @@ class TDTestCase: tdSql.checkData(0, 0, last_ts4) tdSql.checkData(0, 1, 5 * maxRange - 1) #tdSql.checkData(0, 2, last_ts4) - tdSql.checkData(0, 4, 4 * maxRange + 1) + tdSql.checkData(0, 3, 4 * maxRange + 1) explain_res = self.explain_sql(sql) self.check_explain_res_has_row("Last Row Scan", explain_res, sql) From deff7a76b1f8b9cf397b43bccd8eb854f122ce55 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 11 Apr 2024 18:07:10 +0800 Subject: [PATCH 25/34] add test case --- source/libs/parser/src/parTranslater.c | 15 +- tests/parallel_test/cases.task | 4 + tests/system-test/2-query/tbname.py | 194 +++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 tests/system-test/2-query/tbname.py diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8910325912..32503203e5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2472,7 +2472,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) == QUERY_NODE_REAL_TABLE) { SRealTableNode* pRealTable = (SRealTableNode*)((SSelectStmt*)pCxt->pCurrStmt)->pFromTable; if (strcmp(((SValueNode*)pParam)->literal, pRealTable->table.tableName) == 0) { - nodesClearList((*pFunc)->pParameterList); + NODES_DESTORY_LIST((*pFunc)->pParameterList); (*pFunc)->pParameterList = NULL; } } @@ -2724,11 +2724,14 @@ static bool hasTbnameFunction(SNodeList* pPartitionByList) { return false; } -static bool fromSubtable(SNode* table) { +static bool fromSingleTable(SNode* table) { if (NULL == table) return false; - if (table->type == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)table)->pMeta && - ((SRealTableNode*)table)->pMeta->tableType == TSDB_CHILD_TABLE) { - return true; + if (table->type == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)table)->pMeta) { + int8_t type = ((SRealTableNode*)table)->pMeta->tableType; + if(type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE + || type == TSDB_SYSTEM_TABLE) { + return true; + } } return false; } @@ -2827,7 +2830,7 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { } SNode* pPartKey = NULL; bool partionByTbname = false; - if (fromSubtable(((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pFromTable) || + if (fromSingleTable(((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pFromTable) || hasTbnameFunction(((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pPartitionByList)) { partionByTbname = true; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 2402f6a94a..43d7c2b765 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -105,6 +105,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py diff --git a/tests/system-test/2-query/tbname.py b/tests/system-test/2-query/tbname.py new file mode 100644 index 0000000000..bb2bb43d14 --- /dev/null +++ b/tests/system-test/2-query/tbname.py @@ -0,0 +1,194 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def td29092(self, dbname="db"): + tdSql.execute(f'use {dbname}') + tdSql.execute('CREATE STABLE `st` (`ts` TIMESTAMP, `v1` INT) TAGS (`t1` INT);') + tdSql.execute('CREATE STABLE `st2` (`ts` TIMESTAMP, `v1` INT) TAGS (`t1` INT);') + tdSql.execute('CREATE TABLE `t1` USING `st` (`t1`) TAGS (1);') + tdSql.execute('CREATE TABLE `t2` USING `st` (`t1`) TAGS (2);') + tdSql.execute('CREATE TABLE `t21` USING `st2` (`t1`) TAGS (21);') + tdSql.execute('CREATE TABLE `nt` (`ts` TIMESTAMP, `v1` INT);') + + now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + for i in range(3): + tdSql.execute( + f"insert into {dbname}.t1 values ( { now_time + i * 1000 }, {i} )" + ) + tdSql.execute( + f"insert into {dbname}.t2 values ( { now_time + i * 1000 }, {i} )" + ) + tdSql.execute( + f"insert into {dbname}.nt values ( { now_time + i * 1000 }, {i} )" + ) + + tdLog.debug(f"-------------- step1: normal table test ------------------") + tdSql.query("select tbname, count(*) from nt;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "nt") + tdSql.checkData(0, 1, 3) + + tdSql.query("select nt.tbname, count(*) from nt;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "nt") + tdSql.checkData(0, 1, 3) + + tdSql.query("select tbname, count(*) from nt group by tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "nt") + tdSql.checkData(0, 1, 3) + + tdSql.query("select nt.tbname, count(*) from nt group by tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "nt") + tdSql.checkData(0, 1, 3) + + tdSql.query("select nt.tbname, count(*) from nt group by nt.tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "nt") + tdSql.checkData(0, 1, 3) + + tdLog.debug(f"-------------- step2: system table test ------------------") + tdSql.query("select tbname, count(*) from information_schema.ins_dnodes") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + tdSql.query("select ins_dnodes.tbname, count(*) from information_schema.ins_dnodes") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + tdSql.query("select tbname, count(*) from information_schema.ins_dnodes group by tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + tdSql.query("select ins_dnodes.tbname, count(*) from information_schema.ins_dnodes group by tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + tdSql.query("select ins_dnodes.tbname, count(*) from information_schema.ins_dnodes group by ins_dnodes.tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + tdLog.debug(f"-------------- step3: subtable test ------------------") + tdSql.query("select tbname, count(*) from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 3) + + tdSql.query("select t1.tbname, count(*) from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 3) + + tdSql.query("select tbname, count(*) from t1 group by tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 3) + + tdSql.query("select t1.tbname, count(*) from t1 group by tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 3) + + tdSql.query("select t1.tbname, count(*) from t1 group by t1.tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 3) + + tdSql.error("select t1.tbname, count(*) from t2 group by t1.tbname") + tdSql.error("select t1.tbname, count(*) from t1 group by t2.tbname") + tdSql.error("select t2.tbname, count(*) from t1 group by t1.tbname") + + tdLog.debug(f"-------------- step4: super table test ------------------") + tdSql.query("select tbname, count(*) from st group by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 3) + tdSql.checkData(1, 1, 3) + + tdSql.query("select tbname, count(*) from st partition by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 3) + tdSql.checkData(1, 1, 3) + + tdSql.query("select ts, t1 from st where st.tbname=\"t1\"") + tdSql.checkRows(3) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 1) + + tdSql.query("select tbname, ts from st where tbname=\"t2\"") + tdSql.checkRows(3) + + tdSql.query("select tbname, ts from st where tbname=\"t2\" order by tbname") + tdSql.checkRows(3) + + tdSql.query("select tbname, ts from st where tbname=\"t2\" order by st.tbname") + tdSql.checkRows(3) + + tdSql.query("select tbname, count(*) from st where tbname=\"t2\" group by tbname order by tbname") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 3) + + tdSql.query("select tbname, count(*) from st group by tbname order by tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 3) + tdSql.checkData(1, 1, 3) + + tdSql.query("select tbname, count(*) from st group by st.tbname order by st.tbname") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 3) + tdSql.checkData(1, 1, 3) + + tdLog.debug(f"-------------- step4: join test ------------------") + tdSql.query("select t1.tbname, t2.tbname from t1, t2 where t1.ts=t2.ts and t1.tbname!=t2.tbname") + tdSql.checkRows(3) + + tdSql.query("select t1.tbname, t2.tbname from t1, t2 where t1.ts=t2.ts and t1.tbname!=t2.tbname order by t1.tbname") + tdSql.checkRows(3) + + tdSql.query("select st.tbname, st2.tbname from st, st2 where st.ts=st2.ts and st.tbname!=st2.tbname order by st.tbname") + tdSql.checkRows(0) + + tdSql.execute(f"insert into t21 values ( { now_time + 1000 }, 1 )") + tdSql.query("select st.tbname, st2.tbname from st, st2 where st.ts=st2.ts and st.tbname!=st2.tbname order by st.tbname") + tdSql.checkRows(2) + + tdSql.query("select t1.tbname, st2.tbname from t1, st2 where t1.ts=st2.ts and t1.tbname!=st2.tbname order by t1.tbname") + tdSql.checkRows(1) + + tdSql.query("select nt.ts, st.tbname from nt, st where nt.ts=st.ts order by st.tbname") + tdSql.checkRows(6) + + tdSql.query("select nt.ts, t1.tbname from nt, t1 where nt.ts=t1.ts order by t1.tbname") + tdSql.checkRows(3) + + def run(self): + tdSql.prepare() + + self.td29092() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From d4ebd2ec71cb7a1cb10928d95baada6c7bd132ee Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 11 Apr 2024 18:16:24 +0800 Subject: [PATCH 26/34] add ci --- tests/script/tsim/query/cache_last_tag.sim | 189 +++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 tests/script/tsim/query/cache_last_tag.sim diff --git a/tests/script/tsim/query/cache_last_tag.sim b/tests/script/tsim/query/cache_last_tag.sim new file mode 100644 index 0000000000..458254625f --- /dev/null +++ b/tests/script/tsim/query/cache_last_tag.sim @@ -0,0 +1,189 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql alter local "multiResultFunctionStarReturnTags" "0"; + +print step1===================== +sql drop database if exists test; +sql create database test vgroups 4 CACHEMODEL 'both'; +sql use test; +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); +sql create table t4 using st tags(NULL,4,4); + +sql insert into t1 values(1648791211000,1,1,1); +sql insert into t1 values(1648791211001,2,2,2); +sql insert into t2 values(1648791211002,3,3,3); +sql insert into t2 values(1648791211003,4,4,4); +sql insert into t3 values(1648791211004,5,5,5); +sql insert into t3 values(1648791211005,6,6,6); +sql insert into t4 values(1648791211007,NULL,NULL,NULL); + +sql select last(*),last_row(*) from st; + +if $cols != 8 then + print ======cols=$cols + return -1 +endi + +sql alter local "multiResultFunctionStarReturnTags" "1"; + +sql select last(*),last_row(*) from st; + +if $cols != 14 then + print ======cols=$cols + return -1 +endi + +sql select last(*) from st; + +if $cols != 7 then + return -1 +endi + +sql select last_row(*) from st; + +if $cols != 7 then + return -1 +endi + +sql select last(*),last_row(*) from t1; + +if $cols != 8 then + return -1 +endi + +print step2===================== + +sql drop database if exists test1; +sql create database test1 vgroups 4 CACHEMODEL 'last_row'; +sql use test1; +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); +sql create table t4 using st tags(NULL,4,4); + +sql insert into t1 values(1648791211000,1,1,1); +sql insert into t1 values(1648791211001,2,2,2); +sql insert into t2 values(1648791211002,3,3,3); +sql insert into t2 values(1648791211003,4,4,4); +sql insert into t3 values(1648791211004,5,5,5); +sql insert into t3 values(1648791211005,6,6,6); +sql insert into t4 values(1648791211007,NULL,NULL,NULL); + +sql select last(*),last_row(*) from st; + +if $cols != 14 then + return -1 +endi + +sql select last(*) from st; + +if $cols != 7 then + return -1 +endi + +return -1 + +sql select last_row(*) from st; + +if $cols != 7 then + return -1 +endi + +sql select last(*),last_row(*) from t1; + +if $cols != 8 then + return -1 +endi + +print step3===================== + +sql drop database if exists test2; +sql create database test2 vgroups 4 CACHEMODEL 'last_value'; +sql use test2; +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); +sql create table t4 using st tags(NULL,4,4); + +sql insert into t1 values(1648791211000,1,1,1); +sql insert into t1 values(1648791211001,2,2,2); +sql insert into t2 values(1648791211002,3,3,3); +sql insert into t2 values(1648791211003,4,4,4); +sql insert into t3 values(1648791211004,5,5,5); +sql insert into t3 values(1648791211005,6,6,6); +sql insert into t4 values(1648791211007,NULL,NULL,NULL); + +sql select last(*),last_row(*) from st; + +if $cols != 14 then + return -1 +endi + +sql select last(*) from st; + +if $cols != 7 then + return -1 +endi + +sql select last_row(*) from st; + +if $cols != 7 then + return -1 +endi + +sql select last(*),last_row(*) from t1; + +if $cols != 8 then + return -1 +endi + +sql drop database if exists test4; +sql create database test4 vgroups 4; +sql use test4; +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); +sql create table t4 using st tags(NULL,4,4); + +sql insert into t1 values(1648791211000,1,1,1); +sql insert into t1 values(1648791211001,2,2,2); +sql insert into t2 values(1648791211002,3,3,3); +sql insert into t2 values(1648791211003,4,4,4); +sql insert into t3 values(1648791211004,5,5,5); +sql insert into t3 values(1648791211005,6,6,6); +sql insert into t4 values(1648791211007,NULL,NULL,NULL); + +sql select last(*),last_row(*) from st; + +if $cols != 14 then + return -1 +endi + +sql select last(*) from st; + +if $cols != 7 then + return -1 +endi + +sql select last_row(*) from st; + +if $cols != 7 then + return -1 +endi + +sql select last(*),last_row(*) from t1; + +if $cols != 8 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From d9aaa3d083bfeb190f29ae92506efd5156435c83 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:06:41 +0800 Subject: [PATCH 27/34] Update index.md --- docs/zh/14-reference/12-config/index.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 4d47f0771c..607b5881bd 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -230,6 +230,16 @@ taos -C | 缺省值 | 1 | | 补充说明 | 该参数设置为 1 时，如果查询中含有 GROUP BY，PARTITION BY 以及 INTERVAL 子句且相应的组或窗口内数据为空或者NULL， 对应的组或窗口将不返回查询结果 | +### multiResultFunctionStarReturnTags + +| 属性 | 说明 | +| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| 适用范围 | 仅客户端适用 | +| 含义 | 查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列 | +| 取值范围 | 0：不返回标签列，1：返回标签列 | +| 缺省值 | 0 | +| 补充说明 | 该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列 | + ## 区域相关 ### timezone From 678b3126e9fcc893a8047e3778e48512af98b355 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:15:36 +0800 Subject: [PATCH 28/34] Update index.md --- docs/zh/14-reference/12-config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 607b5881bd..1bada64431 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -235,7 +235,7 @@ taos -C | 属性 | 说明 | | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | | 适用范围 | 仅客户端适用 | -| 含义 | 查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列 | +| 含义 | 查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列；查询普通表、子表时，不受该参数影响。 | | 取值范围 | 0：不返回标签列，1：返回标签列 | | 缺省值 | 0 | | 补充说明 | 该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列 | From f0a7f53dc8dde5710619dbf2ec22d378bdff8809 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:21:14 +0800 Subject: [PATCH 29/34] Update 10-function.md --- docs/zh/12-taos-sql/10-function.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 0482022d95..81b6732f6b 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -954,7 +954,7 @@ FIRST(expr) **使用说明**: -- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)； +- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 时，这个是默认值，FIRST(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。 - 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL； - 如果结果集中所有列全部为 NULL 值，则不返回结果。 @@ -1006,7 +1006,7 @@ LAST(expr) **使用说明**: -- 如果要返回各个列的最后（时间戳最大）一个非 NULL 值，可以使用 LAST(\*)； +- 如果要返回各个列的最后（时间戳最大）一个非 NULL 值，可以使用 LAST(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 时，这个是默认值，LAST(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。 - 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL；如果结果集中所有列全部为 NULL 值，则不返回结果。 - 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。 @@ -1026,7 +1026,7 @@ LAST_ROW(expr) **适用于**：表和超级表。 **使用说明**： - +- 如果要返回各个列的最后一条记录（时间戳最大），可以使用 LAST_ROW(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 时，这个是默认值，LAST_ROW(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。 - 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。 - 不能与 INTERVAL 一起使用。 From e14d4e172d390165c1a910cb50fd73616f314a04 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 15 Apr 2024 09:30:24 +0800 Subject: [PATCH 30/34] cos/multi-writing: fix remove with different version --- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index a2cdd9f381..0820f53117 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -386,7 +386,11 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f fobj2->f[0] = fobj1->f[0]; } } else { - tsdbTFileObjRemoveUpdateLC(fobj2); + if (fobj1->f->cid != fobj2->f->cid) { + tsdbTFileObjRemove(fobj2); + } else { + tsdbTFileObjRemoveUpdateLC(fobj2); + } code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]); if (code) return code; } From 34a15d1431eefb8aaf6d1923f9337ed0dc0c43a8 Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:07:09 +0800 Subject: [PATCH 31/34] Update 10-function.md --- docs/zh/12-taos-sql/10-function.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 81b6732f6b..71accc6322 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -954,7 +954,7 @@ FIRST(expr) **使用说明**: -- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 时，这个是默认值，FIRST(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。 +- 如果要返回各个列的首个（时间戳最小）非 NULL 值，可以使用 FIRST(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 (默认值) 时，FIRST(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。 - 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL； - 如果结果集中所有列全部为 NULL 值，则不返回结果。 @@ -1006,7 +1006,7 @@ LAST(expr) **使用说明**: -- 如果要返回各个列的最后（时间戳最大）一个非 NULL 值，可以使用 LAST(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 时，这个是默认值，LAST(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。 +- 如果要返回各个列的最后（时间戳最大）一个非 NULL 值，可以使用 LAST(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 (默认值) 时，LAST(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。 - 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL；如果结果集中所有列全部为 NULL 值，则不返回结果。 - 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。 @@ -1026,7 +1026,7 @@ LAST_ROW(expr) **适用于**：表和超级表。 **使用说明**： -- 如果要返回各个列的最后一条记录（时间戳最大），可以使用 LAST_ROW(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 时，这个是默认值，LAST_ROW(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。 +- 如果要返回各个列的最后一条记录（时间戳最大），可以使用 LAST_ROW(\*)；查询超级表，且multiResultFunctionStarReturnTags设置为 0 (默认值) 时，LAST_ROW(\*)只返回超级表的普通列；设置为 1 时，返回超级表的普通列和标签列。 - 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。 - 不能与 INTERVAL 一起使用。 From af5af1c6ea6e5eef4f2337f8df4c670fb01f7c47 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:28:29 +0800 Subject: [PATCH 32/34] Update index.md --- docs/en/14-reference/12-config/index.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index af88978603..0bdf143a60 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -231,6 +231,16 @@ Please note the `taoskeeper` needs to be installed and running to create the `lo | Default Value | 0 | | Notes | When multiple of the above functions act on the same column at the same time and no alias is specified, if the order by clause refers to the column name, column selection ambiguous will occur because the aliases of multiple columns are the same. | +### multiResultFunctionStarReturnTags + +| Attribute | Description | +| ------------- | --------------------------------------------------------------------------------------------------------------- | +| Applicable | Client only | +| Meaning | When querying a super table, whether last(\*)/last_row(\*)/first(\*) returns tags is affected by this parameter. When querying a normal table or subtable, this parameter has no effect. | +| Value Range | 0: do not return tags, 1: return tags | +| Default Value | 0 | +| Notes | When this parameter is set to 0, last(\*)/last_row(\*)/first(\*) only returns the columns of the super table; When it is 1, return the columns and tags of the super table. | + ## Locale Parameters ### timezone From fd9ed4fb70f19b98422bf6a63eeb2f6422af4bd4 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:37:25 +0800 Subject: [PATCH 33/34] Update 10-function.md --- docs/en/12-taos-sql/10-function.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index b4f1cf65da..89ff8aeb5e 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -952,7 +952,7 @@ FIRST(expr) **More explanation**: -- FIRST(\*) can be used to get the first non-null value of all columns +- FIRST(\*) can be used to get the first non-null value of all columns;When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), FIRST(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table. - NULL will be returned if all the values of the specified column are all NULL - A result will NOT be returned if all the columns in the result set are all NULL @@ -1014,7 +1014,7 @@ LAST(expr) **More explanation**: -- LAST(\*) can be used to get the last non-NULL value of all columns +- LAST(\*) can be used to get the last non-NULL value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), LAST(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table. - If the values of a column in the result set are all NULL, NULL is returned for that column; if all columns in the result are all NULL, no result will be returned. - When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times. @@ -1035,6 +1035,7 @@ LAST_ROW(expr) **More explanations**: +- LAST_ROW(\*) can be used to get the last value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), LAST_ROW(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table. - When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times. - Can't be used with `INTERVAL`. From 01cbd7fb4994510624dad1fddee750422e99be62 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:38:08 +0800 Subject: [PATCH 34/34] Update 10-function.md --- docs/en/12-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 89ff8aeb5e..07be7ae5ce 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -952,7 +952,7 @@ FIRST(expr) **More explanation**: -- FIRST(\*) can be used to get the first non-null value of all columns;When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), FIRST(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table. +- FIRST(\*) can be used to get the first non-null value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), FIRST(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table. - NULL will be returned if all the values of the specified column are all NULL - A result will NOT be returned if all the columns in the result set are all NULL