chore: merge 3.0
This commit is contained in:
Haojun Liao
commit
35c790d02f
|
|
@ -84,12 +84,9 @@ TDengine 采用“一个数据采集点一张表”的设计虽然有利于高
|
|||
为了解决这个问题，TDengine 引入虚拟表（Virtual Table，简称为 VTable）的概念。虚拟表是一种不存储实际数据而可以用于分析计算的表，它的数据来源为其它真实存储数据的子表、普通表，通过将各个原始表的不同列的数据按照时间戳排序、对齐、合并的方式来生成虚拟表。同真实表类似，虚拟表也可以分为虚拟超级表、虚拟子表、虚拟普通表。虚拟超级表可以是一个设备或一组分析计算所需数据的完整集合，每个虚拟子表可以根据需要引用相同或不同的列，因此可以灵活地根据业务需要进行定义，最终达到“千人千面”的效果。虚拟表不能写入、删除数据，在查询使用上同真实表相同。TDengine 支持虚拟超级表、虚拟子表、虚拟普通表上的任何查询。唯一的区别在于虚拟表的数据是每次查询计算时动态生成的，只有一个查询中引用的列才会被合并进虚拟表中，因此同一个虚拟表在不同的查询中所呈现以及扫描的数据可能是完全不同的。
|
||||
|
||||
虚拟超级表的主要功能特点包括：
|
||||
1. 列选择与拼接 <br />
|
||||
用户可以从多个原始表中选择指定的列，按需组合到一张虚拟表中，形成统一的数据视图。
|
||||
2. 基于时间戳对齐 <br />
|
||||
以时间戳为依据对数据进行对齐，如果多个表在相同时间戳下存在数据，则对应列的值组合成同一行；若部分表在该时间戳下无数据，则对应列填充为 NULL。
|
||||
3. 动态更新 <br />
|
||||
虚拟表根据原始表的数据变化自动更新，确保数据的实时性。虚拟表不需实际存储，计算在生成时动态完成。
|
||||
1. 列选择与拼接：用户可以从多个原始表中选择指定的列，按需组合到一张虚拟表中，形成统一的数据视图。
|
||||
2. 基于时间戳对齐：以时间戳为依据对数据进行对齐，如果多个表在相同时间戳下存在数据，则对应列的值组合成同一行；若部分表在该时间戳下无数据，则对应列填充为 NULL。
|
||||
3. 动态更新：虚拟表根据原始表的数据变化自动更新，确保数据的实时性。虚拟表不需实际存储，计算在生成时动态完成。
|
||||
|
||||
通过引入虚拟表的概念，TDengine 可以非常方便的管理更大更复杂的设备数据。无论每个采集点如何建模（单列 or 多列），无论这些采集点的数据是分布在一个或多个库中，都可以通过定义虚拟表的方式跨库跨表任意指定数据源，通过虚拟超级表的方式进行跨数据采集点、跨分析的聚合运算，从此“一个设备一张表”彻底成为现实。
|
||||
|
||||
|
|
|
|||
|
|
@ -10784,6 +10784,18 @@ static const SSchema* getNormalColSchema(const STableMeta* pTableMeta, const cha
|
|||
return NULL;
|
||||
}
|
||||
|
||||
static const col_id_t getNormalColSchemaIndex(const STableMeta* pTableMeta, const char* pColName) {
|
||||
int32_t numOfCols = getNumOfColumns(pTableMeta);
|
||||
SSchema* pColsSchema = getTableColumnSchema(pTableMeta);
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
const SSchema* pSchema = pColsSchema + i;
|
||||
if (0 == strcmp(pColName, pSchema->name)) {
|
||||
return (col_id_t)i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static SSchema* getTagSchema(const STableMeta* pTableMeta, const char* pTagName) {
|
||||
int32_t numOfTags = getNumOfTags(pTableMeta);
|
||||
SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
|
||||
|
|
@ -16065,11 +16077,12 @@ static int32_t buildVirtualSubTableBatchReq(const SCreateVSubTableStmt* pStmt, S
|
|||
if (pStmt->pSpecificColRefs) {
|
||||
FOREACH(pCol, pStmt->pSpecificColRefs) {
|
||||
SColumnRefNode* pColRef = (SColumnRefNode*)pCol;
|
||||
const SSchema* pSchema = getColSchema(pStbMeta, pColRef->colName);
|
||||
if (pSchema == NULL) {
|
||||
col_id_t schemaIdx = getNormalColSchemaIndex(pStbMeta, pColRef->colName);
|
||||
if (schemaIdx == -1) {
|
||||
PAR_ERR_JRET(TSDB_CODE_PAR_INVALID_COLUMN);
|
||||
}
|
||||
PAR_ERR_JRET(setColRef(&req.colRef.pColRef[pSchema->colId - 1], pSchema->colId, pColRef->refColName, pColRef->refTableName, pColRef->refDbName));
|
||||
const SSchema* pSchema = getTableColumnSchema(pStbMeta) + schemaIdx;
|
||||
PAR_ERR_JRET(setColRef(&req.colRef.pColRef[schemaIdx], pSchema->colId, pColRef->refColName, pColRef->refTableName, pColRef->refDbName));
|
||||
}
|
||||
} else if (pStmt->pColRefs){
|
||||
col_id_t index = 1; // start from second column, don't set column ref for ts column
|
||||
|
|
|
|||
|
|
@ -22,7 +22,10 @@ class TDTestCase(TBase):
|
|||
|
||||
def prepare_vtables(self):
|
||||
tdSql.execute("drop table if exists vtb_virtual_stb;")
|
||||
tdSql.execute("drop table if exists vtb_virtual_stb_1;")
|
||||
tdSql.execute("drop table if exists vtb_virtual_ctb0;")
|
||||
tdSql.execute("drop table if exists vtb_virtual_ctb_after_modified_1;")
|
||||
tdSql.execute("drop table if exists vtb_virtual_ctb_after_modified_2;")
|
||||
tdSql.execute("drop table if exists vtb_virtual_ntb0;")
|
||||
tdLog.info(f"prepare virtual super tables.")
|
||||
tdSql.execute(f"CREATE STABLE `vtb_virtual_stb` ("
|
||||
|
|
@ -55,6 +58,36 @@ class TDTestCase(TBase):
|
|||
"binary_32_tag binary(32))"
|
||||
"VIRTUAL 1")
|
||||
|
||||
tdSql.execute(f"CREATE STABLE `vtb_virtual_stb_1` ("
|
||||
"ts timestamp, "
|
||||
"u_tinyint_col tinyint unsigned, "
|
||||
"u_smallint_col smallint unsigned, "
|
||||
"u_int_col int unsigned, "
|
||||
"u_bigint_col bigint unsigned, "
|
||||
"tinyint_col tinyint, "
|
||||
"smallint_col smallint, "
|
||||
"int_col int, "
|
||||
"bigint_col bigint, "
|
||||
"float_col float, "
|
||||
"double_col double, "
|
||||
"bool_col bool, "
|
||||
"binary_16_col binary(16),"
|
||||
"binary_32_col binary(32),"
|
||||
"nchar_16_col nchar(16),"
|
||||
"nchar_32_col nchar(32),"
|
||||
"varbinary_16_col varbinary(16),"
|
||||
"varbinary_32_col varbinary(32),"
|
||||
"geo_16_col geometry(16),"
|
||||
"geo_32_col geometry(32)"
|
||||
") TAGS ("
|
||||
"int_tag int,"
|
||||
"bool_tag bool,"
|
||||
"float_tag float,"
|
||||
"double_tag double,"
|
||||
"nchar_32_tag nchar(32),"
|
||||
"binary_32_tag binary(32))"
|
||||
"VIRTUAL 1")
|
||||
|
||||
tdLog.info(f"prepare virtual child tables.")
|
||||
tdSql.execute("CREATE VTABLE `vtb_virtual_ctb0`("
|
||||
"u_tinyint_col FROM vtb_org_child_0.u_tinyint_col, "
|
||||
|
|
@ -301,6 +334,27 @@ class TDTestCase(TBase):
|
|||
tdSql.query(f"select tag_type from information_schema.ins_tags where db_name='test_vtable_alter' and table_name='vtb_virtual_ctb0' and tag_name='nchar_32_tag'")
|
||||
tdSql.checkData(0, 0, "NCHAR(64)")
|
||||
|
||||
def test_alter_virtual_super_table_and_create_child(self):
|
||||
tdLog.info(f"test alter virtual super tables and create child.")
|
||||
|
||||
tdSql.execute("use test_vtable_alter;")
|
||||
tdSql.execute("select database();")
|
||||
|
||||
# 1. add column
|
||||
# 1.1. add column without column reference
|
||||
tdSql.execute("alter stable vtb_virtual_stb_1 add column extra_boolcol bool")
|
||||
|
||||
# 1.2. create child table using modified super table
|
||||
tdSql.execute("CREATE VTABLE `vtb_virtual_ctb_after_modified_1`(bool_col from vtb_org_child_18.bool_col, extra_boolcol from vtb_org_child_19.bool_col) USING vtb_virtual_stb_1 TAGS (0, false, 0, 0, 'vchild0', 'vchild0')")
|
||||
tdSql.execute("select * from vtb_virtual_ctb_after_modified_1")
|
||||
|
||||
# 2. drop column
|
||||
# 2.1. drop column from stb
|
||||
tdSql.execute("alter stable vtb_virtual_stb_1 drop column bool_col;")
|
||||
tdSql.execute("CREATE VTABLE `vtb_virtual_ctb_after_modified_2`(extra_boolcol from vtb_org_child_17.bool_col) USING vtb_virtual_stb_1 TAGS (0, false, 0, 0, 'vchild0', 'vchild0')")
|
||||
|
||||
tdSql.execute("select * from vtb_virtual_ctb_after_modified_2")
|
||||
|
||||
|
||||
def test_error_cases(self):
|
||||
tdLog.info(f"test alter virtual super tables.")
|
||||
|
|
@ -360,6 +414,7 @@ class TDTestCase(TBase):
|
|||
self.test_alter_virtual_normal_table()
|
||||
self.test_alter_virtual_child_table()
|
||||
self.test_alter_virtual_super_table()
|
||||
self.test_alter_virtual_super_table_and_create_child()
|
||||
self.test_error_cases()
|
||||
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
|
|
|||
Loading…
Reference in New Issue