xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Enh(insert): use mini cache in stmt to improve insert speed.

This commit is contained in:
xiao-77 2025-03-04 20:10:40 +08:00
parent 97286faa8d
commit 7f5d687882
4 changed files with 28 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/libs/nodes/querynodes.h
View File

@ -571,6 +571,7 @@ typedef struct SVnodeModifyOpStmt {
SHashObj* pVgroupsHashObj; // SHashObj<vgId, SVgInfo> SHashObj* pVgroupsHashObj; // SHashObj<vgId, SVgInfo>
SHashObj* pTableBlockHashObj; // SHashObj<tuid, STableDataCxt*> SHashObj* pTableBlockHashObj; // SHashObj<tuid, STableDataCxt*>
SHashObj* pSubTableHashObj; // SHashObj<table_name, STableMeta*> SHashObj* pSubTableHashObj; // SHashObj<table_name, STableMeta*>
SHashObj* pSuperTableHashObj; // SHashObj<table_name, STableMeta*>
SHashObj* pTableNameHashObj; // set of table names for refreshing meta, sync mode SHashObj* pTableNameHashObj; // set of table names for refreshing meta, sync mode
SHashObj* pDbFNameHashObj; // set of db names for refreshing meta, sync mode SHashObj* pDbFNameHashObj; // set of db names for refreshing meta, sync mode
SHashObj* pTableCxtHashObj; // temp SHashObj<tuid, STableDataCxt*> for single request SHashObj* pTableCxtHashObj; // temp SHashObj<tuid, STableDataCxt*> for single request

1
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -1325,6 +1325,7 @@ void nodesDestroyNode(SNode* pNode) {
taosArrayDestroy(pStmt->pTableTag); taosArrayDestroy(pStmt->pTableTag);
taosHashCleanup(pStmt->pVgroupsHashObj); taosHashCleanup(pStmt->pVgroupsHashObj);
taosHashCleanup(pStmt->pSubTableHashObj); taosHashCleanup(pStmt->pSubTableHashObj);
taosHashCleanup(pStmt->pSuperTableHashObj);
taosHashCleanup(pStmt->pTableNameHashObj); taosHashCleanup(pStmt->pTableNameHashObj);
taosHashCleanup(pStmt->pDbFNameHashObj); taosHashCleanup(pStmt->pDbFNameHashObj);
taosHashCleanup(pStmt->pTableCxtHashObj); taosHashCleanup(pStmt->pTableCxtHashObj);

17
source/libs/parser/src/parInsertSql.c
View File

@ -1325,8 +1325,22 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (!pCxt->missCache) { if (!pCxt->missCache) {
char tbFName[TSDB_TABLE_FNAME_LEN];
code = tNameExtractFullName(&pStmt->usingTableName, tbFName);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
STableMeta** ppStableMeta = taosHashGet(pStmt->pSuperTableHashObj, tbFName, strlen(tbFName));
if (NULL != ppStableMeta) {
pStableMeta = *ppStableMeta;
}
if (NULL == pStableMeta) {
bool bUsingTable = true; bool bUsingTable = true;
code = getTableMeta(pCxt, &pStmt->usingTableName, &pStableMeta, &pCxt->missCache, bUsingTable); code = getTableMeta(pCxt, &pStmt->usingTableName, &pStableMeta, &pCxt->missCache, bUsingTable);
if (TSDB_CODE_SUCCESS == code) {
code = taosHashPut(pStmt->pSuperTableHashObj, tbFName, strlen(tbFName), &pStableMeta, POINTER_BYTES);
}
}
} }
if (pCxt->isStmtBind) { if (pCxt->isStmtBind) {
goto _no_ctb_cache; goto _no_ctb_cache;
@ -1349,7 +1363,6 @@ _no_ctb_cache:
code = cloneTableMeta(pStableMeta, &pStmt->pTableMeta); code = cloneTableMeta(pStableMeta, &pStmt->pTableMeta);
} }
} }
taosMemoryFree(pStableMeta);
taosMemoryFree(pCtableMeta); taosMemoryFree(pCtableMeta);
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
code = getTargetTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache); code = getTargetTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache);
@ -2847,6 +2860,7 @@ static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, S
} }
} }
pStmt->pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); pStmt->pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
pStmt->pSuperTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
pStmt->pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); pStmt->pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
pStmt->pDbFNameHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); pStmt->pDbFNameHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
if ((!reentry && (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj)) || if ((!reentry && (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj)) ||
@ -2856,6 +2870,7 @@ static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, S
} }
taosHashSetFreeFp(pStmt->pSubTableHashObj, destroySubTableHashElem); taosHashSetFreeFp(pStmt->pSubTableHashObj, destroySubTableHashElem);
taosHashSetFreeFp(pStmt->pSuperTableHashObj, destroySubTableHashElem);
*pOutput = (SNode*)pStmt; *pOutput = (SNode*)pStmt;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

16
tests/army/insert/auto_create_bench.py
View File

@ -100,15 +100,15 @@ if __name__ == "__main__":
precreate_tables() precreate_tables()
# 测试场景1：自动建表插入 # 测试场景1：自动建表插入
auto_create_time = test_auto_create_tables() auto_create_time = test_auto_create_tables()
# 清理环境并重新初始化 # # 清理环境并重新初始化
prepare_database() # prepare_database()
# 预创建所有子表 # # 预创建所有子表
precreate_tables() # precreate_tables()
# 测试场景2：直接插入 # # 测试场景2：直接插入
direct_insert_time = test_direct_insert() # direct_insert_time = test_direct_insert()
# 打印最终结果 # 打印最终结果
print("\n测试结果对比:") print("\n测试结果对比:")
print(f"自动建表插入耗时: {auto_create_time:.2f} 秒") print(f"自动建表插入耗时: {auto_create_time:.2f} 秒")
print(f"直接插入耗时: {direct_insert_time:.2f} 秒") # print(f"直接插入耗时: {direct_insert_time:.2f} 秒")
print(f"性能差异: {auto_create_time/direct_insert_time:.1f} 倍") # print(f"性能差异: {auto_create_time/direct_insert_time:.1f} 倍")