xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: desciribe view and memory issues

This commit is contained in:
dapan1121 2023-10-27 19:28:22 +08:00
parent 099f72e84a
commit 9801d5d7b7
13 changed files with 192 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tmsg.h
View File

@ -1809,7 +1809,7 @@ int32_t tDeserializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp);
void tFreeSSTbHbRsp(SSTbHbRsp* pRsp);
typedef struct {
SArray* pViewRsp; // Array of SViewMetaRsp;
SArray* pViewRsp; // Array of SViewMetaRsp*;
} SViewHbRsp;
int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp);

1
include/libs/catalog/catalog.h
View File

@ -37,6 +37,7 @@ enum {
CTG_DBG_DB_NUM = 1,
CTG_DBG_META_NUM,
CTG_DBG_STB_NUM,
CTG_DBG_VIEW_NUM,
CTG_DBG_DB_RENT_NUM,
CTG_DBG_STB_RENT_NUM,
CTG_DBG_VIEW_RENT_NUM,

14
source/client/src/clientHb.c
View File

@ -289,22 +289,34 @@ static int32_t hbProcessDynViewRsp(void *value, int32_t valueLen, struct SCatalo
return catalogUpdateDynViewVer(pCatalog, (SDynViewVersion*)value);
}
static void hbFreeSViewMetaInRsp(void* p) {
if (NULL == p || NULL == *(void**)p) {
return;
}
SViewMetaRsp *pRsp = *(SViewMetaRsp**)p;
tFreeSViewMetaRsp(pRsp);
taosMemoryFreeClear(pRsp);
}
static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
int32_t code = 0;
SViewHbRsp hbRsp = {0};
if (tDeserializeSViewHbRsp(value, valueLen, &hbRsp) != 0) {
taosArrayDestroyEx(hbRsp.pViewRsp, hbFreeSViewMetaInRsp);
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
int32_t numOfMeta = taosArrayGetSize(hbRsp.pViewRsp);
for (int32_t i = 0; i < numOfMeta; ++i) {
SViewMetaRsp *rsp = taosArrayGet(hbRsp.pViewRsp, i);
SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i);
if (rsp->numOfCols < 0) {
tscDebug("hb to remove view, db:%s, view:%s", rsp->dbFName, rsp->name);
catalogRemoveViewMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->name, rsp->viewId);
tFreeSViewMetaRsp(rsp);
taosMemoryFreeClear(rsp);
} else {
tscDebug("hb to update view, db:%s, view:%s", rsp->dbFName, rsp->name);
catalogUpdateViewMeta(pCatalog, rsp);

12
source/common/src/tmsg.c
View File

@ -9034,7 +9034,7 @@ int32_t tSerializeSViewHbRsp(void *buf, int32_t bufLen, SViewHbRsp *pRsp) {
int32_t numOfMeta = taosArrayGetSize(pRsp->pViewRsp);
if (tEncodeI32(&encoder, numOfMeta) < 0) return -1;
for (int32_t i = 0; i < numOfMeta; ++i) {
SViewMetaRsp *pMetaRsp = taosArrayGet(pRsp->pViewRsp, i);
SViewMetaRsp *pMetaRsp = taosArrayGetP(pRsp->pViewRsp, i);
if (tEncodeSViewMetaRsp(&encoder, pMetaRsp) < 0) return -1;
}
@ -9053,15 +9053,16 @@ int32_t tDeserializeSViewHbRsp(void *buf, int32_t bufLen, SViewHbRsp *pRsp) {
int32_t numOfMeta = 0;
if (tDecodeI32(&decoder, &numOfMeta) < 0) return -1;
pRsp->pViewRsp = taosArrayInit(numOfMeta, sizeof(SViewMetaRsp));
pRsp->pViewRsp = taosArrayInit(numOfMeta, POINTER_BYTES);
if (pRsp->pViewRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < numOfMeta; ++i) {
SViewMetaRsp metaRsp = {0};
if (tDecodeSViewMetaRsp(&decoder, &metaRsp) < 0) return -1;
SViewMetaRsp* metaRsp = taosMemoryCalloc(1, sizeof(SViewMetaRsp));
if (NULL == metaRsp) return -1;
if (tDecodeSViewMetaRsp(&decoder, metaRsp) < 0) return -1;
taosArrayPush(pRsp->pViewRsp, &metaRsp);
}
@ -9074,8 +9075,9 @@ int32_t tDeserializeSViewHbRsp(void *buf, int32_t bufLen, SViewHbRsp *pRsp) {
void tFreeSViewHbRsp(SViewHbRsp *pRsp) {
int32_t numOfMeta = taosArrayGetSize(pRsp->pViewRsp);
for (int32_t i = 0; i < numOfMeta; ++i) {
SViewMetaRsp *pMetaRsp = taosArrayGet(pRsp->pViewRsp, i);
SViewMetaRsp *pMetaRsp = taosArrayGetP(pRsp->pViewRsp, i);
tFreeSViewMetaRsp(pMetaRsp);
taosMemoryFree(pMetaRsp);
}
taosArrayDestroy(pRsp->pViewRsp);

1
source/libs/catalog/src/catalog.c
View File

@ -1735,6 +1735,7 @@ int32_t catalogUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pMsg) {
tFreeSViewMetaRsp(pMsg);
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}

2
source/libs/catalog/src/ctgCache.c
View File

@ -1294,7 +1294,7 @@ int32_t ctgUpdateViewMetaEnqueue(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncOp
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateViewMetaMsg));
taosMemoryFree(op);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
char *p = strchr(pRsp->dbFName, '.');

18
source/libs/catalog/src/ctgDbg.c
View File

@ -372,6 +372,10 @@ int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) {
return dbCache->tbCache ? (int32_t)taosHashGetSize(dbCache->tbCache) : 0;
}
int32_t ctgdGetViewNum(SCtgDBCache *dbCache) {
return dbCache->viewCache ? (int32_t)taosHashGetSize(dbCache->viewCache) : 0;
}
int32_t ctgdGetStbNum(SCtgDBCache *dbCache) {
return dbCache->stbCache ? (int32_t)taosHashGetSize(dbCache->stbCache) : 0;
}
@ -420,6 +424,8 @@ int32_t ctgdGetClusterCacheNum(SCatalog *pCtg, int32_t type) {
case CTG_DBG_STB_NUM:
num += ctgdGetStbNum(dbCache);
break;
case CTG_DBG_VIEW_NUM:
num += ctgdGetViewNum(dbCache);
default:
ctgError("invalid type:%d", type);
break;
@ -472,6 +478,7 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
dbFName = taosHashGetKey(pIter, &len);
int32_t metaNum = dbCache->tbCache ? taosHashGetSize(dbCache->tbCache) : 0;
int32_t viewNum = dbCache->viewCache ? taosHashGetSize(dbCache->viewCache) : 0;
int32_t stbNum = dbCache->stbCache ? taosHashGetSize(dbCache->stbCache) : 0;
int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION;
int32_t hashMethod = -1;
@ -492,8 +499,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
}
ctgDebug("[%d] db [%.*s][0x%" PRIx64
"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, stateTs:%" PRId64 ", hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d",
i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion, stateTs,
"] %s: metaNum:%d, viewNum:%d, stbNum:%d, vgVersion:%d, stateTs:%" PRId64 ", hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d",
i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, viewNum, stbNum, vgVersion, stateTs,
hashMethod, hashPrefix, hashSuffix, vgNum);
if (dbCache->vgCache.vgInfo) {
@ -543,9 +550,10 @@ void ctgdShowClusterCache(SCatalog *pCtg) {
}
ctgDebug("## cluster 0x%" PRIx64 " %p cache Info BEGIN ##", pCtg->clusterId, pCtg);
ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));
ctgDebug("db:%d tbmeta:%d viewmeta:%d stb:%d dbRent:%d stbRent:%d viewRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_VIEW_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_VIEW_RENT_NUM));
ctgdShowDBCache(pCtg, pCtg->dbCache);

1
source/libs/parser/inc/parUtil.h
View File

@ -111,6 +111,7 @@ int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTab
int32_t reserveDnodeRequiredInCache(SParseMetaCache* pMetaCache);
int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta);
int32_t getViewMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta);
int32_t buildTableMetaFromViewMeta(STableMeta** pMeta, SViewMeta* pViewMeta);
int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo);
int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup);
int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,

7
source/libs/parser/src/parAuthenticator.c
View File

@ -164,17 +164,16 @@ static EDealRes authSelectImpl(SNode* pNode, void* pContext) {
if (QUERY_NODE_REAL_TABLE == nodeType(pNode)) {
SNode* pTagCond = NULL;
STableNode* pTable = (STableNode*)pNode;
#ifdef TD_ENTERPRISE
SName name;
STableMeta* pTableMeta = NULL;
int32_t code = getTargetMetaImpl(
pAuthCxt->pParseCxt, pAuthCxt->pMetaCache, toName(pAuthCxt->pParseCxt->acctId, pTable->dbName, pTable->tableName, &name), &pTableMeta, true);
if (TSDB_CODE_SUCCESS != code) {
pAuthCxt->errCode = code;
return DEAL_RES_ERROR;
} else if (TSDB_VIEW_TABLE == pTableMeta->tableType) {
if (TSDB_CODE_SUCCESS == code && TSDB_VIEW_TABLE == pTableMeta->tableType) {
isView = true;
}
taosMemoryFree(pTableMeta);
#endif
if (!isView) {
pAuthCxt->errCode = checkAuth(pAuthCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_READ, &pTagCond);
if (TSDB_CODE_SUCCESS != pAuthCxt->errCode && NULL != pAuthCxt->pParseCxt->pEffectiveUser) {

45
source/libs/parser/src/parTranslater.c
View File

@ -455,15 +455,6 @@ static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName,
code = catalogRefreshGetTableMeta(pParCxt->pCatalog, &conn, &name, pMeta, false);
}
#ifdef TD_ENTERPRISE
if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) {
int32_t origCode = code;
code = getViewMetaFromCache(pCxt->pMetaCache, &name, pMeta);
if (TSDB_CODE_SUCCESS != code) {
code = origCode;
}
}
#endif
if (TSDB_CODE_SUCCESS != code) {
parserError("0x%" PRIx64 " catalogRefreshGetTableMeta error, code:%s, dbName:%s, tbName:%s",
pCxt->pParseCxt->requestId, tstrerror(code), pDbName, pTableName);
@ -6653,7 +6644,41 @@ static int32_t translateExplain(STranslateContext* pCxt, SExplainStmt* pStmt) {
}
static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) {
return refreshGetTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta);
int32_t code = refreshGetTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta);
#ifdef TD_ENTERPRISE
if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) {
int32_t origCode = code;
SName name;
toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name);
SViewMeta* pMeta = NULL;
code = getViewMetaFromMetaCache(pCxt, &name, &pMeta);
if (TSDB_CODE_SUCCESS != code) {
code = origCode;
} else {
SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA};
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&name, dbFName);
code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, name.dbname, pMeta->querySql, false, pMeta->user, &res);
if (TSDB_CODE_SUCCESS == code) {
code = collectUseTable(&name, pCxt->pTargetTables);
}
if (TSDB_CODE_SUCCESS == code) {
SViewMeta viewMeta = {0};
viewMeta.viewId = pMeta->viewId;
viewMeta.precision = res.schemaRes.precision;
viewMeta.type = pMeta->type;
viewMeta.version = pMeta->version;
viewMeta.numOfCols = res.schemaRes.numOfCols;
viewMeta.pSchema = res.schemaRes.pSchema;
code = buildTableMetaFromViewMeta(&pStmt->pMeta, &viewMeta);
parserDebug("rebuild view meta, view:%s.%s, numOfCols:%d, code:0x%x", dbFName, pStmt->tableName, viewMeta.numOfCols, code);
}
taosMemoryFree(res.schemaRes.pSchema);
}
}
#endif
return code;
}
static int32_t translateCompactRange(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt, SCompactDbReq* pReq) {

19
source/libs/parser/src/parUtil.c
View File

@ -905,15 +905,10 @@ int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, S
return code;
}
int32_t getViewMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) {
char fullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pName, fullName);
SViewMeta* pViewMeta = NULL;
int32_t code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pViews, (void**)&pViewMeta);
if (TSDB_CODE_SUCCESS == code) {
int32_t buildTableMetaFromViewMeta(STableMeta** pMeta, SViewMeta* pViewMeta) {
*pMeta = taosMemoryCalloc(1, sizeof(STableMeta) + pViewMeta->numOfCols * sizeof(SSchema));
if (NULL == *pMeta) {
code = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pMeta)->uid = pViewMeta->viewId;
(*pMeta)->vgId = MNODE_HANDLE;
@ -927,6 +922,16 @@ int32_t getViewMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, ST
for (int32_t i = 0; i < pViewMeta->numOfCols; ++i) {
(*pMeta)->tableInfo.rowSize += (*pMeta)->schema[i].bytes;
}
return TSDB_CODE_SUCCESS;
}
int32_t getViewMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) {
char fullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pName, fullName);
SViewMeta* pViewMeta = NULL;
int32_t code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pViews, (void**)&pViewMeta);
if (TSDB_CODE_SUCCESS == code) {
code = buildTableMetaFromViewMeta(pMeta, pViewMeta);
}
return code;
}

72
tests/script/tsim/view/same_name_tb_view.sim Normal file
View File

@ -0,0 +1,72 @@
sql connect
sql use testa;
print == create view sta1
sql create view sta1 as select * from stv;
sql select * from sta1;
if $rows != 4 then
return -1
endi
sql desc sta1;
if $rows != 4 then
return -1
endi
sql show create table sta1;
sql show create view sta1;
sql create view view1 as select * from sta1;
sql select * from view1;
if $rows != 4 then
return -1
endi
print == drop view sta1
sql reset query cache
sql drop view sta1;
sql select * from sta1;
if $rows != 4 then
return -1
endi
sql desc sta1;
if $rows != 4 then
return -1
endi
sql show create table sta1;
sql_error show create view sta1;
sql select * from view1;
if $rows != 4 then
return -1
endi
print == create view sta1, drop table sta1
sql reset query cache
sql create view sta1 as select * from stv;
sql drop table sta1;
sql select * from sta1;
if $rows != 1 then
return -1
endi
sql desc sta1;
if $rows != 3 then
print $rows
return -1
endi
sql_error show create table sta1;
sql show create view sta1;
sql select * from view1;
if $rows != 1 then
return -1
endi
sql desc view1;
if $rows != 3 then
return -1
endi
print == restore data
sql drop view sta1;
sql drop view view1;
sql create table sta1(ts timestamp, f int, g int) tags (t int);
sql insert into cta11 using sta1 tags(1) values('2023-10-16 09:10:11', 100111, 1001110);
sql insert into cta12 using sta1 tags(2) values('2023-10-16 09:10:12', 100112, 1001120);
sql insert into cta13 using sta1 tags(3) values('2023-10-16 09:10:13', 100113, 1001130);
sql insert into cta14 using sta1 tags(4) values('2023-10-16 09:10:14', 100114, 1001140);

41
tests/script/tsim/view/view.sim
View File

@ -21,6 +21,9 @@ sql insert into cta24 using st2 tags(4) values('2023-10-16 09:10:14', 100224, 10
sql create table stt(ts timestamp, f int, g int) tags (t int);
sql create table tt using stt tags(99);
sql create table stv(ts timestamp, h int) tags (t1 int);
sql insert into ctv1 using stv tags(1) values('2023-10-16 10:10:10', 1);
sql drop database if exists testb
sql create database testb vgroups 1;
sql use testb;
@ -37,25 +40,27 @@ sql insert into ctb22 using st2 tags(2) values('2023-10-16 09:10:12', 110222, 11
sql insert into ctb23 using st2 tags(3) values('2023-10-16 09:10:13', 110223, 1102230);
sql insert into ctb24 using st2 tags(4) values('2023-10-16 09:10:14', 110224, 1102240);
#run tsim/view/privilege_basic_view.sim
#run tsim/view/privilege_nested_view.sim
#run tsim/view/create_drop_view.sim
run tsim/view/privilege_basic_view.sim
run tsim/view/privilege_nested_view.sim
run tsim/view/create_drop_view.sim
run tsim/view/query_view.sim
#run tsim/view/insert_view.sim
#run tsim/view/stream_view.sim
#run tsim/view/show_desc_view.sim
run tsim/view/insert_view.sim
run tsim/view/stream_view.sim
run tsim/view/show_desc_view.sim
run tsim/view/same_name_tb_view.sim
#print ================== restart server to commit data into disk
#system sh/exec.sh -n dnode1 -s stop -x SIGINT
#system sh/exec.sh -n dnode1 -s start
#print ================== server restart completed
print ================== restart server to commit data into disk
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
print ================== server restart completed
#run tsim/view/privilege_basic_view.sim
#run tsim/view/privilege_nested_view.sim
#run tsim/view/create_drop_view.sim
#run tsim/view/query_view.sim
#run tsim/view/insert_view.sim
#run tsim/view/stream_view.sim
#run tsim/view/show_desc_view.sim
run tsim/view/privilege_basic_view.sim
run tsim/view/privilege_nested_view.sim
run tsim/view/create_drop_view.sim
run tsim/view/query_view.sim
run tsim/view/insert_view.sim
run tsim/view/stream_view.sim
run tsim/view/show_desc_view.sim
run tsim/view/same_name_tb_view.sim
#system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s stop -x SIGINT