From fcc87f09feaf48ea8cea03e469d7c1b872feb6a8 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 15 Jul 2022 21:33:30 +0800 Subject: [PATCH 1/8] feat: add user tags --- include/common/systable.h | 1 + source/common/src/systable.c | 11 ++ source/libs/executor/src/scanoperator.c | 167 +++++++++++++++++++++++- 3 files changed, 178 insertions(+), 1 deletion(-) diff --git a/include/common/systable.h b/include/common/systable.h index faea4dd3f8..6fd40072fb 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -35,6 +35,7 @@ extern "C" { #define TSDB_INS_TABLE_USER_INDEXES "user_indexes" #define TSDB_INS_TABLE_USER_STABLES "user_stables" #define TSDB_INS_TABLE_USER_TABLES "user_tables" +#define TSDB_INS_TABLE_USER_TAGS "user_tags" #define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed" #define TSDB_INS_TABLE_USER_USERS "user_users" #define TSDB_INS_TABLE_LICENCES "grants" diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 08997bcaf8..1015ae7fda 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -153,6 +153,16 @@ static const SSysDbTableSchema userTblsSchema[] = { {.name = "type", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; +static const SSysDbTableSchema userTagsSchema[] = { + {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "tag_name", .bytes = TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "tag_type", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tag_value", .bytes = TSDB_MAX_TAGS_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + static const SSysDbTableSchema userTblDistSchema[] = { {.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, @@ -255,6 +265,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)}, {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)}, + {TSDB_INS_TABLE_USER_TAGS, userTagsSchema, tListLen(userTagsSchema)}, // {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6aacb0ee01..0ab64ade3a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1765,6 +1765,172 @@ static SSDataBlock* buildSysTableMetaBlock() { return pBlock; } +static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + const char* name = tNameGetTableName(&pInfo->name); + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SSDataBlock* p = buildSysTableMetaBlock(); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + + int32_t ret = 0; + while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); + + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + + int32_t tableType = pInfo->pCur->mr.me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + metaReaderClear(&mr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + longjmp(pTaskInfo->env, terrno); + } + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); + + // super table name + STR_TO_VARSTR(n, mr.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, n, false); + metaReaderClear(&mr); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "CHILD_TABLE"); + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "NORMAL_TABLE"); + } + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + if (++numOfRows >= pOperator->resultInfo.capacity) { + break; + } + } + + // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + doSetOperatorCompleted(pOperator); + } + + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); + doFilterResult(pInfo); + + blockDataDestroy(p); + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { // build message and send to mnode to fetch the content of system tables. SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2906,4 +3072,3 @@ _error: taosMemoryFree(pOperator); return NULL; } - From 9fccdce05a2152c049be29845fecaa82e3f40533 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 16 Jul 2022 14:38:03 +0800 Subject: [PATCH 2/8] feat: add user tags to information schema --- include/common/systable.h | 2 +- source/common/src/systable.c | 5 +- source/libs/executor/src/scanoperator.c | 541 +++++++++++------------- 3 files changed, 258 insertions(+), 290 deletions(-) diff --git a/include/common/systable.h b/include/common/systable.h index 6fd40072fb..ca097cd8d2 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -35,7 +35,7 @@ extern "C" { #define TSDB_INS_TABLE_USER_INDEXES "user_indexes" #define TSDB_INS_TABLE_USER_STABLES "user_stables" #define TSDB_INS_TABLE_USER_TABLES "user_tables" -#define TSDB_INS_TABLE_USER_TAGS "user_tags" +#define TSDB_INS_TABLE_USER_TAGS "user_tags" #define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed" #define TSDB_INS_TABLE_USER_USERS "user_users" #define TSDB_INS_TABLE_LICENCES "grants" diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 1015ae7fda..1bd739cc2e 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -155,11 +155,10 @@ static const SSysDbTableSchema userTblsSchema[] = { static const SSysDbTableSchema userTagsSchema[] = { {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "tag_name", .bytes = TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "tag_type", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tag_type", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "tag_value", .bytes = TSDB_MAX_TAGS_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0ab64ade3a..0a864bbac8 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1742,14 +1742,14 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } -static SSDataBlock* buildSysTableMetaBlock() { +static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { size_t size = 0; const SSysTableMeta* pMeta = NULL; getInfosDbMeta(&pMeta, &size); int32_t index = 0; for (int32_t i = 0; i < size; ++i) { - if (strcmp(pMeta[i].name, TSDB_INS_TABLE_USER_TABLES) == 0) { + if (strcmp(pMeta[i].name, tableName) == 0) { index = i; break; } @@ -1768,7 +1768,6 @@ static SSDataBlock* buildSysTableMetaBlock() { static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; - const char* name = tNameGetTableName(&pInfo->name); if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -1791,123 +1790,85 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { tNameGetDbName(&sn, varDataVal(dbname)); varDataSetLen(dbname, strlen(varDataVal(dbname))); - SSDataBlock* p = buildSysTableMetaBlock(); + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_USER_TAGS); blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t ret = 0; while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { - STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); - - // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // vgId - pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); - - int32_t tableType = pInfo->pCur->mr.me.type; - if (tableType == TSDB_CHILD_TABLE) { - // create time - int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); - - SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); - - uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), - GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - longjmp(pTaskInfo->env, terrno); - } - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); - - // super table name - STR_TO_VARSTR(n, mr.me.name); - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "CHILD_TABLE"); - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); - - // super table name - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "NORMAL_TABLE"); + if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { + continue; } - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); - if (++numOfRows >= pOperator->resultInfo.capacity) { + SMetaReader smr = {0}; + metaReaderInit(&smr, pInfo->readHandle.meta, 0); + + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&smr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + metaReaderClear(&smr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + longjmp(pTaskInfo->env, terrno); + } + + char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(stableName, smr.me.name); + + int32_t numOfTags = smr.me.stbEntry.schemaTag.nCols; + for (int32_t i = 0; i < numOfTags; ++i) { + SColumnInfoData* pColInfoData = NULL; + + // table name + pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, tableName, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, stableName, false); + + char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tagName, smr.me.stbEntry.schemaTag.pSchema[i].name); + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, tagName, false); + + int8_t tagType = smr.me.stbEntry.schemaTag.pSchema[i].type; + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, (char*)&tagType, false); + + STagVal tagVal = {0}; + tagVal.cid = smr.me.stbEntry.schemaTag.pSchema[i].colId; + const char* pVal = metaGetTableTagVal(&pInfo->pCur->mr.me, tagType, &tagVal); + + char* tagData = NULL; + if (tagType != TSDB_DATA_TYPE_JSON && p != NULL) { + tagData = tTagValToData((const STagVal*)pVal, false); + } else { + tagData = (char*)pVal; + } + + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, tagData, + (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); + + if (tagType != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && + tagData != NULL) { + taosMemoryFree(tagData); + } + + ++numOfRows; + } + metaReaderClear(&smr); + + if (numOfRows >= pOperator->resultInfo.capacity) { break; } } @@ -1931,185 +1892,193 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } +static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. + if (pInfo->readHandle.mnd != NULL) { + buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); + + doFilterResult(pInfo); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + + doSetOperatorCompleted(pOperator); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } else { + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_USER_TABLES); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + + int32_t ret = 0; + while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); + + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + + int32_t tableType = pInfo->pCur->mr.me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + metaReaderClear(&mr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + longjmp(pTaskInfo->env, terrno); + } + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); + + // super table name + STR_TO_VARSTR(n, mr.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, n, false); + metaReaderClear(&mr); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "CHILD_TABLE"); + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "NORMAL_TABLE"); + } + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + if (++numOfRows >= pOperator->resultInfo.capacity) { + break; + } + } + + // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + doSetOperatorCompleted(pOperator); + } + + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); + doFilterResult(pInfo); + + blockDataDestroy(p); + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } + +} + static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { // build message and send to mnode to fetch the content of system tables. SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; - // retrieve local table list info from vnode const char* name = tNameGetTableName(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. - if (pInfo->readHandle.mnd != NULL) { - buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); - - doFilterResult(pInfo); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - - doSetOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } else { - if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); - } - - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SSDataBlock* p = buildSysTableMetaBlock(); - blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - - char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - - int32_t ret = 0; - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { - STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); - - // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // vgId - pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); - - int32_t tableType = pInfo->pCur->mr.me.type; - if (tableType == TSDB_CHILD_TABLE) { - // create time - int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); - - SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); - - uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), - GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - longjmp(pTaskInfo->env, terrno); - } - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); - - // super table name - STR_TO_VARSTR(n, mr.me.name); - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "CHILD_TABLE"); - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); - - // super table name - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "NORMAL_TABLE"); - } - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - if (++numOfRows >= pOperator->resultInfo.capacity) { - break; - } - } - - // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found - if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - doSetOperatorCompleted(pOperator); - } - - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); - doFilterResult(pInfo); - - blockDataDestroy(p); - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } + return sysTableScanUserTables(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + return sysTableScanUserTags(pOperator); } else { // load the meta from mnode of the given epset if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2189,7 +2158,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { - SSDataBlock* p = buildSysTableMetaBlock(); + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_USER_TABLES); blockDataEnsureCapacity(p, capacity); size_t size = 0; From 96495363d73c28e7a77f6f612fa288a6006ff542 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 16 Jul 2022 16:03:45 +0800 Subject: [PATCH 3/8] feat: add user tags system table --- include/common/tmsg.h | 1 + source/dnode/mnode/impl/src/mndShow.c | 2 ++ source/libs/executor/src/scanoperator.c | 7 +++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c8e13fce3d..cb80ba92ba 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -99,6 +99,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_STB, TSDB_MGMT_TABLE_STREAMS, TSDB_MGMT_TABLE_TABLE, + TSDB_MGMT_TABLE_TAG, TSDB_MGMT_TABLE_USER, TSDB_MGMT_TABLE_GRANTS, TSDB_MGMT_TABLE_VGROUP, diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 2a8cbc4425..0711f7e06d 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -76,6 +76,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_STB; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) { type = TSDB_MGMT_TABLE_TABLE; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, len) == 0) { + type = TSDB_MGMT_TABLE_TAG; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) { // type = TSDB_MGMT_TABLE_DIST; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, len) == 0) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0a864bbac8..c7b9a26a97 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1595,7 +1595,9 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) { blockDataDestroy(pInfo->pRes); const char* name = tNameGetTableName(&pInfo->name); - if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || + pInfo->pCur != NULL) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; } @@ -2247,7 +2249,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan tNameAssign(&pInfo->name, &pScanNode->tableName); const char* name = tNameGetTableName(&pInfo->name); - if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = *(SReadHandle*)readHandle; blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); } else { From 97d30c68585db7e4c28b71e0899560c4e3d94291 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 16 Jul 2022 17:25:05 +0800 Subject: [PATCH 4/8] feat: add user tags to system table --- include/libs/nodes/nodes.h | 1 + source/common/src/systable.c | 2 +- source/libs/executor/src/scanoperator.c | 80 ++++++++++++++++++++-- source/libs/parser/src/parAstParser.c | 15 ++++ source/libs/parser/src/parTranslater.c | 10 ++- source/libs/planner/src/planPhysiCreater.c | 3 +- 6 files changed, 101 insertions(+), 10 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index c453ce98f8..1a1ae00ad1 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -170,6 +170,7 @@ typedef enum ENodeType { QUERY_NODE_SHOW_STABLES_STMT, QUERY_NODE_SHOW_STREAMS_STMT, QUERY_NODE_SHOW_TABLES_STMT, + QUERY_NODE_SHOW_TAGS_STMT, QUERY_NODE_SHOW_USERS_STMT, QUERY_NODE_SHOW_LICENCE_STMT, QUERY_NODE_SHOW_VGROUPS_STMT, diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 1bd739cc2e..af81e47204 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -157,7 +157,7 @@ static const SSysDbTableSchema userTagsSchema[] = { {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "tag_name", .bytes = TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "tag_name", .bytes = TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "tag_type", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "tag_value", .bytes = TSDB_MAX_TAGS_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c7b9a26a97..0cb8eaa82e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1596,8 +1596,7 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) { const char* name = tNameGetTableName(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || - pInfo->pCur != NULL) { + strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; } @@ -1767,6 +1766,76 @@ static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { return pBlock; } +//TODO: check more datatype, json? and return detailed error when len is not enough +static int32_t convertTagDataToVarchar(int8_t tagType, char* tagVal, char* str, size_t len) { + switch (tagType) { + case TSDB_DATA_TYPE_TINYINT: + snprintf(str, len, "%d", *((int8_t*)tagVal)); + break; + + case TSDB_DATA_TYPE_UTINYINT: + snprintf(str, len, "%u", *((uint8_t*)tagVal)); + break; + + case TSDB_DATA_TYPE_SMALLINT: + snprintf(str, len, "%d", *((int16_t*)tagVal)); + break; + + case TSDB_DATA_TYPE_USMALLINT: + snprintf(str, len, "%u", *((uint16_t*)tagVal)); + break; + + case TSDB_DATA_TYPE_INT: + snprintf(str, len, "%d", *((int32_t*)tagVal)); + break; + + case TSDB_DATA_TYPE_UINT: + snprintf(str, len, "%u", *((uint32_t*)tagVal)); + break; + + case TSDB_DATA_TYPE_BIGINT: + snprintf(str, len, "%" PRId64, *((int64_t*)tagVal)); + break; + + case TSDB_DATA_TYPE_UBIGINT: + snprintf(str, len, "%" PRIu64, *((uint64_t*)tagVal)); + break; + + case TSDB_DATA_TYPE_FLOAT: { + float fv = 0; + fv = GET_FLOAT_VAL(tagVal); + snprintf(str, len, "%f", fv); + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double dv = 0; + dv = GET_DOUBLE_VAL(tagVal); + snprintf(str, len, "%lf", dv); + break; + } + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: { + memcpy(str, tagVal, len); + break; + } + + case TSDB_DATA_TYPE_TIMESTAMP: + snprintf(str, len, "%" PRId64, *((int64_t*)tagVal)); + break; + + case TSDB_DATA_TYPE_BOOL: + snprintf(str, len, "%d", *((int8_t*)tagVal)); + break; + default: + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; @@ -1856,9 +1925,11 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } else { tagData = (char*)pVal; } - + //TODO: memory allocation for varchar size and make sure enough memory + char tagStr[1024]; + convertTagDataToVarchar(tagType, tagData, tagStr, 1024); pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, tagData, + colDataAppend(pColInfoData, numOfRows, tagStr, (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); if (tagType != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && @@ -2068,7 +2139,6 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } - } static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index d24da2f83c..fd518f89af 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -371,6 +371,19 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt* return code; } +static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { + int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, + TSDB_INS_TABLE_USER_TAGS, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pStmt->pDbName) { + code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); + } else { + code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, pCxt->pMetaCache); + } + } + return code; +} + static int32_t collectMetaKeyFromShowUsers(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_USERS, pCxt->pMetaCache); @@ -537,6 +550,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromShowStreams(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TABLES_STMT: return collectMetaKeyFromShowTables(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_SHOW_TAGS_STMT: + return collectMetaKeyFromShowTags(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_USERS_STMT: return collectMetaKeyFromShowUsers(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_LICENCE_STMT: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c88e38ddff..3e9ca2676b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1677,7 +1677,8 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { static bool sysTableFromVnode(const char* pTable) { return (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLES)) || - (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED)); + (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) || + (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } @@ -1693,7 +1694,7 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, code = getDBVgInfoImpl(pCxt, pName, &vgroupList); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) { code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); } @@ -1782,7 +1783,8 @@ static bool isSingleTable(SRealTableNode* pRealTable) { int8_t tableType = pRealTable->pMeta->tableType; if (TSDB_SYSTEM_TABLE == tableType) { return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED); + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) && + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TAGS); } return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); } @@ -5025,6 +5027,8 @@ static const char* getSysTableName(ENodeType type) { return TSDB_INS_TABLE_USER_DATABASES; case QUERY_NODE_SHOW_TABLES_STMT: return TSDB_INS_TABLE_USER_TABLES; + case QUERY_NODE_SHOW_TAGS_STMT: + return TSDB_INS_TABLE_USER_TAGS; case QUERY_NODE_SHOW_STABLES_STMT: return TSDB_INS_TABLE_USER_STABLES; case QUERY_NODE_SHOW_USERS_STMT: diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 7b4a9c97ad..8250fb462f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -570,7 +570,8 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pScan->showRewrite = pScanLogicNode->showRewrite; pScan->accountId = pCxt->pPlanCxt->acctId; if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLES) || - 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED)) { + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) || + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TAGS)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } else { pSubplan->execNode.nodeId = MNODE_HANDLE; From 7570a4495db882383f06a7c2ab1a953d01bbf22b Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 17 Jul 2022 04:57:16 +0800 Subject: [PATCH 5/8] feat: add user tags --- source/libs/executor/src/scanoperator.c | 86 ++++++++++++++++--------- 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0cb8eaa82e..be6b36fb7f 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1766,73 +1766,80 @@ static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { return pBlock; } -//TODO: check more datatype, json? and return detailed error when len is not enough -static int32_t convertTagDataToVarchar(int8_t tagType, char* tagVal, char* str, size_t len) { +// TODO: check more datatype, json? and return detailed error when len is not enough +static int32_t convertTagDataToTagVarchar(int8_t tagType, char* tagVal, uint32_t tagLen, char* varData, + int32_t bufSize) { + int outputLen = -1; switch (tagType) { case TSDB_DATA_TYPE_TINYINT: - snprintf(str, len, "%d", *((int8_t*)tagVal)); + outputLen = snprintf(varDataVal(varData), bufSize, "%d", *((int8_t*)tagVal)); break; case TSDB_DATA_TYPE_UTINYINT: - snprintf(str, len, "%u", *((uint8_t*)tagVal)); + outputLen = snprintf(varDataVal(varData), bufSize, "%u", *((uint8_t*)tagVal)); break; case TSDB_DATA_TYPE_SMALLINT: - snprintf(str, len, "%d", *((int16_t*)tagVal)); + outputLen = snprintf(varDataVal(varData), bufSize, "%d", *((int16_t*)tagVal)); break; case TSDB_DATA_TYPE_USMALLINT: - snprintf(str, len, "%u", *((uint16_t*)tagVal)); + outputLen = snprintf(varDataVal(varData), bufSize, "%u", *((uint16_t*)tagVal)); break; case TSDB_DATA_TYPE_INT: - snprintf(str, len, "%d", *((int32_t*)tagVal)); + outputLen = snprintf(varDataVal(varData), bufSize, "%d", *((int32_t*)tagVal)); break; case TSDB_DATA_TYPE_UINT: - snprintf(str, len, "%u", *((uint32_t*)tagVal)); + outputLen = snprintf(varDataVal(varData), bufSize, "%u", *((uint32_t*)tagVal)); break; case TSDB_DATA_TYPE_BIGINT: - snprintf(str, len, "%" PRId64, *((int64_t*)tagVal)); + outputLen = snprintf(varDataVal(varData), bufSize, "%" PRId64, *((int64_t*)tagVal)); break; case TSDB_DATA_TYPE_UBIGINT: - snprintf(str, len, "%" PRIu64, *((uint64_t*)tagVal)); + outputLen = snprintf(varDataVal(varData), bufSize, "%" PRIu64, *((uint64_t*)tagVal)); break; case TSDB_DATA_TYPE_FLOAT: { float fv = 0; fv = GET_FLOAT_VAL(tagVal); - snprintf(str, len, "%f", fv); + outputLen = snprintf(varDataVal(varData), bufSize, "%f", fv); break; } case TSDB_DATA_TYPE_DOUBLE: { double dv = 0; dv = GET_DOUBLE_VAL(tagVal); - snprintf(str, len, "%lf", dv); + outputLen = snprintf(varDataVal(varData), bufSize, "%lf", dv); break; } case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_JSON: { - memcpy(str, tagVal, len); + memcpy(varDataVal(varData), tagVal, tagLen); + outputLen = tagLen; break; } case TSDB_DATA_TYPE_TIMESTAMP: - snprintf(str, len, "%" PRId64, *((int64_t*)tagVal)); + outputLen = snprintf(varDataVal(varData), bufSize, "%" PRId64, *((int64_t*)tagVal)); break; case TSDB_DATA_TYPE_BOOL: - snprintf(str, len, "%d", *((int8_t*)tagVal)); + outputLen = snprintf(varDataVal(varData), bufSize, "%d", *((int8_t*)tagVal)); break; default: return TSDB_CODE_FAILED; } + if (outputLen < 0 || outputLen == bufSize && !IS_VAR_DATA_TYPE(tagType) || outputLen > bufSize) { + return TSDB_CODE_FAILED; + } + varDataSetLen(varData, outputLen); return TSDB_CODE_SUCCESS; } @@ -1917,25 +1924,44 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STagVal tagVal = {0}; tagVal.cid = smr.me.stbEntry.schemaTag.pSchema[i].colId; - const char* pVal = metaGetTableTagVal(&pInfo->pCur->mr.me, tagType, &tagVal); - - char* tagData = NULL; - if (tagType != TSDB_DATA_TYPE_JSON && p != NULL) { - tagData = tTagValToData((const STagVal*)pVal, false); + char* tagData = NULL; + uint32_t tagLen = 0; + if (tagType == TSDB_DATA_TYPE_JSON) { + // TODO: json type?+varheader+data + tagData = varDataVal(pInfo->pCur->mr.me.ctbEntry.pTags + 1); + tagLen = varDataLen(pInfo->pCur->mr.me.ctbEntry.pTags + 1); } else { - tagData = (char*)pVal; + bool exist = tTagGet((STag*)pInfo->pCur->mr.me.ctbEntry.pTags, &tagVal); + if (exist) { + if (IS_VAR_DATA_TYPE(tagType)) { + tagData = (char*)tagVal.pData; + tagLen = tagVal.nData; + } else { + tagData = (char*)&tagVal.i64; + tagLen = tDataTypes[tagType].bytes; + } + } } - //TODO: memory allocation for varchar size and make sure enough memory - char tagStr[1024]; - convertTagDataToVarchar(tagType, tagData, tagStr, 1024); - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, tagStr, - (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); - if (tagType != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && - tagData != NULL) { - taosMemoryFree(tagData); + int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? tagLen : 1024; + char* tagVarChar = NULL; + if (tagData != NULL) { + tagVarChar = taosMemoryMalloc(bufSize); + code = convertTagDataToTagVarchar(tagType, tagData, tagLen, tagVarChar, bufSize); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + taosMemoryFree(tagVarChar); + metaReaderClear(&smr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + longjmp(pTaskInfo->env, terrno); + } } + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, tagVarChar, + (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); + taosMemoryFree(tagVarChar); ++numOfRows; } From e7c4b72f775cf549146ddc76c533ef3a946ad966 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 17 Jul 2022 05:06:58 +0800 Subject: [PATCH 6/8] fix: the system table number is 31 instead of 30 since user_tags is added --- tests/script/tsim/show/basic.sim | 4 ++-- tests/script/tsim/valgrind/checkError1.sim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index d0643bc7f2..4d646f39e3 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -99,7 +99,7 @@ if $rows != 1 then endi #sql select * from information_schema.`streams` sql select * from information_schema.user_tables -if $rows != 30 then +if $rows != 31 then return -1 endi #sql select * from information_schema.user_table_distributed @@ -197,7 +197,7 @@ if $rows != 1 then endi #sql select * from performance_schema.`streams` sql select * from information_schema.user_tables -if $rows != 30 then +if $rows != 31 then return -1 endi #sql select * from information_schema.user_table_distributed diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index fe4f4654b1..94557562c1 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -96,7 +96,7 @@ if $rows != 1 then endi sql select * from information_schema.user_tables -if $rows != 30 then +if $rows != 31 then return -1 endi From 76f38ee03a06807f42e5fd9280c3cd3ec134fc51 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 17 Jul 2022 05:19:01 +0800 Subject: [PATCH 7/8] feat: change bufSize to leave space for var string header --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 77427680af..b9bd90fa73 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1978,7 +1978,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } } - int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? tagLen : 1024; + int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? tagLen + VARSTR_HEADER_SIZE : 1024; char* tagVarChar = NULL; if (tagData != NULL) { tagVarChar = taosMemoryMalloc(bufSize); From 110aef1f1215fc986e4010e944cd8f0f3e22dec3 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 17 Jul 2022 05:30:20 +0800 Subject: [PATCH 8/8] feat: change bufSize for double digits --- source/libs/executor/src/scanoperator.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b9bd90fa73..41e41f9f5d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1143,7 +1143,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC); // must check update info first. bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); - if ( (update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup)) ) && out) { + if ((update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup))) && out) { taosArrayPush(pInfo->tsArray, &rowId); } } @@ -1596,7 +1596,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateRes = createResDataBlock(pDescNode); pInfo->pCondition = pScanPhyNode->node.pConditions; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; + pInfo->sessionSup = + (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; pInfo->groupId = 0; pInfo->pPullDataRes = createPullDataBlock(); pInfo->pStreamScanOp = pOperator; @@ -1978,7 +1979,8 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } } - int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? tagLen + VARSTR_HEADER_SIZE : 1024; + int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? (tagLen + VARSTR_HEADER_SIZE) + : (3 + DBL_MANT_DIG - DBL_MIN_EXP + VARSTR_HEADER_SIZE); char* tagVarChar = NULL; if (tagData != NULL) { tagVarChar = taosMemoryMalloc(bufSize);