xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-32] refactor the tablemeta

This commit is contained in:
hjxilinx 2020-03-19 14:36:01 +08:00
parent 03346d0ccc
commit 06e89a5eef
16 changed files with 130 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/client/CMakeLists.txt
View File

@ -6,6 +6,7 @@ INCLUDE_DIRECTORIES(jni)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
AUX_SOURCE_DIRECTORY(src SRC)

16
src/client/inc/tschemautil.h
View File

@ -89,17 +89,19 @@ bool isValidSchema(struct SSchema *pSchema, int32_t numOfCols);
* get the schema for the "tbname" column. it is a built column
* @return
*/
SSchema tsGetTbnameColumnSchema();
SSchema tscGetTbnameColumnSchema();
/**
* create the table meta from the msg
* @param pTableMetaMsg
* @param size size of the table meta
* @return
*/
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size);
//todo tags value as well as the table id structure needs refactor
char *tsGetTagsValue(STableMeta *pMeta);
bool tsMeterMetaIdentical(STableMeta *p1, STableMeta *p2);
void extractTableName(char *tableId, char *name);
SSQLToken extractDBName(char *tableId, char *name);
void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable);
#ifdef __cplusplus

9
src/client/src/tscSQLParser.c
View File

@ -29,6 +29,8 @@
#include "tsclient.h"
#include "ttokendef.h"
#include "name.h"
#define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0"
// -1 is tbname column index, so here use the -2 as the initial value
@ -2470,7 +2472,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
STableMeta* pTableMeta = NULL;
SSchema* pSchema = NULL;
SSchema s = tsGetTbnameColumnSchema();
SSchema s = tscGetTbnameColumnSchema();
int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL;
@ -3538,8 +3540,9 @@ static int32_t setTableCondForMetricQuery(SQueryInfo* pQueryInfo, const char* ac
}
num = j;
SSQLToken dbToken = extractDBName(pTableMetaInfo->name, db);
char* name = extractDBName(pTableMetaInfo->name, db);
SSQLToken dbToken = {.type = TK_STRING, .z = name, .n = strlen(name)};
for (int32_t i = 0; i < num; ++i) {
if (i >= 1) {
taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1);

58
src/client/src/tscSchemaUtil.c
View File

@ -45,8 +45,6 @@ int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) {
// table created according to super table, use data from super table
STableInfo tinfo = tscGetTableInfo(pTableMeta);
assert(tinfo.numOfColumns >= 2);
return tinfo.numOfColumns;
}
@ -141,7 +139,7 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t startCol)
return &pSchema[startCol];
}
struct SSchema tsGetTbnameColumnSchema() {
struct SSchema tscGetTbnameColumnSchema() {
struct SSchema s = {
.colId = TSDB_TBNAME_COLUMN_INDEX,
.type = TSDB_DATA_TYPE_BINARY,
@ -152,6 +150,22 @@ struct SSchema tsGetTbnameColumnSchema() {
return s;
}
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) {
assert(pTableMetaMsg != NULL);
int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema);
STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize);
memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize);
if (size != NULL) {
*size = sizeof(STableMeta) + schemaSize;
}
return pTableMeta;
}
/**
* the TableMeta data format in memory is as follows:
*
@ -174,26 +188,6 @@ char* tsGetTagsValue(STableMeta* pTableMeta) {
return ((char*)pTableMeta + offset);
}
bool tsMeterMetaIdentical(STableMeta* p1, STableMeta* p2) {
if (p1 == NULL || p2 == NULL || p1->uid != p2->uid || p1->sversion != p2->sversion) {
return false;
}
//
// if (p1 == p2) {
// return true;
// }
//
// size_t size = sizeof(STableMeta) + p1->numOfColumns * sizeof(SSchema);
//
// for (int32_t i = 0; i < p1->numOfTags; ++i) {
// SSchema* pColSchema = tscGetTableColumnSchema(p1, i + p1->numOfColumns);
// size += pColSchema->bytes;
// }
// return memcmp(p1, p2, size) == 0;
return true;
}
// todo refactor
static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) {
for (int32_t i = 0; i < num; ++i) {
@ -213,24 +207,6 @@ static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) {
return len;
}
/**
* extract table name from meterid, which the format of userid.dbname.metername
* @param tableId
* @return
*/
void extractTableName(char* tableId, char* name) {
char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 2);
copy(name, r, TS_PATH_DELIMITER[0]);
}
SSQLToken extractDBName(char* tableId, char* name) {
char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 1);
size_t len = copy(name, r, TS_PATH_DELIMITER[0]);
SSQLToken token = {.z = name, .n = len, .type = TK_STRING};
return token;
}
/*
* tablePrefix.columnName
* extract table name and save it in pTable, with only column name in pToken

24
src/client/src/tscServer.c
View File

@ -2247,7 +2247,7 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_SUCCESS;
}
int tscBuildMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMTableInfoMsg *pInfoMsg;
char * pMsg;
int msgLen = 0;
@ -2878,7 +2878,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); //?
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0);
@ -2891,7 +2891,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns);
pSchema = (SSchema *)((char *)pMetaMsg + sizeof(STableMeta));
pSchema = pMetaMsg->schema;
pMetaMsg->sid = ntohs(pMetaMsg->sid);
for (int i = 0; i < pMetaMsg->numOfColumns; ++i) {
pSchema->bytes = htons(pSchema->bytes);
@ -2902,12 +2902,14 @@ int tscProcessShowRsp(SSqlObj *pSql) {
strcpy(key + 1, "showlist");
taosCacheRelease(tscCacheHandle, (void *)&(pTableMetaInfo->pTableMeta), false);
int32_t size = pMetaMsg->numOfColumns * sizeof(SSchema) + sizeof(STableMeta);
size_t size = 0;
STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size);
pTableMetaInfo->pTableMeta =
(STableMeta *)taosCachePut(tscCacheHandle, key, (char *)pMetaMsg, size, tsMeterMetaKeepTimer);
(STableMeta *)taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer);
pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols;
SSchema *pMeterSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
tscColumnBaseInfoReserve(&pQueryInfo->colList, pMetaMsg->numOfColumns);
SColumnIndex index = {0};
@ -2915,13 +2917,15 @@ int tscProcessShowRsp(SSqlObj *pSql) {
for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i) {
index.columnIndex = i;
tscColumnBaseInfoInsert(pQueryInfo, &index);
tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pMeterSchema[i]);
tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pTableSchema[i]);
pQueryInfo->fieldsInfo.pSqlExpr[i] = tscSqlExprInsert(pQueryInfo, i, TSDB_FUNC_TS_DUMMY, &index,
pMeterSchema[i].type, pMeterSchema[i].bytes, pMeterSchema[i].bytes);
pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes);
}
tscFieldInfoCalOffset(pQueryInfo);
tfree(pTableMeta);
return 0;
}
@ -3379,7 +3383,7 @@ void tscInitMsgs() {
tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg;
tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg;
tscBuildMsg[TSDB_SQL_META] = tscBuildMeterMetaMsg;
tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg;
tscBuildMsg[TSDB_SQL_METRIC] = tscBuildMetricMetaMsg;
tscBuildMsg[TSDB_SQL_MULTI_META] = tscBuildMultiMeterMetaMsg;

8
src/common/inc/name.h Normal file
View File

@ -0,0 +1,8 @@
#ifndef TDENGINE_NAME_H
#define TDENGINE_NAME_H
int32_t extractTableName(const char *tableId, char *name);
char* extractDBName(const char *tableId, char *name);
#endif // TDENGINE_NAME_H

42
src/common/src/name.c Normal file
View File

@ -0,0 +1,42 @@
#include "os.h"
#include "tutil.h"
#include "name.h"
#include "tstoken.h"
#include "ttokendef.h"
// todo refactor
static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) {
for (int32_t i = 0; i < num; ++i) {
while (*input != 0 && *input++ != delim) {
};
}
return input;
}
static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) {
size_t len = 0;
while (*src != delimiter && *src != 0) {
*dst++ = *src++;
len++;
}
return len;
}
int32_t extractTableName(const char* tableId, char* name) {
size_t offset = strcspn(tableId, &TS_PATH_DELIMITER[0]);
offset = strcspn(&tableId[offset], &TS_PATH_DELIMITER[0]);
return strncpy(name, &tableId[offset], TSDB_TABLE_NAME_LEN);
// char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 2);
// return copy(name, r, TS_PATH_DELIMITER[0]);
}
char* extractDBName(const char* tableId, char* name) {
size_t offset1 = strcspn(tableId, &TS_PATH_DELIMITER[0]);
size_t len = strcspn(&tableId[offset1 + 1], &TS_PATH_DELIMITER[0]);
return strncpy(name, &tableId[offset1 + 1], len);
}

5
src/inc/taosmsg.h
View File

@ -223,7 +223,7 @@ typedef struct {
typedef struct SSchema {
uint8_t type;
char name[TSDB_COL_NAME_LEN + 1];
char name[TSDB_COL_NAME_LEN];
int16_t colId;
int16_t bytes;
} SSchema;
@ -684,10 +684,9 @@ typedef struct STableMetaMsg {
uint8_t numOfTags;
uint8_t precision;
uint8_t tableType;
// uint8_t index : 4; // used locally
int16_t numOfColumns;
// int16_t rowSize; // used locally, calculated in client
int16_t sversion;
int8_t numOfVpeers;
SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT];
int32_t sid;

28
src/mnode/src/mgmtDb.c
View File

@ -15,23 +15,26 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tstatus.h"
#include "tutil.h"
#include "mnode.h"
#include "mgmtDb.h"
#include "mgmtAcct.h"
#include "mgmtBalance.h"
#include "mgmtDb.h"
#include "mgmtDnode.h"
#include "mgmtMnode.h"
#include "mgmtGrant.h"
#include "mgmtShell.h"
#include "mgmtNormalTable.h"
#include "mgmtChildTable.h"
#include "mgmtDnode.h"
#include "mgmtGrant.h"
#include "mgmtMnode.h"
#include "mgmtNormalTable.h"
#include "mgmtShell.h"
#include "mgmtSuperTable.h"
#include "mgmtTable.h"
#include "mgmtUser.h"
#include "mgmtVgroup.h"
#include "mnode.h"
#include "taoserror.h"
#include "tstatus.h"
#include "tutil.h"
#include "name.h"
static void *tsDbSdb = NULL;
static int32_t tsDbUpdateSize;
@ -294,8 +297,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
bool mgmtCheckIsMonitorDB(char *db, char *monitordb) {
char dbName[TSDB_DB_NAME_LEN + 1] = {0};
assert(0);
// extractDBName(db, dbName);
extractDBName(db, dbName);
size_t len = strlen(dbName);
return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb));
@ -436,7 +438,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "created time");
strcpy(pSchema[cols].name, "created_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;

2
src/mnode/src/mgmtDnode.c
View File

@ -331,7 +331,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
pShow->bytes[cols] = 12;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "sync status");
strcpy(pSchema[cols].name, "sync_status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;

2
src/mnode/src/mgmtSuperTable.c
View File

@ -508,7 +508,7 @@ static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow,
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create time");
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;

6
src/mnode/src/mgmtTable.c
View File

@ -198,13 +198,13 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "table name");
strcpy(pSchema[cols].name, "table_name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create time");
strcpy(pSchema[cols].name, "created_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -216,7 +216,7 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "super table name");
strcpy(pSchema[cols].name, "stable_name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;

2
src/mnode/src/mgmtUser.c
View File

@ -193,7 +193,7 @@ static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCon
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "created time");
strcpy(pSchema[cols].name, "created_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;

6
src/mnode/src/mgmtVgroup.c
View File

@ -233,7 +233,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
pShow->bytes[cols] = 9;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "vgroup status");
strcpy(pSchema[cols].name, "vgroup_status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -273,13 +273,13 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
pShow->bytes[cols] = 9;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "vnode status");
strcpy(pSchema[cols].name, "vnode_status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "public ip");
strcpy(pSchema[cols].name, "public_ip");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
}

2
src/query/CMakeLists.txt
View File

@ -9,5 +9,5 @@ INCLUDE_DIRECTORIES(inc)
IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(query ${SRC})
TARGET_LINK_LIBRARIES(query util m rt)
TARGET_LINK_LIBRARIES(query tutil m rt)
ENDIF ()

11
src/util/src/tcache.c
View File

@ -206,9 +206,10 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pObj, SCacheDataNode *p
return;
}
int32_t size = pNode->size;
taosHashRemove(pObj->pHashTable, pNode->key, pNode->keySize);
pTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pObj->totalSize, pObj->totalSize);
pTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pObj->totalSize, size);
free(pNode);
}
@ -418,10 +419,10 @@ void *taosCachePut(void *handle, char *key, char *pData, int dataSize, int durat
if (pOld == NULL) { // do addedTime to cache
pNode = taosAddToCacheImpl(pObj, key, keyLen, pData, dataSize, duration * 1000L);
if (NULL != pNode) {
pTrace("key:%s %p added into cache, addedTime:%" PRIu64 ", expireTime:%" PRIu64 ", cache total:%d, size:%" PRId64
" bytes, collision:%d",
key, pNode, pNode->addedTime, pNode->expiredTime, dataSize, pObj->totalSize,
pObj->statistics.numOfCollision);
pObj->totalSize += pNode->size;
pTrace("key:%s %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%d, size:%" PRId64 " bytes",
key, pNode, pNode->addedTime, pNode->expiredTime, pObj->totalSize, dataSize);
}
} else { // old data exists, update the node
pNode = taosUpdateCacheImpl(pObj, pOld, key, keyLen, pData, dataSize, duration * 1000L);