floraachy
/
pcm-coordinator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into upmaster 

# Conflicts:
#	api/internal/types/types.go


Former-commit-id: 844ae53a2ede50fd681dfc373b97ca50c07ec24d
This commit is contained in:
qiwang 2024-03-22 15:56:35 +08:00
parent c6ad5637be d5dc85dca5
commit 1d1b024e09
26 changed files with 692 additions and 395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

327
api/desc/core/pcm-core.api
View File

@ -23,10 +23,11 @@ type (
CentersIndex []CenterIndex `json:"centersIndex"` CentersIndex []CenterIndex `json:"centersIndex"`
} }
CenterIndex { CenterIndex {
id int64 `json:"id"`
name string `json:"name"` name string `json:"name"`
cpu float32 `json:"cpu"` cpu string `json:"cpu"`
memory float32 `json:"memory"` memory string `json:"memory"`
storage float32 `json:"storage"` storage string `json:"storage"`
centerType string `json:"centerType"` centerType string `json:"centerType"`
} }
) )
@ -35,9 +36,16 @@ type (
clusterLoadRecords []ClusterLoadRecord `json:"clusterLoadRecords"` clusterLoadRecords []ClusterLoadRecord `json:"clusterLoadRecords"`
} }
ClusterLoadRecord { ClusterLoadRecord {
AdapterId int64 `json:"adapterId"`
ClusterName string `json:"clusterName"` ClusterName string `json:"clusterName"`
CpuAvail float64 `json:"cpuAvail"`
CpuTotal float64 `json:"cpuTotal"`
CpuUsage float64 `json:"cpuUsage"` CpuUsage float64 `json:"cpuUsage"`
MemoryAvail float64 `json:"memoryAvail"`
MemoryUsage float64 `json:"memoryUsage"` MemoryUsage float64 `json:"memoryUsage"`
MemoryTotal float64 `json:"memoryTotal"`
DiskAvail float64 `json:"diskAvail"`
DiskTotal float64 `json:"diskTotal"`
DiskUsage float64 `json:"diskUsage"` DiskUsage float64 `json:"diskUsage"`
} }
) )
@ -79,7 +87,6 @@ type commitTaskReq {
MatchLabels map[string]string `json:"matchLabels,optional"` MatchLabels map[string]string `json:"matchLabels,optional"`
YamlList []string `json:"yamlList"` YamlList []string `json:"yamlList"`
ClusterName string `json:"clusterName"` ClusterName string `json:"clusterName"`
} }
type ( type (
@ -536,7 +543,7 @@ type (
type ( type (
AdapterQueryReq { AdapterQueryReq {
Id string `form:"id,optional" db:"id"` Id string `form:"id,optional" db:"id"`
Name string `form:"name,optional"` Name string `form:"name,optional"`
Type string `form:"type,optional"` Type string `form:"type,optional"`
Nickname string `form:"nickname,optional"` Nickname string `form:"nickname,optional"`
@ -544,8 +551,16 @@ type (
Server string `form:"server,optional"` Server string `form:"server,optional"`
PageInfo PageInfo
} }
AdapterRelationQueryReq {
Id string `form:"id,optional" db:"id"`
Name string `form:"name,optional"`
Type string `form:"type,optional"`
Nickname string `form:"nickname,optional"`
Version string `form:"version,optional"`
Server string `form:"server,optional"`
}
AdapterReq { AdapterReq {
Id string `json:"id,optional" db:"id"` Id string `json:"id,optional" db:"id"`
Name string `json:"name,optional"` Name string `json:"name,optional"`
Type string `json:"type,optional"` Type string `json:"type,optional"`
Nickname string `json:"nickname,optional"` Nickname string `json:"nickname,optional"`
@ -553,7 +568,7 @@ type (
Server string `json:"server,optional"` Server string `json:"server,optional"`
} }
AdapterCreateReq { AdapterCreateReq {
Id string `json:"id,optional" db:"id"` Id string `json:"id,optional" db:"id"`
Name string `json:"name"` Name string `json:"name"`
Type string `json:"type"` Type string `json:"type"`
Nickname string `json:"nickname"` Nickname string `json:"nickname"`
@ -561,7 +576,7 @@ type (
Server string `json:"server"` Server string `json:"server"`
} }
AdapterDelReq { AdapterDelReq {
Id string `form:"id,optional" db:"id"` Id string `form:"id,optional" db:"id"`
} }
AdapterInfo { AdapterInfo {
Id string `json:"id,omitempty" db:"id"` Id string `json:"id,omitempty" db:"id"`
@ -595,79 +610,79 @@ type (
type ( type (
ClusterReq { ClusterReq {
Id string `form:"id,optional"` Id string `form:"id,optional"`
AdapterId string `form:"adapterId,optional"` AdapterId string `form:"adapterId,optional"`
Name string `form:"name,optional"` Name string `form:"name,optional"`
Nickname string `form:"nickname,optional"` Nickname string `form:"nickname,optional"`
Description string `form:"description,optional"` Description string `form:"description,optional"`
Server string `form:"server,optional"` Server string `form:"server,optional"`
MonitorServer string `form:"monitorServer,optional"` MonitorServer string `form:"monitorServer,optional"`
Username string `form:"username,optional"` Username string `form:"username,optional"`
Password string `form:"password,optional"` Password string `form:"password,optional"`
Token string `form:"token,optional"` Token string `form:"token,optional"`
Ak string `form:"ak,optional"` Ak string `form:"ak,optional"`
Sk string `form:"sk,optional"` Sk string `form:"sk,optional"`
Region string `form:"region,optional"` Region string `form:"region,optional"`
ProjectId string `form:"projectId,optional"` ProjectId string `form:"projectId,optional"`
Version string `form:"version,optional"` Version string `form:"version,optional"`
Label string `form:"label,optional"` Label string `form:"label,optional"`
OwnerId string `form:"ownerId,omitempty,optional"` OwnerId string `form:"ownerId,omitempty,optional"`
AuthType string `form:"authType,optional"` AuthType string `form:"authType,optional"`
Type string `form:"type,optional"` Type string `form:"type,optional"`
producerDict string `form:"producerDict,optional"` producerDict string `form:"producerDict,optional"`
regionDict string `form:"regionDict,optional"` regionDict string `form:"regionDict,optional"`
PageInfo PageInfo
} }
ClusterCreateReq { ClusterCreateReq {
Id string `json:"id,optional"` Id string `json:"id,optional"`
AdapterId string `json:"adapterId,optional"` AdapterId string `json:"adapterId,optional"`
Name string `json:"name,optional"` Name string `json:"name,optional"`
Nickname string `json:"nickname,optional"` Nickname string `json:"nickname,optional"`
Description string `json:"description,optional"` Description string `json:"description,optional"`
Server string `json:"server,optional"` Server string `json:"server,optional"`
MonitorServer string `json:"monitorServer,optional"` MonitorServer string `json:"monitorServer,optional"`
Username string `json:"username,optional"` Username string `json:"username,optional"`
Password string `json:"password,optional"` Password string `json:"password,optional"`
Token string `json:"token,optional"` Token string `json:"token,optional"`
Ak string `json:"ak,optional"` Ak string `json:"ak,optional"`
Sk string `json:"sk,optional"` Sk string `json:"sk,optional"`
Region string `json:"region,optional"` Region string `json:"region,optional"`
ProjectId string `json:"projectId,optional"` ProjectId string `json:"projectId,optional"`
Version string `json:"version,optional"` Version string `json:"version,optional"`
Label string `json:"label,optional"` Label string `json:"label,optional"`
OwnerId string `json:"ownerId,omitempty,optional"` OwnerId string `json:"ownerId,omitempty,optional"`
AuthType string `json:"authType,optional"` AuthType string `json:"authType,optional"`
producerDict string `json:"producerDict,optional"` producerDict string `json:"producerDict,optional"`
regionDict string `json:"regionDict,optional"` regionDict string `json:"regionDict,optional"`
} }
ClusterInfo { ClusterInfo {
Id string `json:"id,omitempty" db:"id"` Id string `json:"id,omitempty" db:"id"`
AdapterId string `json:"adapterId,omitempty" db:"adapter_id"` AdapterId string `json:"adapterId,omitempty" db:"adapter_id"`
Name string `json:"name,omitempty" db:"name"` Name string `json:"name,omitempty" db:"name"`
Nickname string `json:"nickname,omitempty" db:"nickname"` Nickname string `json:"nickname,omitempty" db:"nickname"`
Description string `json:"description,omitempty" db:"description"` Description string `json:"description,omitempty" db:"description"`
Server string `json:"server,omitempty" db:"server"` Server string `json:"server,omitempty" db:"server"`
MonitorServer string `json:"monitorServer,omitempty" db:"monitor_server"` MonitorServer string `json:"monitorServer,omitempty" db:"monitor_server"`
Username string `json:"username,omitempty" db:"username"` Username string `json:"username,omitempty" db:"username"`
Password string `json:"password,omitempty" db:"password"` Password string `json:"password,omitempty" db:"password"`
Token string `json:"token,omitempty" db:"token"` Token string `json:"token,omitempty" db:"token"`
Ak string `json:"ak,omitempty" db:"ak"` Ak string `json:"ak,omitempty" db:"ak"`
Sk string `json:"sk,omitempty" db:"sk"` Sk string `json:"sk,omitempty" db:"sk"`
Region string `json:"region,omitempty" db:"region"` Region string `json:"region,omitempty" db:"region"`
ProjectId string `json:"projectId,omitempty" db:"project_id"` ProjectId string `json:"projectId,omitempty" db:"project_id"`
Version string `json:"version,omitempty" db:"version"` Version string `json:"version,omitempty" db:"version"`
Label string `json:"label,omitempty" db:"label"` Label string `json:"label,omitempty" db:"label"`
OwnerId string `json:"ownerId,omitempty" db:"owner_id"` OwnerId string `json:"ownerId,omitempty" db:"owner_id"`
AuthType string `json:"authType,omitempty" db:"auth_type"` AuthType string `json:"authType,omitempty" db:"auth_type"`
producerDict string `json:"producerDict,omitempty" db:"producer_dict"` producerDict string `json:"producerDict,omitempty" db:"producer_dict"`
regionDict string `json:"regionDict,omitempty" db:"region_dict"` regionDict string `json:"regionDict,omitempty" db:"region_dict"`
CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"` CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"`
} }
) )
type ClusterDelReq { type ClusterDelReq {
Id string `form:"id,optional"` Id string `form:"id,optional"`
} }
type ClusterResp { type ClusterResp {
@ -696,66 +711,66 @@ type ClusterRelationInfo {
Version string `json:"version,omitempty" db:"version"` Version string `json:"version,omitempty" db:"version"`
Server string `json:"server,omitempty" db:"server"` Server string `json:"server,omitempty" db:"server"`
CreateTime string `json:"createTime,omitempty" db:"create_time" gorm:"autoCreateTime"` CreateTime string `json:"createTime,omitempty" db:"create_time" gorm:"autoCreateTime"`
CId string `json:"cId,omitempty" db:"id"` CId string `json:"cId,omitempty" db:"id"`
CAdapterId string `json:"cAdapterId,omitempty" db:"adapter_id"` CAdapterId string `json:"cAdapterId,omitempty" db:"adapter_id"`
CName string `json:"cName,omitempty" db:"name"` CName string `json:"cName,omitempty" db:"name"`
CNickname string `json:"cNickname,omitempty" db:"nickname"` CNickname string `json:"cNickname,omitempty" db:"nickname"`
CDescription string `json:"cDescription,omitempty" db:"description"` CDescription string `json:"cDescription,omitempty" db:"description"`
CServer string `json:"cServer,omitempty" db:"server"` CServer string `json:"cServer,omitempty" db:"server"`
CMonitorServer string `json:"cMonitorServer,omitempty" db:"monitor_server"` CMonitorServer string `json:"cMonitorServer,omitempty" db:"monitor_server"`
CUsername string `json:"cUsername,omitempty" db:"username"` CUsername string `json:"cUsername,omitempty" db:"username"`
CPassword string `json:"cPassword,omitempty" db:"password"` CPassword string `json:"cPassword,omitempty" db:"password"`
CToken string `json:"cToken,omitempty" db:"token"` CToken string `json:"cToken,omitempty" db:"token"`
CAk string `json:"cAk,omitempty" db:"ak"` CAk string `json:"cAk,omitempty" db:"ak"`
CSk string `json:"cSk,omitempty" db:"sk"` CSk string `json:"cSk,omitempty" db:"sk"`
CRegion string `json:"cRegion,omitempty" db:"region"` CRegion string `json:"cRegion,omitempty" db:"region"`
CProjectId string `json:"cProjectId,omitempty" db:"project_id"` CProjectId string `json:"cProjectId,omitempty" db:"project_id"`
CVersion string `json:"cVersion,omitempty" db:"version"` CVersion string `json:"cVersion,omitempty" db:"version"`
CLabel string `json:"cLabel,omitempty" db:"label"` CLabel string `json:"cLabel,omitempty" db:"label"`
COwnerId string `json:"cOwnerId,omitempty" db:"owner_id"` COwnerId string `json:"cOwnerId,omitempty" db:"owner_id"`
CAuthType string `json:"cAuthType,omitempty" db:"auth_type"` CAuthType string `json:"cAuthType,omitempty" db:"auth_type"`
CCreateTime string `json:"cCreateTime,omitempty" db:"created_time" gorm:"autoCreateTime"` CCreateTime string `json:"cCreateTime,omitempty" db:"created_time" gorm:"autoCreateTime"`
} }
type ( type (
DictInfo { DictInfo {
Id string `json:"id,omitempty"` Id string `json:"id,omitempty"`
DictName string `json:"dictName,omitempty"` DictName string `json:"dictName,omitempty"`
DictCode string `json:"dictCode,omitempty"` DictCode string `json:"dictCode,omitempty"`
Description string `json:"description,omitempty"` Description string `json:"description,omitempty"`
Type string `json:"type,omitempty" db:"type"` Type string `json:"type,omitempty" db:"type"`
Status string `json:"status,omitempty" db:"status"` Status string `json:"status,omitempty" db:"status"`
CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"` CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"`
} }
DictReq { DictReq {
Id string `form:"id,optional"` Id string `form:"id,optional"`
DictName string `form:"dictName,optional"` DictName string `form:"dictName,optional"`
DictCode string `form:"dictCode,optional"` DictCode string `form:"dictCode,optional"`
Description string `form:"description,optional"` Description string `form:"description,optional"`
Type string `form:"type,optional"` Type string `form:"type,optional"`
Status string `form:"status,optional"` Status string `form:"status,optional"`
PageInfo PageInfo
} }
DictEditReq { DictEditReq {
Id string `json:"id,optional"` Id string `json:"id,optional"`
DictName string `json:"dictName,optional"` DictName string `json:"dictName,optional"`
DictCode string `json:"dictCode,optional"` DictCode string `json:"dictCode,optional"`
Description string `json:"description,optional"` Description string `json:"description,optional"`
Type string `json:"type,optional"` Type string `json:"type,optional"`
Status string `json:"status,optional"` Status string `json:"status,optional"`
} }
DictResp { DictResp {
Id string `json:"id,omitempty"` Id string `json:"id,omitempty"`
DictName string `json:"dictName,omitempty"` DictName string `json:"dictName,omitempty"`
DictCode string `json:"dictCode,omitempty"` DictCode string `json:"dictCode,omitempty"`
Description string `json:"description,omitempty"` Description string `json:"description,omitempty"`
Type string `json:"type,omitempty"` Type string `json:"type,omitempty"`
Status string `json:"status,omitempty"` Status string `json:"status,omitempty"`
CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"` CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"`
DictItemInfo []*DictItemInfo `json:"dictItemInfo,omitempty"` DictItemInfo []*DictItemInfo `json:"dictItemInfo,omitempty"`
} }
Dicts { Dicts {
@ -764,55 +779,55 @@ type (
DictItemInfo { DictItemInfo {
Id string `json:"id,omitempty"` Id string `json:"id,omitempty"`
DictId string `json:"dictId,omitempty"` DictId string `json:"dictId,omitempty"`
ItemText string `json:"itemText,omitempty"` ItemText string `json:"itemText,omitempty"`
ItemValue string `json:"itemValue,omitempty"` ItemValue string `json:"itemValue,omitempty"`
Description string `json:"description,omitempty"` Description string `json:"description,omitempty"`
SortOrder string `json:"sortOrder,omitempty"` SortOrder string `json:"sortOrder,omitempty"`
Type string `json:"type,omitempty" db:"type"` Type string `json:"type,omitempty" db:"type"`
ParentId string `json:"parentId,omitempty"` ParentId string `json:"parentId,omitempty"`
Status string `json:"status,omitempty" db:"status"` Status string `json:"status,omitempty" db:"status"`
CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"` CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"`
} }
DictItemReq { DictItemReq {
Id string `form:"id,optional"` Id string `form:"id,optional"`
DictId string `form:"dictId,optional"` DictId string `form:"dictId,optional"`
ItemText string `form:"itemText,optional"` ItemText string `form:"itemText,optional"`
ItemValue string `form:"itemValue,optional"` ItemValue string `form:"itemValue,optional"`
Description string `form:"description,optional"` Description string `form:"description,optional"`
SortOrder string `form:"sortOrder,optional"` SortOrder string `form:"sortOrder,optional"`
Type string `form:"type,optional"` Type string `form:"type,optional"`
ParentId string `form:"parentId,optional"` ParentId string `form:"parentId,optional"`
Status string `form:"status,optional"` Status string `form:"status,optional"`
PageInfo PageInfo
} }
DictItemEditReq { DictItemEditReq {
Id string `json:"id,optional"` Id string `json:"id,optional"`
DictId string `json:"dictId,optional"` DictId string `json:"dictId,optional"`
ItemText string `json:"itemText,optional"` ItemText string `json:"itemText,optional"`
ItemValue string `json:"itemValue,optional"` ItemValue string `json:"itemValue,optional"`
Description string `json:"description,optional"` Description string `json:"description,optional"`
SortOrder string `json:"sortOrder,optional"` SortOrder string `json:"sortOrder,optional"`
Type string `json:"type,optional"` Type string `json:"type,optional"`
ParentId string `json:"parentId,optional"` ParentId string `json:"parentId,optional"`
Status string `json:"status,optional"` Status string `json:"status,optional"`
} }
DictItemResp { DictItemResp {
Id string `json:"id,omitempty"` Id string `json:"id,omitempty"`
DictId string `json:"dictId,omitempty"` DictId string `json:"dictId,omitempty"`
ItemText string `json:"itemText,omitempty"` ItemText string `json:"itemText,omitempty"`
ItemValue string `json:"itemValue,omitempty"` ItemValue string `json:"itemValue,omitempty"`
Description string `json:"description,omitempty"` Description string `json:"description,omitempty"`
SortOrder string `json:"sortOrder,omitempty"` SortOrder string `json:"sortOrder,omitempty"`
Type string `json:"type,omitempty"` Type string `json:"type,omitempty"`
ParentId string `json:"parentId,omitempty"` ParentId string `json:"parentId,omitempty"`
Status string `json:"status,omitempty"` Status string `json:"status,omitempty"`
CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"` CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"`
DictInfo *DictInfo `json:"dictInfo,omitempty"` DictInfo *DictInfo `json:"dictInfo,omitempty"`
} }
DictItems { DictItems {
@ -840,14 +855,14 @@ type (
type ( type (
PageInfo { PageInfo {
PageNum int `form:"pageNum"` PageNum int `form:"pageNum"`
PageSize int `form:"pageSize"` PageSize int `form:"pageSize"`
} }
PageResult { PageResult {
List interface{} `json:"list,omitempty"` List interface{} `json:"list,omitempty"`
Total int64 `json:"total,omitempty"` Total int64 `json:"total,omitempty"`
PageNum int `json:"pageNum,omitempty"` PageNum int `json:"pageNum,omitempty"`
PageSize int `json:"pageSize,omitempty"` PageSize int `json:"pageSize,omitempty"`
} }
) )

2
api/desc/pcm.api
View File

@ -852,7 +852,7 @@ service pcm {
get /adapter/cluster/get (FId) returns (ClusterResp) get /adapter/cluster/get (FId) returns (ClusterResp)
@handler GetAdapterRelationHandler @handler GetAdapterRelationHandler
get /adapter/relation (AdapterQueryReq) returns (PageResult) get /adapter/relation (AdapterRelationQueryReq) returns (PageResult)
@handler GetClusterSumHandler @handler GetClusterSumHandler
get /adapter/clusterSum (clusterSumReq) returns (clusterSumReqResp) get /adapter/clusterSum (clusterSumReq) returns (clusterSumReqResp)

2
api/etc/pcm.yaml
View File

@ -16,6 +16,8 @@ Cache:
- Host: 10.206.0.12:6379 - Host: 10.206.0.12:6379
Pass: redisPW123 Pass: redisPW123
PromUrl: http://47.92.39.128:30877
# k8s rpc # k8s rpc
K8sNativeConf: K8sNativeConf:
# target: nacos://10.206.0.12:8848/pcm.kubenative.rpc?timeout=30s&namespaceid=test&groupname=DEFAULT_GROUP&appName=pcm.core.api # target: nacos://10.206.0.12:8848/pcm.kubenative.rpc?timeout=30s&namespaceid=test&groupname=DEFAULT_GROUP&appName=pcm.core.api

1
api/internal/config/config.go
View File

@ -49,6 +49,7 @@ type Config struct {
Password string Password string
} }
SnowflakeConf SnowflakeConf SnowflakeConf SnowflakeConf
PromUrl string
} }
// SnowflakeConf 雪花算法机器id配置 // SnowflakeConf 雪花算法机器id配置

2
api/internal/cron/participant.go
View File

@ -33,7 +33,7 @@ func SyncParticipantRpc(svc *svc.ServiceContext) {
if err != nil { if err != nil {
return return
} }
svc.PromClient[participant.Id] = promClient svc.MonitorClient[participant.Id] = promClient
} }
} }
} }

2
api/internal/handler/adapters/getadapterrelationhandler.go
View File

@ -11,7 +11,7 @@ import (
func GetAdapterRelationHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { func GetAdapterRelationHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
var req types.AdapterQueryReq var req types.AdapterRelationQueryReq
if err := httpx.Parse(r, &req); err != nil { if err := httpx.Parse(r, &req); err != nil {
result.ParamErrorResult(r, w, err) result.ParamErrorResult(r, w, err)
return return

2
api/internal/logic/adapters/getadapterrelationlogic.go
View File

@ -24,7 +24,7 @@ func NewGetAdapterRelationLogic(ctx context.Context, svcCtx *svc.ServiceContext)
} }
} }
func (l *GetAdapterRelationLogic) GetAdapterRelation(req *types.AdapterQueryReq) (resp *types.PageResult, err error) { func (l *GetAdapterRelationLogic) GetAdapterRelation(req *types.AdapterRelationQueryReq) (resp *types.PageResult, err error) {
resp = &types.PageResult{} resp = &types.PageResult{}
var list []types.AdapterInfo var list []types.AdapterInfo
db := l.svcCtx.DbEngin.Model(&types.AdapterInfo{}).Table("t_adapter") db := l.svcCtx.DbEngin.Model(&types.AdapterInfo{}).Table("t_adapter")

6
api/internal/logic/cloud/controllermetricslogic.go
View File

@ -27,14 +27,14 @@ func NewControllerMetricsLogic(ctx context.Context, svcCtx *svc.ServiceContext)
func (l *ControllerMetricsLogic) ControllerMetrics(req *types.ControllerMetricsReq) (resp *types.ControllerMetricsResp, err error) { func (l *ControllerMetricsLogic) ControllerMetrics(req *types.ControllerMetricsReq) (resp *types.ControllerMetricsResp, err error) {
resp = &types.ControllerMetricsResp{} resp = &types.ControllerMetricsResp{}
if _, ok := l.svcCtx.PromClient[req.ParticipantId]; ok { if _, ok := l.svcCtx.MonitorClient[req.ParticipantId]; ok {
if len(req.Pod) != 0 { if len(req.Pod) != 0 {
resp.Data = l.svcCtx.PromClient[req.ParticipantId].GetNamedMetricsByTime(req.Metrics, req.Start, req.End, 60*time.Minute, tracker.PodOption{ resp.Data = l.svcCtx.MonitorClient[req.ParticipantId].GetNamedMetricsByTime(req.Metrics, req.Start, req.End, 60*time.Minute, tracker.PodOption{
PodName: req.Pod, PodName: req.Pod,
}) })
} else { } else {
resp.Data = l.svcCtx.PromClient[req.ParticipantId].GetNamedMetricsByTime(req.Metrics, req.Start, req.End, 60*time.Minute, tracker.ControllerOption{ resp.Data = l.svcCtx.MonitorClient[req.ParticipantId].GetNamedMetricsByTime(req.Metrics, req.Start, req.End, 60*time.Minute, tracker.ControllerOption{
WorkloadName: req.WorkloadName, WorkloadName: req.WorkloadName,
}) })
} }

67
api/internal/logic/core/centerresourceslogic.go
View File

@ -2,6 +2,9 @@ package core
import ( import (
"context" "context"
"github.com/prometheus/common/model"
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/tracker"
"strconv"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types"
@ -24,31 +27,45 @@ func NewCenterResourcesLogic(ctx context.Context, svcCtx *svc.ServiceContext) *C
} }
func (l *CenterResourcesLogic) CenterResources() (resp *types.CenterResourcesResp, err error) { func (l *CenterResourcesLogic) CenterResources() (resp *types.CenterResourcesResp, err error) {
// todo: add your logic here and delete this line
centerIndex1 := types.CenterIndex{
Name: "阿里云",
Cpu: float32(12.33),
Memory: float32(64.55),
Storage: float32(33.90),
CenterType: "cloud",
}
centerIndex2 := types.CenterIndex{
Name: "A超算中心",
Cpu: float32(42.36),
Memory: float32(66.55),
Storage: float32(23.231),
CenterType: "hpc",
}
centerIndex3 := types.CenterIndex{
Name: "智算中心",
Cpu: float32(78.33),
Memory: float32(36.55),
Storage: float32(88.93),
CenterType: "ai",
}
resp = &types.CenterResourcesResp{} resp = &types.CenterResourcesResp{}
resp.CentersIndex = append(resp.CentersIndex, centerIndex1) rawData, err := l.svcCtx.PromClient.GetRawData("center_top3", tracker.AdapterOption{})
resp.CentersIndex = append(resp.CentersIndex, centerIndex2) if err != nil {
resp.CentersIndex = append(resp.CentersIndex, centerIndex3) return nil, err
}
var centersIndex []*types.CenterIndex
data := rawData.(model.Vector)
for _, d := range data {
for _, v := range d.Metric {
num, err := strconv.ParseInt(string(v), 10, 64)
if err != nil {
return nil, err
}
centersIndex = append(centersIndex, &types.CenterIndex{Id: num})
}
}
for _, centerIndex := range centersIndex {
// Query the types of resource centers
l.svcCtx.DbEngin.Raw("select name,type as CenterType from t_adapter where id = ?", centerIndex.Id).Scan(&centerIndex)
cpuRawData, err := l.svcCtx.PromClient.GetRawData("center_cpu_usage", tracker.AdapterOption{AdapterId: centerIndex.Id})
cpuData := cpuRawData.(model.Vector)
if err != nil {
return nil, err
}
centerIndex.Cpu = cpuData[0].Value.String()
memoryRawData, err := l.svcCtx.PromClient.GetRawData("center_memory_usage", tracker.AdapterOption{AdapterId: centerIndex.Id})
if err != nil {
return nil, err
}
memoryData := memoryRawData.(model.Vector)
centerIndex.Memory = memoryData[0].Value.String()
diskRawData, err := l.svcCtx.PromClient.GetRawData("center_disk_usage", tracker.AdapterOption{AdapterId: centerIndex.Id})
if err != nil {
return nil, err
}
diskData := diskRawData.(model.Vector)
centerIndex.Storage = diskData[0].Value.String()
resp.CentersIndex = append(resp.CentersIndex, *centerIndex)
}
return resp, nil return resp, nil
} }

15
api/internal/logic/core/syncclusterloadlogic.go
View File

@ -5,6 +5,7 @@ import (
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types"
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/tracker" "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/tracker"
"strconv"
"github.com/zeromicro/go-zero/core/logx" "github.com/zeromicro/go-zero/core/logx"
) )
@ -26,9 +27,17 @@ func NewSyncClusterLoadLogic(ctx context.Context, svcCtx *svc.ServiceContext) *S
func (l *SyncClusterLoadLogic) SyncClusterLoad(req *types.SyncClusterLoadReq) error { func (l *SyncClusterLoadLogic) SyncClusterLoad(req *types.SyncClusterLoadReq) error {
if len(req.ClusterLoadRecords) != 0 { if len(req.ClusterLoadRecords) != 0 {
for _, record := range req.ClusterLoadRecords { for _, record := range req.ClusterLoadRecords {
tracker.ClusterCpuGauge.WithLabelValues(record.ClusterName).Set(record.CpuUsage) tracker.ClusterCpuUsageGauge.WithLabelValues(record.ClusterName, strconv.FormatInt(record.AdapterId, 10)).Set(record.CpuUsage)
tracker.ClusterMemoryGauge.WithLabelValues(record.ClusterName).Set(record.MemoryUsage) tracker.ClusterCpuAvailGauge.WithLabelValues(record.ClusterName, strconv.FormatInt(record.AdapterId, 10)).Set(record.CpuAvail)
tracker.ClusterDiskGauge.WithLabelValues(record.ClusterName).Set(record.DiskUsage) tracker.ClusterCpuTotalGauge.WithLabelValues(record.ClusterName, strconv.FormatInt(record.AdapterId, 10)).Set(record.CpuTotal)
tracker.ClusterMemoryUsageGauge.WithLabelValues(record.ClusterName, strconv.FormatInt(record.AdapterId, 10)).Set(record.MemoryUsage)
tracker.ClusterMemoryAvailGauge.WithLabelValues(record.ClusterName, strconv.FormatInt(record.AdapterId, 10)).Set(record.MemoryAvail)
tracker.ClusterMemoryTotalGauge.WithLabelValues(record.ClusterName, strconv.FormatInt(record.AdapterId, 10)).Set(record.MemoryTotal)
tracker.ClusterDiskUsageGauge.WithLabelValues(record.ClusterName, strconv.FormatInt(record.AdapterId, 10)).Set(record.DiskUsage)
tracker.ClusterDiskAvailGauge.WithLabelValues(record.ClusterName, strconv.FormatInt(record.AdapterId, 10)).Set(record.DiskAvail)
tracker.ClusterDiskTotalGauge.WithLabelValues(record.ClusterName, strconv.FormatInt(record.AdapterId, 10)).Set(record.DiskTotal)
} }
} }
return nil return nil

14
api/internal/scheduler/common/common.go
View File

@ -15,18 +15,11 @@
package common package common
import ( import (
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/strategy" "math"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/pkg/response"
"math/rand" "math/rand"
"time" "time"
) )
type SubSchedule interface {
GetNewStructForDb(task *response.TaskInfo, resource string, participantId int64) (interface{}, error)
PickOptimalStrategy() (strategy.Strategy, error)
AssignTask(clusters []*strategy.AssignedCluster) error
}
// 求交集 // 求交集
func Intersect(slice1, slice2 []int64) []int64 { func Intersect(slice1, slice2 []int64) []int64 {
m := make(map[int64]int) m := make(map[int64]int)
@ -90,3 +83,8 @@ func MicsSlice(origin []int64, count int) []int64 {
} }
return result return result
} }
func RoundFloat(val float64, precision uint) float64 {
ratio := math.Pow(10, float64(precision))
return math.Round(val*ratio) / ratio
}

13
api/internal/scheduler/scheduler.go
View File

@ -22,6 +22,7 @@ import (
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/database" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/database"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/service/collector" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/service/collector"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/service/executor" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/service/executor"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/strategy"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/pkg/response" "gitlink.org.cn/JointCloud/pcm-coordinator/api/pkg/response"
"gitlink.org.cn/JointCloud/pcm-coordinator/rpc/client/participantservice" "gitlink.org.cn/JointCloud/pcm-coordinator/rpc/client/participantservice"
"gorm.io/gorm" "gorm.io/gorm"
@ -33,7 +34,7 @@ import (
type Scheduler struct { type Scheduler struct {
task *response.TaskInfo task *response.TaskInfo
participantIds []int64 participantIds []int64
subSchedule common.SubSchedule subSchedule SubSchedule
dbEngin *gorm.DB dbEngin *gorm.DB
result []string //pID:子任务yamlstring 键值对 result []string //pID:子任务yamlstring 键值对
participantRpc participantservice.ParticipantService participantRpc participantservice.ParticipantService
@ -43,7 +44,13 @@ type Scheduler struct {
mu sync.RWMutex mu sync.RWMutex
} }
func NewScheduler(subSchedule common.SubSchedule, val string, dbEngin *gorm.DB, participantRpc participantservice.ParticipantService) (*Scheduler, error) { type SubSchedule interface {
GetNewStructForDb(task *response.TaskInfo, resource string, participantId int64) (interface{}, error)
PickOptimalStrategy() (strategy.Strategy, error)
AssignTask(clusters []*strategy.AssignedCluster) error
}
func NewScheduler(subSchedule SubSchedule, val string, dbEngin *gorm.DB, participantRpc participantservice.ParticipantService) (*Scheduler, error) {
var task *response.TaskInfo var task *response.TaskInfo
err := json.Unmarshal([]byte(val), &task) err := json.Unmarshal([]byte(val), &task)
if err != nil { if err != nil {
@ -123,7 +130,7 @@ func (s *Scheduler) TempAssign() error {
return nil return nil
} }
func (s *Scheduler) AssignAndSchedule(ss common.SubSchedule) error { func (s *Scheduler) AssignAndSchedule(ss SubSchedule) error {
//// 已指定 ParticipantId //// 已指定 ParticipantId
//if s.task.ParticipantId != 0 { //if s.task.ParticipantId != 0 {
// return nil // return nil

3
api/internal/scheduler/schedulers/aiScheduler.go
View File

@ -74,6 +74,9 @@ func (as *AiScheduler) PickOptimalStrategy() (strategy.Strategy, error) {
case strategy.RESOURCES_PRICING: case strategy.RESOURCES_PRICING:
strategy := strategy.NewPricingStrategy(&param.ResourcePricingParams{Params: params, Replicas: 1}) strategy := strategy.NewPricingStrategy(&param.ResourcePricingParams{Params: params, Replicas: 1})
return strategy, nil return strategy, nil
case strategy.DYNAMIC_RESOURCES:
strategy := strategy.NewDynamicResourcesStrategy(resources, as.option, 1)
return strategy, nil
} }
return nil, errors.New("no strategy has been chosen") return nil, errors.New("no strategy has been chosen")

5
api/internal/scheduler/schedulers/option/aiOption.go
View File

@ -4,6 +4,7 @@ type AiOption struct {
AiClusterId string // shuguangAi /octopus ClusterId AiClusterId string // shuguangAi /octopus ClusterId
TaskName string TaskName string
ResourceType string // cpu/gpu/compute card ResourceType string // cpu/gpu/compute card
CpuCoreNum int64
TaskType string // pytorch/tensorflow/mindspore TaskType string // pytorch/tensorflow/mindspore
DatasetsName string // mnist/imageNet/iris DatasetsName string // mnist/imageNet/iris
StrategyName string StrategyName string
@ -29,3 +30,7 @@ type AiOption struct {
Image string Image string
Model interface{} Model interface{}
} }
func (a AiOption) GetOptionType() string {
return AI
}

10
api/internal/scheduler/schedulers/option/option.go
View File

@ -1,5 +1,11 @@
package option package option
type Option struct { const (
Name string AI = "ai"
CLOUD = "cloud"
HPC = "hpc"
)
type Option interface {
GetOptionType() string
} }

12
api/internal/scheduler/service/collector/collector.go
View File

@ -8,20 +8,22 @@ type AiCollector interface {
type ResourceStats struct { type ResourceStats struct {
ParticipantId int64 ParticipantId int64
Name string Name string
CpuAvail float64 CpuCoreAvail int64
MemAvail float64 MemAvail float64
DiskAvail float64 DiskAvail float64
GpuAvail float64 GpuAvail int64
CardToHours map[Card]float64 CardsAvail []*Card
CpuToHours map[int]float64 CpuCoreHours float64
Balance float64 Balance float64
} }
type Card struct { type Card struct {
Platform string
Type string Type string
Name string Name string
TOpsAtFp16 float64 TOpsAtFp16 float64
Price int32 CardHours float64
Num int32
} }
type DatasetsSpecs struct { type DatasetsSpecs struct {

64
api/internal/scheduler/strategy/dynamicResources.go
View File

@ -1,8 +1,70 @@
package strategy package strategy
import (
"errors"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/common"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/schedulers/option"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/service/collector"
)
type DynamicResourcesStrategy struct { type DynamicResourcesStrategy struct {
replicas int32
resources []*collector.ResourceStats
opt option.Option
}
func NewDynamicResourcesStrategy(resources []*collector.ResourceStats, opt option.Option, replica int32) *DynamicResourcesStrategy {
return &DynamicResourcesStrategy{resources: resources, opt: opt, replicas: replica}
} }
func (ps *DynamicResourcesStrategy) Schedule() ([]*AssignedCluster, error) { func (ps *DynamicResourcesStrategy) Schedule() ([]*AssignedCluster, error) {
return nil, nil if ps.replicas < 1 {
return nil, errors.New("replicas must be greater than 0")
}
switch ps.opt.GetOptionType() {
case option.AI:
opt := (interface{})(ps.opt).(*option.AiOption)
var maxCardHoursAvailable float64
var maxCpuCoreHoursAvailable float64
var assignedCluster *AssignedCluster
var results []*AssignedCluster
for _, res := range ps.resources {
if opt.ResourceType == "" {
if res.CpuCoreHours <= 0 {
cluster := &AssignedCluster{ParticipantId: res.ParticipantId, Name: res.Name, Replicas: ps.replicas}
results = append(results, cluster)
return results, nil
}
if res.CpuCoreHours > maxCpuCoreHoursAvailable {
maxCpuCoreHoursAvailable = res.CpuCoreHours
assignedCluster.Name = res.Name
assignedCluster.ParticipantId = res.ParticipantId
assignedCluster.Replicas = ps.replicas
}
}
if opt.ResourceType == "" {
var maxCurrentCardHours float64
for _, card := range res.CardsAvail {
cardHours := common.RoundFloat(card.TOpsAtFp16*card.CardHours, 3)
if cardHours > maxCurrentCardHours {
maxCurrentCardHours = cardHours
}
}
if maxCurrentCardHours > maxCardHoursAvailable {
maxCardHoursAvailable = maxCurrentCardHours
assignedCluster.Name = res.Name
assignedCluster.ParticipantId = res.ParticipantId
assignedCluster.Replicas = ps.replicas
}
}
}
results = append(results, assignedCluster)
return results, nil
}
return nil, errors.New("failed to apply DynamicResourcesStrategy")
} }

2
api/internal/scheduler/strategy/param/resourcePricing.go
View File

@ -23,7 +23,7 @@ func (r *ResourcePricingParams) GetProviders() []*providerPricing.Provider {
for _, resource := range r.Resources { for _, resource := range r.Resources {
provider := providerPricing.NewProvider( provider := providerPricing.NewProvider(
resource.ParticipantId, resource.ParticipantId,
resource.CpuAvail, float64(resource.CpuCoreAvail),
resource.MemAvail, resource.MemAvail,
resource.DiskAvail, 0.0, 0.0, 0.0) resource.DiskAvail, 0.0, 0.0, 0.0)
providerList = append(providerList, provider) providerList = append(providerList, provider)

83
api/internal/storeLink/octopus.go
View File

@ -60,6 +60,10 @@ var (
MLU: CAMBRICON, MLU: CAMBRICON,
GCU: ENFLAME, GCU: ENFLAME,
} }
cardTopsMap = map[string]float64{
MLU: CAMBRICONMLU290,
GCU: EnflameT20,
}
) )
func NewOctopusLink(ctx context.Context, svcCtx *svc.ServiceContext, name string, id int64) *OctopusLink { func NewOctopusLink(ctx context.Context, svcCtx *svc.ServiceContext, name string, id int64) *OctopusLink {
@ -245,13 +249,49 @@ func (o *OctopusLink) GetResourceStats() (*collector.ResourceStats, error) {
return nil, errors.New(balanceResp.Error.Message) return nil, errors.New(balanceResp.Error.Message)
} }
//resourceStat := collector.ResourceStats{} var cards []*collector.Card
// balance := float64(balanceResp.Payload.BillingUser.Amount)
//for _, spec := range specResp.TrainResourceSpecs { var cpuHours float64
// for _, spec := range specResp.TrainResourceSpecs {
//} if spec.Price == 0 {
ns := strings.Split(spec.Name, COMMA)
if len(ns) == 2 {
nss := strings.Split(ns[0], COLON)
if nss[0] == CPU {
cpuHours = -1
}
}
}
return nil, nil if spec.Price == 1 {
ns := strings.Split(spec.Name, COMMA)
cardSpecs := strings.Split(ns[0], STAR)
cardTops, isMapContainsKey := cardTopsMap[cardSpecs[1]]
if !isMapContainsKey {
continue
}
card := &collector.Card{
Platform: OCTOPUS,
Type: CARD,
Name: cardSpecs[1],
TOpsAtFp16: cardTops,
CardHours: balance / spec.Price,
}
cards = append(cards, card)
}
}
resourceStats := &collector.ResourceStats{
ParticipantId: o.participantId,
Name: o.platform,
Balance: balance,
CardsAvail: cards,
CpuCoreHours: cpuHours,
}
return resourceStats, nil
} }
func (o *OctopusLink) GetDatasetsSpecs() ([]*collector.DatasetsSpecs, error) { func (o *OctopusLink) GetDatasetsSpecs() ([]*collector.DatasetsSpecs, error) {
@ -349,6 +389,7 @@ func (o *OctopusLink) generateResourceId(option *option.AiOption) error {
if err != nil { if err != nil {
return err return err
} }
return nil
} }
return errors.New("failed to get ResourceId") return errors.New("failed to get ResourceId")
@ -433,7 +474,14 @@ func (o *OctopusLink) generateImageId(option *option.AiOption) error {
func (o *OctopusLink) generateAlgorithmId(option *option.AiOption) error { func (o *OctopusLink) generateAlgorithmId(option *option.AiOption) error {
// temporarily set algorithm to cnn // temporarily set algorithm to cnn
option.AlgorithmName = "cnn" if option.AlgorithmName == "" {
switch option.DatasetsName {
case "cifar10":
option.AlgorithmName = "cnn"
case "mnist":
option.AlgorithmName = "fcn"
}
}
req := &octopus.GetMyAlgorithmListReq{ req := &octopus.GetMyAlgorithmListReq{
Platform: o.platform, Platform: o.platform,
@ -457,14 +505,26 @@ func (o *OctopusLink) generateAlgorithmId(option *option.AiOption) error {
if ns[1] != option.AlgorithmName { if ns[1] != option.AlgorithmName {
continue continue
} }
if ns[2] != option.ResourceType { switch option.ResourceType {
continue case CPU:
if ns[2] != CPU {
continue
}
case CARD:
if ns[2] != strings.ToLower(option.ComputeCard) {
continue
}
} }
option.AlgorithmId = algorithm.AlgorithmId option.AlgorithmId = algorithm.AlgorithmId
return nil return nil
} }
} }
if option.AlgorithmId == "" {
return errors.New("Algorithm does not exist")
}
return errors.New("failed to get AlgorithmId") return errors.New("failed to get AlgorithmId")
} }
@ -487,7 +547,10 @@ func (o *OctopusLink) generateEnv(option *option.AiOption) error {
} }
func (o *OctopusLink) generateParams(option *option.AiOption) error { func (o *OctopusLink) generateParams(option *option.AiOption) error {
if len(option.Params) == 0 {
epoch := "epoch" + COMMA + "1"
option.Params = append(option.Params, epoch)
}
return nil return nil
} }

84
api/internal/storeLink/shuguangai.go
View File

@ -17,6 +17,7 @@ package storeLink
import ( import (
"context" "context"
"errors" "errors"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/common"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/schedulers/option" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/schedulers/option"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/service/collector" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/service/collector"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc"
@ -42,6 +43,8 @@ const (
DATASETS_DIR = "/work/home/acgnnmfbwo/pcmv1/dataset" DATASETS_DIR = "/work/home/acgnnmfbwo/pcmv1/dataset"
ALGORITHM_DIR = "/work/home/acgnnmfbwo/pcmv1/algorithm" ALGORITHM_DIR = "/work/home/acgnnmfbwo/pcmv1/algorithm"
TRAIN_FILE = "train.py" TRAIN_FILE = "train.py"
CPUCOREPRICEPERHOUR = 0.09
DCUPRICEPERHOUR = 2.0
) )
var RESOURCESGAIMAP = map[string]ResourceSpecSGAI{ var RESOURCESGAIMAP = map[string]ResourceSpecSGAI{
@ -197,9 +200,9 @@ func (s *ShuguangAi) SubmitTensorflowTask(imageId string, cmd string, envs []str
} }
func (s *ShuguangAi) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) { func (s *ShuguangAi) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) {
// set algorithmId temporarily // set algorithmId temporarily for storelink submit
if algorithmId == "" { if algorithmId == "" {
algorithmId = "pytorch-mnist-fully_connected_network" algorithmId = "pytorch-mnist-fcn"
} }
// shuguangAi提交任务 // shuguangAi提交任务
@ -268,24 +271,41 @@ func (s *ShuguangAi) GetResourceStats() (*collector.ResourceStats, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
limitReq := &hpcAC.QueueReq{}
_, err = s.svcCtx.ACRpc.QueryUserQuotasLimit(s.ctx, limitReq)
if err != nil {
return nil, err
}
diskReq := &hpcAC.ParaStorQuotaReq{}
_, err = s.svcCtx.ACRpc.ParaStorQuota(s.ctx, diskReq)
if err != nil {
return nil, err
}
//limitReq := &hpcAC.QueueReq{}
//limitResp, err := s.svcCtx.ACRpc.QueryUserQuotasLimit(s.ctx, limitReq)
//if err != nil {
// return nil, err
//}
//diskReq := &hpcAC.ParaStorQuotaReq{}
//diskResp, err := s.svcCtx.ACRpc.ParaStorQuota(s.ctx, diskReq)
//if err != nil {
// return nil, err
//}
var cards []*collector.Card
balance, _ := strconv.ParseFloat(userinfo.Data.AccountBalance, 64) balance, _ := strconv.ParseFloat(userinfo.Data.AccountBalance, 64)
_ = &collector.ResourceStats{ cardHours := common.RoundFloat(balance/DCUPRICEPERHOUR, 3)
cpuHours := common.RoundFloat(balance/CPUCOREPRICEPERHOUR, 3)
dcu := &collector.Card{
Platform: SHUGUANGAI,
Type: CARD,
Name: DCU,
TOpsAtFp16: DCU_TOPS,
CardHours: cardHours,
}
cards = append(cards, dcu)
resourceStats := &collector.ResourceStats{
ParticipantId: s.participantId, ParticipantId: s.participantId,
Name: s.platform, Name: s.platform,
Balance: balance, Balance: balance,
CardsAvail: cards,
CpuCoreHours: cpuHours,
} }
return nil, nil
return resourceStats, nil
} }
func (s *ShuguangAi) GetDatasetsSpecs() ([]*collector.DatasetsSpecs, error) { func (s *ShuguangAi) GetDatasetsSpecs() ([]*collector.DatasetsSpecs, error) {
@ -413,6 +433,7 @@ func (s *ShuguangAi) generateAlgorithmId(option *option.AiOption) error {
if option.DatasetsName == "" { if option.DatasetsName == "" {
return errors.New("DatasetsName not set") return errors.New("DatasetsName not set")
} }
req := &hpcAC.GetFileListReq{Limit: 100, Path: ALGORITHM_DIR + FORWARD_SLASH + option.TaskType, Start: 0} req := &hpcAC.GetFileListReq{Limit: 100, Path: ALGORITHM_DIR + FORWARD_SLASH + option.TaskType, Start: 0}
list, err := s.svcCtx.ACRpc.GetFileList(s.ctx, req) list, err := s.svcCtx.ACRpc.GetFileList(s.ctx, req)
if err != nil { if err != nil {
@ -426,13 +447,34 @@ func (s *ShuguangAi) generateAlgorithmId(option *option.AiOption) error {
for _, file := range list.Data.FileList { for _, file := range list.Data.FileList {
ns := strings.Split(file.Name, DASH) ns := strings.Split(file.Name, DASH)
if ns[0] == option.DatasetsName { if ns[0] == option.DatasetsName {
algorithmId = option.TaskType + DASH + file.Name algoName := ns[1]
option.AlgorithmId = algorithmId if option.AlgorithmName == "" {
option.AlgorithmName = ns[1] switch option.DatasetsName {
return nil case "cifar10":
algorithmId = option.TaskType + DASH + option.DatasetsName + DASH + "cnn"
option.AlgorithmId = algorithmId
option.AlgorithmName = algoName
return nil
case "mnist":
algorithmId = option.TaskType + DASH + option.DatasetsName + DASH + "fcn"
option.AlgorithmId = algorithmId
option.AlgorithmName = algoName
return nil
}
} else {
if algoName == option.AlgorithmName {
algorithmId = option.TaskType + DASH + option.DatasetsName + DASH + algoName
option.AlgorithmId = algorithmId
return nil
}
}
} }
} }
if algorithmId == "" {
return errors.New("Algorithm does not exist")
}
return errors.New("failed to get AlgorithmId") return errors.New("failed to get AlgorithmId")
} }
@ -451,8 +493,10 @@ func (s *ShuguangAi) generateParams(option *option.AiOption) error {
return errors.New("ResourceType not set") return errors.New("ResourceType not set")
} }
//epoch := "epoch" + COMMA + "1" if len(option.Params) == 0 {
//option.Params = append(option.Params, epoch) epoch := "epoch" + COMMA + "1"
option.Params = append(option.Params, epoch)
}
switch option.ResourceType { switch option.ResourceType {
case CPU: case CPU:

14
api/internal/svc/servicecontext.go
View File

@ -59,8 +59,9 @@ type ServiceContext struct {
Downloader *s3manager.Downloader Downloader *s3manager.Downloader
Uploader *s3manager.Uploader Uploader *s3manager.Uploader
K8sRpc kubernetesclient.Kubernetes K8sRpc kubernetesclient.Kubernetes
PromClient map[int64]tracker.Prometheus MonitorClient map[int64]tracker.Prometheus
ParticipantRpc participantservice.ParticipantService ParticipantRpc participantservice.ParticipantService
PromClient tracker.Prometheus
} }
func NewServiceContext(c config.Config) *ServiceContext { func NewServiceContext(c config.Config) *ServiceContext {
@ -72,8 +73,14 @@ func NewServiceContext(c config.Config) *ServiceContext {
DisableSSL: aws.Bool(false), //是否禁用https,这里表示不禁用,即使用HTTPS DisableSSL: aws.Bool(false), //是否禁用https,这里表示不禁用,即使用HTTPS
S3ForcePathStyle: aws.Bool(true), //使用路径样式而非虚拟主机样式,区别请参考:https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html S3ForcePathStyle: aws.Bool(true), //使用路径样式而非虚拟主机样式,区别请参考:https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html
}) })
promClient, err := tracker.NewPrometheus(c.PromUrl)
if err != nil {
logx.Errorf("InitPrometheus err: %v", err)
panic("InitSnowflake err")
}
//添加snowflake支持 //添加snowflake支持
err := utils.InitSnowflake(c.SnowflakeConf.MachineId) err = utils.InitSnowflake(c.SnowflakeConf.MachineId)
if err != nil { if err != nil {
logx.Errorf("InitSnowflake err: %v", err) logx.Errorf("InitSnowflake err: %v", err)
panic("InitSnowflake err") panic("InitSnowflake err")
@ -122,10 +129,11 @@ func NewServiceContext(c config.Config) *ServiceContext {
OctopusRpc: octopusclient.NewOctopus(zrpc.MustNewClient(c.OctopusRpcConf)), OctopusRpc: octopusclient.NewOctopus(zrpc.MustNewClient(c.OctopusRpcConf)),
OpenstackRpc: openstackclient.NewOpenstack(zrpc.MustNewClient(c.OpenstackRpcConf)), OpenstackRpc: openstackclient.NewOpenstack(zrpc.MustNewClient(c.OpenstackRpcConf)),
K8sRpc: kubernetesclient.NewKubernetes(zrpc.MustNewClient(c.K8sNativeConf)), K8sRpc: kubernetesclient.NewKubernetes(zrpc.MustNewClient(c.K8sNativeConf)),
PromClient: make(map[int64]tracker.Prometheus), MonitorClient: make(map[int64]tracker.Prometheus),
ParticipantRpc: participantservice.NewParticipantService(zrpc.MustNewClient(c.PcmCoreRpcConf)), ParticipantRpc: participantservice.NewParticipantService(zrpc.MustNewClient(c.PcmCoreRpcConf)),
DockerClient: dockerClient, DockerClient: dockerClient,
Downloader: downloader, Downloader: downloader,
Uploader: uploader, Uploader: uploader,
PromClient: promClient,
} }
} }

176
api/internal/types/types.go
View File

@ -17,11 +17,12 @@ type CenterResourcesResp struct {
} }
type CenterIndex struct { type CenterIndex struct {
Name string `json:"name"` Id int64 `json:"id"`
Cpu float32 `json:"cpu"` Name string `json:"name"`
Memory float32 `json:"memory"` Cpu string `json:"cpu"`
Storage float32 `json:"storage"` Memory string `json:"memory"`
CenterType string `json:"centerType"` Storage string `json:"storage"`
CenterType string `json:"centerType"`
} }
type SyncClusterLoadReq struct { type SyncClusterLoadReq struct {
@ -29,9 +30,16 @@ type SyncClusterLoadReq struct {
} }
type ClusterLoadRecord struct { type ClusterLoadRecord struct {
AdapterId int64 `json:"adapterId"`
ClusterName string `json:"clusterName"` ClusterName string `json:"clusterName"`
CpuAvail float64 `json:"cpuAvail"`
CpuTotal float64 `json:"cpuTotal"`
CpuUsage float64 `json:"cpuUsage"` CpuUsage float64 `json:"cpuUsage"`
MemoryAvail float64 `json:"memoryAvail"`
MemoryUsage float64 `json:"memoryUsage"` MemoryUsage float64 `json:"memoryUsage"`
MemoryTotal float64 `json:"memoryTotal"`
DiskAvail float64 `json:"diskAvail"`
DiskTotal float64 `json:"diskTotal"`
DiskUsage float64 `json:"diskUsage"` DiskUsage float64 `json:"diskUsage"`
} }
@ -518,6 +526,15 @@ type AdapterQueryReq struct {
PageInfo PageInfo
} }
type AdapterRelationQueryReq struct {
Id string `form:"id,optional" db:"id"`
Name string `form:"name,optional"`
Type string `form:"type,optional"`
Nickname string `form:"nickname,optional"`
Version string `form:"version,optional"`
Server string `form:"server,optional"`
}
type AdapterReq struct { type AdapterReq struct {
Id string `json:"id,optional" db:"id"` Id string `json:"id,optional" db:"id"`
Name string `json:"name,optional"` Name string `json:"name,optional"`
@ -2569,7 +2586,7 @@ type Limits struct {
} }
type GetComputeLimitsReq struct { type GetComputeLimitsReq struct {
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type GetComputeLimitsResp struct { type GetComputeLimitsResp struct {
@ -2601,7 +2618,7 @@ type VolumeLimits struct {
} }
type GetVolumeLimitsReq struct { type GetVolumeLimitsReq struct {
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type GetVolumeLimitsResp struct { type GetVolumeLimitsResp struct {
@ -2614,7 +2631,7 @@ type GetVolumeLimitsResp struct {
type ListServersReq struct { type ListServersReq struct {
Limit int32 `form:"limit,optional"` Limit int32 `form:"limit,optional"`
OffSet int32 `form:"offSet,optional"` OffSet int32 `form:"offSet,optional"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListServersResp struct { type ListServersResp struct {
@ -2643,7 +2660,7 @@ type Servers_links struct {
} }
type ListServersDetailedReq struct { type ListServersDetailedReq struct {
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListServersDetailedResp struct { type ListServersDetailedResp struct {
@ -2667,7 +2684,7 @@ type ServersDetailed struct {
type GetServersDetailedByIdReq struct { type GetServersDetailedByIdReq struct {
ServerId string `form:"server_id" copier:"ServerId"` ServerId string `form:"server_id" copier:"ServerId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type GetServersDetailedByIdResp struct { type GetServersDetailedByIdResp struct {
@ -2755,7 +2772,7 @@ type ExtraSpecs struct {
type UpdateServerReq struct { type UpdateServerReq struct {
ServerId string `form:"server_id" copier:"ServerId"` ServerId string `form:"server_id" copier:"ServerId"`
ServerUpdate ServerUpdate `json:"server_update" copier:"ServerUpdate"` ServerUpdate ServerUpdate `json:"server_update" copier:"ServerUpdate"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ServerUpdate struct { type ServerUpdate struct {
@ -2773,7 +2790,7 @@ type StartServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Action []map[string]string `json:"action,optional" copier:"Action"` Action []map[string]string `json:"action,optional" copier:"Action"`
Start_action string `json:"start_action" copier:"start_action"` Start_action string `json:"start_action" copier:"start_action"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type StartServerResp struct { type StartServerResp struct {
@ -2786,7 +2803,7 @@ type StopServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Action []map[string]string `json:"action,optional" copier:"Action"` Action []map[string]string `json:"action,optional" copier:"Action"`
Stop_action string `json:"stop_action" copier:"stop_action"` Stop_action string `json:"stop_action" copier:"stop_action"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type StopServerResp struct { type StopServerResp struct {
@ -2798,7 +2815,7 @@ type StopServerResp struct {
type RebootServerReq struct { type RebootServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Reboot Reboot `json:"reboot" copier:"Reboot"` Reboot Reboot `json:"reboot" copier:"Reboot"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type RebootServerResp struct { type RebootServerResp struct {
@ -2815,7 +2832,7 @@ type PauseServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Action []map[string]string `json:"Action,optional" copier:"Action"` Action []map[string]string `json:"Action,optional" copier:"Action"`
Pause_action string `json:"pause_action" copier:"pause_action"` Pause_action string `json:"pause_action" copier:"pause_action"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type PauseServerResp struct { type PauseServerResp struct {
@ -2828,7 +2845,7 @@ type UnpauseServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Action []map[string]string `json:"Action,optional" copier:"Action"` Action []map[string]string `json:"Action,optional" copier:"Action"`
Unpause_action string `json:"unpause_action" copier:"unpause_action"` Unpause_action string `json:"unpause_action" copier:"unpause_action"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type UnpauseServerResp struct { type UnpauseServerResp struct {
@ -2839,7 +2856,7 @@ type UnpauseServerResp struct {
type DeleteServerReq struct { type DeleteServerReq struct {
ServerId string `form:"server_id" copier:"ServerId"` ServerId string `form:"server_id" copier:"ServerId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type DeleteServerResp struct { type DeleteServerResp struct {
@ -2850,7 +2867,7 @@ type DeleteServerResp struct {
type CreateServerReq struct { type CreateServerReq struct {
Server Server `json:"server" copier:"Server"` Server Server `json:"server" copier:"Server"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type CreateServerResp struct { type CreateServerResp struct {
@ -2894,7 +2911,7 @@ type ServerResp struct {
type RebuildServerReq struct { type RebuildServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
Rebuild Rebuild `json:"rebuild" copier:"Rebuild"` Rebuild Rebuild `json:"rebuild" copier:"Rebuild"`
} }
@ -2969,7 +2986,7 @@ type ShelveServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Action []map[string]string `json:"Action,optional" copier:"Action"` Action []map[string]string `json:"Action,optional" copier:"Action"`
Shelve_action string `json:"shelve_action" copier:"shelve_action"` Shelve_action string `json:"shelve_action" copier:"shelve_action"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ShelveServerResp struct { type ShelveServerResp struct {
@ -2980,7 +2997,7 @@ type ShelveServerResp struct {
type RescueServerReq struct { type RescueServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
Rescue Rescue `json:"rescue" copier:"Rescue"` Rescue Rescue `json:"rescue" copier:"Rescue"`
} }
@ -2998,7 +3015,7 @@ type Rescue struct {
type UnRescueServerReq struct { type UnRescueServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Action []map[string]string `json:"Action,optional" copier:"Action"` Action []map[string]string `json:"Action,optional" copier:"Action"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
UnRescue_action string `json:"UnRescue_action" copier:"UnRescue_action"` UnRescue_action string `json:"UnRescue_action" copier:"UnRescue_action"`
Rescue Rescue `json:"rescue" copier:"Rescue"` Rescue Rescue `json:"rescue" copier:"Rescue"`
} }
@ -3012,7 +3029,7 @@ type UnRescueServerResp struct {
type ChangeAdministrativePasswordReq struct { type ChangeAdministrativePasswordReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Changepassword string `json:"changePassword" copier:"Changepassword"` Changepassword string `json:"changePassword" copier:"Changepassword"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
ChangePassword ChangePassword `json:"changepassword" copier:"ChangePassword"` ChangePassword ChangePassword `json:"changepassword" copier:"ChangePassword"`
} }
@ -3029,7 +3046,7 @@ type ChangePassword struct {
type SuspendServerReq struct { type SuspendServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Action []map[string]string `json:"Action,optional" copier:"Action"` Action []map[string]string `json:"Action,optional" copier:"Action"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
UnRescue_action string `json:"UnRescue_action" copier:"UnRescue_action"` UnRescue_action string `json:"UnRescue_action" copier:"UnRescue_action"`
Rescue Rescue `json:"rescue" copier:"Rescue"` Rescue Rescue `json:"rescue" copier:"Rescue"`
} }
@ -3043,7 +3060,7 @@ type SuspendServerResp struct {
type AddSecurityGroupToServerReq struct { type AddSecurityGroupToServerReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Action []map[string]string `json:"Action,optional" copier:"Action"` Action []map[string]string `json:"Action,optional" copier:"Action"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
UnRescue_action string `json:"UnRescue_action" copier:"UnRescue_action"` UnRescue_action string `json:"UnRescue_action" copier:"UnRescue_action"`
AddSecurityGroup AddSecurityGroup `json:"addSecurityGroup" copier:"AddSecurityGroup"` AddSecurityGroup AddSecurityGroup `json:"addSecurityGroup" copier:"AddSecurityGroup"`
} }
@ -3060,7 +3077,7 @@ type AddSecurityGroup struct {
type RemoveSecurityGroupReq struct { type RemoveSecurityGroupReq struct {
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
RemoveSecurityGroup RemoveSecurityGroup `json:"removeSecurityGroup" copier:"RemoveSecurityGroup"` RemoveSecurityGroup RemoveSecurityGroup `json:"removeSecurityGroup" copier:"RemoveSecurityGroup"`
} }
@ -3096,7 +3113,7 @@ type FlavorServer struct {
} }
type DeleteFlavorReq struct { type DeleteFlavorReq struct {
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
ServerId string `json:"server_id" copier:"ServerId"` ServerId string `json:"server_id" copier:"ServerId"`
FlavorId string `json:"flavor_id" copier:"FlavorId"` FlavorId string `json:"flavor_id" copier:"FlavorId"`
} }
@ -3108,7 +3125,8 @@ type DeleteFlavorResp struct {
} }
type ListImagesReq struct { type ListImagesReq struct {
Platform string `form:"platform,optional"` Limit int32 `form:"limit,optional"`
Platform string `json:"platform,optional"`
} }
type ListImagesResp struct { type ListImagesResp struct {
@ -3178,7 +3196,7 @@ type CreateImageResp struct {
type UploadOsImageReq struct { type UploadOsImageReq struct {
ImageId string `form:"image_id" copier:"ImageId"` ImageId string `form:"image_id" copier:"ImageId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type UploadOsImageResp struct { type UploadOsImageResp struct {
@ -3189,7 +3207,7 @@ type UploadOsImageResp struct {
type DeleteImageReq struct { type DeleteImageReq struct {
ImageId string `form:"image_id" copier:"ImageId"` ImageId string `form:"image_id" copier:"ImageId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type DeleteImageResp struct { type DeleteImageResp struct {
@ -3213,7 +3231,7 @@ type ImageNum struct {
} }
type ListNetworksReq struct { type ListNetworksReq struct {
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListNetworksResp struct { type ListNetworksResp struct {
@ -3253,7 +3271,7 @@ type Network struct {
type DeleteNetworkReq struct { type DeleteNetworkReq struct {
NetworkId string `form:"network_id" copier:"NetworkId"` NetworkId string `form:"network_id" copier:"NetworkId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type DeleteNetworkResp struct { type DeleteNetworkResp struct {
@ -3264,7 +3282,7 @@ type DeleteNetworkResp struct {
type CreateNetworkReq struct { type CreateNetworkReq struct {
Network CreateNetwork `json:"network" copier:"Network"` Network CreateNetwork `json:"network" copier:"Network"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type CreateNetworkResp struct { type CreateNetworkResp struct {
@ -3282,7 +3300,7 @@ type CreateNetwork struct {
type CreateSubnetReq struct { type CreateSubnetReq struct {
Subnet Subnet `json:"subnet" copier:"Subnet"` Subnet Subnet `json:"subnet" copier:"Subnet"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type CreateSubnetResp struct { type CreateSubnetResp struct {
@ -3337,7 +3355,7 @@ type Allocation_pools struct {
type ShowNetworkDetailsReq struct { type ShowNetworkDetailsReq struct {
NetworkId string `form:"network_id" copier:"NetworkId"` NetworkId string `form:"network_id" copier:"NetworkId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ShowNetworkDetailsResp struct { type ShowNetworkDetailsResp struct {
@ -3376,7 +3394,7 @@ type Networkdetail struct {
type UpdateNetworkReq struct { type UpdateNetworkReq struct {
NetworkId string `form:"network_id" copier:"NetworkId"` NetworkId string `form:"network_id" copier:"NetworkId"`
Network Network `json:"network" copier:"Network"` Network Network `json:"network" copier:"Network"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type UpdateNetworkResp struct { type UpdateNetworkResp struct {
@ -3401,7 +3419,7 @@ type BulkCreateNetworksResp struct {
type ListSubnetsReq struct { type ListSubnetsReq struct {
Limit int32 `json:"limit,optional"` Limit int32 `json:"limit,optional"`
OffSet int32 `json:"offSet,optional"` OffSet int32 `json:"offSet,optional"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListSubnetsResp struct { type ListSubnetsResp struct {
@ -3443,7 +3461,7 @@ type Allocation_pool struct {
type DeleteSubnetReq struct { type DeleteSubnetReq struct {
SubnetId string `json:"subnet_id,optional" copier:"subnetId"` SubnetId string `json:"subnet_id,optional" copier:"subnetId"`
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
} }
type DeleteSubnetResp struct { type DeleteSubnetResp struct {
@ -3454,7 +3472,7 @@ type DeleteSubnetResp struct {
type UpdateSubnetReq struct { type UpdateSubnetReq struct {
SubnetId string `json:"subnet_id,optional" copier:"subnetId"` SubnetId string `json:"subnet_id,optional" copier:"subnetId"`
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
} }
type UpdateSubnetResp struct { type UpdateSubnetResp struct {
@ -3499,7 +3517,7 @@ type Used struct {
type ListNetworkSegmentRangesReq struct { type ListNetworkSegmentRangesReq struct {
Limit int32 `json:"limit,optional"` Limit int32 `json:"limit,optional"`
OffSet int32 `json:"offSet,optional"` OffSet int32 `json:"offSet,optional"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListNetworkSegmentRangesResp struct { type ListNetworkSegmentRangesResp struct {
@ -3529,7 +3547,7 @@ type Network_segment_ranges struct {
type DeleteNetworkSegmentRangesReq struct { type DeleteNetworkSegmentRangesReq struct {
Network_segment_range_id string `json:"network_segment_range_id,optional" copier:"network_segment_range_id"` Network_segment_range_id string `json:"network_segment_range_id,optional" copier:"network_segment_range_id"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type DeleteNetworkSegmentRangesResp struct { type DeleteNetworkSegmentRangesResp struct {
@ -3541,7 +3559,7 @@ type DeleteNetworkSegmentRangesResp struct {
type UpdateNetworkSegmentRangesReq struct { type UpdateNetworkSegmentRangesReq struct {
Network_segment_range_id string `json:"network_segment_range_id,optional" copier:"network_segment_range_id"` Network_segment_range_id string `json:"network_segment_range_id,optional" copier:"network_segment_range_id"`
NetworkSegmentRange Network_segment_range `json:"network_segment_range,optional" copier:"NetworkSegmentRange"` NetworkSegmentRange Network_segment_range `json:"network_segment_range,optional" copier:"NetworkSegmentRange"`
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
} }
type UpdateNetworkSegmentRangesResp struct { type UpdateNetworkSegmentRangesResp struct {
@ -3613,7 +3631,7 @@ type CreatePortResp struct {
type ListPortsReq struct { type ListPortsReq struct {
Limit int32 `json:"limit,optional"` Limit int32 `json:"limit,optional"`
OffSet int32 `json:"offSet,optional"` OffSet int32 `json:"offSet,optional"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListPortsResp struct { type ListPortsResp struct {
@ -3672,7 +3690,7 @@ type Fixed_ips struct {
type DeletePortReq struct { type DeletePortReq struct {
Port_id string `json:"port_id,optional" copier:"port_id"` Port_id string `json:"port_id,optional" copier:"port_id"`
Platform string `form:"platform,optional" copier:"platform"` Platform string `json:"platform,optional" copier:"platform"`
} }
type DeletePortResp struct { type DeletePortResp struct {
@ -3684,7 +3702,7 @@ type DeletePortResp struct {
type UpdatePortReq struct { type UpdatePortReq struct {
Port_id string `json:"port_id,optional" copier:"port_id"` Port_id string `json:"port_id,optional" copier:"port_id"`
Port Port `json:"port,optional" copier:"port"` Port Port `json:"port,optional" copier:"port"`
Platform string `form:"platform,optional" copier:"platform"` Platform string `json:"platform,optional" copier:"platform"`
} }
type UpdatePortResp struct { type UpdatePortResp struct {
@ -3697,7 +3715,7 @@ type UpdatePortResp struct {
type ShowPortDetailsReq struct { type ShowPortDetailsReq struct {
Port_id string `json:"port_id,optional" copier:"port_id"` Port_id string `json:"port_id,optional" copier:"port_id"`
Fields string `json:"fields,optional" copier:"fields"` Fields string `json:"fields,optional" copier:"fields"`
Platform string `form:"platform,optional" copier:"platform"` Platform string `json:"platform,optional" copier:"platform"`
} }
type ShowPortDetailsResp struct { type ShowPortDetailsResp struct {
@ -3710,7 +3728,7 @@ type ShowPortDetailsResp struct {
type ListRoutersReq struct { type ListRoutersReq struct {
Limit int32 `json:"limit,optional"` Limit int32 `json:"limit,optional"`
OffSet int32 `json:"offSet,optional"` OffSet int32 `json:"offSet,optional"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListRoutersResp struct { type ListRoutersResp struct {
@ -3804,7 +3822,7 @@ type Router struct {
type UpdateRouterReq struct { type UpdateRouterReq struct {
Router Router `json:"router,optional" copier:"router"` Router Router `json:"router,optional" copier:"router"`
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
} }
type UpdateRouterResp struct { type UpdateRouterResp struct {
@ -3816,7 +3834,7 @@ type UpdateRouterResp struct {
type ShowRouterDetailsReq struct { type ShowRouterDetailsReq struct {
Router Router `json:"router,optional" copier:"router"` Router Router `json:"router,optional" copier:"router"`
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
} }
type ShowRouterDetailsResp struct { type ShowRouterDetailsResp struct {
@ -3827,7 +3845,7 @@ type ShowRouterDetailsResp struct {
} }
type DeleteRouterReq struct { type DeleteRouterReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Router_id string `json:"router_id,optional" copier:"router_id"` Router_id string `json:"router_id,optional" copier:"router_id"`
} }
@ -3838,7 +3856,7 @@ type DeleteRouterResp struct {
} }
type ListFloatingIPsReq struct { type ListFloatingIPsReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Limit int32 `json:"limit,optional" copier:"Limit"` Limit int32 `json:"limit,optional" copier:"Limit"`
OffSet int32 `json:"offSet,optional" copier:"OffSet"` OffSet int32 `json:"offSet,optional" copier:"OffSet"`
} }
@ -3925,7 +3943,7 @@ type Floatingip struct {
} }
type UpdateFloatingIPReq struct { type UpdateFloatingIPReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Floatingip_id string `json:"floatingip_id,optional" copier:"floatingip_id"` Floatingip_id string `json:"floatingip_id,optional" copier:"floatingip_id"`
Floatingip Floatingip `json:"floatingip,optional" copier:"floatingip"` Floatingip Floatingip `json:"floatingip,optional" copier:"floatingip"`
} }
@ -3938,7 +3956,7 @@ type UpdateFloatingIPResp struct {
} }
type DeleteFloatingIPReq struct { type DeleteFloatingIPReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Floatingip_id string `json:"floatingip_id,optional" copier:"floatingip_id"` Floatingip_id string `json:"floatingip_id,optional" copier:"floatingip_id"`
} }
@ -3961,7 +3979,7 @@ type ShowFloatingIPDetailsResp struct {
} }
type ListFirewallGroupsReq struct { type ListFirewallGroupsReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Fields string `json:"fields,optional" copier:"fields"` Fields string `json:"fields,optional" copier:"fields"`
} }
@ -3988,7 +4006,7 @@ type Firewall_groups struct {
type DeleteFirewallGroupReq struct { type DeleteFirewallGroupReq struct {
Firewall_group_id string `json:"firewall_group_id,optional" copier:"firewall_group_id"` Firewall_group_id string `json:"firewall_group_id,optional" copier:"firewall_group_id"`
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
} }
type DeleteFirewallGroupResp struct { type DeleteFirewallGroupResp struct {
@ -4025,7 +4043,7 @@ type Firewall_group struct {
type UpdateFirewallGroupReq struct { type UpdateFirewallGroupReq struct {
Firewall_group Firewall_group `json:"firewall_group,optional" copier:"firewall_group"` Firewall_group Firewall_group `json:"firewall_group,optional" copier:"firewall_group"`
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Firewall_group_id string `json:"firewall_group_id,optional" copier:"firewall_group_id"` Firewall_group_id string `json:"firewall_group_id,optional" copier:"firewall_group_id"`
} }
@ -4037,7 +4055,7 @@ type UpdateFirewallGroupResp struct {
} }
type ShowFirewallGroupDetailsReq struct { type ShowFirewallGroupDetailsReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Firewall_group_id string `json:"firewall_group_id,optional" copier:"firewall_group_id"` Firewall_group_id string `json:"firewall_group_id,optional" copier:"firewall_group_id"`
} }
@ -4072,7 +4090,7 @@ type Firewall_policy struct {
} }
type DeleteFirewallPolicyReq struct { type DeleteFirewallPolicyReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Firewall_policy_id string `json:"firewall_policy_id,optional" copier:"firewall_policy_id"` Firewall_policy_id string `json:"firewall_policy_id,optional" copier:"firewall_policy_id"`
} }
@ -4083,7 +4101,7 @@ type DeleteFirewallPolicyResp struct {
} }
type ListFirewallPoliciesReq struct { type ListFirewallPoliciesReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Fields string `json:"fields,optional" copier:"fields"` Fields string `json:"fields,optional" copier:"fields"`
} }
@ -4149,7 +4167,7 @@ type Firewall_rule struct {
} }
type DeleteFirewallRuleReq struct { type DeleteFirewallRuleReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Firewall_rule_id string `json:"firewall_rule_id,optional" copier:"firewall_rule_id"` Firewall_rule_id string `json:"firewall_rule_id,optional" copier:"firewall_rule_id"`
} }
@ -4160,7 +4178,7 @@ type DeleteFirewallRuleResp struct {
} }
type ListFirewallRulesReq struct { type ListFirewallRulesReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Fields string `json:"fields,optional" copier:"fields"` Fields string `json:"fields,optional" copier:"fields"`
} }
@ -4191,7 +4209,7 @@ type Firewall_rules struct {
} }
type ShowFirewallRuleDetailsReq struct { type ShowFirewallRuleDetailsReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Firewall_rule_id string `json:"firewall_rule_id,optional" copier:"firewall_rule_id"` Firewall_rule_id string `json:"firewall_rule_id,optional" copier:"firewall_rule_id"`
} }
@ -4203,7 +4221,7 @@ type ShowFirewallRuleDetailsResp struct {
} }
type ListSecurityGroupsReq struct { type ListSecurityGroupsReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Fields string `json:"firewall_rule_id,optional" copier:"firewall_rule_id"` Fields string `json:"firewall_rule_id,optional" copier:"firewall_rule_id"`
} }
@ -4229,7 +4247,7 @@ type Security_groups struct {
} }
type CreateSecurityGroupReq struct { type CreateSecurityGroupReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Firewall_rule_id string `json:"firewall_rule_id,optional" copier:"firewall_rule_id"` Firewall_rule_id string `json:"firewall_rule_id,optional" copier:"firewall_rule_id"`
} }
@ -4276,7 +4294,7 @@ type Security_group_rules struct {
} }
type DeleteSecurityGroupReq struct { type DeleteSecurityGroupReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Security_group_id string `json:"security_group_id,optional" copier:"security_group_id"` Security_group_id string `json:"security_group_id,optional" copier:"security_group_id"`
} }
@ -4287,7 +4305,7 @@ type DeleteSecurityGroupResp struct {
} }
type UpdateSecurityGroupReq struct { type UpdateSecurityGroupReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Security_group_id string `json:"security_group_id,optional" copier:"security_group_id"` Security_group_id string `json:"security_group_id,optional" copier:"security_group_id"`
} }
@ -4299,7 +4317,7 @@ type UpdateSecurityGroupResp struct {
} }
type ShowSecurityGroupReq struct { type ShowSecurityGroupReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Security_group_id string `json:"security_group_id,optional" copier:"security_group_id"` Security_group_id string `json:"security_group_id,optional" copier:"security_group_id"`
Fields string `json:"fields,optional" copier:"fields"` Fields string `json:"fields,optional" copier:"fields"`
Verbose string `json:"verbose,optional" copier:"verbose"` Verbose string `json:"verbose,optional" copier:"verbose"`
@ -4313,7 +4331,7 @@ type ShowSecurityGroupResp struct {
} }
type ListSecurityGroupRulesReq struct { type ListSecurityGroupRulesReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
} }
type ListSecurityGroupRulesResp struct { type ListSecurityGroupRulesResp struct {
@ -4355,7 +4373,7 @@ type Security_group_rule struct {
} }
type ShowSecurityGroupRuleReq struct { type ShowSecurityGroupRuleReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Security_group_rule_id string `json:"security_group_rule_id,optional" copier:"security_group_rule_id"` Security_group_rule_id string `json:"security_group_rule_id,optional" copier:"security_group_rule_id"`
} }
@ -4367,7 +4385,7 @@ type ShowSecurityGroupRuleResp struct {
} }
type DeleteSecurityGroupRuleReq struct { type DeleteSecurityGroupRuleReq struct {
Platform string `form:"platform,optional" copier:"Platform"` Platform string `json:"platform,optional" copier:"Platform"`
Security_group_rule_id string `json:"security_group_rule_id,optional" copier:"security_group_rule_id"` Security_group_rule_id string `json:"security_group_rule_id,optional" copier:"security_group_rule_id"`
} }
@ -4379,7 +4397,7 @@ type DeleteSecurityGroupRuleResp struct {
} }
type ListVolumesDetailReq struct { type ListVolumesDetailReq struct {
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListVolumesDetailResp struct { type ListVolumesDetailResp struct {
@ -4413,7 +4431,7 @@ type DeleteVolumeReq struct {
VolumeId string `form:"volume_id" copier:"VolumeId"` VolumeId string `form:"volume_id" copier:"VolumeId"`
Cascade bool `json:"cascade" copier:"Cascade"` Cascade bool `json:"cascade" copier:"Cascade"`
Force bool `json:"force" copier:"force"` Force bool `json:"force" copier:"force"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type DeleteVolumeResp struct { type DeleteVolumeResp struct {
@ -4463,7 +4481,7 @@ type VolumeResp struct {
} }
type ListVolumeTypesReq struct { type ListVolumeTypesReq struct {
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListVolumeTypesResp struct { type ListVolumeTypesResp struct {
@ -4490,7 +4508,7 @@ type Extra_specs struct {
type UpdateVolumeReq struct { type UpdateVolumeReq struct {
Volume Volume `json:"volume" copier:"Volume"` Volume Volume `json:"volume" copier:"Volume"`
VolumeId string `json:"volume_id" copier:"VolumeId"` VolumeId string `json:"volume_id" copier:"VolumeId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type UpdateVolumeResp struct { type UpdateVolumeResp struct {
@ -4502,7 +4520,7 @@ type UpdateVolumeResp struct {
type GetVolumeDetailedByIdReq struct { type GetVolumeDetailedByIdReq struct {
VolumeId string `form:"volume_id" copier:"VolumeId"` VolumeId string `form:"volume_id" copier:"VolumeId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type GetVolumeDetailedByIdResp struct { type GetVolumeDetailedByIdResp struct {
@ -4554,7 +4572,7 @@ type VolumeType struct {
type DeleteVolumeTypeReq struct { type DeleteVolumeTypeReq struct {
VolumeTypeId string `json:"volume_type_id" copier:"VolumeTypeId"` VolumeTypeId string `json:"volume_type_id" copier:"VolumeTypeId"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type DeleteVolumeTypeResp struct { type DeleteVolumeTypeResp struct {
@ -4574,7 +4592,7 @@ type ListVolumesReq struct {
CreatedAt string `json:"created_at" copier:"CreatedAt"` CreatedAt string `json:"created_at" copier:"CreatedAt"`
ConsumesQuota bool `json:"consumes_quota" copier:"ConsumesQuota"` ConsumesQuota bool `json:"consumes_quota" copier:"ConsumesQuota"`
UpdatedAt string `json:"updated_at" copier:"UpdatedAt"` UpdatedAt string `json:"updated_at" copier:"UpdatedAt"`
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListVolumesResp struct { type ListVolumesResp struct {
@ -4591,7 +4609,7 @@ type VolumesList struct {
} }
type ListFlavorsDetailReq struct { type ListFlavorsDetailReq struct {
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
} }
type ListFlavorsDetailResp struct { type ListFlavorsDetailResp struct {
@ -4616,7 +4634,7 @@ type Flavors struct {
} }
type ListNodesReq struct { type ListNodesReq struct {
Platform string `form:"platform,optional"` Platform string `json:"platform,optional"`
Limit int64 `json:"limit" copier:"Limit"` Limit int64 `json:"limit" copier:"Limit"`
Marker string `json:"marker" copier:"Marker"` Marker string `json:"marker" copier:"Marker"`
SortDir string `json:"sort_dir" copier:"SortDir"` SortDir string `json:"sort_dir" copier:"SortDir"`

6
pkg/tracker/interface.go
View File

@ -14,7 +14,10 @@
package tracker package tracker
import "time" import (
"github.com/prometheus/common/model"
"time"
)
type Interface interface { type Interface interface {
//GetMetric(expr string, time time.Time) Metric //GetMetric(expr string, time time.Time) Metric
@ -27,4 +30,5 @@ type Interface interface {
//// meter //// meter
//GetNamedMeters(meters []string, time time.Time, opts []QueryOption) []Metric //GetNamedMeters(meters []string, time time.Time, opts []QueryOption) []Metric
//GetNamedMetersOverTime(metrics []string, start, end time.Time, step time.Duration, opts []QueryOption) []Metric //GetNamedMetersOverTime(metrics []string, start, end time.Time, step time.Duration, opts []QueryOption) []Metric
GetRawData(expr string, o QueryOption) (model.Value, error)
} }

78
pkg/tracker/promql.go
View File

@ -27,37 +27,18 @@ const (
var promQLTemplates = map[string]string{ var promQLTemplates = map[string]string{
//namespace "cluster_cpu_usage": "sum by (cluster_name)(cluster_cpu_usage{$1})",
"namespace_cpu_usage": `round(namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", $1}, 0.001)`, "cluster_memory_usage": "sum by (cluster_name)(cluster_memory_usage{$1})",
"namespace_memory_usage": `namespace:container_memory_usage_bytes:sum{namespace!="", $1}`, "cluster_disk_usage": "sum by (cluster_name)(cluster_disk_usage{$1})",
"namespace_memory_usage_wo_cache": `namespace:container_memory_usage_bytes_wo_cache:sum{namespace!="", $1}`, "center_cpu_usage": "(sum by (adapter_id)(cluster_cpu_total{$1})-sum by (adapter_id)(cluster_cpu_avail{$1}))/sum by (adapter_id)(cluster_cpu_total{$1})",
"namespace_net_bytes_transmitted": `sum by (namespace) (irate(container_network_transmit_bytes_total{namespace!="", pod!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m]) * on (namespace) group_left(workspace) kube_namespace_labels{$1}) or on(namespace) max by(namespace) (kube_namespace_labels{$1} * 0)`, "center_memory_usage": "(sum by (adapter_id)(cluster_memory_total{$1})-sum by (adapter_id)(cluster_memory_avail{$1}))/sum by (adapter_id)(cluster_memory_total{$1})",
"namespace_net_bytes_received": `sum by (namespace) (irate(container_network_receive_bytes_total{namespace!="", pod!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m]) * on (namespace) group_left(workspace) kube_namespace_labels{$1}) or on(namespace) max by(namespace) (kube_namespace_labels{$1} * 0)`, "center_disk_usage": "(sum by (adapter_id)(cluster_disk_total{$1})-sum by (adapter_id)(cluster_disk_avail{$1}))/sum by (adapter_id)(cluster_disk_total{$1})",
"namespace_pod_count": `sum by (namespace) (kube_pod_status_phase{phase!~"Failed|Succeeded", namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1}) or on(namespace) max by(namespace) (kube_namespace_labels{$1} * 0)`, "center_top3": "topk(3,((sum by (adapter_id)(cluster_cpu_total)-sum by (adapter_id)(cluster_cpu_avail))/sum by (adapter_id)(cluster_cpu_total) + (sum by (adapter_id)(cluster_memory_total) - sum by (adapter_id)(cluster_memory_avail))/sum by (adapter_id)(cluster_memory_total) + (sum by (adapter_id)(cluster_disk_total)-sum by (adapter_id)(cluster_disk_avail))/sum by (adapter_id)(cluster_disk_total))/3)",
"namespace_pod_running_count": `sum by (namespace) (kube_pod_status_phase{phase="Running", namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1}) or on(namespace) max by(namespace) (kube_namespace_labels{$1} * 0)`, "namespace_cpu_usage": `round(namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", $1}, 0.001)`,
"namespace_pod_succeeded_count": `sum by (namespace) (kube_pod_status_phase{phase="Succeeded", namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1}) or on(namespace) max by(namespace) (kube_namespace_labels{$1} * 0)`, "namespace_memory_usage": `namespace:container_memory_usage_bytes:sum{namespace!="", $1}`,
"namespace_pod_abnormal_count": `namespace:pod_abnormal:count{namespace!="", $1}`, "namespace_memory_usage_wo_cache": `namespace:container_memory_usage_bytes_wo_cache:sum{namespace!="", $1}`,
"namespace_pod_abnormal_ratio": `namespace:pod_abnormal:ratio{namespace!="", $1}`, "controller_cpu_usage_rate": `sum( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{}* on(namespace,pod) group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{$1}) by (workload)/sum( kube_pod_container_resource_limits{job="kube-state-metrics", resource="cpu"}* on(namespace,pod) group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{ }) by (workload)`,
"namespace_memory_limit_hard": `min by (namespace) (kube_resourcequota{resourcequota!="quota", type="hard", namespace!="", resource="limits.memory"} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`, "controller_memory_usage_rate": `sum( container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", container!="", image!=""} * on(namespace,pod) group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{$1}) by (workload)/sum( kube_pod_container_resource_limits{job="kube-state-metrics", resource="memory"}* on(namespace,pod) group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{ }) by (workload)`,
"namespace_cpu_limit_hard": `min by (namespace) (kube_resourcequota{resourcequota!="quota", type="hard", namespace!="", resource="limits.cpu"} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_pod_count_hard": `min by (namespace) (kube_resourcequota{resourcequota!="quota", type="hard", namespace!="", resource="count/pods"} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_cronjob_count": `sum by (namespace) (kube_cronjob_labels{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_pvc_count": `sum by (namespace) (kube_persistentvolumeclaim_info{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_daemonset_count": `sum by (namespace) (kube_daemonset_labels{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_deployment_count": `sum by (namespace) (kube_deployment_labels{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_endpoint_count": `sum by (namespace) (kube_endpoint_labels{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_hpa_count": `sum by (namespace) (kube_horizontalpodautoscaler_labels{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_job_count": `sum by (namespace) (kube_job_labels{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_statefulset_count": `sum by (namespace) (kube_statefulset_labels{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_replicaset_count": `count by (namespace) (kube_replicaset_labels{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_service_count": `sum by (namespace) (kube_service_info{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_secret_count": `sum by (namespace) (kube_secret_info{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_configmap_count": `sum by (namespace) (kube_configmap_info{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_ingresses_extensions_count": `sum by (namespace) (kube_ingress_labels{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"namespace_s2ibuilder_count": `sum by (namespace) (s2i_s2ibuilder_created{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`,
"controller_cpu_usage_rate": `sum( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{}* on(namespace,pod) group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{$1}) by (workload)/sum( kube_pod_container_resource_limits{job="kube-state-metrics", resource="cpu"}* on(namespace,pod) group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{ }) by (workload)`,
"controller_memory_usage_rate": `sum( container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", container!="", image!=""} * on(namespace,pod) group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{$1}) by (workload)/sum( kube_pod_container_resource_limits{job="kube-state-metrics", resource="memory"}* on(namespace,pod) group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{ }) by (workload)`,
// pod // pod
"pod_cpu_usage": `round(sum by (namespace, pod) (irate(container_cpu_usage_seconds_total{job="kubelet", pod!="", image!=""}[5m])) * on (namespace, pod) group_left(owner_kind,owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}, 0.001)`, "pod_cpu_usage": `round(sum by (namespace, pod) (irate(container_cpu_usage_seconds_total{job="kubelet", pod!="", image!=""}[5m])) * on (namespace, pod) group_left(owner_kind,owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}, 0.001)`,
"pod_cpu_usage_rate": `sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{ $1}) by (pod) / sum(kube_pod_container_resource_limits{ $1,unit="core"}) by (pod)`, "pod_cpu_usage_rate": `sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{ $1}) by (pod) / sum(kube_pod_container_resource_limits{ $1,unit="core"}) by (pod)`,
@ -80,8 +61,10 @@ var promQLTemplates = map[string]string{
func makeExpr(metric string, opts QueryOptions) string { func makeExpr(metric string, opts QueryOptions) string {
tmpl := promQLTemplates[metric] tmpl := promQLTemplates[metric]
switch opts.Level { switch opts.Level {
case LevelAdapter:
return makeAdapterMetricExpr(tmpl, opts)
case LevelCluster: case LevelCluster:
return tmpl return makeClusterMetricExpr(tmpl, opts)
case LevelNode: case LevelNode:
return makeNodeMetricExpr(tmpl, opts) return makeNodeMetricExpr(tmpl, opts)
case LevelWorkspace: case LevelWorkspace:
@ -105,6 +88,24 @@ func makeExpr(metric string, opts QueryOptions) string {
} }
} }
func makeClusterMetricExpr(tmpl string, o QueryOptions) string {
var clusterSelector string
if o.ClusterName != "" {
clusterSelector = fmt.Sprintf(`cluster_name="%s"`, o.ClusterName)
}
return strings.Replace(tmpl, "$1", clusterSelector, -1)
}
func makeAdapterMetricExpr(tmpl string, o QueryOptions) string {
var adapterSelector string
if o.AdapterId != 0 {
adapterSelector = fmt.Sprintf(`adapter_id="%d"`, o.AdapterId)
}
return strings.Replace(tmpl, "$1", adapterSelector, -1)
}
func makeNodeMetricExpr(tmpl string, o QueryOptions) string { func makeNodeMetricExpr(tmpl string, o QueryOptions) string {
var nodeSelector string var nodeSelector string
if o.NodeName != "" { if o.NodeName != "" {
@ -177,19 +178,12 @@ func makePVCMetricExpr(tmpl string, o QueryOptions) string {
// GET /namespaces/{namespace}/persistentvolumeclaims/{persistentvolumeclaim} or // GET /namespaces/{namespace}/persistentvolumeclaims/{persistentvolumeclaim} or
// GET /namespaces/{namespace}/persistentvolumeclaims // GET /namespaces/{namespace}/persistentvolumeclaims
if o.Namespace != "" { if o.Namespace != "" {
if o.PersistentVolumeClaimName != "" {
pvcSelector = fmt.Sprintf(`namespace="%s", persistentvolumeclaim="%s"`, o.Namespace, o.PersistentVolumeClaimName) pvcSelector = fmt.Sprintf(`namespace="%s", persistentvolumeclaim=~"%s"`, o.Namespace, o.ResourceFilter)
} else {
pvcSelector = fmt.Sprintf(`namespace="%s", persistentvolumeclaim=~"%s"`, o.Namespace, o.ResourceFilter)
}
return strings.Replace(tmpl, "$1", pvcSelector, -1) return strings.Replace(tmpl, "$1", pvcSelector, -1)
} }
// For monitoring persistentvolumeclaims of the specific storageclass
// GET /storageclasses/{storageclass}/persistentvolumeclaims
if o.StorageClassName != "" {
pvcSelector = fmt.Sprintf(`storageclass="%s", persistentvolumeclaim=~"%s"`, o.StorageClassName, o.ResourceFilter)
}
return strings.Replace(tmpl, "$1", pvcSelector, -1) return strings.Replace(tmpl, "$1", pvcSelector, -1)
} }

38
pkg/tracker/queryoptions.go
View File

@ -28,7 +28,6 @@ const (
LevelWorkspace LevelWorkspace
LevelNamespace LevelNamespace
LevelApplication LevelApplication
LevelOpenpitrix
LevelController LevelController
LevelService LevelService
LevelPod LevelPod
@ -36,9 +35,11 @@ const (
LevelPVC LevelPVC
LevelComponent LevelComponent
LevelIngress LevelIngress
LevelAdapter
) )
var MeteringLevelMap = map[string]int{ var MeteringLevelMap = map[string]int{
"LevelAdapter": LevelAdapter,
"LevelCluster": LevelCluster, "LevelCluster": LevelCluster,
"LevelNode": LevelNode, "LevelNode": LevelNode,
"LevelWorkspace": LevelWorkspace, "LevelWorkspace": LevelWorkspace,
@ -68,6 +69,7 @@ type QueryOptions struct {
NamespacedResourcesFilter string NamespacedResourcesFilter string
QueryType string QueryType string
ResourceFilter string ResourceFilter string
ClusterName string
NodeName string NodeName string
WorkspaceName string WorkspaceName string
Namespace string Namespace string
@ -77,10 +79,7 @@ type QueryOptions struct {
PodName string PodName string
PodsName string PodsName string
ContainerName string ContainerName string
StorageClassName string AdapterId int64
PersistentVolumeClaimName string
PVCFilter string
ApplicationName string
ServiceName string ServiceName string
Ingress string Ingress string
Job string Job string
@ -92,10 +91,22 @@ func NewQueryOptions() *QueryOptions {
return &QueryOptions{} return &QueryOptions{}
} }
type ClusterOption struct{} type AdapterOption struct {
AdapterId int64
}
func (_ ClusterOption) Apply(o *QueryOptions) { func (a AdapterOption) Apply(o *QueryOptions) {
o.Level = LevelAdapter
o.AdapterId = a.AdapterId
}
type ClusterOption struct {
ClusterName string
}
func (c ClusterOption) Apply(o *QueryOptions) {
o.Level = LevelCluster o.Level = LevelCluster
o.ClusterName = c.ClusterName
} }
type NodeOption struct { type NodeOption struct {
@ -110,8 +121,6 @@ func (no NodeOption) Apply(o *QueryOptions) {
o.Level = LevelNode o.Level = LevelNode
o.ResourceFilter = no.ResourceFilter o.ResourceFilter = no.ResourceFilter
o.NodeName = no.NodeName o.NodeName = no.NodeName
o.PVCFilter = no.PVCFilter
o.StorageClassName = no.StorageClassName
o.QueryType = no.QueryType o.QueryType = no.QueryType
} }
@ -126,8 +135,6 @@ func (wo WorkspaceOption) Apply(o *QueryOptions) {
o.Level = LevelWorkspace o.Level = LevelWorkspace
o.ResourceFilter = wo.ResourceFilter o.ResourceFilter = wo.ResourceFilter
o.WorkspaceName = wo.WorkspaceName o.WorkspaceName = wo.WorkspaceName
o.PVCFilter = wo.PVCFilter
o.StorageClassName = wo.StorageClassName
} }
type NamespaceOption struct { type NamespaceOption struct {
@ -143,8 +150,6 @@ func (no NamespaceOption) Apply(o *QueryOptions) {
o.ResourceFilter = no.ResourceFilter o.ResourceFilter = no.ResourceFilter
o.WorkspaceName = no.WorkspaceName o.WorkspaceName = no.WorkspaceName
o.Namespace = no.NamespaceName o.Namespace = no.NamespaceName
o.PVCFilter = no.PVCFilter
o.StorageClassName = no.StorageClassName
} }
type ApplicationsOption struct { type ApplicationsOption struct {
@ -183,8 +188,6 @@ type ApplicationOption struct {
func (ao ApplicationOption) Apply(o *QueryOptions) { func (ao ApplicationOption) Apply(o *QueryOptions) {
o.Level = LevelApplication o.Level = LevelApplication
o.Namespace = ao.NamespaceName o.Namespace = ao.NamespaceName
o.ApplicationName = ao.Application
o.StorageClassName = ao.StorageClassName
app_components := strings.Join(ao.ApplicationComponents[:], "|") app_components := strings.Join(ao.ApplicationComponents[:], "|")
@ -303,11 +306,6 @@ func (po PVCOption) Apply(o *QueryOptions) {
o.Level = LevelPVC o.Level = LevelPVC
o.ResourceFilter = po.ResourceFilter o.ResourceFilter = po.ResourceFilter
o.Namespace = po.NamespaceName o.Namespace = po.NamespaceName
o.StorageClassName = po.StorageClassName
o.PersistentVolumeClaimName = po.PersistentVolumeClaimName
// for meter
o.PVCFilter = po.PersistentVolumeClaimName
} }
type IngressOption struct { type IngressOption struct {

59
pkg/tracker/tracker.go
View File

@ -27,22 +27,53 @@ import (
) )
var ( var (
ClusterCpuGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ClusterCpuUsageGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_cpu_usage", Name: "cluster_cpu_usage",
Help: "Cluster CPU Utilization Rate.", Help: "Cluster CPU Utilization Rate.",
}, []string{"cluster_name"}) }, []string{"cluster_name", "adapter_id"})
ClusterMemoryGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ClusterCpuAvailGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_cpu_avail",
Help: "Cluster CPU Available.",
}, []string{"cluster_name", "adapter_id"})
ClusterCpuTotalGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_cpu_total",
Help: "Cluster CPU Total.",
}, []string{"cluster_name", "adapter_id"})
ClusterMemoryUsageGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_memory_usage", Name: "cluster_memory_usage",
Help: "Cluster Memory Utilization Rate.", Help: "Cluster Memory Utilization Rate.",
}, []string{"cluster_name"}) }, []string{"cluster_name", "adapter_id"})
ClusterDiskGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ ClusterMemoryAvailGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_memory_avail",
Help: "Cluster Memory Available.",
}, []string{"cluster_name", "adapter_id"})
ClusterMemoryTotalGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_memory_total",
Help: "Cluster Memory Total.",
}, []string{"cluster_name", "adapter_id"})
ClusterDiskUsageGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_disk_usage", Name: "cluster_disk_usage",
Help: "Cluster Disk Utilization Rate.", Help: "Cluster Disk Utilization Rate.",
}, []string{"cluster_name"}) }, []string{"cluster_name", "adapter_id"})
ClusterDiskAvailGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_disk_avail",
Help: "Cluster Disk Available.",
}, []string{"cluster_name", "adapter_id"})
ClusterDiskTotalGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cluster_disk_total",
Help: "Cluster Disk Total.",
}, []string{"cluster_name", "adapter_id"})
metrics = []prometheus.Collector{ metrics = []prometheus.Collector{
ClusterCpuGauge, ClusterCpuUsageGauge,
ClusterMemoryGauge, ClusterCpuAvailGauge,
ClusterDiskGauge, ClusterCpuTotalGauge,
ClusterMemoryUsageGauge,
ClusterMemoryAvailGauge,
ClusterMemoryTotalGauge,
ClusterDiskUsageGauge,
ClusterDiskAvailGauge,
ClusterDiskTotalGauge,
} }
) )
@ -225,3 +256,13 @@ func genMetricFilter(o QueryOption) func(metric model.Metric) bool {
return true return true
} }
} }
func (p Prometheus) GetRawData(expr string, o QueryOption) (model.Value, error) {
opts := NewQueryOptions()
o.Apply(opts)
value, _, err := p.client.Query(context.Background(), makeExpr(expr, *opts), time.Now())
if err != nil {
return nil, err
}
return value, nil
}