floraachy
/
pcm-coordinator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'add task details' (#128) from devad/pcm-coordinator:master into master 

Former-commit-id: 32190d9b2b216318bd5913fc56cfa5f3d7a0e5cd
This commit is contained in:
devad 2024-04-29 11:07:53 +08:00
parent b9ee118d40 8aba7624a6
commit 9ecd0190d5
14 changed files with 179 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/desc/cloud/pcm-cloud.api
View File

@ -59,9 +59,9 @@ type (
Type int64 `json:"type"` // 租户所属(0数算，1超算，2智算）
DeletedFlag int64 `json:"deletedFlag"` // 是否删除
CreatedBy int64 `json:"createdBy"` // 创建人
CreatedTime string `json:"createdTime"` // 创建时间
CreateTime string `json:"createdTime"` // 创建时间
UpdatedBy int64 `json:"updatedBy"` // 更新人
UpdatedTime string `json:"updated_time"` // 更新时间
UpdateTime string `json:"updated_time"` // 更新时间
}
UpdateTenantReq {
@ -115,6 +115,6 @@ type Cloud {
StartTime string `json:"startTime"` // 开始时间
RunningTime int64 `json:"runningTime"` // 运行时长
CreatedBy int64 `json:"createdBy"` // 创建人
CreatedTime string `json:"createdTime"` // 创建时间
CreateTime string `json:"createdTime"` // 创建时间
Result string `json:"result"`
}

35
api/desc/core/pcm-core.api
View File

@ -112,24 +112,13 @@ type (
type (
GeneralTaskReq {
Name string `json:"name"`
ComputeType string `json:"computeType"`
TemplateId string `json:"templateId"`
AdapterId string `json:"adapterId"`
ClusterIds []string `json:"clusterIds"`
Strategy Strategy `json:"strategy"`
Name string `json:"name"`
AdapterIds []string `json:"adapterIds"`
ClusterIds []string `json:"clusterIds"`
Strategy string `json:"strategy"`
StaticWeightMap map[string]int32 `json:"staticWeightMap,optional"`
ReqBody []string `json:"reqBody"`
}
Strategy {
Name string `json:"name"`
StaticWeightList []StaticWeightList `json:"staticWeightList"`
}
StaticWeightList {
ClusterName string `json:"clusterName"`
Weight int `json:"weight"`
}
)
type deleteTaskReq {
@ -958,9 +947,9 @@ type (
Environment string `json:"environment"`
DeletedFlag int64 `json:"deleted_flag"` // 是否删除（0-否，1-是）
CreatedBy int64 `json:"created_by"` // 创建人
CreatedTime string `json:"created_time"` // 创建时间
CreateTime string `json:"created_time"` // 创建时间
UpdatedBy int64 `json:"updated_by"` // 更新人
UpdatedTime string `json:"updated_time"` // 更新时间
UpdateTime string `json:"updated_time"` // 更新时间
}
CloudInfo {
@ -1110,4 +1099,14 @@ type TaskStatusResp {
Failed int `json:"Failed"`
Running int `json:"Running"`
Pause int `json:"Pause"`
}
type TaskDetailsResp {
Name string `json:"name"`
description string `json:"description"`
StartTime string `json:"startTime"`
EndTime string `json:"endTime"`
Strategy int64 `json:"strategy,string"`
SynergyStatus int64 `json:"synergyStatus,string"`
ClusterInfos []*ClusterInfo `json:"clusterInfos"`
}

18
api/desc/pcm.api
View File

@ -137,11 +137,14 @@ service pcm {
@doc "Statistical task status"
@handler countTaskStatus
get /core/task/countTaskStatus () returns (TaskStatusResp)
get /core/task/countTaskStatus () returns(TaskStatusResp)
@doc "Home Page Overview"
@handler homeOverviewHandler
get /core/homeOverview (HomeOverviewReq) returns (HomeOverviewResp)
@doc "task details"
@handler taskDetails
get /core/task/details (FId) returns(TaskDetailsResp)
}
//hpc二级接口
@ -215,7 +218,7 @@ service pcm {
@doc "Create cloud computing common tasks"
@handler commitGeneralTask
post /cloud/task/create (GeneralTaskReq) returns()
post /cloud/task/create (GeneralTaskReq) returns ()
}
//智算二级接口
@ -927,6 +930,9 @@ service pcm {
@handler ScheduleGetAlgorithmsHandler
get /schedule/ai/getAlgorithms/:adapterId/:resourceType/:taskType/:dataset (AiAlgorithmsReq) returns (AiAlgorithmsResp)
@handler ScheduleGetAiJobLogLogHandler
get /schedule/ai/getJobLog/:adapterId/:clusterId/:taskId/:instanceNum (AiJobLogReq) returns (AiJobLogResp)
@handler ScheduleSubmitHandler
post /schedule/submit (ScheduleReq) returns (ScheduleResp)
}
@ -980,7 +986,7 @@ service pcm {
@doc "alert rules"
@handler alertRulesHandler
get /monitoring/alert/rule (AlertRulesReq)returns (AlertRulesResp)
get /monitoring/alert/rule (AlertRulesReq) returns (AlertRulesResp)
@doc "cluster resource load"
@handler clustersLoadHandler
@ -997,4 +1003,10 @@ service pcm {
@doc "Synchronize Cluster alert Information"
@handler syncClusterAlertHandler
post /core/syncClusterAlert (SyncClusterAlertReq)
@handler taskNumHandler
get /monitoring/task/num (taskNumReq) returns (taskNumResp)
@handler adapterInfoHandler
get /monitoring/adapter/info (adapterInfoReq) returns (adapterInfoResp)
}

2
api/etc/pcm.yaml
View File

@ -5,7 +5,7 @@ Port: 8999
Timeout: 50000
DB:
DataSource: root:uJpLd6u-J?HC1@(10.206.0.12:3306)/pcm?parseTime=true&loc=Local
DataSource: root:uJpLd6u-J?HC1@(47.92.88.143:3306)/pcm?parseTime=true&loc=Local
# DataSource: root:uJpLd6u-J?HC1@(47.92.88.143:3306)/pcm?parseTime=true&loc=Local
Redis:
Host: 10.206.0.12:6379

24
api/internal/handler/core/taskdetailshandler.go Normal file
View File

@ -0,0 +1,24 @@
package core
import (
"github.com/zeromicro/go-zero/rest/httpx"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/logic/core"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types"
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/repository/result"
"net/http"
)
func TaskDetailsHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
var req types.FId
if err := httpx.Parse(r, &req); err != nil {
result.ParamErrorResult(r, w, err)
return
}
l := core.NewTaskDetailsLogic(r.Context(), svcCtx)
resp, err := l.TaskDetails(&req)
result.HttpResult(r, w, resp, err)
}
}

7
api/internal/handler/routes.go
View File

@ -170,6 +170,11 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) {
Path: "/core/homeOverview",
Handler: core.HomeOverviewHandler(serverCtx),
},
{
Method: http.MethodGet,
Path: "/core/task/details",
Handler: core.TaskDetailsHandler(serverCtx),
},
},
rest.WithPrefix("/pcm/v1"),
)
@ -1254,7 +1259,7 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) {
},
{
Method: http.MethodPost,
Path: "/monitoring/syncClusterAlert",
Path: "/core/syncClusterAlert",
Handler: monitoring.SyncClusterAlertHandler(serverCtx),
},
{

15
api/internal/logic/cloud/commitgeneraltasklogic.go
View File

@ -7,6 +7,7 @@ import (
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/constants"
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models"
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models/cloud"
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/utils"
"io"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
@ -52,7 +53,6 @@ func (l *CommitGeneralTaskLogic) CommitGeneralTask(req *types.GeneralTaskReq) er
Status: constants.Saved,
Name: req.Name,
CommitTime: time.Now(),
NsID: "ns-admin",
YamlString: "[" + result + "]",
}
// Save the task data to the database
@ -62,7 +62,7 @@ func (l *CommitGeneralTaskLogic) CommitGeneralTask(req *types.GeneralTaskReq) er
}
var clusters []*models.CloudModel
err := l.svcCtx.DbEngin.Raw("SELECT * FROM `t_cluster` where adapter_id = ? and id in ?", req.AdapterId, req.ClusterIds).Scan(&clusters).Error
err := l.svcCtx.DbEngin.Raw("SELECT * FROM `t_cluster` where adapter_id in ? and id in ?", req.AdapterIds, req.ClusterIds).Scan(&clusters).Error
if err != nil {
logx.Errorf("CommitGeneralTask() => sql execution error: %v", err)
return errors.Errorf("the cluster does not match the drive resources. Check the data")
@ -73,11 +73,12 @@ func (l *CommitGeneralTaskLogic) CommitGeneralTask(req *types.GeneralTaskReq) er
for _, s := range req.ReqBody {
sStruct := UnMarshalK8sStruct(s)
unString, _ := sStruct.MarshalJSON()
taskCloud.Id = utils.GenSnowflakeIDUint()
taskCloud.TaskId = uint(taskModel.Id)
taskCloud.AdapterId = c.AdapterId
taskCloud.ClusterId = c.Id
taskCloud.ClusterName = c.Name
taskCloud.Status = "Saved"
taskCloud.Status = "Pending"
taskCloud.YamlString = string(unString)
taskCloud.Kind = sStruct.GetKind()
taskCloud.Namespace = sStruct.GetNamespace()
@ -113,6 +114,14 @@ func UnMarshalK8sStruct(yamlString string) *unstructured.Unstructured {
if len(unstructuredObj.GetNamespace()) == 0 {
unstructuredObj.SetNamespace("default")
}
//设置副本数
if unstructuredObj.GetKind() == "Deployment" || unstructuredObj.GetKind() == "StatefulSet" {
unstructured.SetNestedField(
unstructuredObj.Object,
int64(6),
"spec", "replicas",
)
}
}
return unstructuredObj
}

61
api/internal/logic/core/taskdetailslogic.go Normal file
View File

@ -0,0 +1,61 @@
package core
import (
"context"
"github.com/pkg/errors"
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models"
"gorm.io/gorm"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc"
"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types"
"github.com/zeromicro/go-zero/core/logx"
)
type TaskDetailsLogic struct {
logx.Logger
ctx context.Context
svcCtx *svc.ServiceContext
}
func NewTaskDetailsLogic(ctx context.Context, svcCtx *svc.ServiceContext) *TaskDetailsLogic {
return &TaskDetailsLogic{
Logger: logx.WithContext(ctx),
ctx: ctx,
svcCtx: svcCtx,
}
}
func (l *TaskDetailsLogic) TaskDetails(req *types.FId) (resp *types.TaskDetailsResp, err error) {
var task models.Task
if errors.Is(l.svcCtx.DbEngin.Where("id", req.Id).First(&task).Error, gorm.ErrRecordNotFound) {
return nil, errors.New("记录不存在")
}
clusterIds := make([]int64, 0)
var cList []*types.ClusterInfo
switch task.TaskTypeDict {
case 0:
l.svcCtx.DbEngin.Table("task_cloud").Select("cluster_id").Where("task_id", task.Id).Scan(&clusterIds)
case 1:
l.svcCtx.DbEngin.Table("task_ai").Select("cluster_id").Where("task_id", task.Id).Scan(&clusterIds)
case 2:
l.svcCtx.DbEngin.Table("task_hpc").Select("cluster_id").Where("task_id", task.Id).Scan(&clusterIds)
case 3:
l.svcCtx.DbEngin.Table("task_vm").Select("cluster_id").Where("task_id", task.Id).Find(&clusterIds)
}
err = l.svcCtx.DbEngin.Table("t_cluster").Where("id in ?", clusterIds).Scan(&cList).Error
if err != nil {
return resp, err
}
resp = &types.TaskDetailsResp{
Name: task.Name,
Description: task.Description,
StartTime: task.StartTime,
EndTime: task.EndTime,
Strategy: task.Strategy,
SynergyStatus: task.SynergyStatus,
ClusterInfos: cList,
}
return
}

7
api/internal/scheduler/schedulers/option/cloudOption.go
View File

@ -1,7 +1,12 @@
package option
type CloudOption struct {
task interface{}
Name string `json:"name"`
AdapterIds []string `json:"adapterIds"`
ClusterIds []string `json:"clusterIds"`
Strategy string `json:"strategy"`
StaticWeightMap map[string]int32 `json:"staticWeightMap,optional"`
ReqBody []string `json:"reqBody"`
}
func (c CloudOption) GetOptionType() string {

43
api/internal/types/types.go
View File

@ -99,23 +99,12 @@ type Region struct {
}
type GeneralTaskReq struct {
Name string `json:"name"`
ComputeType string `json:"computeType"`
TemplateId string `json:"templateId"`
AdapterId string `json:"adapterId"`
ClusterIds []string `json:"clusterIds"`
Strategy Strategy `json:"strategy"`
ReqBody []string `json:"reqBody"`
}
type Strategy struct {
Name string `json:"name"`
StaticWeightList []StaticWeightList `json:"staticWeightList"`
}
type StaticWeightList struct {
ClusterName string `json:"clusterName"`
Weight int `json:"weight"`
Name string `json:"name"`
AdapterIds []string `json:"adapterIds"`
ClusterIds []string `json:"clusterIds"`
Strategy string `json:"strategy"`
StaticWeightMap map[string]int32 `json:"staticWeightMap,optional"`
ReqBody []string `json:"reqBody"`
}
type DeleteTaskReq struct {
@ -928,9 +917,9 @@ type HpcInfo struct {
Environment string `json:"environment"`
DeletedFlag int64 `json:"deleted_flag"` // 是否删除（0-否，1-是）
CreatedBy int64 `json:"created_by"` // 创建人
CreatedTime string `json:"created_time"` // 创建时间
CreateTime string `json:"created_time"` // 创建时间
UpdatedBy int64 `json:"updated_by"` // 更新人
UpdatedTime string `json:"updated_time"` // 更新时间
UpdateTime string `json:"updated_time"` // 更新时间
}
type CloudInfo struct {
@ -1075,6 +1064,16 @@ type TaskStatusResp struct {
Pause int `json:"Pause"`
}
type TaskDetailsResp struct {
Name string `json:"name"`
Description string `json:"description"`
StartTime string `json:"startTime"`
EndTime string `json:"endTime"`
Strategy int64 `json:"strategy,string"`
SynergyStatus int64 `json:"synergyStatus,string"`
ClusterInfos []*ClusterInfo `json:"clusterInfos"`
}
type CommitHpcTaskReq struct {
Name string `json:"name"` // paratera:jobName
Description string `json:"description,optional"`
@ -5305,9 +5304,9 @@ type TenantInfo struct {
Type int64 `json:"type"` // 租户所属(0数算，1超算，2智算）
DeletedFlag int64 `json:"deletedFlag"` // 是否删除
CreatedBy int64 `json:"createdBy"` // 创建人
CreatedTime string `json:"createdTime"` // 创建时间
CreateTime string `json:"createdTime"` // 创建时间
UpdatedBy int64 `json:"updatedBy"` // 更新人
UpdatedTime string `json:"updated_time"` // 更新时间
UpdateTime string `json:"updated_time"` // 更新时间
}
type UpdateTenantReq struct {
@ -5361,7 +5360,7 @@ type Cloud struct {
StartTime string `json:"startTime"` // 开始时间
RunningTime int64 `json:"runningTime"` // 运行时长
CreatedBy int64 `json:"createdBy"` // 创建人
CreatedTime string `json:"createdTime"` // 创建时间
CreateTime string `json:"createdTime"` // 创建时间
Result string `json:"result"`
}

10
pkg/models/base/base_model.go
View File

@ -6,9 +6,9 @@ import (
)
type BaseModel struct {
DeletedAt gorm.DeletedAt `gorm:"index;comment:删除时间" json:"-"` // 删除时间
CreatedBy uint `gorm:"created_by;comment:创建人" json:"createdBy"` //创建人
CreatedTime time.Time `gorm:"comment:创建时间" json:"-"` // 创建时间
UpdatedBy uint `gorm:"updated_by;comment:更新人" json:"UpdatedBy"` //创建人
UpdatedTime time.Time `gorm:"comment:更新时间" json:"-"` // 更新时间
DeletedAt gorm.DeletedAt `gorm:"index;comment:删除时间" json:"-"` // 删除时间
CreatedBy uint `gorm:"created_by;comment:创建人" json:"createdBy"` //创建人
CreateTime time.Time `gorm:"autoCreateTime:nano;comment:创建时间" json:"-"` // 创建时间
UpdatedBy uint `gorm:"updated_by;comment:更新人" json:"UpdatedBy"` //创建人
UpdateTime time.Time `gorm:"autoUpdateTime:nano;;comment:更新时间" json:"-"` // 更新时间
}

4
pkg/models/file.go
View File

@ -37,9 +37,9 @@ type File struct {
Status string `gorm:"column:status" json:"Status"` //type:string comment:hash version:2023-05-06 09:58
DeletedFlag *int `gorm:"column:deleted_flag" json:"DeletedFlag"` //type:*int comment:是否删除 version:2023-05-06 09:58
CreatedBy *int `gorm:"column:created_by" json:"CreatedBy"` //type:*int comment:创建人 version:2023-05-06 09:58
CreatedTime *time.Time `gorm:"column:created_time" json:"CreatedTime"` //type:*time.Time comment:创建时间 version:2023-05-06 09:58
CreatedTime *time.Time `gorm:"column:created_time" json:"CreateTime"` //type:*time.Time comment:创建时间 version:2023-05-06 09:58
UpdatedBy *int `gorm:"column:updated_by" json:"UpdatedBy"` //type:*int comment:更新人 version:2023-05-06 09:58
UpdatedTime *time.Time `gorm:"column:updated_time" json:"UpdatedTime"` //type:*time.Time comment:更新时间 version:2023-05-06 09:58
UpdatedTime *time.Time `gorm:"column:updated_time" json:"UpdateTime"` //type:*time.Time comment:更新时间 version:2023-05-06 09:58
}
// TableName 表名:data_set，。

1
pkg/models/taskmodel_gen.go
View File

@ -49,6 +49,7 @@ type (
Result string `db:"result"` // 作业结果
DeletedAt gorm.DeletedAt `gorm:"index"`
NsID string `db:"ns_id"`
TaskTypeDict int `db:"task_type_dict"` //任务类型（对应字典表的值）
}
)

5
pkg/utils/snowflake.go
View File

@ -41,3 +41,8 @@ func GenSnowflakeID() int64 {
func GenSnowflakeIDStr() string {
return node.Generate().String()
}
// GenSnowflakeIDStr 工作id
func GenSnowflakeIDUint() uint {
return uint(node.Generate().Int64())
}