From 571e7603c7910208cd83520eff3ce3216414532e Mon Sep 17 00:00:00 2001 From: jagger Date: Sun, 7 Apr 2024 18:11:11 +0800 Subject: [PATCH 1/6] fix bugs Signed-off-by: jagger Former-commit-id: 8902acbd35bd41440625c0ac972f46e06f5305f7 --- api/desc/core/pcm-core.api | 22 +++ api/desc/pcm.api | 4 + .../handler/cloud/commitgeneraltaskhandler.go | 17 +++ api/internal/handler/routes.go | 5 + .../logic/cloud/commitgeneraltasklogic.go | 66 +++++++++ api/internal/types/types.go | 137 ++++++++++++++++++ go.sum | 8 - 7 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 api/internal/handler/cloud/commitgeneraltaskhandler.go create mode 100644 api/internal/logic/cloud/commitgeneraltasklogic.go diff --git a/api/desc/core/pcm-core.api b/api/desc/core/pcm-core.api index 4cdefe5e..79758887 100644 --- a/api/desc/core/pcm-core.api +++ b/api/desc/core/pcm-core.api @@ -93,6 +93,28 @@ type ( } ) +type ( + GeneralTaskReq { + Name string `json:"name"` + ComputeType string `json:"computeType"` + TemplateId string `json:"templateId"` + AdapterId string `json:"adapterId"` + ClusterIds []string `json:"clusterIds"` + Strategy Strategy `json:"strategy"` + ReqBody []string `json:"reqBody"` + } + + Strategy { + Name string `json:"name"` + StaticWeightList []StaticWeightList `json:"staticWeightList"` + } + + StaticWeightList { + ClusterName string `json:"clusterName"` + Weight int `json:"weight"` + } +) + type deleteTaskReq { Id int64 `path:"id"` } diff --git a/api/desc/pcm.api b/api/desc/pcm.api index 24fd3043..9040a551 100644 --- a/api/desc/pcm.api +++ b/api/desc/pcm.api @@ -192,6 +192,10 @@ service pcm { @doc "Obtain cluster list information according to adapterId" @handler getClusterListHandler get /core/clusterList (getClusterListReq) returns (getClusterListResp) + + @doc "Create cloud computing common tasks" + @handler commitGeneralTask + post /cloud/task/create (GeneralTaskReq) returns() } //智算二级接口 diff --git a/api/internal/handler/cloud/commitgeneraltaskhandler.go b/api/internal/handler/cloud/commitgeneraltaskhandler.go new file mode 100644 index 00000000..a1d591eb --- /dev/null +++ b/api/internal/handler/cloud/commitgeneraltaskhandler.go @@ -0,0 +1,17 @@ +package cloud + +import ( + "github.com/zeromicro/go-zero/rest/httpx" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/logic/cloud" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/repository/result" + "net/http" +) + +func CommitGeneralTaskHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := cloud.NewCommitGeneralTaskLogic(r.Context(), svcCtx) + err := l.CommitGeneralTask() + result.HttpResult(r, w, nil, err) + } +} diff --git a/api/internal/handler/routes.go b/api/internal/handler/routes.go index 45e701c2..ff1df60e 100644 --- a/api/internal/handler/routes.go +++ b/api/internal/handler/routes.go @@ -227,6 +227,11 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { Path: "/core/clusterList", Handler: cloud.GetClusterListHandler(serverCtx), }, + { + Method: http.MethodPost, + Path: "/cloud/task/create", + Handler: cloud.CommitGeneralTaskHandler(serverCtx), + }, }, rest.WithPrefix("/pcm/v1"), ) diff --git a/api/internal/logic/cloud/commitgeneraltasklogic.go b/api/internal/logic/cloud/commitgeneraltasklogic.go new file mode 100644 index 00000000..0dd34a54 --- /dev/null +++ b/api/internal/logic/cloud/commitgeneraltasklogic.go @@ -0,0 +1,66 @@ +package cloud + +import ( + "context" + "github.com/pkg/errors" + "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/constants" + "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models" + "sigs.k8s.io/yaml" + "strings" + "time" + + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type CommitGeneralTaskLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewCommitGeneralTaskLogic(ctx context.Context, svcCtx *svc.ServiceContext) *CommitGeneralTaskLogic { + return &CommitGeneralTaskLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *CommitGeneralTaskLogic) CommitGeneralTask(req *types.GeneralTaskReq) error { + var yamlStr []string + for _, s := range req.ReqBody { + j2, err := yaml.YAMLToJSON([]byte(s)) + if err != nil { + logx.Errorf("Failed to convert yaml to JSON, err: %v", err) + return err + } + yamlStr = append(yamlStr, string(j2)) + } + result := strings.Join(yamlStr, ",") + //TODO The namespace is fixed to ns-admin for the time being. Later, the namespace is obtained based on the user + taskModel := models.Task{ + Status: constants.Saved, + Name: req.Name, + CommitTime: time.Now(), + NsID: "ns-admin", + YamlString: "[" + result + "]", + } + // Save the task data to the database + tx := l.svcCtx.DbEngin.Create(&taskModel) + if tx.Error != nil { + return tx.Error + } + + var clusterIds []int64 + l.svcCtx.DbEngin.Raw("SELECT id FROM `t_cluster` where adapter_id = ? and id in ?", req.AdapterId, req.ClusterIds).Scan(&clusterIds) + + if len(clusterIds) == 0 || clusterIds == nil { + return errors.Errorf("the cluster does not match the drive resources. Check the data") + } + //保存数据到cloud表 + + return nil +} diff --git a/api/internal/types/types.go b/api/internal/types/types.go index bc35feeb..9406bc6b 100644 --- a/api/internal/types/types.go +++ b/api/internal/types/types.go @@ -80,6 +80,26 @@ type Region struct { RunningJobs int64 `json:"runningJobs"` } +type GeneralTaskReq struct { + Name string `json:"name"` + ComputeType string `json:"computeType"` + TemplateId string `json:"templateId"` + AdapterId string `json:"adapterId"` + ClusterIds []string `json:"clusterIds"` + Strategy Strategy `json:"strategy"` + ReqBody []string `json:"reqBody"` +} + +type Strategy struct { + Name string `json:"name"` + StaticWeightList []StaticWeightList `json:"staticWeightList"` +} + +type StaticWeightList struct { + ClusterName string `json:"clusterName"` + Weight int `json:"weight"` +} + type DeleteTaskReq struct { Id int64 `path:"id"` } @@ -5271,6 +5291,123 @@ type AiAlgorithmsResp struct { Algorithms []string `json:"algorithms"` } +type PullTaskInfoReq struct { + AdapterId int64 `form:"adapterId"` +} + +type PullTaskInfoResp struct { + HpcInfoList []*HpcInfo `json:"HpcInfoList,omitempty"` + CloudInfoList []*CloudInfo `json:"CloudInfoList,omitempty"` + AiInfoList []*AiInfo `json:"AiInfoList,omitempty"` + VmInfoList []*VmInfo `json:"VmInfoList,omitempty"` +} + +type HpcInfo struct { + Id int64 `json:"id"` // id + TaskId int64 `json:"task_id"` // 任务id + JobId string `json:"job_id"` // 作业id(在第三方系统中的作业id) + AdapterId int64 `json:"adapter_id"` // 执行任务的适配器id + ClusterId int64 `json:"cluster_id"` // 执行任务的集群id + ClusterType string `json:"cluster_type"` // 执行任务的集群类型 + Name string `json:"name"` // 名称 + Status string `json:"status"` // 状态 + CmdScript string `json:"cmd_script"` + StartTime string `json:"start_time"` // 开始时间 + RunningTime int64 `json:"running_time"` // 运行时间 + DerivedEs string `json:"derived_es"` + Cluster string `json:"cluster"` + BlockId int64 `json:"block_id"` + AllocNodes int64 `json:"alloc_nodes"` + AllocCpu int64 `json:"alloc_cpu"` + CardCount int64 `json:"card_count"` // 卡数 + Version string `json:"version"` + Account string `json:"account"` + WorkDir string `json:"work_dir"` // 工作路径 + AssocId int64 `json:"assoc_id"` + ExitCode int64 `json:"exit_code"` + WallTime string `json:"wall_time"` // 最大运行时间 + Result string `json:"result"` // 运行结果 + DeletedAt string `json:"deleted_at"` // 删除时间 + YamlString string `json:"yaml_string"` + AppType string `json:"app_type"` // 应用类型 + AppName string `json:"app_name"` // 应用名称 + Queue string `json:"queue"` // 队列名称 + SubmitType string `json:"submit_type"` // cmd（命令行模式） + NNode string `json:"n_node"` // 节点个数（当指定该参数时，GAP_NODE_STRING必须为""） + StdOutFile string `json:"std_out_file"` // 工作路径/std.err.%j + StdErrFile string `json:"std_err_file"` // 工作路径/std.err.%j + StdInput string `json:"std_input"` + Environment string `json:"environment"` + DeletedFlag int64 `json:"deleted_flag"` // 是否删除（0-否，1-是） + CreatedBy int64 `json:"created_by"` // 创建人 + CreatedTime string `json:"created_time"` // 创建时间 + UpdatedBy int64 `json:"updated_by"` // 更新人 + UpdatedTime string `json:"updated_time"` // 更新时间 +} + +type CloudInfo struct { + Participant int64 `json:"participant,omitempty"` + Id int64 `json:"id,omitempty"` + TaskId int64 `json:"taskId,omitempty"` + ApiVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + StartTime string `json:"startTime,omitempty"` + RunningTime int64 `json:"runningTime,omitempty"` + Result string `json:"result,omitempty"` + YamlString string `json:"yamlString,omitempty"` +} + +type AiInfo struct { + ParticipantId int64 `json:"participantId,omitempty"` + TaskId int64 `json:"taskId,omitempty"` + ProjectId string `json:"project_id,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + StartTime string `json:"startTime,omitempty"` + RunningTime int64 `json:"runningTime,omitempty"` + Result string `json:"result,omitempty"` + JobId string `json:"jobId,omitempty"` + CreateTime string `json:"createTime,omitempty"` + ImageUrl string `json:"imageUrl,omitempty"` + Command string `json:"command,omitempty"` + FlavorId string `json:"flavorId,omitempty"` + SubscriptionId string `json:"subscriptionId,omitempty"` + ItemVersionId string `json:"itemVersionId,omitempty"` +} + +type VmInfo struct { + ParticipantId int64 `json:"participantId,omitempty"` + TaskId int64 `json:"taskId,omitempty"` + Name string `json:"name,omitempty"` + FlavorRef string `json:"flavor_ref,omitempty"` + ImageRef string `json:"image_ref,omitempty"` + NetworkUuid string `json:"network_uuid,omitempty"` + BlockUuid string `json:"block_uuid,omitempty"` + SourceType string `json:"source_type,omitempty"` + DeleteOnTermination bool `json:"delete_on_termination,omitempty"` + State string `json:"state,omitempty"` +} + +type PushTaskInfoReq struct { + AdapterId int64 `json:"adapterId"` + HpcInfoList []*HpcInfo `json:"hpcInfoList"` + CloudInfoList []*CloudInfo `json:"cloudInfoList"` + AiInfoList []*AiInfo `json:"aiInfoList"` + VmInfoList []*VmInfo `json:"vmInfoList"` +} + +type PushTaskInfoResp struct { + Code int64 `json:"code"` + Msg string `json:"msg"` +} + +type PushResourceInfoReq struct { + AdapterId int64 `json:"adapterId"` +} + type CreateAlertRuleReq struct { ClusterName string `json:"clusterName"` Namespace string `json:"namespace"` diff --git a/go.sum b/go.sum index 4c2e07cc..d00af510 100644 --- a/go.sum +++ b/go.sum @@ -1079,8 +1079,6 @@ github.com/zeromicro/go-zero v1.6.3 h1:OL0NnHD5LdRNDolfcK9vUkJt7K8TcBE3RkzfM8poO github.com/zeromicro/go-zero v1.6.3/go.mod h1:XZL435ZxVi9MSXXtw2MRQhHgx6OoX3++MRMOE9xU70c= gitlink.org.cn/JointCloud/pcm-kubernetes v0.0.0-20240301071143-347480abff2c h1:2Wl/hvaSFjh6fmCSIQhjkr9llMRREQeqcXNLZ/HPY18= gitlink.org.cn/JointCloud/pcm-kubernetes v0.0.0-20240301071143-347480abff2c/go.mod h1:lSRfGs+PxFvw7CcndHWRd6UlLlGrZn0b0hp5cfaMNGw= -gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240328020739-cbdd8f5b226b h1:suRANMHQPhKKmgdJOZcbFYDJ0NUQkUGgVvMKxw75BQI= -gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240328020739-cbdd8f5b226b/go.mod h1:i2rrbMQ+Fve345BY9Heh4MUqVTAimZQElQhzzRee5B8= gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240401022404-2f1425735f0d h1:ZX/Kg8eKdaAfDsTd+Y+TrJsUvxp/DpbWUp+Ij4CtR+s= gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240401022404-2f1425735f0d/go.mod h1:i2rrbMQ+Fve345BY9Heh4MUqVTAimZQElQhzzRee5B8= gitlink.org.cn/JointCloud/pcm-slurm v0.0.0-20240301080743-8b94bbaf57f5 h1:+/5vnzkJBfMRnya1NrhOzlroUtRa5ePiYbPKlHLoLV0= @@ -1094,18 +1092,12 @@ gitlink.org.cn/jcce-pcm/pcm-participant-modelarts v0.0.0-20231101085149-724c7c4c gitlink.org.cn/jcce-pcm/pcm-participant-octopus v0.0.0-20240222124813-e275cfa342f4 h1:NrxKAZ5uAzshB9EHcPw+XTOTzpxb5HslNRMYBrFC1Qo= gitlink.org.cn/jcce-pcm/pcm-participant-octopus v0.0.0-20240222124813-e275cfa342f4/go.mod h1:uyvpVqG1jHDXX+ubXI0RBwnWXzVykD/mliqGQIDvRoo= go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= -go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= -go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4= go.etcd.io/etcd/api/v3 v3.5.13/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c= go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= -go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= -go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= go.etcd.io/etcd/client/pkg/v3 v3.5.13 h1:RVZSAnWWWiI5IrYAXjQorajncORbS0zI48LQlE2kQWg= go.etcd.io/etcd/client/pkg/v3 v3.5.13/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8= go.etcd.io/etcd/client/v3 v3.5.7/go.mod h1:sOWmj9DZUMyAngS7QQwCyAXXAL6WhgTOPLNS/NabQgw= -go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= -go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= go.etcd.io/etcd/client/v3 v3.5.13 h1:o0fHTNJLeO0MyVbc7I3fsCf6nrOqn5d+diSarKnB2js= go.etcd.io/etcd/client/v3 v3.5.13/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= From 5d03296353a9733a72179280be8c42b71eb2a4d9 Mon Sep 17 00:00:00 2001 From: jagger Date: Wed, 10 Apr 2024 09:17:47 +0800 Subject: [PATCH 2/6] fix bugs Signed-off-by: jagger Former-commit-id: 0dfcf5d24e7865fd86bc81278a150c1740da623f --- .../handler/cloud/commitgeneraltaskhandler.go | 9 ++- .../logic/cloud/commitgeneraltasklogic.go | 62 +++++++++++++++++-- pkg/models/base/base_model.go | 14 +++++ pkg/models/cloud/task_cloud.go | 26 ++++++++ pkg/models/cloud_model.go | 27 ++++++++ 5 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 pkg/models/base/base_model.go create mode 100644 pkg/models/cloud/task_cloud.go create mode 100644 pkg/models/cloud_model.go diff --git a/api/internal/handler/cloud/commitgeneraltaskhandler.go b/api/internal/handler/cloud/commitgeneraltaskhandler.go index a1d591eb..692b7d56 100644 --- a/api/internal/handler/cloud/commitgeneraltaskhandler.go +++ b/api/internal/handler/cloud/commitgeneraltaskhandler.go @@ -4,14 +4,21 @@ import ( "github.com/zeromicro/go-zero/rest/httpx" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/logic/cloud" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types" "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/repository/result" "net/http" ) func CommitGeneralTaskHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var req types.GeneralTaskReq + if err := httpx.Parse(r, &req); err != nil { + result.ParamErrorResult(r, w, err) + return + } + l := cloud.NewCommitGeneralTaskLogic(r.Context(), svcCtx) - err := l.CommitGeneralTask() + err := l.CommitGeneralTask(&req) result.HttpResult(r, w, nil, err) } } diff --git a/api/internal/logic/cloud/commitgeneraltasklogic.go b/api/internal/logic/cloud/commitgeneraltasklogic.go index 0dd34a54..9412e23e 100644 --- a/api/internal/logic/cloud/commitgeneraltasklogic.go +++ b/api/internal/logic/cloud/commitgeneraltasklogic.go @@ -1,10 +1,17 @@ package cloud import ( + "bytes" "context" "github.com/pkg/errors" "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/constants" "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models" + "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models/cloud" + "io" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + syaml "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + kyaml "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/yaml" "strings" "time" @@ -54,13 +61,58 @@ func (l *CommitGeneralTaskLogic) CommitGeneralTask(req *types.GeneralTaskReq) er return tx.Error } - var clusterIds []int64 - l.svcCtx.DbEngin.Raw("SELECT id FROM `t_cluster` where adapter_id = ? and id in ?", req.AdapterId, req.ClusterIds).Scan(&clusterIds) - - if len(clusterIds) == 0 || clusterIds == nil { + var clusters []*models.CloudModel + err := l.svcCtx.DbEngin.Raw("SELECT * FROM `t_cluster` where adapter_id = ? and id in ?", req.AdapterId, req.ClusterIds).Scan(&clusters).Error + if err != nil { + logx.Errorf("CommitGeneralTask() => sql execution error: %v", err) return errors.Errorf("the cluster does not match the drive resources. Check the data") } - //保存数据到cloud表 + taskCloud := cloud.TaskCloudModel{} + //TODO 执行策略返回集群跟 Replica + for _, c := range clusters { + for _, s := range req.ReqBody { + sStruct := UnMarshalK8sStruct(s) + unString, _ := sStruct.MarshalJSON() + taskCloud.TaskId = uint(taskModel.Id) + taskCloud.AdapterId = c.AdapterId + taskCloud.ClusterId = c.Id + taskCloud.ClusterName = c.Name + taskCloud.Status = "Saved" + taskCloud.YamlString = string(unString) + taskCloud.Kind = sStruct.GetKind() + taskCloud.Namespace = sStruct.GetNamespace() + tx = l.svcCtx.DbEngin.Create(&taskCloud) + if tx.Error != nil { + logx.Errorf("CommitGeneralTask() create taskCloud => sql execution error: %v", err) + return tx.Error + } + } + } return nil } + +func UnMarshalK8sStruct(yamlString string) *unstructured.Unstructured { + unstructuredObj := &unstructured.Unstructured{} + d := kyaml.NewYAMLOrJSONDecoder(bytes.NewBufferString(yamlString), 4096) + var err error + for { + var rawObj runtime.RawExtension + err = d.Decode(&rawObj) + if err == io.EOF { + break + } + obj := &unstructured.Unstructured{} + syaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).Decode(rawObj.Raw, nil, obj) + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + logx.Errorf("UnMarshalK8sStruct() => Execution failure err:%v", err) + } + unstructuredObj = &unstructured.Unstructured{Object: unstructuredMap} + // 命名空间为空 设置默认值 + if len(unstructuredObj.GetNamespace()) == 0 { + unstructuredObj.SetNamespace("default") + } + } + return unstructuredObj +} diff --git a/pkg/models/base/base_model.go b/pkg/models/base/base_model.go new file mode 100644 index 00000000..c7c34df0 --- /dev/null +++ b/pkg/models/base/base_model.go @@ -0,0 +1,14 @@ +package base + +import ( + "gorm.io/gorm" + "time" +) + +type BaseModel struct { + DeletedAt gorm.DeletedAt `gorm:"index;comment:删除时间" json:"-"` // 删除时间 + CreatedBy uint `gorm:"created_by;comment:创建人" json:"createdBy"` //创建人 + CreatedTime time.Time `gorm:"comment:创建时间" json:"-"` // 创建时间 + UpdatedBy uint `gorm:"updated_by;comment:更新人" json:"UpdatedBy"` //创建人 + UpdatedTime time.Time `gorm:"comment:更新时间" json:"-"` // 更新时间 +} diff --git a/pkg/models/cloud/task_cloud.go b/pkg/models/cloud/task_cloud.go new file mode 100644 index 00000000..13e8c045 --- /dev/null +++ b/pkg/models/cloud/task_cloud.go @@ -0,0 +1,26 @@ +package cloud + +import ( + "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models/base" + "time" +) + +type TaskCloudModel struct { + Id uint `json:"id" gorm:"primarykey;not null;comment:id"` + TaskId uint `json:"taskId" gorm:"not null;comment:task表id"` + AdapterId uint `json:"adapterId" gorm:"not null;comment:适配器id"` + ClusterId uint `json:"clusterId" gorm:"not null;comment:集群id"` + ClusterName string `json:"clusterName" gorm:"not null;comment:集群名称"` + Kind string `json:"kind" gorm:"comment:种类"` + Status string `json:"status" gorm:"comment:状态"` + StartTime time.Time `json:"startTime" gorm:"comment:开始时间"` + YamlString string `json:"yamlString" gorm:"not null;comment:入参"` + Result string `json:"result" gorm:"comment:运行结果"` + Namespace string `json:"namespace" gorm:"comment:命名空间"` + Replica int `json:"replica" gorm:"not null;comment:副本数"` + base.BaseModel +} + +func (TaskCloudModel) TableName() string { + return "task_cloud" +} diff --git a/pkg/models/cloud_model.go b/pkg/models/cloud_model.go new file mode 100644 index 00000000..200b6852 --- /dev/null +++ b/pkg/models/cloud_model.go @@ -0,0 +1,27 @@ +package models + +import "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models/base" + +type CloudModel struct { + Id uint `json:"id,omitempty" gorm:"id"` + AdapterId uint `json:"adapterId,omitempty" gorm:"adapter_id"` + Name string `json:"name,omitempty" gorm:"name"` + Nickname string `json:"nickname,omitempty" gorm:"nickname"` + Description string `json:"description,omitempty" gorm:"description"` + Server string `json:"server,omitempty" gorm:"server"` + MonitorServer string `json:"monitorServer,omitempty" gorm:"monitor_server"` + Username string `json:"username,omitempty" gorm:"username"` + Password string `json:"password,omitempty" gorm:"password"` + Token string `json:"token,omitempty" gorm:"token"` + Ak string `json:"ak,omitempty" gorm:"ak"` + Sk string `json:"sk,omitempty" gorm:"sk"` + Region string `json:"region,omitempty" gorm:"region"` + ProjectId string `json:"projectId,omitempty" gorm:"project_id"` + Version string `json:"version,omitempty" gorm:"version"` + Label string `json:"label,omitempty" gorm:"label"` + OwnerId uint `json:"ownerId,omitempty" gorm:"owner_id"` + AuthType int `json:"authType,omitempty" gorm:"auth_type"` + ProducerDict string `json:"producerDict,omitempty" gorm:"producer_dict"` + RegionDict string `json:"regionDict,omitempty" gorm:"region_dict"` + base.BaseModel +} From 6f9cf73cd47c5e601b008a5f10981b0fda91fa99 Mon Sep 17 00:00:00 2001 From: jagger Date: Fri, 19 Apr 2024 15:40:01 +0800 Subject: [PATCH 3/6] fix Signed-off-by: jagger Former-commit-id: 836d8f7f97db13bab73ba5a9575ef0654caea401 --- api/internal/logic/adapters/clusterlistlogic.go | 4 ++-- api/internal/logic/dictionary/listdictitemlogic.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/internal/logic/adapters/clusterlistlogic.go b/api/internal/logic/adapters/clusterlistlogic.go index 764878b2..78f5b500 100644 --- a/api/internal/logic/adapters/clusterlistlogic.go +++ b/api/internal/logic/adapters/clusterlistlogic.go @@ -27,9 +27,9 @@ func (l *ClusterListLogic) ClusterList(req *types.ClusterReq) (resp *types.PageR offset := req.PageSize * (req.PageNum - 1) resp = &types.PageResult{} var list []types.ClusterInfo - db := l.svcCtx.DbEngin.Model(&types.AdapterInfo{}).Table("t_adapter") + db := l.svcCtx.DbEngin.Model(&types.AdapterInfo{}).Table("t_cluster") - db = db.Joins("left join t_cluster on t_adapter.id = t_cluster.adapter_id"). + db = db.Joins("left join t_adapter on t_adapter.id = t_cluster.adapter_id"). Where("t_cluster.deleted_at is null") if req.Name != "" { db = db.Where("t_cluster.name LIKE ?", "%"+req.Name+"%") diff --git a/api/internal/logic/dictionary/listdictitemlogic.go b/api/internal/logic/dictionary/listdictitemlogic.go index 3c209ad8..a0dc04ee 100644 --- a/api/internal/logic/dictionary/listdictitemlogic.go +++ b/api/internal/logic/dictionary/listdictitemlogic.go @@ -54,7 +54,7 @@ func (l *ListDictItemLogic) ListDictItem(req *types.DictItemReq) (resp *types.Pa return resp, err } db = db.Limit(limit).Offset(offset) - err = db.Order("create_time desc").Find(&dictList).Error + err = db.Order("sort_order").Find(&dictList).Error resp.List = dictList resp.PageSize = req.PageSize From 58263827d12fe223633fa71c972646d5e6dba685 Mon Sep 17 00:00:00 2001 From: zhouqunjie Date: Fri, 19 Apr 2024 16:04:36 +0800 Subject: [PATCH 4/6] hpc resource overview Former-commit-id: cfb5c293ce9cb59518c3a202a7519b794cff2aeb --- api/desc/hpc/pcm-hpc.api | 224 ++++++++++++------------ api/internal/logic/hpc/resourcelogic.go | 32 ++-- api/internal/types/types.go | 18 +- pkg/models/tclusterresourcemodel.go | 29 +++ pkg/models/tclusterresourcemodel_gen.go | 93 ++++++++++ 5 files changed, 264 insertions(+), 132 deletions(-) create mode 100644 pkg/models/tclusterresourcemodel.go create mode 100644 pkg/models/tclusterresourcemodel_gen.go diff --git a/api/desc/hpc/pcm-hpc.api b/api/desc/hpc/pcm-hpc.api index d86f262c..d5f15c65 100644 --- a/api/desc/hpc/pcm-hpc.api +++ b/api/desc/hpc/pcm-hpc.api @@ -1,137 +1,137 @@ syntax = "v1" info( - title: "type title here" - desc: "type desc here" - author: "type author here" - email: "type email here" - version: "type version here" + title: "type title here" + desc: "type desc here" + author: "type author here" + email: "type email here" + version: "type version here" ) type ( - commitHpcTaskReq { - Name string `json:"name"` // paratera:jobName - Description string `json:"description,optional"` - tenantId int64 `json:"tenantId,optional"` - TaskId int64 `json:"taskId,optional"` - AdapterId string `json:"adapterId,optional"` - MatchLabels map[string]string `json:"matchLabels,optional"` - CardCount int64 `json:"cardCount,optional"` - WorkDir string `json:"workDir,optional"` //paratera:workingDir - WallTime string `json:"wallTime,optional"` - CmdScript string `json:"cmdScript,optional"` // paratera:bootScript - AppType string `json:"appType,optional"` - AppName string `json:"appName,optional"` // paratera:jobGroupName ac:appname - Queue string `json:"queue,optional"` - NNode string `json:"nNode,optional"` - SubmitType string `json:"submitType,optional"` - StdOutFile string `json:"stdOutFile,optional"` - StdErrFile string `json:"stdErrFile,optional"` - StdInput string `json:"stdInput,optional"` - Environment map[string]string `json:"environment,optional"` - ClusterType string `json:"clusterType,optional"` - } + commitHpcTaskReq { + Name string `json:"name"` // paratera:jobName + Description string `json:"description,optional"` + TenantId int64 `json:"tenantId,optional"` + TaskId int64 `json:"taskId,optional"` + AdapterId string `json:"adapterId,optional"` + MatchLabels map[string]string `json:"matchLabels,optional"` + CardCount int64 `json:"cardCount,optional"` + WorkDir string `json:"workDir,optional"` //paratera:workingDir + WallTime string `json:"wallTime,optional"` + CmdScript string `json:"cmdScript,optional"` // paratera:bootScript + AppType string `json:"appType,optional"` + AppName string `json:"appName,optional"` // paratera:jobGroupName ac:appname + Queue string `json:"queue,optional"` + NNode string `json:"nNode,optional"` + SubmitType string `json:"submitType,optional"` + StdOutFile string `json:"stdOutFile,optional"` + StdErrFile string `json:"stdErrFile,optional"` + StdInput string `json:"stdInput,optional"` + Environment map[string]string `json:"environment,optional"` + ClusterType string `json:"clusterType,optional"` + } - commitHpcTaskResp { - TaskId int64 `json:"taskId"` - Code int32 `json:"code"` - Msg string `json:"msg"` - } + commitHpcTaskResp { + TaskId int64 `json:"taskId"` + Code int32 `json:"code"` + Msg string `json:"msg"` + } ) type ( - hpcOverViewReq { - } - hpcOverViewResp { - Code int32 `json:"code"` - Msg string `json:"msg"` - Data HPCOverView `json:"data"` - } - HPCOverView { - AdapterCount int32 `json:"adapterCount"` - StackCount int32 `json:"stackCount"` - ClusterCount int32 `json:"clusterCount"` - TaskCount int32 `json:"taskCount"` - } + hpcOverViewReq { + } + hpcOverViewResp { + Code int32 `json:"code"` + Msg string `json:"msg"` + Data HPCOverView `json:"data"` + } + HPCOverView { + AdapterCount int32 `json:"adapterCount"` + StackCount int32 `json:"stackCount"` + ClusterCount int32 `json:"clusterCount"` + TaskCount int32 `json:"taskCount"` + } ) type ( - hpcAdapterSummaryReq { - } - hpcAdapterSummaryResp { - Code int32 `json:"code"` - Msg string `json:"msg"` - Data []HPCAdapterSummary `json:"data"` - } - HPCAdapterSummary { - AdapterName string `json:"adapterName"` - StackCount int32 `json:"stackCount"` - ClusterCount int32 `json:"clusterCount"` - TaskCount int32 `json:"taskCount"` - } + hpcAdapterSummaryReq { + } + hpcAdapterSummaryResp { + Code int32 `json:"code"` + Msg string `json:"msg"` + Data []HPCAdapterSummary `json:"data"` + } + HPCAdapterSummary { + AdapterName string `json:"adapterName"` + StackCount int32 `json:"stackCount"` + ClusterCount int32 `json:"clusterCount"` + TaskCount int32 `json:"taskCount"` + } ) type ( - hpcJobReq { - } - hpcJobResp { - Code int32 `json:"code"` - Msg string `json:"msg"` - Data []Job `json:"data"` - } - Job { - JobName string `json:"jobName"` - JobDesc string `json:"jobDesc"` - SubmitTime string `json:"submitTime"` - JobStatus string `json:"jobStatus"` - AdapterName string `json:"adapterName"` - ClusterName string `json:"clusterName"` - ClusterType string `json:"clusterType"` - } + hpcJobReq { + } + hpcJobResp { + Code int32 `json:"code"` + Msg string `json:"msg"` + Data []Job `json:"data"` + } + Job { + JobName string `json:"jobName"` + JobDesc string `json:"jobDesc"` + SubmitTime string `json:"submitTime"` + JobStatus string `json:"jobStatus"` + AdapterName string `json:"adapterName"` + ClusterName string `json:"clusterName"` + ClusterType string `json:"clusterType"` + } ) type ( - hpcResourceReq { - } - hpcResourceResp { - Code int32 `json:"code"` - Msg string `json:"msg"` - HPCResource HPCResource `json:"hpcResource"` - } - HPCResource { - GPUCardsTotal int32 `json:"gpuCoresTotal"` - CPUCoresTotal int32 `json:"cpuCoresTotal"` - RAMTotal int32 `json:"ramTotal"` - GPUCardsUsed int32 `json:"gpuCoresUsed"` - CPUCoresUsed int32 `json:"cpuCoresUsed"` - RAMUsed int32 `json:"ramUsed"` - GPURate float32 `json:"gpuRate"` - CPURate float32 `json:"cpuRate"` - RAMRate float32 `json:"ramRate"` - } + hpcResourceReq { + } + hpcResourceResp { + Code int32 `json:"code"` + Msg string `json:"msg"` + HPCResource HPCResource `json:"hpcResource"` + } + HPCResource { + GPUCardsTotal float64 `json:"gpuCoresTotal"` + CPUCoresTotal float64 `json:"cpuCoresTotal"` + RAMTotal float64 `json:"ramTotal"` + GPUCardsUsed float64 `json:"gpuCoresUsed"` + CPUCoresUsed float64 `json:"cpuCoresUsed"` + RAMUsed float64 `json:"ramUsed"` + GPURate float64 `json:"gpuRate"` + CPURate float64 `json:"cpuRate"` + RAMRate float64 `json:"ramRate"` + } ) type QueueAssetsResp { - QueueAssets []QueueAsset `json:"queueAsset"` + QueueAssets []QueueAsset `json:"queueAsset"` } type QueueAsset { - TenantName string `json:"tenantName"` //租户名称 - ParticipantId int64 `json:"participantId"` - AclHosts string `json:"aclHosts"` // 可用节点，多个节点用逗号隔开 - QueNodes string `json:"queNodes"` //队列节点总数 - QueMinNodect string `json:"queMinNodect,omitempty"` //队列最小节点数 - QueMaxNgpus string `json:"queMaxNgpus,omitempty"` //队列最大GPU卡数 - QueMaxPPN string `json:"queMaxPPN,omitempty"` //使用该队列作业最大CPU核心数 - QueChargeRate string `json:"queChargeRate,omitempty"` //费率 - QueMaxNcpus string `json:"queMaxNcpus,omitempty"` //用户最大可用核心数 - QueMaxNdcus string `json:"queMaxNdcus,omitempty"` //队列总DCU卡数 - QueueName string `json:"queueName,omitempty"` //队列名称 - QueMinNcpus string `json:"queMinNcpus,omitempty"` //队列最小CPU核数 - QueFreeNodes string `json:"queFreeNodes,omitempty"` //队列空闲节点数 - QueMaxNodect string `json:"queMaxNodect,omitempty"` //队列作业最大节点数 - QueMaxGpuPN string `json:"queMaxGpuPN,omitempty"` //队列单作业最大GPU卡数 - QueMaxWalltime string `json:"queMaxWalltime,omitempty"` //队列最大运行时间 - QueMaxDcuPN string `json:"queMaxDcuPN,omitempty"` //队列单作业最大DCU卡数 - QueFreeNcpus string `json:"queFreeNcpus"` //队列空闲cpu数 - QueNcpus string `json:"queNcpus"` //队列cpu数 + TenantName string `json:"tenantName"` //租户名称 + ParticipantId int64 `json:"participantId"` + AclHosts string `json:"aclHosts"` // 可用节点，多个节点用逗号隔开 + QueNodes string `json:"queNodes"` //队列节点总数 + QueMinNodect string `json:"queMinNodect,omitempty"` //队列最小节点数 + QueMaxNgpus string `json:"queMaxNgpus,omitempty"` //队列最大GPU卡数 + QueMaxPPN string `json:"queMaxPPN,omitempty"` //使用该队列作业最大CPU核心数 + QueChargeRate string `json:"queChargeRate,omitempty"` //费率 + QueMaxNcpus string `json:"queMaxNcpus,omitempty"` //用户最大可用核心数 + QueMaxNdcus string `json:"queMaxNdcus,omitempty"` //队列总DCU卡数 + QueueName string `json:"queueName,omitempty"` //队列名称 + QueMinNcpus string `json:"queMinNcpus,omitempty"` //队列最小CPU核数 + QueFreeNodes string `json:"queFreeNodes,omitempty"` //队列空闲节点数 + QueMaxNodect string `json:"queMaxNodect,omitempty"` //队列作业最大节点数 + QueMaxGpuPN string `json:"queMaxGpuPN,omitempty"` //队列单作业最大GPU卡数 + QueMaxWalltime string `json:"queMaxWalltime,omitempty"` //队列最大运行时间 + QueMaxDcuPN string `json:"queMaxDcuPN,omitempty"` //队列单作业最大DCU卡数 + QueFreeNcpus string `json:"queFreeNcpus"` //队列空闲cpu数 + QueNcpus string `json:"queNcpus"` //队列cpu数 } \ No newline at end of file diff --git a/api/internal/logic/hpc/resourcelogic.go b/api/internal/logic/hpc/resourcelogic.go index b88e94a1..fd620904 100644 --- a/api/internal/logic/hpc/resourcelogic.go +++ b/api/internal/logic/hpc/resourcelogic.go @@ -2,7 +2,6 @@ package hpc import ( "context" - "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types" @@ -25,18 +24,29 @@ func NewResourceLogic(ctx context.Context, svcCtx *svc.ServiceContext) *Resource func (l *ResourceLogic) Resource(req *types.HpcResourceReq) (resp *types.HpcResourceResp, err error) { - l.svcCtx.DbEngin.Raw("SELECT th.NAME as job_name,t.description as job_desc,t.commit_time as submit_time,th.STATUS as job_status,ta.name as adapter_name,tc.name as cluster_name,tc.label as cluster_type FROM task_hpc th LEFT JOIN task t ON t.id = th.task_id JOIN t_cluster tc on th.cluster_id = tc.id JOIN t_adapter ta on tc.adapter_id = ta.id") + type hpcResourceOV struct { + CpuAvail float64 `json:"cpu_avail"` + CpuTotal float64 `json:"cpu_total"` + MemAvail float64 `json:"mem_avail"` + MemTotal float64 `json:"mem_total"` + DiskAvail float64 `json:"disk_avail"` + DiskTotal float64 `json:"disk_total"` + GpuAvail float64 `json:"gpu_avail"` + GpuTotal float64 `json:"gpu_total"` + } + var hrov hpcResourceOV + l.svcCtx.DbEngin.Raw("SELECT sum(cpu_avail) as cpu_avail,sum(cpu_total) as cpu_total,sum(mem_avail) as mem_avail,sum(mem_total) as mem_total,sum(disk_avail) as disk_avail,sum(disk_total) as disk_total,sum(gpu_avail) as gpu_avail,sum(gpu_total) as gpu_total FROM t_cluster_resource where cluster_type = 2").Scan(&hrov) hpcResource := types.HPCResource{ - GPUCardsTotal: 0, - CPUCoresTotal: 0, - RAMTotal: 0, - GPUCardsUsed: 0, - CPUCoresUsed: 0, - RAMUsed: 0, - GPURate: 0, - CPURate: 0, - RAMRate: 0, + GPUCardsTotal: hrov.GpuTotal, + CPUCoresTotal: hrov.CpuTotal, + RAMTotal: hrov.MemTotal, + GPUCardsUsed: hrov.GpuTotal - hrov.GpuAvail, + CPUCoresUsed: hrov.CpuTotal - hrov.CpuAvail, + RAMUsed: hrov.MemTotal - hrov.MemAvail, + GPURate: (hrov.GpuTotal - hrov.GpuAvail) / hrov.GpuTotal, + CPURate: (hrov.CpuTotal - hrov.CpuAvail) / hrov.CpuTotal, + RAMRate: (hrov.MemTotal - hrov.MemAvail) / hrov.MemTotal, } resp = &types.HpcResourceResp{ diff --git a/api/internal/types/types.go b/api/internal/types/types.go index 55ad50af..cb489383 100644 --- a/api/internal/types/types.go +++ b/api/internal/types/types.go @@ -937,15 +937,15 @@ type HpcResourceResp struct { } type HPCResource struct { - GPUCardsTotal int32 `json:"gpuCoresTotal"` - CPUCoresTotal int32 `json:"cpuCoresTotal"` - RAMTotal int32 `json:"ramTotal"` - GPUCardsUsed int32 `json:"gpuCoresUsed"` - CPUCoresUsed int32 `json:"cpuCoresUsed"` - RAMUsed int32 `json:"ramUsed"` - GPURate float32 `json:"gpuRate"` - CPURate float32 `json:"cpuRate"` - RAMRate float32 `json:"ramRate"` + GPUCardsTotal float64 `json:"gpuCoresTotal"` + CPUCoresTotal float64 `json:"cpuCoresTotal"` + RAMTotal float64 `json:"ramTotal"` + GPUCardsUsed float64 `json:"gpuCoresUsed"` + CPUCoresUsed float64 `json:"cpuCoresUsed"` + RAMUsed float64 `json:"ramUsed"` + GPURate float64 `json:"gpuRate"` + CPURate float64 `json:"cpuRate"` + RAMRate float64 `json:"ramRate"` } type QueueAssetsResp struct { diff --git a/pkg/models/tclusterresourcemodel.go b/pkg/models/tclusterresourcemodel.go new file mode 100644 index 00000000..42c1fb2e --- /dev/null +++ b/pkg/models/tclusterresourcemodel.go @@ -0,0 +1,29 @@ +package models + +import "github.com/zeromicro/go-zero/core/stores/sqlx" + +var _ TClusterResourceModel = (*customTClusterResourceModel)(nil) + +type ( + // TClusterResourceModel is an interface to be customized, add more methods here, + // and implement the added methods in customTClusterResourceModel. + TClusterResourceModel interface { + tClusterResourceModel + withSession(session sqlx.Session) TClusterResourceModel + } + + customTClusterResourceModel struct { + *defaultTClusterResourceModel + } +) + +// NewTClusterResourceModel returns a model for the database table. +func NewTClusterResourceModel(conn sqlx.SqlConn) TClusterResourceModel { + return &customTClusterResourceModel{ + defaultTClusterResourceModel: newTClusterResourceModel(conn), + } +} + +func (m *customTClusterResourceModel) withSession(session sqlx.Session) TClusterResourceModel { + return NewTClusterResourceModel(sqlx.NewSqlConnFromSession(session)) +} diff --git a/pkg/models/tclusterresourcemodel_gen.go b/pkg/models/tclusterresourcemodel_gen.go new file mode 100644 index 00000000..bf4068e6 --- /dev/null +++ b/pkg/models/tclusterresourcemodel_gen.go @@ -0,0 +1,93 @@ +// Code generated by goctl. DO NOT EDIT. + +package models + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/zeromicro/go-zero/core/stores/builder" + "github.com/zeromicro/go-zero/core/stores/sqlc" + "github.com/zeromicro/go-zero/core/stores/sqlx" + "github.com/zeromicro/go-zero/core/stringx" +) + +var ( + tClusterResourceFieldNames = builder.RawFieldNames(&TClusterResource{}) + tClusterResourceRows = strings.Join(tClusterResourceFieldNames, ",") + tClusterResourceRowsExpectAutoSet = strings.Join(stringx.Remove(tClusterResourceFieldNames, "`create_at`", "`create_time`", "`created_at`", "`update_at`", "`update_time`", "`updated_at`"), ",") + tClusterResourceRowsWithPlaceHolder = strings.Join(stringx.Remove(tClusterResourceFieldNames, "`cluster_id`", "`create_at`", "`create_time`", "`created_at`", "`update_at`", "`update_time`", "`updated_at`"), "=?,") + "=?" +) + +type ( + tClusterResourceModel interface { + Insert(ctx context.Context, data *TClusterResource) (sql.Result, error) + FindOne(ctx context.Context, clusterId int64) (*TClusterResource, error) + Update(ctx context.Context, data *TClusterResource) error + Delete(ctx context.Context, clusterId int64) error + } + + defaultTClusterResourceModel struct { + conn sqlx.SqlConn + table string + } + + TClusterResource struct { + ClusterId int64 `db:"cluster_id"` + ClusterName string `db:"cluster_name"` + ClusterType int64 `db:"cluster_type"` // 类型0->容器，1->智算，2->超算，3-虚拟机 + CpuAvail float64 `db:"cpu_avail"` + CpuTotal float64 `db:"cpu_total"` + MemAvail float64 `db:"mem_avail"` + MemTotal float64 `db:"mem_total"` + DiskAvail float64 `db:"disk_avail"` + DiskTotal float64 `db:"disk_total"` + GpuAvail float64 `db:"gpu_avail"` + GpuTotal float64 `db:"gpu_total"` + } +) + +func newTClusterResourceModel(conn sqlx.SqlConn) *defaultTClusterResourceModel { + return &defaultTClusterResourceModel{ + conn: conn, + table: "`t_cluster_resource`", + } +} + +func (m *defaultTClusterResourceModel) Delete(ctx context.Context, clusterId int64) error { + query := fmt.Sprintf("delete from %s where `cluster_id` = ?", m.table) + _, err := m.conn.ExecCtx(ctx, query, clusterId) + return err +} + +func (m *defaultTClusterResourceModel) FindOne(ctx context.Context, clusterId int64) (*TClusterResource, error) { + query := fmt.Sprintf("select %s from %s where `cluster_id` = ? limit 1", tClusterResourceRows, m.table) + var resp TClusterResource + err := m.conn.QueryRowCtx(ctx, &resp, query, clusterId) + switch err { + case nil: + return &resp, nil + case sqlc.ErrNotFound: + return nil, ErrNotFound + default: + return nil, err + } +} + +func (m *defaultTClusterResourceModel) Insert(ctx context.Context, data *TClusterResource) (sql.Result, error) { + query := fmt.Sprintf("insert into %s (%s) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", m.table, tClusterResourceRowsExpectAutoSet) + ret, err := m.conn.ExecCtx(ctx, query, data.ClusterId, data.ClusterName, data.ClusterType, data.CpuAvail, data.CpuTotal, data.MemAvail, data.MemTotal, data.DiskAvail, data.DiskTotal, data.GpuAvail, data.GpuTotal) + return ret, err +} + +func (m *defaultTClusterResourceModel) Update(ctx context.Context, data *TClusterResource) error { + query := fmt.Sprintf("update %s set %s where `cluster_id` = ?", m.table, tClusterResourceRowsWithPlaceHolder) + _, err := m.conn.ExecCtx(ctx, query, data.ClusterName, data.ClusterType, data.CpuAvail, data.CpuTotal, data.MemAvail, data.MemTotal, data.DiskAvail, data.DiskTotal, data.GpuAvail, data.GpuTotal, data.ClusterId) + return err +} + +func (m *defaultTClusterResourceModel) tableName() string { + return m.table +} From a6fed8326770f2a592f088ac71777122aa858c88 Mon Sep 17 00:00:00 2001 From: zhouqunjie Date: Fri, 19 Apr 2024 16:23:21 +0800 Subject: [PATCH 5/6] amend:hpc overview resp Former-commit-id: 64b50119aa8ce1921bb84ff018df85684b6e9bcb --- api/internal/logic/hpc/resourcelogic.go | 6 +++--- api/internal/types/types.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/internal/logic/hpc/resourcelogic.go b/api/internal/logic/hpc/resourcelogic.go index fd620904..1556c265 100644 --- a/api/internal/logic/hpc/resourcelogic.go +++ b/api/internal/logic/hpc/resourcelogic.go @@ -50,9 +50,9 @@ func (l *ResourceLogic) Resource(req *types.HpcResourceReq) (resp *types.HpcReso } resp = &types.HpcResourceResp{ - Code: 200, - Msg: "success", - HPCResource: hpcResource, + Code: 200, + Msg: "success", + Data: hpcResource, } return resp, nil } diff --git a/api/internal/types/types.go b/api/internal/types/types.go index cb489383..8d38e1f2 100644 --- a/api/internal/types/types.go +++ b/api/internal/types/types.go @@ -931,9 +931,9 @@ type HpcResourceReq struct { } type HpcResourceResp struct { - Code int32 `json:"code"` - Msg string `json:"msg"` - HPCResource HPCResource `json:"hpcResource"` + Code int32 `json:"code"` + Msg string `json:"msg"` + Data HPCResource `json:"data"` } type HPCResource struct { From 70fdfe2e953626ee7cabf8f499b30a48c55fa45a Mon Sep 17 00:00:00 2001 From: jagger Date: Fri, 19 Apr 2024 16:47:26 +0800 Subject: [PATCH 6/6] feat: Task status statistics Signed-off-by: jagger Former-commit-id: 158b74d8ae362e8e46100ca4712812216d4a7a1c --- api/desc/core/pcm-core.api | 3 + api/desc/pcm.api | 4 + .../handler/core/counttaskstatushandler.go | 16 + api/internal/handler/routes.go | 5 + .../logic/core/counttaskstatuslogic.go | 41 +++ api/internal/types/types.go | 332 ++++++++++-------- 6 files changed, 251 insertions(+), 150 deletions(-) create mode 100644 api/internal/handler/core/counttaskstatushandler.go create mode 100644 api/internal/logic/core/counttaskstatuslogic.go diff --git a/api/desc/core/pcm-core.api b/api/desc/core/pcm-core.api index 07a20722..f103dc78 100644 --- a/api/desc/core/pcm-core.api +++ b/api/desc/core/pcm-core.api @@ -894,6 +894,9 @@ type ( PageNum int `json:"pageNum,omitempty"` PageSize int `json:"pageSize,omitempty"` } + ListResult{ + List interface{} `json:"list,omitempty"` + } ) type ( diff --git a/api/desc/pcm.api b/api/desc/pcm.api index 8e54a96d..a9640874 100644 --- a/api/desc/pcm.api +++ b/api/desc/pcm.api @@ -133,6 +133,10 @@ service pcm { @doc "paging queries the task list" @handler pageListTaskHandler get /core/task/list (pageTaskReq) returns(PageResult) + + @doc "Statistical task status" + @handler countTaskStatus + get /core/task/countTaskStatus () returns(ListResult) } //hpc二级接口 diff --git a/api/internal/handler/core/counttaskstatushandler.go b/api/internal/handler/core/counttaskstatushandler.go new file mode 100644 index 00000000..6f46929d --- /dev/null +++ b/api/internal/handler/core/counttaskstatushandler.go @@ -0,0 +1,16 @@ +package core + +import ( + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/logic/core" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/repository/result" + "net/http" +) + +func CountTaskStatusHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := core.NewCountTaskStatusLogic(r.Context(), svcCtx) + resp, err := l.CountTaskStatus() + result.HttpResult(r, w, resp, err) + } +} diff --git a/api/internal/handler/routes.go b/api/internal/handler/routes.go index d588b7ed..b7c3a31e 100644 --- a/api/internal/handler/routes.go +++ b/api/internal/handler/routes.go @@ -160,6 +160,11 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { Path: "/core/task/list", Handler: core.PageListTaskHandler(serverCtx), }, + { + Method: http.MethodGet, + Path: "/core/task/countTaskStatus", + Handler: core.CountTaskStatusHandler(serverCtx), + }, }, rest.WithPrefix("/pcm/v1"), ) diff --git a/api/internal/logic/core/counttaskstatuslogic.go b/api/internal/logic/core/counttaskstatuslogic.go new file mode 100644 index 00000000..7f8cc190 --- /dev/null +++ b/api/internal/logic/core/counttaskstatuslogic.go @@ -0,0 +1,41 @@ +package core + +import ( + "context" + "github.com/pkg/errors" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type CountTaskStatusLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewCountTaskStatusLogic(ctx context.Context, svcCtx *svc.ServiceContext) *CountTaskStatusLogic { + return &CountTaskStatusLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +type taskStatus struct { + Quantity string `json:"quantity"` + Status string `json:"status"` +} + +func (l *CountTaskStatusLogic) CountTaskStatus() (resp *types.ListResult, err error) { + resp = &types.ListResult{} + var taskStatusList []*taskStatus + err = l.svcCtx.DbEngin.Raw("select count(*) quantity, status from task group by status").Scan(&taskStatusList).Error + if err != nil { + logx.Errorf("CountTaskStatus() => sql execution error: %v", err) + return nil, errors.Errorf("Description Failed to collect statistics on the status of a task. Please try again later") + } + resp.List = &taskStatusList + return +} diff --git a/api/internal/types/types.go b/api/internal/types/types.go index 850791ba..5dc9105a 100644 --- a/api/internal/types/types.go +++ b/api/internal/types/types.go @@ -867,6 +867,188 @@ type PageResult struct { PageSize int `json:"pageSize,omitempty"` } +type ListResult struct { + List interface{} `json:"list,omitempty"` +} + +type HpcInfo struct { + Id int64 `json:"id"` // id + TaskId int64 `json:"task_id"` // 任务id + JobId string `json:"job_id"` // 作业id(在第三方系统中的作业id) + AdapterId int64 `json:"adapter_id"` // 执行任务的适配器id + ClusterId int64 `json:"cluster_id"` // 执行任务的集群id + ClusterType string `json:"cluster_type"` // 执行任务的集群类型 + Name string `json:"name"` // 名称 + Status string `json:"status"` // 状态 + CmdScript string `json:"cmd_script"` + StartTime string `json:"start_time"` // 开始时间 + RunningTime int64 `json:"running_time"` // 运行时间 + DerivedEs string `json:"derived_es"` + Cluster string `json:"cluster"` + BlockId int64 `json:"block_id"` + AllocNodes int64 `json:"alloc_nodes"` + AllocCpu int64 `json:"alloc_cpu"` + CardCount int64 `json:"card_count"` // 卡数 + Version string `json:"version"` + Account string `json:"account"` + WorkDir string `json:"work_dir"` // 工作路径 + AssocId int64 `json:"assoc_id"` + ExitCode int64 `json:"exit_code"` + WallTime string `json:"wall_time"` // 最大运行时间 + Result string `json:"result"` // 运行结果 + DeletedAt string `json:"deleted_at"` // 删除时间 + YamlString string `json:"yaml_string"` + AppType string `json:"app_type"` // 应用类型 + AppName string `json:"app_name"` // 应用名称 + Queue string `json:"queue"` // 队列名称 + SubmitType string `json:"submit_type"` // cmd（命令行模式） + NNode string `json:"n_node"` // 节点个数（当指定该参数时，GAP_NODE_STRING必须为""） + StdOutFile string `json:"std_out_file"` // 工作路径/std.err.%j + StdErrFile string `json:"std_err_file"` // 工作路径/std.err.%j + StdInput string `json:"std_input"` + Environment string `json:"environment"` + DeletedFlag int64 `json:"deleted_flag"` // 是否删除（0-否，1-是） + CreatedBy int64 `json:"created_by"` // 创建人 + CreatedTime string `json:"created_time"` // 创建时间 + UpdatedBy int64 `json:"updated_by"` // 更新人 + UpdatedTime string `json:"updated_time"` // 更新时间 +} + +type CloudInfo struct { + Participant int64 `json:"participant,omitempty"` + Id int64 `json:"id,omitempty"` + TaskId int64 `json:"taskId,omitempty"` + ApiVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + StartTime string `json:"startTime,omitempty"` + RunningTime int64 `json:"runningTime,omitempty"` + Result string `json:"result,omitempty"` + YamlString string `json:"yamlString,omitempty"` +} + +type AiInfo struct { + ParticipantId int64 `json:"participantId,omitempty"` + TaskId int64 `json:"taskId,omitempty"` + ProjectId string `json:"project_id,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + StartTime string `json:"startTime,omitempty"` + RunningTime int64 `json:"runningTime,omitempty"` + Result string `json:"result,omitempty"` + JobId string `json:"jobId,omitempty"` + CreateTime string `json:"createTime,omitempty"` + ImageUrl string `json:"imageUrl,omitempty"` + Command string `json:"command,omitempty"` + FlavorId string `json:"flavorId,omitempty"` + SubscriptionId string `json:"subscriptionId,omitempty"` + ItemVersionId string `json:"itemVersionId,omitempty"` +} + +type VmInfo struct { + ParticipantId int64 `json:"participantId,omitempty"` + TaskId int64 `json:"taskId,omitempty"` + Name string `json:"name,omitempty"` + FlavorRef string `json:"flavor_ref,omitempty"` + ImageRef string `json:"image_ref,omitempty"` + NetworkUuid string `json:"network_uuid,omitempty"` + BlockUuid string `json:"block_uuid,omitempty"` + SourceType string `json:"source_type,omitempty"` + DeleteOnTermination bool `json:"delete_on_termination,omitempty"` + Status string `json:"status,omitempty"` + MinCount string `json:"min_count,omitempty"` + Platform string `json:"platform,omitempty"` + Uuid string `json:"uuid,omitempty"` +} + +type PullTaskInfoReq struct { + AdapterId int64 `form:"adapterId"` +} + +type PullTaskInfoResp struct { + HpcInfoList []*HpcInfo `json:"HpcInfoList,omitempty"` + CloudInfoList []*CloudInfo `json:"CloudInfoList,omitempty"` + AiInfoList []*AiInfo `json:"AiInfoList,omitempty"` + VmInfoList []*VmInfo `json:"VmInfoList,omitempty"` +} + +type PushTaskInfoReq struct { + AdapterId int64 `json:"adapterId"` + HpcInfoList []*HpcInfo `json:"hpcInfoList"` + CloudInfoList []*CloudInfo `json:"cloudInfoList"` + AiInfoList []*AiInfo `json:"aiInfoList"` + VmInfoList []*VmInfo `json:"vmInfoList"` +} + +type PushTaskInfoResp struct { + Code int64 `json:"code"` + Msg string `json:"msg"` +} + +type PushResourceInfoReq struct { + AdapterId int64 `json:"adapterId"` + ResourceStats []ResourceStats `json:"resourceStats"` +} + +type PushResourceInfoResp struct { + Code int64 `json:"code"` + Msg string `json:"msg"` +} + +type NoticeInfo struct { + AdapterId int64 `json:"adapterId"` + AdapterName string `json:"adapterName"` + ClusterId int64 `json:"clusterId"` + ClusterName string `json:"clusterName"` + NoticeType string `json:"noticeType"` + TaskName string `json:"taskName"` + Incident string `json:"incident"` +} + +type ListNoticeReq struct { +} + +type ListNoticeResp struct { + Code int64 `json:"code"` + Msg string `json:"msg"` + Data []NoticeInfo `json:"data"` +} + +type PushNoticeReq struct { + NoticeInfo NoticeInfo `json:"noticeInfo"` +} + +type PushNoticeResp struct { + Code int64 `json:"code"` + Msg string `json:"msg"` +} + +type ResourceStats struct { + ClusterId int64 `json:"clusterId"` + Name string `json:"name"` + CpuCoreAvail int64 `json:"cpuCoreAvail"` + CpuCoreTotal int64 `json:"cpuCoreTotal"` + MemAvail float64 `json:"memAvail"` + MemTotal float64 `json:"memTotal"` + DiskAvail float64 `json:"diskAvail"` + DiskTotal float64 `json:"diskTotal"` + GpuAvail int64 `json:"gpuAvail"` + CardsAvail []*Card `json:"cardsAvail"` + CpuCoreHours float64 `json:"cpuCoreHours"` + Balance float64 `json:"balance"` +} + +type Card struct { + Platform string `json:"platform"` + Type string `json:"type"` + Name string `json:"name"` + TOpsAtFp16 float64 `json:"TOpsAtFp16"` + CardHours float64 `json:"cardHours"` + CardNum int32 `json:"cardNum"` +} + type CommitHpcTaskReq struct { Name string `json:"name"` // paratera:jobName Description string `json:"description,optional"` @@ -5343,156 +5525,6 @@ type AiAlgorithmsResp struct { Algorithms []string `json:"algorithms"` } -type PullTaskInfoReq struct { - AdapterId int64 `form:"adapterId"` -} - -type PullTaskInfoResp struct { - HpcInfoList []*HpcInfo `json:"HpcInfoList,omitempty"` - CloudInfoList []*CloudInfo `json:"CloudInfoList,omitempty"` - AiInfoList []*AiInfo `json:"AiInfoList,omitempty"` - VmInfoList []*VmInfo `json:"VmInfoList,omitempty"` -} - -type HpcInfo struct { - Id int64 `json:"id"` // id - TaskId int64 `json:"task_id"` // 任务id - JobId string `json:"job_id"` // 作业id(在第三方系统中的作业id) - AdapterId int64 `json:"adapter_id"` // 执行任务的适配器id - ClusterId int64 `json:"cluster_id"` // 执行任务的集群id - ClusterType string `json:"cluster_type"` // 执行任务的集群类型 - Name string `json:"name"` // 名称 - Status string `json:"status"` // 状态 - CmdScript string `json:"cmd_script"` - StartTime string `json:"start_time"` // 开始时间 - RunningTime int64 `json:"running_time"` // 运行时间 - DerivedEs string `json:"derived_es"` - Cluster string `json:"cluster"` - BlockId int64 `json:"block_id"` - AllocNodes int64 `json:"alloc_nodes"` - AllocCpu int64 `json:"alloc_cpu"` - CardCount int64 `json:"card_count"` // 卡数 - Version string `json:"version"` - Account string `json:"account"` - WorkDir string `json:"work_dir"` // 工作路径 - AssocId int64 `json:"assoc_id"` - ExitCode int64 `json:"exit_code"` - WallTime string `json:"wall_time"` // 最大运行时间 - Result string `json:"result"` // 运行结果 - DeletedAt string `json:"deleted_at"` // 删除时间 - YamlString string `json:"yaml_string"` - AppType string `json:"app_type"` // 应用类型 - AppName string `json:"app_name"` // 应用名称 - Queue string `json:"queue"` // 队列名称 - SubmitType string `json:"submit_type"` // cmd（命令行模式） - NNode string `json:"n_node"` // 节点个数（当指定该参数时，GAP_NODE_STRING必须为""） - StdOutFile string `json:"std_out_file"` // 工作路径/std.err.%j - StdErrFile string `json:"std_err_file"` // 工作路径/std.err.%j - StdInput string `json:"std_input"` - Environment string `json:"environment"` - DeletedFlag int64 `json:"deleted_flag"` // 是否删除（0-否，1-是） - CreatedBy int64 `json:"created_by"` // 创建人 - CreatedTime string `json:"created_time"` // 创建时间 - UpdatedBy int64 `json:"updated_by"` // 更新人 - UpdatedTime string `json:"updated_time"` // 更新时间 -} - -type CloudInfo struct { - Participant int64 `json:"participant,omitempty"` - Id int64 `json:"id,omitempty"` - TaskId int64 `json:"taskId,omitempty"` - ApiVersion string `json:"apiVersion,omitempty"` - Kind string `json:"kind,omitempty"` - Namespace string `json:"namespace,omitempty"` - Name string `json:"name,omitempty"` - Status string `json:"status,omitempty"` - StartTime string `json:"startTime,omitempty"` - RunningTime int64 `json:"runningTime,omitempty"` - Result string `json:"result,omitempty"` - YamlString string `json:"yamlString,omitempty"` -} - -type AiInfo struct { - ParticipantId int64 `json:"participantId,omitempty"` - TaskId int64 `json:"taskId,omitempty"` - ProjectId string `json:"project_id,omitempty"` - Name string `json:"name,omitempty"` - Status string `json:"status,omitempty"` - StartTime string `json:"startTime,omitempty"` - RunningTime int64 `json:"runningTime,omitempty"` - Result string `json:"result,omitempty"` - JobId string `json:"jobId,omitempty"` - CreateTime string `json:"createTime,omitempty"` - ImageUrl string `json:"imageUrl,omitempty"` - Command string `json:"command,omitempty"` - FlavorId string `json:"flavorId,omitempty"` - SubscriptionId string `json:"subscriptionId,omitempty"` - ItemVersionId string `json:"itemVersionId,omitempty"` -} - -type VmInfo struct { - ParticipantId int64 `json:"participantId,omitempty"` - TaskId int64 `json:"taskId,omitempty"` - Name string `json:"name,omitempty"` - FlavorRef string `json:"flavor_ref,omitempty"` - ImageRef string `json:"image_ref,omitempty"` - NetworkUuid string `json:"network_uuid,omitempty"` - BlockUuid string `json:"block_uuid,omitempty"` - SourceType string `json:"source_type,omitempty"` - DeleteOnTermination bool `json:"delete_on_termination,omitempty"` - Status string `json:"status,omitempty"` - MinCount string `json:"min_count,omitempty"` - Platform string `json:"platform,omitempty"` - Uuid string `json:"uuid,omitempty"` -} - -type PushTaskInfoReq struct { - AdapterId int64 `json:"adapterId"` - HpcInfoList []*HpcInfo `json:"hpcInfoList"` - CloudInfoList []*CloudInfo `json:"cloudInfoList"` - AiInfoList []*AiInfo `json:"aiInfoList"` - VmInfoList []*VmInfo `json:"vmInfoList"` -} - -type PushTaskInfoResp struct { - Code int64 `json:"code"` - Msg string `json:"msg"` -} - -type PushResourceInfoReq struct { - AdapterId int64 `json:"adapterId"` - ResourceStats []ResourceStats `json:"resourceStats"` -} - -type PushResourceInfoResp struct { - Code int64 `json:"code"` - Msg string `json:"msg"` -} - -type ResourceStats struct { - ClusterId int64 `json:"clusterId"` - Name string `json:"name"` - CpuCoreAvail int64 `json:"cpuCoreAvail"` - CpuCoreTotal int64 `json:"cpuCoreTotal"` - MemAvail float64 `json:"memAvail"` - MemTotal float64 `json:"memTotal"` - DiskAvail float64 `json:"diskAvail"` - DiskTotal float64 `json:"diskTotal"` - GpuAvail int64 `json:"gpuAvail"` - CardsAvail []*Card `json:"cardsAvail"` - CpuCoreHours float64 `json:"cpuCoreHours"` - Balance float64 `json:"balance"` -} - -type Card struct { - Platform string `json:"platform"` - Type string `json:"type"` - Name string `json:"name"` - TOpsAtFp16 float64 `json:"TOpsAtFp16"` - CardHours float64 `json:"cardHours"` - CardNum int32 `json:"cardNum"` -} - type CreateAlertRuleReq struct { CLusterId int64 `json:"clusterId"` ClusterName string `json:"clusterName"`