From 571e7603c7910208cd83520eff3ce3216414532e Mon Sep 17 00:00:00 2001 From: jagger Date: Sun, 7 Apr 2024 18:11:11 +0800 Subject: [PATCH] fix bugs Signed-off-by: jagger Former-commit-id: 8902acbd35bd41440625c0ac972f46e06f5305f7 --- api/desc/core/pcm-core.api | 22 +++ api/desc/pcm.api | 4 + .../handler/cloud/commitgeneraltaskhandler.go | 17 +++ api/internal/handler/routes.go | 5 + .../logic/cloud/commitgeneraltasklogic.go | 66 +++++++++ api/internal/types/types.go | 137 ++++++++++++++++++ go.sum | 8 - 7 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 api/internal/handler/cloud/commitgeneraltaskhandler.go create mode 100644 api/internal/logic/cloud/commitgeneraltasklogic.go diff --git a/api/desc/core/pcm-core.api b/api/desc/core/pcm-core.api index 4cdefe5e..79758887 100644 --- a/api/desc/core/pcm-core.api +++ b/api/desc/core/pcm-core.api @@ -93,6 +93,28 @@ type ( } ) +type ( + GeneralTaskReq { + Name string `json:"name"` + ComputeType string `json:"computeType"` + TemplateId string `json:"templateId"` + AdapterId string `json:"adapterId"` + ClusterIds []string `json:"clusterIds"` + Strategy Strategy `json:"strategy"` + ReqBody []string `json:"reqBody"` + } + + Strategy { + Name string `json:"name"` + StaticWeightList []StaticWeightList `json:"staticWeightList"` + } + + StaticWeightList { + ClusterName string `json:"clusterName"` + Weight int `json:"weight"` + } +) + type deleteTaskReq { Id int64 `path:"id"` } diff --git a/api/desc/pcm.api b/api/desc/pcm.api index 24fd3043..9040a551 100644 --- a/api/desc/pcm.api +++ b/api/desc/pcm.api @@ -192,6 +192,10 @@ service pcm { @doc "Obtain cluster list information according to adapterId" @handler getClusterListHandler get /core/clusterList (getClusterListReq) returns (getClusterListResp) + + @doc "Create cloud computing common tasks" + @handler commitGeneralTask + post /cloud/task/create (GeneralTaskReq) returns() } //智算二级接口 diff --git a/api/internal/handler/cloud/commitgeneraltaskhandler.go b/api/internal/handler/cloud/commitgeneraltaskhandler.go new file mode 100644 index 00000000..a1d591eb --- /dev/null +++ b/api/internal/handler/cloud/commitgeneraltaskhandler.go @@ -0,0 +1,17 @@ +package cloud + +import ( + "github.com/zeromicro/go-zero/rest/httpx" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/logic/cloud" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/repository/result" + "net/http" +) + +func CommitGeneralTaskHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := cloud.NewCommitGeneralTaskLogic(r.Context(), svcCtx) + err := l.CommitGeneralTask() + result.HttpResult(r, w, nil, err) + } +} diff --git a/api/internal/handler/routes.go b/api/internal/handler/routes.go index 45e701c2..ff1df60e 100644 --- a/api/internal/handler/routes.go +++ b/api/internal/handler/routes.go @@ -227,6 +227,11 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { Path: "/core/clusterList", Handler: cloud.GetClusterListHandler(serverCtx), }, + { + Method: http.MethodPost, + Path: "/cloud/task/create", + Handler: cloud.CommitGeneralTaskHandler(serverCtx), + }, }, rest.WithPrefix("/pcm/v1"), ) diff --git a/api/internal/logic/cloud/commitgeneraltasklogic.go b/api/internal/logic/cloud/commitgeneraltasklogic.go new file mode 100644 index 00000000..0dd34a54 --- /dev/null +++ b/api/internal/logic/cloud/commitgeneraltasklogic.go @@ -0,0 +1,66 @@ +package cloud + +import ( + "context" + "github.com/pkg/errors" + "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/constants" + "gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models" + "sigs.k8s.io/yaml" + "strings" + "time" + + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type CommitGeneralTaskLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewCommitGeneralTaskLogic(ctx context.Context, svcCtx *svc.ServiceContext) *CommitGeneralTaskLogic { + return &CommitGeneralTaskLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *CommitGeneralTaskLogic) CommitGeneralTask(req *types.GeneralTaskReq) error { + var yamlStr []string + for _, s := range req.ReqBody { + j2, err := yaml.YAMLToJSON([]byte(s)) + if err != nil { + logx.Errorf("Failed to convert yaml to JSON, err: %v", err) + return err + } + yamlStr = append(yamlStr, string(j2)) + } + result := strings.Join(yamlStr, ",") + //TODO The namespace is fixed to ns-admin for the time being. Later, the namespace is obtained based on the user + taskModel := models.Task{ + Status: constants.Saved, + Name: req.Name, + CommitTime: time.Now(), + NsID: "ns-admin", + YamlString: "[" + result + "]", + } + // Save the task data to the database + tx := l.svcCtx.DbEngin.Create(&taskModel) + if tx.Error != nil { + return tx.Error + } + + var clusterIds []int64 + l.svcCtx.DbEngin.Raw("SELECT id FROM `t_cluster` where adapter_id = ? and id in ?", req.AdapterId, req.ClusterIds).Scan(&clusterIds) + + if len(clusterIds) == 0 || clusterIds == nil { + return errors.Errorf("the cluster does not match the drive resources. Check the data") + } + //保存数据到cloud表 + + return nil +} diff --git a/api/internal/types/types.go b/api/internal/types/types.go index bc35feeb..9406bc6b 100644 --- a/api/internal/types/types.go +++ b/api/internal/types/types.go @@ -80,6 +80,26 @@ type Region struct { RunningJobs int64 `json:"runningJobs"` } +type GeneralTaskReq struct { + Name string `json:"name"` + ComputeType string `json:"computeType"` + TemplateId string `json:"templateId"` + AdapterId string `json:"adapterId"` + ClusterIds []string `json:"clusterIds"` + Strategy Strategy `json:"strategy"` + ReqBody []string `json:"reqBody"` +} + +type Strategy struct { + Name string `json:"name"` + StaticWeightList []StaticWeightList `json:"staticWeightList"` +} + +type StaticWeightList struct { + ClusterName string `json:"clusterName"` + Weight int `json:"weight"` +} + type DeleteTaskReq struct { Id int64 `path:"id"` } @@ -5271,6 +5291,123 @@ type AiAlgorithmsResp struct { Algorithms []string `json:"algorithms"` } +type PullTaskInfoReq struct { + AdapterId int64 `form:"adapterId"` +} + +type PullTaskInfoResp struct { + HpcInfoList []*HpcInfo `json:"HpcInfoList,omitempty"` + CloudInfoList []*CloudInfo `json:"CloudInfoList,omitempty"` + AiInfoList []*AiInfo `json:"AiInfoList,omitempty"` + VmInfoList []*VmInfo `json:"VmInfoList,omitempty"` +} + +type HpcInfo struct { + Id int64 `json:"id"` // id + TaskId int64 `json:"task_id"` // 任务id + JobId string `json:"job_id"` // 作业id(在第三方系统中的作业id) + AdapterId int64 `json:"adapter_id"` // 执行任务的适配器id + ClusterId int64 `json:"cluster_id"` // 执行任务的集群id + ClusterType string `json:"cluster_type"` // 执行任务的集群类型 + Name string `json:"name"` // 名称 + Status string `json:"status"` // 状态 + CmdScript string `json:"cmd_script"` + StartTime string `json:"start_time"` // 开始时间 + RunningTime int64 `json:"running_time"` // 运行时间 + DerivedEs string `json:"derived_es"` + Cluster string `json:"cluster"` + BlockId int64 `json:"block_id"` + AllocNodes int64 `json:"alloc_nodes"` + AllocCpu int64 `json:"alloc_cpu"` + CardCount int64 `json:"card_count"` // 卡数 + Version string `json:"version"` + Account string `json:"account"` + WorkDir string `json:"work_dir"` // 工作路径 + AssocId int64 `json:"assoc_id"` + ExitCode int64 `json:"exit_code"` + WallTime string `json:"wall_time"` // 最大运行时间 + Result string `json:"result"` // 运行结果 + DeletedAt string `json:"deleted_at"` // 删除时间 + YamlString string `json:"yaml_string"` + AppType string `json:"app_type"` // 应用类型 + AppName string `json:"app_name"` // 应用名称 + Queue string `json:"queue"` // 队列名称 + SubmitType string `json:"submit_type"` // cmd(命令行模式) + NNode string `json:"n_node"` // 节点个数(当指定该参数时,GAP_NODE_STRING必须为"") + StdOutFile string `json:"std_out_file"` // 工作路径/std.err.%j + StdErrFile string `json:"std_err_file"` // 工作路径/std.err.%j + StdInput string `json:"std_input"` + Environment string `json:"environment"` + DeletedFlag int64 `json:"deleted_flag"` // 是否删除(0-否,1-是) + CreatedBy int64 `json:"created_by"` // 创建人 + CreatedTime string `json:"created_time"` // 创建时间 + UpdatedBy int64 `json:"updated_by"` // 更新人 + UpdatedTime string `json:"updated_time"` // 更新时间 +} + +type CloudInfo struct { + Participant int64 `json:"participant,omitempty"` + Id int64 `json:"id,omitempty"` + TaskId int64 `json:"taskId,omitempty"` + ApiVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + StartTime string `json:"startTime,omitempty"` + RunningTime int64 `json:"runningTime,omitempty"` + Result string `json:"result,omitempty"` + YamlString string `json:"yamlString,omitempty"` +} + +type AiInfo struct { + ParticipantId int64 `json:"participantId,omitempty"` + TaskId int64 `json:"taskId,omitempty"` + ProjectId string `json:"project_id,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + StartTime string `json:"startTime,omitempty"` + RunningTime int64 `json:"runningTime,omitempty"` + Result string `json:"result,omitempty"` + JobId string `json:"jobId,omitempty"` + CreateTime string `json:"createTime,omitempty"` + ImageUrl string `json:"imageUrl,omitempty"` + Command string `json:"command,omitempty"` + FlavorId string `json:"flavorId,omitempty"` + SubscriptionId string `json:"subscriptionId,omitempty"` + ItemVersionId string `json:"itemVersionId,omitempty"` +} + +type VmInfo struct { + ParticipantId int64 `json:"participantId,omitempty"` + TaskId int64 `json:"taskId,omitempty"` + Name string `json:"name,omitempty"` + FlavorRef string `json:"flavor_ref,omitempty"` + ImageRef string `json:"image_ref,omitempty"` + NetworkUuid string `json:"network_uuid,omitempty"` + BlockUuid string `json:"block_uuid,omitempty"` + SourceType string `json:"source_type,omitempty"` + DeleteOnTermination bool `json:"delete_on_termination,omitempty"` + State string `json:"state,omitempty"` +} + +type PushTaskInfoReq struct { + AdapterId int64 `json:"adapterId"` + HpcInfoList []*HpcInfo `json:"hpcInfoList"` + CloudInfoList []*CloudInfo `json:"cloudInfoList"` + AiInfoList []*AiInfo `json:"aiInfoList"` + VmInfoList []*VmInfo `json:"vmInfoList"` +} + +type PushTaskInfoResp struct { + Code int64 `json:"code"` + Msg string `json:"msg"` +} + +type PushResourceInfoReq struct { + AdapterId int64 `json:"adapterId"` +} + type CreateAlertRuleReq struct { ClusterName string `json:"clusterName"` Namespace string `json:"namespace"` diff --git a/go.sum b/go.sum index 4c2e07cc..d00af510 100644 --- a/go.sum +++ b/go.sum @@ -1079,8 +1079,6 @@ github.com/zeromicro/go-zero v1.6.3 h1:OL0NnHD5LdRNDolfcK9vUkJt7K8TcBE3RkzfM8poO github.com/zeromicro/go-zero v1.6.3/go.mod h1:XZL435ZxVi9MSXXtw2MRQhHgx6OoX3++MRMOE9xU70c= gitlink.org.cn/JointCloud/pcm-kubernetes v0.0.0-20240301071143-347480abff2c h1:2Wl/hvaSFjh6fmCSIQhjkr9llMRREQeqcXNLZ/HPY18= gitlink.org.cn/JointCloud/pcm-kubernetes v0.0.0-20240301071143-347480abff2c/go.mod h1:lSRfGs+PxFvw7CcndHWRd6UlLlGrZn0b0hp5cfaMNGw= -gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240328020739-cbdd8f5b226b h1:suRANMHQPhKKmgdJOZcbFYDJ0NUQkUGgVvMKxw75BQI= -gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240328020739-cbdd8f5b226b/go.mod h1:i2rrbMQ+Fve345BY9Heh4MUqVTAimZQElQhzzRee5B8= gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240401022404-2f1425735f0d h1:ZX/Kg8eKdaAfDsTd+Y+TrJsUvxp/DpbWUp+Ij4CtR+s= gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240401022404-2f1425735f0d/go.mod h1:i2rrbMQ+Fve345BY9Heh4MUqVTAimZQElQhzzRee5B8= gitlink.org.cn/JointCloud/pcm-slurm v0.0.0-20240301080743-8b94bbaf57f5 h1:+/5vnzkJBfMRnya1NrhOzlroUtRa5ePiYbPKlHLoLV0= @@ -1094,18 +1092,12 @@ gitlink.org.cn/jcce-pcm/pcm-participant-modelarts v0.0.0-20231101085149-724c7c4c gitlink.org.cn/jcce-pcm/pcm-participant-octopus v0.0.0-20240222124813-e275cfa342f4 h1:NrxKAZ5uAzshB9EHcPw+XTOTzpxb5HslNRMYBrFC1Qo= gitlink.org.cn/jcce-pcm/pcm-participant-octopus v0.0.0-20240222124813-e275cfa342f4/go.mod h1:uyvpVqG1jHDXX+ubXI0RBwnWXzVykD/mliqGQIDvRoo= go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= -go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= -go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4= go.etcd.io/etcd/api/v3 v3.5.13/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c= go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= -go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= -go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= go.etcd.io/etcd/client/pkg/v3 v3.5.13 h1:RVZSAnWWWiI5IrYAXjQorajncORbS0zI48LQlE2kQWg= go.etcd.io/etcd/client/pkg/v3 v3.5.13/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8= go.etcd.io/etcd/client/v3 v3.5.7/go.mod h1:sOWmj9DZUMyAngS7QQwCyAXXAL6WhgTOPLNS/NabQgw= -go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= -go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= go.etcd.io/etcd/client/v3 v3.5.13 h1:o0fHTNJLeO0MyVbc7I3fsCf6nrOqn5d+diSarKnB2js= go.etcd.io/etcd/client/v3 v3.5.13/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=