Merge pull request '移除ceph' (#322) from zhangweiii/pcm-coordinator:master into master

Former-commit-id: 5d505e3b8ba3c6e2bdeafe442014d9d2fe824bf8

client/types.go

@@ -98,6 +98,7 @@ type HpcInfo struct {
 	Account     string            `json:"account"`
 	WorkDir     string            `json:"work_dir"` // 工作路径
 	AssocId     int64             `json:"assoc_id"`
+	Partition   string            `json:"partition,omitempty,optional"`
 	ExitCode    int64             `json:"exit_code"`
 	WallTime    string            `json:"wall_time"`  // 最大运行时间
 	Result      string            `json:"result"`     // 运行结果
@@ -111,7 +112,7 @@ type HpcInfo struct {
 	StdOutFile  string            `json:"std_out_file"` // 工作路径/std.err.%j
 	StdErrFile  string            `json:"std_err_file"` // 工作路径/std.err.%j
 	StdInput    string            `json:"std_input"`
-	Environment string       `json:"environment"`
+	Environment map[string]string `json:"environment"`
 	DeletedFlag int64             `json:"deleted_flag"` // 是否删除（0-否，1-是）
 	CreatedBy   int64             `json:"created_by"`   // 创建人
 	CreatedTime time.Time         `json:"created_time"` // 创建时间

desc/core/pcm-core.api

@@ -92,7 +92,6 @@ type (
         Id int `json:"id"`
         PublicNetworkName string `json:"public_netWork_name"`
     }
-
 )
 
 type remoteResp {
@@ -176,7 +175,6 @@ type (
         AdapterName string `json:"adapterName"`
         PodName string `json:"podName"`
         stream bool `json:"stream"`
-	
     }
 )
 
@@ -515,6 +513,14 @@ type ScreenChartResp {
     CenterName string `json:"centerName"`
 }
 
+type getClusterByIdReq {
+    ClusterId int32 `path:"clusterId"`
+}
+
+type getClusterByIdResp {
+    ClusterInfo ClusterInfo `json:"clusterInfo"`
+}
+
 
 
 type ScreenInfoResp {
@@ -867,6 +873,7 @@ type (
         ProducerDict string `json:"producerDict,optional"`
         RegionDict string `json:"regionDict,optional"`
         RegionName string `json:"regionName,optional"`
+        Environment map[string]string `json:"environment,optional"`
     }
     ClusterInfo {
         Id string `json:"id,omitempty" db:"id"`
@@ -891,6 +898,7 @@ type (
         RegionDict string `json:"regionDict,omitempty" db:"region_dict"`
         Location string `json:"location,omitempty" db:"location"`
         CreateTime string `json:"createTime,omitempty" db:"created_time" gorm:"autoCreateTime"`
+        Environment map[string]string `json:"environment,omitempty" db:"environment"`
     }
 )
 
@@ -1225,7 +1233,8 @@ type (
         TaskName string `json:"taskName"`
         Incident string `json:"incident"`
     }
-	ListNoticeReq {}
+    ListNoticeReq{
+    }
 
     ListNoticeResp {
         Code int64 `json:"code"`

desc/hpc/pcm-hpc.api

@@ -10,11 +10,13 @@ info(
 
 type (
     commitHpcTaskReq {
-        Name string `json:"name"`                    // paratera:jobName
+        ClusterId int64 `json:"clusterId,optional"`
+        Name string `json:"name"`
+        Account string `json:"account,optional"`
         Description string `json:"description,optional"`
         TenantId int64 `json:"tenantId,optional"`
         TaskId int64 `json:"taskId,optional"`
-        AdapterIds []string `json:"adapterIds"`
+        AdapterIds []string `json:"adapterIds,optional"`
         MatchLabels map[string]string `json:"matchLabels,optional"`
         CardCount int64 `json:"cardCount,optional"`
         WorkDir string `json:"workDir,optional"`     //paratera:workingDir

desc/pcm.api

@@ -166,6 +166,10 @@ service pcm {
 	@doc "screen"
 	@handler getScreenChartHandler
 	get /core/getScreenChart returns (ScreenChartResp)
+
+	@doc "根据集群id获取集群信息"
+	@handler getClusterByIdHandler
+	get /core/getClusterById (getClusterByIdReq)returns (getClusterByIdResp)
 }
 
 //hpc二级接口
@@ -1060,4 +1064,3 @@ service pcm {
 	@handler scheduleSituationHandler
 	get /monitoring/schedule/situation returns (scheduleSituationResp)
 }
-

go.mod

@@ -23,7 +23,6 @@ require (
 	gitlink.org.cn/JointCloud/pcm-octopus v0.0.0-20240817071412-44397870b110
 	gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240403033338-e7edabad4203
 	gitlink.org.cn/JointCloud/pcm-slurm v0.0.0-20240301080743-8b94bbaf57f5
-	gitlink.org.cn/jcce-pcm/pcm-participant-ceph v0.0.0-20230904090036-24fc730ec87d
 	go.opentelemetry.io/otel/trace v1.30.0
 	gonum.org/v1/gonum v0.11.0
 	google.golang.org/grpc v1.66.2

go.sum

@@ -476,8 +476,6 @@ gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240403033338-e7edabad4203 h1:s6
 gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240403033338-e7edabad4203/go.mod h1:i2rrbMQ+Fve345BY9Heh4MUqVTAimZQElQhzzRee5B8=
 gitlink.org.cn/JointCloud/pcm-slurm v0.0.0-20240301080743-8b94bbaf57f5 h1:+/5vnzkJBfMRnya1NrhOzlroUtRa5ePiYbPKlHLoLV0=
 gitlink.org.cn/JointCloud/pcm-slurm v0.0.0-20240301080743-8b94bbaf57f5/go.mod h1:97AlUXN13g9UN3+9/DzCHpeoU5sbdyv0IQuTEHNexzQ=
-gitlink.org.cn/jcce-pcm/pcm-participant-ceph v0.0.0-20230904090036-24fc730ec87d h1:DHjl/rLuH2gKYtY0MKMGNQDHFT12APg25RlMUQo+tHk=
-gitlink.org.cn/jcce-pcm/pcm-participant-ceph v0.0.0-20230904090036-24fc730ec87d/go.mod h1:r/KLzUpupCV5jdxSfgDhc2pVjP0fBi3VhAWRttsBn30=
 go.etcd.io/etcd/api/v3 v3.5.16 h1:WvmyJVbjWqK4R1E+B12RRHz3bRGy9XVfh++MgbN+6n0=
 go.etcd.io/etcd/api/v3 v3.5.16/go.mod h1:1P4SlIP/VwkDmGo3OlOD7faPeP8KDIFhqvciH5EfN28=
 go.etcd.io/etcd/client/pkg/v3 v3.5.16 h1:ZgY48uH6UvB+/7R9Yf4x574uCO3jIx0TRDyetSfId3Q=

internal/config/config.go

@@ -33,7 +33,6 @@ type Config struct {
 	THRpcConf           zrpc.RpcClientConf
 	ModelArtsRpcConf    zrpc.RpcClientConf
 	ModelArtsImgRpcConf zrpc.RpcClientConf
-	CephRpcConf         zrpc.RpcClientConf
 	OpenstackRpcConf    zrpc.RpcClientConf
 	OctopusRpcConf      zrpc.RpcClientConf
 	PcmCoreRpcConf      zrpc.RpcClientConf

internal/handler/cloud/clusterinfohandler.go

@@ -1,25 +0,0 @@
-package cloud
-
-import (
-	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/repository/result"
-	"net/http"
-
-	"github.com/zeromicro/go-zero/rest/httpx"
-	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/logic/cloud"
-	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/svc"
-	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/types"
-)
-
-func ClusterInfoHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		var req types.ClusterInfoReq
-		if err := httpx.Parse(r, &req); err != nil {
-			httpx.ErrorCtx(r.Context(), w, err)
-			return
-		}
-
-		l := cloud.NewClusterInfoLogic(r.Context(), svcCtx)
-		resp, err := l.ClusterInfo(&req)
-		result.HttpResult(r, w, resp, err)
-	}
-}

internal/handler/core/getclusterbyidhandler.go

@@ -0,0 +1,28 @@
+package core
+
+import (
+	"net/http"
+
+	"github.com/zeromicro/go-zero/rest/httpx"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/logic/core"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/svc"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/types"
+)
+
+func GetClusterByIdHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var req types.GetClusterByIdReq
+		if err := httpx.Parse(r, &req); err != nil {
+			httpx.ErrorCtx(r.Context(), w, err)
+			return
+		}
+
+		l := core.NewGetClusterByIdLogic(r.Context(), svcCtx)
+		resp, err := l.GetClusterById(&req)
+		if err != nil {
+			httpx.ErrorCtx(r.Context(), w, err)
+		} else {
+			httpx.OkJsonCtx(r.Context(), w, resp)
+		}
+	}
+}

internal/handler/core/pulltaskinfohandler.go

@@ -18,7 +18,7 @@ func PullTaskInfoHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
 		}
 
 		l := core.NewPullTaskInfoLogic(r.Context(), svcCtx)
-		resp, err := l.PullTaskInfo((*clientCore.PullTaskInfoReq)(&req))
+		resp, err := l.PullTaskInfo(&req)
 		if err != nil {
 			httpx.ErrorCtx(r.Context(), w, err)
 		} else {

internal/logic/adapters/createclusterlogic.go

@@ -32,6 +32,13 @@ func NewCreateClusterLogic(ctx context.Context, svcCtx *svc.ServiceContext) *Cre
 }
 
 func (l *CreateClusterLogic) CreateCluster(req *types.ClusterCreateReq) (resp *types.ClusterResp, err error) {
+	// 校验集群名称是否唯一
+	var count int64
+	l.svcCtx.DbEngin.Table("t_cluster").Where("name = ?", req.Name).Count(&count)
+	if count > 0 {
+		return nil, errors.New("the cluster name is already in use")
+	}
+	// 校验驱动器是否存在
 	adapter := &types.AdapterInfo{}
 	result := l.svcCtx.DbEngin.Table("t_adapter").First(&adapter, req.AdapterId)
 	if errors.Is(result.Error, gorm.ErrRecordNotFound) {
@@ -45,6 +52,9 @@ func (l *CreateClusterLogic) CreateCluster(req *types.ClusterCreateReq) (resp *t
 	cluster.OwnerId = "0"
 	// 获取集群经纬度
 	//location, err := GeoMap(req.RegionName)
+	//if err != nil {
+	//	return nil, err
+	//}
 	//cluster.Location = location
 
 	cluster.Id = tool.GenSnowflakeIDStr()
@@ -54,14 +64,11 @@ func (l *CreateClusterLogic) CreateCluster(req *types.ClusterCreateReq) (resp *t
 		return nil, errors.New("cluster create failed")
 	}
 
-	if err != nil {
-		return nil, err
-	}
-
 	// push cluster info to adapter
+	go func() {
 		var adapterServer string
 		l.svcCtx.DbEngin.Raw("select server from t_adapter  where id = ?", req.AdapterId).Scan(&adapterServer)
-	response, err := l.svcCtx.HttpClient.R().
+		l.svcCtx.HttpClient.R().
 			SetBody(&types.ClusterInfo{
 				Name:          req.Name,
 				Server:        req.Server,
@@ -70,12 +77,9 @@ func (l *CreateClusterLogic) CreateCluster(req *types.ClusterCreateReq) (resp *t
 			}).
 			ForceContentType("application/json").
 			Post(adapterServer + "/api/v1/cluster/info")
-	if err != nil {
-		return nil, err
-	}
-	if response.IsError() {
-		return nil, errors.New(string(response.Body()))
-	}
+
+	}()
+
 	return
 }
 

internal/logic/adapters/updateclusterlogic.go

@@ -33,6 +33,12 @@ func (l *UpdateClusterLogic) UpdateCluster(req *types.ClusterCreateReq) (resp *t
 		return nil, errors.New("cluster does not exist")
 	}
 	utils.Convert(req, &cluster)
+	// 获取集群经纬度
+	location, err := GeoMap(req.RegionName)
+	if err != nil {
+		return nil, err
+	}
+	cluster.Location = location
 	l.svcCtx.DbEngin.Table("t_cluster").Model(&cluster).Updates(&cluster)
 	return
 }

internal/logic/cloud/clusterinfologic.go

@@ -1,30 +0,0 @@
-package cloud
-
-import (
-	"context"
-
-	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/svc"
-	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/types"
-
-	"github.com/zeromicro/go-zero/core/logx"
-)
-
-type ClusterInfoLogic struct {
-	logx.Logger
-	ctx    context.Context
-	svcCtx *svc.ServiceContext
-}
-
-func NewClusterInfoLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ClusterInfoLogic {
-	return &ClusterInfoLogic{
-		Logger: logx.WithContext(ctx),
-		ctx:    ctx,
-		svcCtx: svcCtx,
-	}
-}
-
-func (l *ClusterInfoLogic) ClusterInfo(req *types.ClusterInfoReq) (resp *types.ClusterInfoResp, err error) {
-	// todo: add your logic here and delete this line
-
-	return
-}

internal/logic/core/getclusterbyidlogic.go

@@ -0,0 +1,35 @@
+package core
+
+import (
+	"context"
+	"github.com/pkg/errors"
+
+	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/svc"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/types"
+
+	"github.com/zeromicro/go-zero/core/logx"
+)
+
+type GetClusterByIdLogic struct {
+	logx.Logger
+	ctx    context.Context
+	svcCtx *svc.ServiceContext
+}
+
+func NewGetClusterByIdLogic(ctx context.Context, svcCtx *svc.ServiceContext) *GetClusterByIdLogic {
+	return &GetClusterByIdLogic{
+		Logger: logx.WithContext(ctx),
+		ctx:    ctx,
+		svcCtx: svcCtx,
+	}
+}
+
+func (l *GetClusterByIdLogic) GetClusterById(req *types.GetClusterByIdReq) (resp *types.GetClusterByIdResp, err error) {
+	resp = &types.GetClusterByIdResp{}
+	tx := l.svcCtx.DbEngin.Raw("select * from t_cluster where id = ?", req.ClusterId).Scan(&resp)
+	if tx.Error != nil {
+		logx.Errorf(tx.Error.Error())
+		return nil, errors.New("cluster create failed")
+	}
+	return resp, nil
+}

internal/logic/core/pulltaskinfologic.go

@@ -53,7 +53,7 @@ func (l *PullTaskInfoLogic) PullTaskInfo(req *clientCore.PullTaskInfoReq) (*clie
 				}
 				var clusterType string
 				l.svcCtx.DbEngin.Raw("SELECT label FROM `t_cluster` where id = ? ", hpcInfo.ClusterId).Scan(&clusterType)
-
+				utils.Convert(hpcInfo.Environment, &resp.HpcInfoList[i].Environment)
 				resp.HpcInfoList[i].ClusterType = clusterType
 			}
 		}

internal/logic/hpc/commithpctasklogic.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	clientCore "gitlink.org.cn/JointCloud/pcm-coordinator/client"
-	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/constants"
 	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models"
 	"k8s.io/apimachinery/pkg/util/json"
 	"math/rand"
@@ -36,10 +35,6 @@ func (l *CommitHpcTaskLogic) CommitHpcTask(req *types.CommitHpcTaskReq) (resp *t
 	taskModel := models.Task{
 		Name:            req.Name,
 		Description:     req.Description,
-		Status:          constants.Saved,
-		Strategy:        0,
-		SynergyStatus:   0,
-		CommitTime:      time.Now(),
 		AdapterTypeDict: "2",
 	}
 
@@ -76,7 +71,6 @@ func (l *CommitHpcTaskLogic) CommitHpcTask(req *types.CommitHpcTaskReq) (resp *t
 		ClusterId:   uint(clusterId),
 		ClusterName: clusterName,
 		Name:        taskModel.Name,
-		Status:      "Saved",
 		CmdScript:   req.CmdScript,
 		StartTime:   time.Now().String(),
 		CardCount:   req.CardCount,
@@ -87,15 +81,11 @@ func (l *CommitHpcTaskLogic) CommitHpcTask(req *types.CommitHpcTaskReq) (resp *t
 		Queue:       req.Queue,
 		SubmitType:  req.SubmitType,
 		NNode:       req.NNode,
+		Account:     req.Account,
 		StdOutFile:  req.StdOutFile,
 		StdErrFile:  req.StdErrFile,
 		StdInput:    req.StdInput,
 		Partition:   req.Partition,
-		DeletedFlag: 0,
-		CreatedBy:   0,
-		CreatedTime: time.Now(),
-		UpdatedBy:   0,
-		UpdatedTime: time.Now(),
 		Environment: string(env),
 	}
 
@@ -111,7 +101,6 @@ func (l *CommitHpcTaskLogic) CommitHpcTask(req *types.CommitHpcTaskReq) (resp *t
 		NoticeType:  "create",
 		TaskName:    req.Name,
 		Incident:    "任务创建中",
-		CreatedTime: time.Now(),
 	}
 	result := l.svcCtx.DbEngin.Table("t_notice").Create(&noticeInfo)
 	if result.Error != nil {

internal/logic/storage/screenstoragelogic.go

@@ -1,61 +0,0 @@
-/*
-
- Copyright (c) [2023] [pcm]
- [pcm-coordinator] is licensed under Mulan PSL v2.
- You can use this software according to the terms and conditions of the Mulan PSL v2.
- You may obtain a copy of Mulan PSL v2 at:
-          http://license.coscl.org.cn/MulanPSL2
- THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
- EITHER EXPaRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
- MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
- See the Mulan PSL v2 for more details.
-
-*/
-
-package storage
-
-import (
-	"context"
-	"github.com/jinzhu/copier"
-	"github.com/pkg/errors"
-	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/svc"
-	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/types"
-	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/helper/xerr"
-	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/repository/result"
-	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/utils"
-	"gitlink.org.cn/jcce-pcm/pcm-participant-ceph/ceph"
-	"k8s.io/apimachinery/pkg/util/json"
-
-	"github.com/zeromicro/go-zero/core/logx"
-)
-
-type ScreenStorageLogic struct {
-	logx.Logger
-	ctx    context.Context
-	svcCtx *svc.ServiceContext
-}
-
-func NewScreenStorageLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ScreenStorageLogic {
-	return &ScreenStorageLogic{
-		Logger: logx.WithContext(ctx),
-		ctx:    ctx,
-		svcCtx: svcCtx,
-	}
-}
-
-func (l *ScreenStorageLogic) ScreenStorage(req *types.StorageScreenReq) (resp *types.StorageScreenResp, err error) {
-	// todo: add your logic here and delete this line
-	storageScreenReq := &ceph.StorageScreenReq{}
-	err = copier.CopyWithOption(storageScreenReq, req, copier.Option{Converters: utils.Converters})
-	StorageScreenResp, err := l.svcCtx.CephRpc.StorageScreen(l.ctx, storageScreenReq)
-	if err != nil {
-		return nil, errors.Wrapf(xerr.NewErrMsg("Failed to get db storage list"), "Failed to get db storage list err : %v ,req:%+v", err, req)
-	}
-	marshal, err := json.Marshal(&StorageScreenResp)
-	if err != nil {
-		return nil, result.NewDefaultError(err.Error())
-	}
-	json.Unmarshal(marshal, &resp)
-	err = copier.CopyWithOption(&resp, &StorageScreenResp, copier.Option{Converters: utils.Converters})
-	return resp, nil
-}

internal/svc/servicecontext.go

@@ -33,7 +33,6 @@ import (
 	"gitlink.org.cn/JointCloud/pcm-octopus/octopusclient"
 	"gitlink.org.cn/JointCloud/pcm-openstack/openstackclient"
 	slurmClient "gitlink.org.cn/JointCloud/pcm-slurm/slurmclient"
-	"gitlink.org.cn/jcce-pcm/pcm-participant-ceph/cephclient"
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
@@ -51,7 +50,6 @@ type ServiceContext struct {
 	ACRpc           hpcacclient.HpcAC
 	THRpc           slurmClient.Slurm
 	OctopusRpc      octopusclient.Octopus
-	CephRpc         cephclient.Ceph
 	OpenstackRpc    openstackclient.Openstack
 	MonitorClient   map[int64]tracker.Prometheus
 	PromClient      tracker.Prometheus
@@ -128,7 +126,6 @@ func NewServiceContext(c config.Config) *ServiceContext {
 		RedisClient:     redisClient,
 		ModelArtsRpc:    modelartsservice.NewModelArtsService(zrpc.MustNewClient(c.ModelArtsRpcConf)),
 		ModelArtsImgRpc: imagesservice.NewImagesService(zrpc.MustNewClient(c.ModelArtsImgRpcConf)),
-		CephRpc:         cephclient.NewCeph(zrpc.MustNewClient(c.CephRpcConf)),
 		ACRpc:           hpcacclient.NewHpcAC(zrpc.MustNewClient(c.ACRpcConf)),
 		OctopusRpc:      octopusclient.NewOctopus(zrpc.MustNewClient(c.OctopusRpcConf)),
 		OpenstackRpc:    openstackclient.NewOpenstack(zrpc.MustNewClient(c.OpenstackRpcConf)),