Merge pull request 'add task details' (#128) from devad/pcm-coordinator:master into master

Former-commit-id: 32190d9b2b216318bd5913fc56cfa5f3d7a0e5cd

api/desc/cloud/pcm-cloud.api

@@ -59,9 +59,9 @@ type (
         Type int64 `json:"type"`                 // 租户所属(0数算，1超算，2智算）
         DeletedFlag int64 `json:"deletedFlag"`   // 是否删除
         CreatedBy int64 `json:"createdBy"`       // 创建人
-        CreatedTime string `json:"createdTime"`  // 创建时间
+        CreateTime string `json:"createdTime"`  // 创建时间
         UpdatedBy int64 `json:"updatedBy"`       // 更新人
-        UpdatedTime string `json:"updated_time"` // 更新时间
+        UpdateTime string `json:"updated_time"` // 更新时间
     }
 
     UpdateTenantReq {
@@ -115,6 +115,6 @@ type Cloud {
     StartTime string `json:"startTime"`        // 开始时间
     RunningTime int64 `json:"runningTime"`     // 运行时长
     CreatedBy int64 `json:"createdBy"`         // 创建人
-    CreatedTime string `json:"createdTime"`    // 创建时间
+    CreateTime string `json:"createdTime"`    // 创建时间
     Result string `json:"result"`
 }

api/desc/core/pcm-core.api

@@ -112,24 +112,13 @@ type (
 
 type (
     GeneralTaskReq {
-        Name        string   `json:"name"`
+        Name        string    `json:"name"`
-        ComputeType string   `json:"computeType"`
+        AdapterIds  []string  `json:"adapterIds"`
-        TemplateId  string   `json:"templateId"`
+        ClusterIds  []string  `json:"clusterIds"`
-        AdapterId   string   `json:"adapterId"`
+        Strategy    string    `json:"strategy"`
-        ClusterIds  []string `json:"clusterIds"`
+		StaticWeightMap map[string]int32 `json:"staticWeightMap,optional"`
-        Strategy    Strategy `json:"strategy"`
         ReqBody     []string `json:"reqBody"`
     }
-
-    Strategy {
-        Name             string `json:"name"`
-        StaticWeightList []StaticWeightList `json:"staticWeightList"`
-    }
-
-    StaticWeightList  {
-        ClusterName string `json:"clusterName"`
-        Weight      int    `json:"weight"`
-    }
 )
 
 type deleteTaskReq {
@@ -958,9 +947,9 @@ type (
 		Environment string `json:"environment"`
 		DeletedFlag int64  `json:"deleted_flag"` // 是否删除（0-否，1-是）
 		CreatedBy   int64  `json:"created_by"`   // 创建人
-		CreatedTime string `json:"created_time"` // 创建时间
+		CreateTime string `json:"created_time"` // 创建时间
 		UpdatedBy   int64  `json:"updated_by"`   // 更新人
-		UpdatedTime string `json:"updated_time"` // 更新时间
+		UpdateTime string `json:"updated_time"` // 更新时间
 	}
 
 	CloudInfo {
@@ -1110,4 +1099,14 @@ type TaskStatusResp {
 	Failed    int `json:"Failed"`
 	Running   int `json:"Running"`
 	Pause     int `json:"Pause"`
+}
+
+type TaskDetailsResp {
+	Name string `json:"name"`
+	description string `json:"description"`
+	StartTime string `json:"startTime"`
+	EndTime string `json:"endTime"`
+	Strategy int64 `json:"strategy,string"`
+	SynergyStatus int64 `json:"synergyStatus,string"`
+	ClusterInfos []*ClusterInfo `json:"clusterInfos"`
 }

api/desc/pcm.api

@@ -137,11 +137,14 @@ service pcm {
 	@doc "Statistical task status"
 	@handler countTaskStatus
 	get /core/task/countTaskStatus () returns (TaskStatusResp)
-	get /core/task/countTaskStatus () returns(TaskStatusResp)
 
 	@doc "Home Page Overview"
 	@handler homeOverviewHandler
 	get /core/homeOverview (HomeOverviewReq) returns (HomeOverviewResp)
+
+	@doc "task details"
+	@handler taskDetails
+	get /core/task/details (FId) returns(TaskDetailsResp)
 }
 
 //hpc二级接口
@@ -215,7 +218,7 @@ service pcm {
 
 	@doc "Create cloud computing common tasks"
 	@handler commitGeneralTask
-	post /cloud/task/create (GeneralTaskReq) returns()
+	post /cloud/task/create (GeneralTaskReq) returns ()
 }
 
 //智算二级接口
@@ -927,6 +930,9 @@ service pcm {
 	@handler ScheduleGetAlgorithmsHandler
 	get /schedule/ai/getAlgorithms/:adapterId/:resourceType/:taskType/:dataset (AiAlgorithmsReq) returns (AiAlgorithmsResp)
 
+	@handler ScheduleGetAiJobLogLogHandler
+	get /schedule/ai/getJobLog/:adapterId/:clusterId/:taskId/:instanceNum (AiJobLogReq) returns (AiJobLogResp)
+
 	@handler ScheduleSubmitHandler
 	post /schedule/submit (ScheduleReq) returns (ScheduleResp)
 }
@@ -980,7 +986,7 @@ service pcm {
 
 	@doc "alert rules"
 	@handler alertRulesHandler
-	get /monitoring/alert/rule (AlertRulesReq)returns (AlertRulesResp)
+	get /monitoring/alert/rule (AlertRulesReq) returns (AlertRulesResp)
 
 	@doc "cluster resource load"
 	@handler clustersLoadHandler
@@ -997,4 +1003,10 @@ service pcm {
 	@doc "Synchronize Cluster alert Information"
 	@handler syncClusterAlertHandler
 	post /core/syncClusterAlert (SyncClusterAlertReq)
+
+	@handler taskNumHandler
+	get /monitoring/task/num (taskNumReq) returns (taskNumResp)
+
+	@handler adapterInfoHandler
+	get /monitoring/adapter/info (adapterInfoReq) returns (adapterInfoResp)
 }

api/etc/pcm.yaml

@@ -5,7 +5,7 @@ Port: 8999
 Timeout: 50000
 
 DB:
-  DataSource: root:uJpLd6u-J?HC1@(10.206.0.12:3306)/pcm?parseTime=true&loc=Local
+  DataSource: root:uJpLd6u-J?HC1@(47.92.88.143:3306)/pcm?parseTime=true&loc=Local
   # DataSource: root:uJpLd6u-J?HC1@(47.92.88.143:3306)/pcm?parseTime=true&loc=Local
 Redis:
   Host: 10.206.0.12:6379

api/internal/handler/core/taskdetailshandler.go

@@ -0,0 +1,24 @@
+package core
+
+import (
+	"github.com/zeromicro/go-zero/rest/httpx"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/logic/core"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/repository/result"
+	"net/http"
+)
+
+func TaskDetailsHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var req types.FId
+		if err := httpx.Parse(r, &req); err != nil {
+			result.ParamErrorResult(r, w, err)
+			return
+		}
+
+		l := core.NewTaskDetailsLogic(r.Context(), svcCtx)
+		resp, err := l.TaskDetails(&req)
+		result.HttpResult(r, w, resp, err)
+	}
+}

api/internal/handler/routes.go

@@ -170,6 +170,11 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) {
 				Path:    "/core/homeOverview",
 				Handler: core.HomeOverviewHandler(serverCtx),
 			},
+			{
+				Method:  http.MethodGet,
+				Path:    "/core/task/details",
+				Handler: core.TaskDetailsHandler(serverCtx),
+			},
 		},
 		rest.WithPrefix("/pcm/v1"),
 	)
@@ -1254,7 +1259,7 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) {
 			},
 			{
 				Method:  http.MethodPost,
-				Path:    "/monitoring/syncClusterAlert",
+				Path:    "/core/syncClusterAlert",
 				Handler: monitoring.SyncClusterAlertHandler(serverCtx),
 			},
 			{

api/internal/logic/cloud/commitgeneraltasklogic.go

@@ -7,6 +7,7 @@ import (
 	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/constants"
 	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models"
 	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models/cloud"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/utils"
 	"io"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -52,7 +53,6 @@ func (l *CommitGeneralTaskLogic) CommitGeneralTask(req *types.GeneralTaskReq) er
 		Status:     constants.Saved,
 		Name:       req.Name,
 		CommitTime: time.Now(),
-		NsID:       "ns-admin",
 		YamlString: "[" + result + "]",
 	}
 	// Save the task data to the database
@@ -62,7 +62,7 @@ func (l *CommitGeneralTaskLogic) CommitGeneralTask(req *types.GeneralTaskReq) er
 	}
 
 	var clusters []*models.CloudModel
-	err := l.svcCtx.DbEngin.Raw("SELECT * FROM `t_cluster` where adapter_id = ? and id in ?", req.AdapterId, req.ClusterIds).Scan(&clusters).Error
+	err := l.svcCtx.DbEngin.Raw("SELECT * FROM `t_cluster` where adapter_id in ? and id in ?", req.AdapterIds, req.ClusterIds).Scan(&clusters).Error
 	if err != nil {
 		logx.Errorf("CommitGeneralTask() => sql execution error: %v", err)
 		return errors.Errorf("the cluster does not match the drive resources. Check the data")
@@ -73,11 +73,12 @@ func (l *CommitGeneralTaskLogic) CommitGeneralTask(req *types.GeneralTaskReq) er
 		for _, s := range req.ReqBody {
 			sStruct := UnMarshalK8sStruct(s)
 			unString, _ := sStruct.MarshalJSON()
+			taskCloud.Id = utils.GenSnowflakeIDUint()
 			taskCloud.TaskId = uint(taskModel.Id)
 			taskCloud.AdapterId = c.AdapterId
 			taskCloud.ClusterId = c.Id
 			taskCloud.ClusterName = c.Name
-			taskCloud.Status = "Saved"
+			taskCloud.Status = "Pending"
 			taskCloud.YamlString = string(unString)
 			taskCloud.Kind = sStruct.GetKind()
 			taskCloud.Namespace = sStruct.GetNamespace()
@@ -113,6 +114,14 @@ func UnMarshalK8sStruct(yamlString string) *unstructured.Unstructured {
 		if len(unstructuredObj.GetNamespace()) == 0 {
 			unstructuredObj.SetNamespace("default")
 		}
+		//设置副本数
+		if unstructuredObj.GetKind() == "Deployment" || unstructuredObj.GetKind() == "StatefulSet" {
+			unstructured.SetNestedField(
+				unstructuredObj.Object,
+				int64(6),
+				"spec", "replicas",
+			)
+		}
 	}
 	return unstructuredObj
 }

api/internal/logic/core/taskdetailslogic.go

@@ -0,0 +1,61 @@
+package core
+
+import (
+	"context"
+	"github.com/pkg/errors"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models"
+	"gorm.io/gorm"
+
+	"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/svc"
+	"gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/types"
+
+	"github.com/zeromicro/go-zero/core/logx"
+)
+
+type TaskDetailsLogic struct {
+	logx.Logger
+	ctx    context.Context
+	svcCtx *svc.ServiceContext
+}
+
+func NewTaskDetailsLogic(ctx context.Context, svcCtx *svc.ServiceContext) *TaskDetailsLogic {
+	return &TaskDetailsLogic{
+		Logger: logx.WithContext(ctx),
+		ctx:    ctx,
+		svcCtx: svcCtx,
+	}
+}
+
+func (l *TaskDetailsLogic) TaskDetails(req *types.FId) (resp *types.TaskDetailsResp, err error) {
+
+	var task models.Task
+	if errors.Is(l.svcCtx.DbEngin.Where("id", req.Id).First(&task).Error, gorm.ErrRecordNotFound) {
+		return nil, errors.New("记录不存在")
+	}
+	clusterIds := make([]int64, 0)
+	var cList []*types.ClusterInfo
+	switch task.TaskTypeDict {
+	case 0:
+		l.svcCtx.DbEngin.Table("task_cloud").Select("cluster_id").Where("task_id", task.Id).Scan(&clusterIds)
+	case 1:
+		l.svcCtx.DbEngin.Table("task_ai").Select("cluster_id").Where("task_id", task.Id).Scan(&clusterIds)
+	case 2:
+		l.svcCtx.DbEngin.Table("task_hpc").Select("cluster_id").Where("task_id", task.Id).Scan(&clusterIds)
+	case 3:
+		l.svcCtx.DbEngin.Table("task_vm").Select("cluster_id").Where("task_id", task.Id).Find(&clusterIds)
+	}
+	err = l.svcCtx.DbEngin.Table("t_cluster").Where("id in ?", clusterIds).Scan(&cList).Error
+	if err != nil {
+		return resp, err
+	}
+	resp = &types.TaskDetailsResp{
+		Name:          task.Name,
+		Description:   task.Description,
+		StartTime:     task.StartTime,
+		EndTime:       task.EndTime,
+		Strategy:      task.Strategy,
+		SynergyStatus: task.SynergyStatus,
+		ClusterInfos:  cList,
+	}
+	return
+}

api/internal/scheduler/schedulers/option/cloudOption.go

@@ -1,7 +1,12 @@
 package option
 
 type CloudOption struct {
-	task interface{}
+	Name            string           `json:"name"`
+	AdapterIds      []string         `json:"adapterIds"`
+	ClusterIds      []string         `json:"clusterIds"`
+	Strategy        string           `json:"strategy"`
+	StaticWeightMap map[string]int32 `json:"staticWeightMap,optional"`
+	ReqBody         []string         `json:"reqBody"`
 }
 
 func (c CloudOption) GetOptionType() string {

api/internal/types/types.go

@@ -99,23 +99,12 @@ type Region struct {
 }
 
 type GeneralTaskReq struct {
-	Name        string   `json:"name"`
+	Name            string           `json:"name"`
-	ComputeType string   `json:"computeType"`
+	AdapterIds      []string         `json:"adapterIds"`
-	TemplateId  string   `json:"templateId"`
+	ClusterIds      []string         `json:"clusterIds"`
-	AdapterId   string   `json:"adapterId"`
+	Strategy        string           `json:"strategy"`
-	ClusterIds  []string `json:"clusterIds"`
+	StaticWeightMap map[string]int32 `json:"staticWeightMap,optional"`
-	Strategy    Strategy `json:"strategy"`
+	ReqBody         []string         `json:"reqBody"`
-	ReqBody     []string `json:"reqBody"`
-}
-
-type Strategy struct {
-	Name             string             `json:"name"`
-	StaticWeightList []StaticWeightList `json:"staticWeightList"`
-}
-
-type StaticWeightList struct {
-	ClusterName string `json:"clusterName"`
-	Weight      int    `json:"weight"`
 }
 
 type DeleteTaskReq struct {
@@ -928,9 +917,9 @@ type HpcInfo struct {
 	Environment string `json:"environment"`
 	DeletedFlag int64  `json:"deleted_flag"` // 是否删除（0-否，1-是）
 	CreatedBy   int64  `json:"created_by"`   // 创建人
-	CreatedTime string `json:"created_time"` // 创建时间
+	CreateTime  string `json:"created_time"` // 创建时间
 	UpdatedBy   int64  `json:"updated_by"`   // 更新人
-	UpdatedTime string `json:"updated_time"` // 更新时间
+	UpdateTime  string `json:"updated_time"` // 更新时间
 }
 
 type CloudInfo struct {
@@ -1075,6 +1064,16 @@ type TaskStatusResp struct {
 	Pause     int `json:"Pause"`
 }
 
+type TaskDetailsResp struct {
+	Name          string         `json:"name"`
+	Description   string         `json:"description"`
+	StartTime     string         `json:"startTime"`
+	EndTime       string         `json:"endTime"`
+	Strategy      int64          `json:"strategy,string"`
+	SynergyStatus int64          `json:"synergyStatus,string"`
+	ClusterInfos  []*ClusterInfo `json:"clusterInfos"`
+}
+
 type CommitHpcTaskReq struct {
 	Name        string            `json:"name"` // paratera:jobName
 	Description string            `json:"description,optional"`
@@ -5305,9 +5304,9 @@ type TenantInfo struct {
 	Type        int64  `json:"type"`         // 租户所属(0数算，1超算，2智算）
 	DeletedFlag int64  `json:"deletedFlag"`  // 是否删除
 	CreatedBy   int64  `json:"createdBy"`    // 创建人
-	CreatedTime string `json:"createdTime"`  // 创建时间
+	CreateTime  string `json:"createdTime"`  // 创建时间
 	UpdatedBy   int64  `json:"updatedBy"`    // 更新人
-	UpdatedTime string `json:"updated_time"` // 更新时间
+	UpdateTime  string `json:"updated_time"` // 更新时间
 }
 
 type UpdateTenantReq struct {
@@ -5361,7 +5360,7 @@ type Cloud struct {
 	StartTime     string `json:"startTime"`   // 开始时间
 	RunningTime   int64  `json:"runningTime"` // 运行时长
 	CreatedBy     int64  `json:"createdBy"`   // 创建人
-	CreatedTime   string `json:"createdTime"` // 创建时间
+	CreateTime    string `json:"createdTime"` // 创建时间
 	Result        string `json:"result"`
 }
 

pkg/models/base/base_model.go

@@ -6,9 +6,9 @@ import (
 )
 
 type BaseModel struct {
-	DeletedAt   gorm.DeletedAt `gorm:"index;comment:删除时间" json:"-"`             // 删除时间
+	DeletedAt  gorm.DeletedAt `gorm:"index;comment:删除时间" json:"-"`                // 删除时间
-	CreatedBy   uint           `gorm:"created_by;comment:创建人" json:"createdBy"` //创建人
+	CreatedBy  uint           `gorm:"created_by;comment:创建人" json:"createdBy"`    //创建人
-	CreatedTime time.Time      `gorm:"comment:创建时间" json:"-"`                   // 创建时间
+	CreateTime time.Time      `gorm:"autoCreateTime:nano;comment:创建时间" json:"-"`  // 创建时间
-	UpdatedBy   uint           `gorm:"updated_by;comment:更新人" json:"UpdatedBy"` //创建人
+	UpdatedBy  uint           `gorm:"updated_by;comment:更新人" json:"UpdatedBy"`    //创建人
-	UpdatedTime time.Time      `gorm:"comment:更新时间" json:"-"`                   // 更新时间
+	UpdateTime time.Time      `gorm:"autoUpdateTime:nano;;comment:更新时间" json:"-"` // 更新时间
 }

pkg/models/file.go

@@ -37,9 +37,9 @@ type File struct {
 	Status      string     `gorm:"column:status" json:"Status"`            //type:string       comment:hash          version:2023-05-06 09:58
 	DeletedFlag *int       `gorm:"column:deleted_flag" json:"DeletedFlag"` //type:*int         comment:是否删除    version:2023-05-06 09:58
 	CreatedBy   *int       `gorm:"column:created_by" json:"CreatedBy"`     //type:*int         comment:创建人      version:2023-05-06 09:58
-	CreatedTime *time.Time `gorm:"column:created_time" json:"CreatedTime"` //type:*time.Time   comment:创建时间    version:2023-05-06 09:58
+	CreatedTime *time.Time `gorm:"column:created_time" json:"CreateTime"`  //type:*time.Time   comment:创建时间    version:2023-05-06 09:58
 	UpdatedBy   *int       `gorm:"column:updated_by" json:"UpdatedBy"`     //type:*int         comment:更新人      version:2023-05-06 09:58
-	UpdatedTime *time.Time `gorm:"column:updated_time" json:"UpdatedTime"` //type:*time.Time   comment:更新时间    version:2023-05-06 09:58
+	UpdatedTime *time.Time `gorm:"column:updated_time" json:"UpdateTime"`  //type:*time.Time   comment:更新时间    version:2023-05-06 09:58
 }
 
 // TableName 表名:data_set，。

pkg/models/taskmodel_gen.go

@@ -49,6 +49,7 @@ type (
 		Result        string         `db:"result"` // 作业结果
 		DeletedAt     gorm.DeletedAt `gorm:"index"`
 		NsID          string         `db:"ns_id"`
+		TaskTypeDict  int            `db:"task_type_dict"` //任务类型（对应字典表的值）
 	}
 )
 

pkg/utils/snowflake.go

@@ -41,3 +41,8 @@ func GenSnowflakeID() int64 {
 func GenSnowflakeIDStr() string {
 	return node.Generate().String()
 }
+
+// GenSnowflakeIDStr 工作id
+func GenSnowflakeIDUint() uint {
+	return uint(node.Generate().Int64())
+}