added GetAlgorithms for ai scheduler

Former-commit-id: 02fabcc30a84ad7b9afbde1a693a34ffacf47d4b

api/desc/pcm.api

@@ -876,6 +876,9 @@ service pcm {
 	@handler ScheduleGetStrategyHandler
 	get /schedule/ai/getStrategies returns (AiStrategyResp)
 
+	@handler ScheduleGetAlgorithmsHandler
+	get /schedule/ai/getAlgorithms (AiAlgorithmsReq) returns (AiAlgorithmsResp)
+
 	@handler ScheduleSubmitHandler
 	post /schedule/submit (ScheduleReq) returns (ScheduleResp)
 }

api/desc/schedule/pcm-schedule.api

@@ -41,4 +41,14 @@ type (
     AiStrategyResp {
         Strategies []string `json:"strategies"`
     }
+
+    AiAlgorithmsReq {
+        ResourceType string `json:"resourceType"`
+        TaskType string `json:"taskType"`
+        Dataset string `json:"dataset"`
+    }
+
+    AiAlgorithmsResp {
+        Algorithms []string `json:"algorithms"`
+    }
 )

api/internal/scheduler/service/collector/collector.go

@@ -3,6 +3,7 @@ package collector
 type AiCollector interface {
 	GetResourceStats() (*ResourceStats, error)
 	GetDatasetsSpecs() ([]*DatasetsSpecs, error)
+	GetAlgorithms() ([]*Algorithm, error)
 }
 
 type ResourceStats struct {
@@ -33,3 +34,9 @@ type DatasetsSpecs struct {
 	Name string
 	Size string
 }
+
+type Algorithm struct {
+	Name     string
+	Platform string
+	TaskType string
+}

api/internal/storeLink/modelarts.go

@@ -157,6 +157,10 @@ func (m *ModelArtsLink) GetDatasetsSpecs() ([]*collector.DatasetsSpecs, error) {
 	return nil, nil
 }
 
+func (m *ModelArtsLink) GetAlgorithms() ([]*collector.Algorithm, error) {
+	return nil, nil
+}
+
 func (m *ModelArtsLink) Execute(option *option.AiOption) (interface{}, error) {
 	err := m.GenerateSubmitParams(option)
 	if err != nil {

api/internal/storeLink/octopus.go

@@ -315,6 +315,29 @@ func (o *OctopusLink) GetDatasetsSpecs() ([]*collector.DatasetsSpecs, error) {
 	return specs, nil
 }
 
+func (o *OctopusLink) GetAlgorithms() ([]*collector.Algorithm, error) {
+	var algorithms []*collector.Algorithm
+
+	req := &octopus.GetMyAlgorithmListReq{
+		Platform:  o.platform,
+		PageIndex: o.pageIndex,
+		PageSize:  o.pageSize,
+	}
+	resp, err := o.svcCtx.OctopusRpc.GetMyAlgorithmList(o.ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	if !resp.Success {
+		return nil, errors.New("failed to get algorithms")
+	}
+
+	for _, a := range resp.Payload.Algorithms {
+		algorithm := &collector.Algorithm{Name: a.AlgorithmName, Platform: OCTOPUS, TaskType: strings.ToLower(a.FrameworkName)}
+		algorithms = append(algorithms, algorithm)
+	}
+	return algorithms, nil
+}
+
 func (o *OctopusLink) Execute(option *option.AiOption) (interface{}, error) {
 	err := o.GenerateSubmitParams(option)
 	if err != nil {

api/internal/storeLink/shuguangai.go

@@ -341,6 +341,26 @@ func (s *ShuguangAi) GetDatasetsSpecs() ([]*collector.DatasetsSpecs, error) {
 	return specs, nil
 }
 
+func (s *ShuguangAi) GetAlgorithms() ([]*collector.Algorithm, error) {
+	var algorithms []*collector.Algorithm
+	for _, t := range GetTaskTypes() {
+		taskType := t
+		req := &hpcAC.GetFileListReq{Limit: 100, Path: ALGORITHM_DIR + FORWARD_SLASH + taskType, Start: 0}
+		list, err := s.svcCtx.ACRpc.GetFileList(s.ctx, req)
+		if err != nil {
+			return nil, err
+		}
+		if list.Code != "0" {
+			return nil, errors.New(list.Msg)
+		}
+		for _, file := range list.Data.FileList {
+			algorithm := &collector.Algorithm{Name: file.Name, Platform: SHUGUANGAI, TaskType: taskType}
+			algorithms = append(algorithms, algorithm)
+		}
+	}
+	return algorithms, nil
+}
+
 func (s *ShuguangAi) Execute(option *option.AiOption) (interface{}, error) {
 	err := s.GenerateSubmitParams(option)
 	if err != nil {

api/internal/storeLink/storeLink.go

@@ -27,6 +27,7 @@ import (
 	"gitlink.org.cn/jcce-pcm/pcm-participant-modelarts/modelarts"
 	"gitlink.org.cn/jcce-pcm/pcm-participant-octopus/octopus"
 	"gorm.io/gorm"
+	"strings"
 )
 
 type Linkage interface {
@@ -152,6 +153,48 @@ func GetDatasetsNames(collectorMap *map[string]collector.AiCollector) ([]string,
 	return names, nil
 }
 
+func GetAlgorithms(collectorMap *map[string]collector.AiCollector, resourceType string, taskType string, dataset string) ([]string, error) {
+	var names []string
+	colMap := *collectorMap
+	for _, col := range colMap {
+		var ns []string
+		algorithms, err := col.GetAlgorithms()
+		if err != nil {
+			return nil, err
+		}
+		for _, algorithm := range algorithms {
+			if algorithm.TaskType != taskType {
+				continue
+			}
+			switch algorithm.Platform {
+			case OCTOPUS:
+				splitns := strings.Split(algorithm.Name, UNDERSCORE)
+				if dataset != splitns[0] || len(splitns) == 1 {
+					continue
+				}
+				ns = append(ns, splitns[1])
+			case SHUGUANGAI:
+				splitns := strings.Split(algorithm.Name, DASH)
+				if dataset != splitns[0] || len(splitns) == 1 {
+					continue
+				}
+				ns = append(ns, splitns[1])
+			}
+		}
+		if len(ns) == 0 {
+			continue
+		}
+		if len(names) == 0 {
+			names = ns
+			continue
+		}
+
+		names = common.IntersectString(names, ns)
+	}
+	names = common.RemoveDuplicates(names)
+	return names, nil
+}
+
 func GetTaskTypes() []string {
 	return taskTypes
 }