From 5c0a3de84912d2a1d2ea5a8af58764a43f7a665a Mon Sep 17 00:00:00 2001 From: tzwang Date: Mon, 8 Apr 2024 17:25:59 +0800 Subject: [PATCH] updated pcm.api and ai storage Former-commit-id: ad231ed21f23a0c39ded7b00ae76393b76ab299f --- api/desc/pcm.api | 2 +- api/desc/schedule/pcm-schedule.api | 6 +-- api/internal/scheduler/database/aiStorage.go | 10 +++++ api/internal/scheduler/service/aiService.go | 40 ++++++++++---------- api/internal/svc/servicecontext.go | 2 +- 5 files changed, 34 insertions(+), 26 deletions(-) diff --git a/api/desc/pcm.api b/api/desc/pcm.api index ee7ffd74..6165299c 100644 --- a/api/desc/pcm.api +++ b/api/desc/pcm.api @@ -897,7 +897,7 @@ service pcm { get /schedule/ai/getStrategies returns (AiStrategyResp) @handler ScheduleGetAlgorithmsHandler - get /schedule/ai/getAlgorithms (AiAlgorithmsReq) returns (AiAlgorithmsResp) + get /schedule/ai/getAlgorithms/:resourceType/:taskType/:dataset (AiAlgorithmsReq) returns (AiAlgorithmsResp) @handler ScheduleSubmitHandler post /schedule/submit (ScheduleReq) returns (ScheduleResp) diff --git a/api/desc/schedule/pcm-schedule.api b/api/desc/schedule/pcm-schedule.api index eafba3de..8b6b2dd0 100644 --- a/api/desc/schedule/pcm-schedule.api +++ b/api/desc/schedule/pcm-schedule.api @@ -55,9 +55,9 @@ type ( } AiAlgorithmsReq { - ResourceType string `json:"resourceType"` - TaskType string `json:"taskType"` - Dataset string `json:"dataset"` + ResourceType string `path:"resourceType"` + TaskType string `path:"taskType"` + Dataset string `path:"dataset"` } AiAlgorithmsResp { diff --git a/api/internal/scheduler/database/aiStorage.go b/api/internal/scheduler/database/aiStorage.go index 670aec91..c0b706f2 100644 --- a/api/internal/scheduler/database/aiStorage.go +++ b/api/internal/scheduler/database/aiStorage.go @@ -23,6 +23,16 @@ func (s *AiStorage) GetParticipants() (*types.ClusterListResp, error) { return &resp, nil } +func (s *AiStorage) GetClustersByAdapterId(id string) (*types.ClusterListResp, error) { + var resp types.ClusterListResp + tx := s.DbEngin.Raw("select * from t_cluster where `deleted_at` IS NULL and `adapter_id` = ? ORDER BY create_time Desc", id).Scan(&resp.List) + if tx.Error != nil { + logx.Errorf(tx.Error.Error()) + return nil, tx.Error + } + return &resp, nil +} + func (s *AiStorage) SaveTask(name string) error { // 构建主任务结构体 taskModel := models.Task{ diff --git a/api/internal/scheduler/service/aiService.go b/api/internal/scheduler/service/aiService.go index 6c7b4096..aa8ded01 100644 --- a/api/internal/scheduler/service/aiService.go +++ b/api/internal/scheduler/service/aiService.go @@ -2,12 +2,14 @@ package service import ( "gitlink.org.cn/JointCloud/pcm-ac/hpcacclient" + "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/database" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/service/collector" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/scheduler/service/executor" "gitlink.org.cn/JointCloud/pcm-coordinator/api/internal/storeLink" "gitlink.org.cn/JointCloud/pcm-octopus/octopusclient" "gitlink.org.cn/jcce-pcm/pcm-participant-modelarts/client/imagesservice" "gitlink.org.cn/jcce-pcm/pcm-participant-modelarts/client/modelartsservice" + "strconv" ) const ( @@ -16,34 +18,30 @@ const ( SHUGUANGAI = "shuguangAi" ) -var ( - AiTypeMap = map[string]string{ - "hanwuji": OCTOPUS, - //"suiyan": OCTOPUS, - //"sailingsi": OCTOPUS, - //"modelarts-CloudBrain2": MODELARTS, - "shuguangAi": SHUGUANGAI, - } -) +func InitAiClusterMap(octopusRpc octopusclient.Octopus, modelArtsRpc modelartsservice.ModelArtsService, modelArtsImgRpc imagesservice.ImagesService, aCRpc hpcacclient.HpcAC, storages *database.AiStorage) (*map[string]executor.AiExecutor, *map[string]collector.AiCollector) { + clusters, _ := storages.GetClustersByAdapterId("1777144940459986944") -func InitAiClusterMap(octopusRpc octopusclient.Octopus, modelArtsRpc modelartsservice.ModelArtsService, modelArtsImgRpc imagesservice.ImagesService, aCRpc hpcacclient.HpcAC) (*map[string]executor.AiExecutor, *map[string]collector.AiCollector) { executorMap := make(map[string]executor.AiExecutor) collectorMap := make(map[string]collector.AiCollector) - for k, v := range AiTypeMap { - switch v { + for _, c := range clusters.List { + switch c.Name { case OCTOPUS: - octopus := storeLink.NewOctopusLink(octopusRpc, k, 0) - collectorMap[k] = octopus - executorMap[k] = octopus + id, _ := strconv.ParseInt(c.Id, 10, 64) + octopus := storeLink.NewOctopusLink(octopusRpc, c.Nickname, id) + collectorMap[c.Nickname] = octopus + executorMap[c.Nickname] = octopus case MODELARTS: - modelarts := storeLink.NewModelArtsLink(modelArtsRpc, modelArtsImgRpc, k, 0) - collectorMap[k] = modelarts - executorMap[k] = modelarts + id, _ := strconv.ParseInt(c.Id, 10, 64) + modelarts := storeLink.NewModelArtsLink(modelArtsRpc, modelArtsImgRpc, c.Nickname, id) + collectorMap[c.Nickname] = modelarts + executorMap[c.Nickname] = modelarts case SHUGUANGAI: - sgai := storeLink.NewShuguangAi(aCRpc, k, 0) - collectorMap[k] = sgai - executorMap[k] = sgai + id, _ := strconv.ParseInt(c.Id, 10, 64) + sgai := storeLink.NewShuguangAi(aCRpc, c.Nickname, id) + collectorMap[c.Nickname] = sgai + executorMap[c.Nickname] = sgai } } + return &executorMap, &collectorMap } diff --git a/api/internal/svc/servicecontext.go b/api/internal/svc/servicecontext.go index ee9510d2..ee6fc50f 100644 --- a/api/internal/svc/servicecontext.go +++ b/api/internal/svc/servicecontext.go @@ -120,8 +120,8 @@ func NewServiceContext(c config.Config) *ServiceContext { aCRpc := hpcacclient.NewHpcAC(zrpc.MustNewClient(c.ACRpcConf)) modelArtsRpc := modelartsservice.NewModelArtsService(zrpc.MustNewClient(c.ModelArtsRpcConf)) modelArtsImgRpc := imagesservice.NewImagesService(zrpc.MustNewClient(c.ModelArtsImgRpcConf)) - aiExecutor, resourceCollector := service.InitAiClusterMap(octopusRpc, modelArtsRpc, modelArtsImgRpc, aCRpc) storage := &database.AiStorage{DbEngin: dbEngin} + aiExecutor, resourceCollector := service.InitAiClusterMap(octopusRpc, modelArtsRpc, modelArtsImgRpc, aCRpc, storage) scheduler := scheduler.NewSchdlr(resourceCollector, storage, aiExecutor) return &ServiceContext{