diff --git a/api/internal/logic/storelink/submitlinktasklogic.go b/api/internal/logic/storelink/submitlinktasklogic.go index 45710841..354e5521 100644 --- a/api/internal/logic/storelink/submitlinktasklogic.go +++ b/api/internal/logic/storelink/submitlinktasklogic.go @@ -67,7 +67,7 @@ func (l *SubmitLinkTaskLogic) SubmitLinkTask(req *types.SubmitLinkTaskReq) (resp envs = append(envs, env) } } - task, err := storelink.ILinkage.SubmitTask(req.ImageId, req.Cmd, envs, params, req.ResourceId, "pytorch") + task, err := storelink.ILinkage.SubmitTask(req.ImageId, req.Cmd, envs, params, req.ResourceId, "", "", "pytorch") if err != nil { return nil, err } diff --git a/api/internal/scheduler/schedulers/option/aiOption.go b/api/internal/scheduler/schedulers/option/aiOption.go index 9512b3e9..acff16de 100644 --- a/api/internal/scheduler/schedulers/option/aiOption.go +++ b/api/internal/scheduler/schedulers/option/aiOption.go @@ -9,11 +9,12 @@ type AiOption struct { ClusterToStaticWeight map[string]int32 CodeType string - ImageId string - SpecId string - //DatasetsId string - CodeId string - ResourceId string + ImageId string + SpecId string + DatasetsId string + CodeId string + ResourceId string + AlgorithmId string Cmd string Envs []string diff --git a/api/internal/scheduler/service/aiService.go b/api/internal/scheduler/service/aiService.go index 01e41685..4ac09867 100644 --- a/api/internal/scheduler/service/aiService.go +++ b/api/internal/scheduler/service/aiService.go @@ -16,11 +16,11 @@ const ( var ( AiTypeMap = map[string]string{ - "hanwuji": OCTOPUS, - "suiyan": OCTOPUS, - "sailingsi": OCTOPUS, - "modelarts-CloudBrain2": MODELARTS, - "shuguangAi": SHUGUANGAI, + "hanwuji": OCTOPUS, + //"suiyan": OCTOPUS, + //"sailingsi": OCTOPUS, + //"modelarts-CloudBrain2": MODELARTS, + "shuguangAi": SHUGUANGAI, } ) diff --git a/api/internal/scheduler/strategy/strategy.go b/api/internal/scheduler/strategy/strategy.go index b73288ef..f5c06d64 100644 --- a/api/internal/scheduler/strategy/strategy.go +++ b/api/internal/scheduler/strategy/strategy.go @@ -1,10 +1,12 @@ package strategy const ( - REPLICATION = "replication" - RESOURCES_PRICING = "resourcesPricing" - STATIC_WEIGHT = "staticWeight" - DYNAMIC_RESOURCES = "dynamicResources" + REPLICATION = "replication" + RESOURCES_PRICING = "resourcesPricing" + STATIC_WEIGHT = "staticWeight" + DYNAMIC_RESOURCES = "dynamicResources" + DATA_LOCALITY = "dataLocality" //感知数据位置,数据调度和计算调度协同,近数据调度 + ENERGY_CONSUMPTION = "energyConsumption" //根据各集群总体能耗水平调度作业,优先选择能耗低的集群调度作业 ) var ( diff --git a/api/internal/storeLink/modelarts.go b/api/internal/storeLink/modelarts.go index c3b4cda1..2d65db43 100644 --- a/api/internal/storeLink/modelarts.go +++ b/api/internal/storeLink/modelarts.go @@ -63,7 +63,7 @@ func (m *ModelArtsLink) QueryImageList() (interface{}, error) { return resp, nil } -func (m *ModelArtsLink) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, aiType string) (interface{}, error) { +func (m *ModelArtsLink) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) { // modelArts提交任务 environments := make(map[string]string) parameters := make([]*modelarts.ParametersTrainJob, 0) @@ -162,7 +162,7 @@ func (m *ModelArtsLink) Execute(option *option.AiOption) (interface{}, error) { if err != nil { return nil, err } - task, err := m.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.TaskType) + task, err := m.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.DatasetsId, option.AlgorithmId, option.TaskType) if err != nil { return nil, err } diff --git a/api/internal/storeLink/octopus.go b/api/internal/storeLink/octopus.go index 9448d897..13fe2b07 100644 --- a/api/internal/storeLink/octopus.go +++ b/api/internal/storeLink/octopus.go @@ -108,7 +108,7 @@ func (o *OctopusLink) QueryImageList() (interface{}, error) { return resp, nil } -func (o *OctopusLink) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, aiType string) (interface{}, error) { +func (o *OctopusLink) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) { // octopus提交任务 // python参数 @@ -227,7 +227,7 @@ func (o *OctopusLink) Execute(option *option.AiOption) (interface{}, error) { if err != nil { return nil, err } - task, err := o.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.TaskType) + task, err := o.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.DatasetsId, option.AlgorithmId, option.TaskType) if err != nil { return nil, err } @@ -239,6 +239,10 @@ func (o *OctopusLink) GenerateSubmitParams(option *option.AiOption) error { if err != nil { return err } + err = o.generateDatasetsId(option) + if err != nil { + return err + } err = o.generateImageId(option) if err != nil { return err @@ -259,10 +263,34 @@ func (o *OctopusLink) GenerateSubmitParams(option *option.AiOption) error { } func (o *OctopusLink) generateResourceId(option *option.AiOption) error { - return nil } +func (o *OctopusLink) generateDatasetsId(option *option.AiOption) error { + if option.DatasetsName == "" { + return errors.New("DatasetsName not set") + } + req := &octopus.GetMyDatasetListReq{ + Platform: o.platform, + PageIndex: o.pageIndex, + PageSize: o.pageSize, + } + resp, err := o.svcCtx.OctopusRpc.GetMyDatasetList(o.ctx, req) + if err != nil { + return err + } + if !resp.Success { + return errors.New("failed to get DatasetsId") + } + for _, dataset := range resp.Payload.Datasets { + if dataset.Name == option.DatasetsName { + option.DatasetsId = dataset.Id + return nil + } + } + return errors.New("failed to get DatasetsId") +} + func (o *OctopusLink) generateImageId(option *option.AiOption) error { return nil diff --git a/api/internal/storeLink/shuguangHpc.go b/api/internal/storeLink/shuguangHpc.go index f7f0af82..85b4c32e 100644 --- a/api/internal/storeLink/shuguangHpc.go +++ b/api/internal/storeLink/shuguangHpc.go @@ -144,7 +144,7 @@ func (s ShuguangHpc) QueryImageList() (interface{}, error) { return nil, nil } -func (s ShuguangHpc) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, aiType string) (interface{}, error) { +func (s ShuguangHpc) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) { // shuguangHpc提交任务 //判断是否resourceId匹配自定义资源Id diff --git a/api/internal/storeLink/shuguangai.go b/api/internal/storeLink/shuguangai.go index eef573bf..c2898513 100644 --- a/api/internal/storeLink/shuguangai.go +++ b/api/internal/storeLink/shuguangai.go @@ -141,7 +141,7 @@ func (s *ShuguangAi) SubmitTensorflowTask(imageId string, cmd string, envs []str return nil, nil } -func (s *ShuguangAi) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, aiType string) (interface{}, error) { +func (s *ShuguangAi) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) { // shuguangAi提交任务 switch aiType { case PYTORCH_TASK: @@ -248,7 +248,7 @@ func (s *ShuguangAi) Execute(option *option.AiOption) (interface{}, error) { if err != nil { return nil, err } - task, err := s.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.TaskType) + task, err := s.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.DatasetsId, option.AlgorithmId, option.TaskType) if err != nil { return nil, err } diff --git a/api/internal/storeLink/storeLink.go b/api/internal/storeLink/storeLink.go index 80714797..b1815758 100644 --- a/api/internal/storeLink/storeLink.go +++ b/api/internal/storeLink/storeLink.go @@ -33,7 +33,7 @@ type Linkage interface { UploadImage(path string) (interface{}, error) DeleteImage(imageId string) (interface{}, error) QueryImageList() (interface{}, error) - SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, aiType string) (interface{}, error) + SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) QueryTask(taskId string) (interface{}, error) QuerySpecs() (interface{}, error) DeleteTask(taskId string) (interface{}, error)