From ceaad285394a42072897081aced45779e604c3d6 Mon Sep 17 00:00:00 2001 From: tzwang Date: Mon, 5 Feb 2024 13:01:45 +0800 Subject: [PATCH 1/3] added schedule apis Former-commit-id: eed1ae30dd41e40fc141401024811f7756d09f2c --- api/desc/pcm.api | 254 +++++++++--------- api/internal/handler/routes.go | 32 +++ .../schedulegetairesourcetypeshandler.go | 21 ++ .../schedule/schedulegetaitasktypeshandler.go | 21 ++ .../schedule/schedulegetdatasetshandler.go | 21 ++ .../schedule/schedulegetstrategyhandler.go | 21 ++ .../handler/schedule/schedulesubmithandler.go | 28 ++ .../schedulegetairesourcetypeslogic.go | 30 +++ .../schedule/schedulegetaitasktypeslogic.go | 30 +++ .../schedule/schedulegetdatasetslogic.go | 30 +++ .../schedule/schedulegetstrategylogic.go | 30 +++ .../logic/schedule/schedulesubmitlogic.go | 30 +++ api/internal/types/types.go | 34 +++ 13 files changed, 455 insertions(+), 127 deletions(-) create mode 100644 api/internal/handler/schedule/schedulegetairesourcetypeshandler.go create mode 100644 api/internal/handler/schedule/schedulegetaitasktypeshandler.go create mode 100644 api/internal/handler/schedule/schedulegetdatasetshandler.go create mode 100644 api/internal/handler/schedule/schedulegetstrategyhandler.go create mode 100644 api/internal/handler/schedule/schedulesubmithandler.go create mode 100644 api/internal/logic/schedule/schedulegetairesourcetypeslogic.go create mode 100644 api/internal/logic/schedule/schedulegetaitasktypeslogic.go create mode 100644 api/internal/logic/schedule/schedulegetdatasetslogic.go create mode 100644 api/internal/logic/schedule/schedulegetstrategylogic.go create mode 100644 api/internal/logic/schedule/schedulesubmitlogic.go diff --git a/api/desc/pcm.api b/api/desc/pcm.api index 8ab8ae7c..cbdc549d 100644 --- a/api/desc/pcm.api +++ b/api/desc/pcm.api @@ -28,87 +28,87 @@ service pcm { @doc "查询P端服务列表" @handler participantListHandler get /core/participantList returns (participantListResp) - + @doc "yaml提交任务" @handler scheduleTaskByYamlHandler post /core/scheduleTaskByYaml (scheduleTaskByYamlReq) - + @doc "提交任务" @handler commitTaskHandler post /core/commitTask (commitTaskReq) - + @doc "提交超算任务" @handler commitHpcTaskHandler post /core/commitHpcTask (commitHpcTaskReq) returns (commitHpcTaskResp) - + @doc "提交虚拟机任务" @handler commitVmTaskHandler post /core/commitVmTask (commitVmTaskReq) returns (commitVmTaskResp) - + @doc "删除任务" @handler deleteTaskHandler delete /core/deleteTask/:id (deleteTaskReq) - + @doc "查询任务列表" @handler TaskListHandler get /core/taskList (taskListReq) returns (taskListResp) - + @doc "查询任务详情" @handler TaskDetailHandler get /core/taskDetail/:taskId (taskDetailReq) returns (taskDetailResp) - + @doc "任务概览" @handler JobTotalHandler get /core/jobTotal returns (jobTotalResp) - + @doc "数据中心概览" @handler listCenterHandler get /core/listCenter returns (listCenterResp) - + @doc "查询集群列表" @handler listClusterHandler get /core/listCluster/:centerId (listClusterReq) returns (listClusterResp) - + @doc "提交任务(超算)" @handler submitJobHandler post /core/submitJob (submitJobReq) returns (submitJobResp) - + @doc "获取region" @handler getRegionHandler get /core/getRegion returns (getRegionResp) - + @doc "获取region列表" @handler listRegionHandler get /core/listRegion returns (listRegionResp) - + @doc "查询算力" @handler getComputingPowerHandler get /core/getComputingPower returns (cpResp) - + @doc "查询通用信息" @handler getGeneralInfoHandler get /core/getGeneralInfo returns (GiResp) - + @doc "查询各域资源信息" @handler listDomainResourceHandler get /core/listDomainResource returns (DomainResourceResp) - + @doc "查询控制面板配置信息" @handler getResourcePanelConfigHandler get /core/getResourcePanelConfigHandler returns (ResourcePanelConfigResp) - + @doc "设置控制面板配置信息" @handler putResourcePanelConfigHandler put /core/resourcePanelConfigHandler (ResourcePanelConfigReq) - + @doc "获取算力统计信息" @handler getComputilityStatisticsHandler get /core/getComputilityStatistics returns (ComputilityStatisticsResp) - + @doc "获取节点资产" @handler nodeAssetsHandler get /core/assets returns (NodeAssetsResp) - + @doc "Center Resources top3" @handler centerResourcesHandler get /core/centerResources returns (centerResourcesResp) @@ -123,11 +123,11 @@ service pcm { @doc "超算查询任务列表" @handler listJobHandler get /hpc/listJob (listJobReq) returns (listJobResp) - + @doc "超算查询历史任务列表" @handler listHistoryJobHandler get /hpc/listHistoryJob (listHistoryJobReq) returns (listHistoryJobResp) - + @doc "超算查询资产列表" @handler queueAssetsHandler get /queue/assets returns (QueueAssetsResp) @@ -142,35 +142,35 @@ service pcm { @doc "云算任务列表" @handler cloudListHandler get /task/list returns (cloudListResp) - + @doc "yaml删除" @handler deleteYamlHandler get /cloud/DeleteYaml (ApplyReq) returns (DeleteResp) - + @doc "控制器监控" @handler controllerMetricsHandler get /cloud/controller/Metrics (ControllerMetricsReq) returns (ControllerMetricsResp) - + @doc "数算集群注册" @handler registerClusterHandler post /cloud/registerCluster (RegisterClusterReq) returns (CloudResp) - + @doc "数算集群删除" @handler deleteClusterHandler post /cloud/deleteCluster (deleteClusterReq) returns (CloudResp) - + @doc "触发租户更新" @handler noticeTenantHandler get /cloud/noticeTenant returns (CloudResp) - + @doc "租户更新" @handler updateTenantHandler post /cloud/updateTenant (UpdateTenantReq) returns (CloudResp) - + @doc "Obtain cluster list information according to adapterId" @handler getClusterListHandler get /core/clusterList (getClusterListReq) returns (getClusterListResp) - + } //智算二级接口 @@ -182,114 +182,114 @@ service pcm { @doc "查询数据集列表" @handler listDataSetHandler get /ai/listDataSet/:projectId (DataSetReq) returns (DataSetResp) - + @doc "创建数据集" @handler CreateDataSetHandler post /ai/createDataSet/:projectId (CreateDataSetReq) returns (CreateDataSetResp) - + @doc "删除数据集" @handler DeleteDataSetHandler delete /ai/deleteDataSet/:projectId/:datasetId (DeleteDataSetReq) returns (DeleteDataSetResp) - + @doc "创建导入任务" @handler CreateTaskHandler post /ai/CreateTask/:projectId/:datasetId (ImportTaskDataReq) returns (ImportTaskDataResp) - + @doc "查询数据集导入任务列表" @handler ListImportHandler get /ai/ListImport/:projectId/:datasetId (ListImportTasksReq) returns (ListImportTasksResp) - + @doc "查询训练作业列表" @handler GetListTrainingJobsHandler get /ai/GetListTrainingJobs/:projectId (ListTrainingJobsreq) returns (ListTrainingJobsresp) - + @doc "删除训练作业" @handler DeleteTrainingJobHandler delete /ai/DeleteTrainingJob/:projectId/:trainingJobId (DeleteTrainingJobReq) returns (DeleteTrainingJobResp) - + @doc "创建算法" @handler CreateAlgorithmHandler post /ai/CreateAlgorithm/:projectId (CreateAlgorithmReq) returns (CreateAlgorithmResp) - + @doc "查询创建算法列表" @handler ListAlgorithms get /ai/ListAlgorithms/:projectId (ListAlgorithmsReq) returns (ListAlgorithmsResp) - + @doc "删除算法" @handler DeleteAlgorithm delete /ai/DeleteAlgorithm/:projectId/:algorithmId (DeleteAlgorithmReq) returns (DeleteAlgorithmResp) - + @doc "创建训练作业" @handler CreateTrainingJobHandler post /ai/CreateTrainingJob/:projectId (CreateTrainingJobReq) returns (CreateTrainingJobResp) - + @doc "展示算法详情" @handler ShowAlgorithmByUuid get /ai/ShowAlgorithmByUuid/:projectId/:algorithmId (ShowAlgorithmByUuidReq) returns (ShowAlgorithmByUuidResp) - + @doc "创建导出任务" @handler CreateExportTaskHandler post /ai/CreateExportTask/:projectId/:datasetId (CreateExportTaskReq) returns (ExportTaskDataResp) - + @doc "获取导出任务数据集" @handler GetExportTasksOfDatasetHandler get /ai/GetExportTasksOfDataset/:projectId/:datasetId (GetExportTasksOfDatasetReq) returns (GetExportTasksOfDatasetResp) - + @doc "获取导出任务数据集状态" @handler GetExportTaskStatusOfDatasetHandler get /ai/GetExportTaskStatusOfDataset/:projectId/:resourceId/:taskId (GetExportTaskStatusOfDatasetReq) returns (GetExportTaskStatusOfDatasetResp) - + @doc "创建处理任务" @handler CreateProcessorTaskHandler post /ai/CreateProcessorTask (CreateProcessorTaskReq) returns (CreateProcessorTaskResp) - + @doc "创建服务" @handler CreateServiceHandler post /ai/CreateService/:projectId (CreateServiceReq) returns (CreateServiceResp) - + @doc "展示服务" @handler ListServicesHandler get /ai/ListServices/:projectId (ListServicesReq) returns (ListServicesResp) - + @doc "展示服务详情" @handler ShowServiceHandler get /ai/ShowService/:projectId/:serviceId (ShowServiceReq) returns (ShowServiceResp) - + @doc "删除服务" @handler DeleteServiceHandler delete /ai/DeleteService/:projectId/:serviceId (DeleteServiceReq) returns (DeleteServiceResp) - + @doc "查询专属资源池列表" @handler ListClustersHandler get /ai/ListClusters (ListClustersReq) returns (ListClustersResp) - + /******************Notebook Method start*************************/ - + @doc "查询notebook列表" @handler listNotebookHandler get /ai/listNotebook (ListNotebookReq) returns (ListNotebookResp) - + @doc "创建notebook" @handler createNotebookHandler post /ai/createNotebook (CreateNotebookReq) returns (CreateNotebookResp) - + @doc "启动notebook" @handler startNotebookHandler post /ai/startNotebook (StartNotebookReq) returns (StartNotebookResp) - + @doc "停止notebook" @handler stopNotebookHandler post /ai/stopNotebook (StopNotebookReq) returns (StopNotebookResp) - + @doc "查询notebook存储" @handler getNotebookStorageHandler get /ai/getNotebookStorage (GetNotebookStorageReq) returns (GetNotebookStorageResp) - + @doc "挂载notebook存储" @handler mountNotebookStorageHandler post /ai/mountNotebookStorage (MountNotebookStorageReq) returns (MountNotebookStorageResp) /******************Notebook Method end*************************/ - + /******************Visualization Job Method start*************************/ @doc "获取虚拟化任务" @handler getVisualizationJobHandler @@ -326,19 +326,19 @@ service pcm { @doc "镜像上传" @handler uploadHandler post /upload - + @doc "镜像分块" @handler chunkHandler post /chunk - + @doc "查询镜像列表" @handler imageListHandler get /image/list returns (imageListResp) - + @doc "数据集检查" @handler dataSetCheckHandler get /dataSet/check/:fileMd5 (checkReq) returns (checkResp) - + @doc "上传数据集" @handler uploadDataSetHandler post /dataSet/upload @@ -353,147 +353,147 @@ service pcm { @doc "openstack计算中心概览" @handler GetComputeLimitsHandler get /vm/getComputeLimits (GetComputeLimitsReq) returns (GetComputeLimitsResp) - + @doc "查询卷列表" @handler GetVolumeLimitsHandler get /vm/getVolumeLimits (GetVolumeLimitsReq) returns (GetVolumeLimitsResp) - + @doc "查询虚拟机列表" @handler ListServerHandler get /vm/listServer (ListServersReq) returns (ListServersResp) - + @doc "查询虚拟机详情列表" @handler ListServersDetailedHandler get /vm/listServersDetailed (ListServersDetailedReq) returns (ListServersDetailedResp) - + @doc "删除虚拟机" @handler DeleteServerHandler delete /vm/deleteServer (DeleteServerReq) returns (DeleteServerResp) - + @doc "创建虚拟机" @handler CreateServerHandler post /vm/createServer (CreateServerReq) returns (CreateServerResp) - + @doc "根据ID查询虚拟机详情" @handler GetServersDetailedByIdHandler get /vm/getServersDetailedById (GetServersDetailedByIdReq) returns (GetServersDetailedByIdResp) - + @doc "更新虚拟机" @handler UpdateServerHandler put /vm/updateServer (UpdateServerReq) returns (UpdateServerResp) - + @doc "启动虚拟机" @handler StartServerHandler post /vm/startServer (StartServerReq) returns (StartServerResp) - + @doc "停止虚拟机" @handler StopServerHandler post /vm/stopServer (StopServerReq) returns (StopServerResp) - + @doc "重启虚拟机" @handler RebootServerHandler post /vm/rebootServer (RebootServerReq) returns (RebootServerResp) - + @doc "暂停虚拟机" @handler PauseServerHandler post /vm/pauseServer (PauseServerReq) returns (PauseServerResp) - + @doc "查询镜像列表" @handler ListImagesHandler get /vm/listImages (ListImagesReq) returns (ListImagesResp) - + @doc "删除镜像" @handler DeleteImageHandler delete /vm/deleteImage (DeleteImageReq) returns (DeleteImageResp) - + @doc "创建镜像" @handler CreateImageHandler post /vm/createImage (CreateImageReq) returns (CreateImageResp) - + @doc "上传镜像" @handler UploadImageHandler put /vm/uploadImage (UploadOsImageReq) returns (UploadOsImageResp) - + @doc "查询网络列表" @handler ListNetworksHandler get /vm/listNetworks (ListNetworksReq) returns (ListNetworksResp) - + @doc "删除网络" @handler DeleteNetworkHandler delete /vm/deleteNetwork (DeleteNetworkReq) returns (DeleteNetworkResp) - + @doc "创建网络" @handler CreateNetworkHandler post /vm/createNetwork (CreateNetworkReq) returns (CreateNetworkResp) - + @doc "创建子网" @handler CreateSubnetHandler post /vm/createSubnet (CreateSubnetReq) returns (CreateSubnetResp) - + @doc "查询网络详情" @handler ShowNetworkDetailsHandler get /vm/showNetworkDetails (ShowNetworkDetailsReq) returns (ShowNetworkDetailsResp) - + @doc "更新网络" @handler UpdateNetworkHandler put /vm/updateNetwork (UpdateNetworkReq) returns (UpdateNetworkResp) - + @doc "批量创建网络" @handler BulkCreateNetworksHandler post /vm/bulkCreateNetworks (BulkCreateNetworksReq) returns (BulkCreateNetworksResp) - + @doc "查询卷详情列表" @handler ListVolumesDetailHandler get /vm/listVolumesDetail (ListVolumesDetailReq) returns (ListVolumesDetailResp) - + @doc "删除卷" @handler DeleteVolumeHandler delete /vm/deleteVolume (DeleteVolumeReq) returns (DeleteVolumeResp) - + @doc "创建卷" @handler CreateVolumeHandler post /vm/createVolume (CreateVolumeReq) returns (CreateVolumeResp) - + @doc "查询规格详情列表" @handler ListFlavorsDetailHandler get /vm/listFlavorsDetail (ListFlavorsDetailReq) returns (ListFlavorsDetailResp) - + @doc "查询规格类型列表" @handler ListVolumeTypesHandler get /vm/listVolumeTypes (ListVolumeTypesReq) returns (ListVolumeTypesResp) - + @doc "更新卷" @handler UpdateVolumeHandler put /vm/updateVolume (UpdateVolumeReq) returns (UpdateVolumeResp) - + @doc "创建卷类型" @handler CreateVolumeTypesHandler post /vm/createVolumeTypes (CreateVolumeTypeReq) returns (CreateVolumeTypeResp) - + @doc "删除卷类型" @handler DeleteVolumeTypeHandler delete /vm/deleteVolumeType (DeleteVolumeTypeReq) returns (DeleteVolumeTypeResp) - + @doc "查询卷列表" @handler ListVolumesHandler get /vm/listVolumes (ListVolumesReq) returns (ListVolumesResp) - + @doc "根据ID获取卷详情" @handler GetVolumeDetailedByIdHandler get /vm/getVolumeDetailedById (GetVolumeDetailedByIdReq) returns (GetVolumeDetailedByIdResp) - + @doc "查询节点列表" @handler ListNodesHandler get /vm/listNodes (ListNodesReq) returns (ListNodesResp) - + @doc "创建节点" @handler CreateNodeHandler post /vm/createNode (CreateNodeReq) returns (CreateNodeResp) - + @doc "删除节点" @handler DeleteNodeHandler delete /vm/deleteNode (DeleteNodeReq) returns (DeleteNodeResp) - + @doc "查询节点详情" @handler ShowNodeDetailsHandler get /vm/showNodeDetails (ShowNodeDetailsReq) returns (ShowNodeDetailsResp) @@ -508,25 +508,25 @@ service pcm { service pcm { @handler UploadLinkImageHandler post /storelink/uploadImage (UploadLinkImageReq) returns (UploadLinkImageResp) - + @handler GetLinkImageListHandler get /storelink/getImageList (GetLinkImageListReq) returns (GetLinkImageListResp) - + @handler DeleteLinkImageHandler delete /storelink/deleteImage (DeleteLinkImageReq) returns (DeleteLinkImageResp) - + @handler SubmitLinkTaskHandler post /storelink/submitTask (SubmitLinkTaskReq) returns (SubmitLinkTaskResp) - + @handler GetLinkTaskHandler get /storelink/getTask (GetLinkTaskReq) returns (GetLinkTaskResp) - + @handler DeleteLinkTaskHandler delete /storelink/deleteTask (DeleteLinkTaskReq) returns (DeleteLinkTaskResp) - + @handler GetParticipantsHandler get /storelink/getParticipants (GetParticipantsReq) returns (GetParticipantsResp) - + @handler GetAISpecsHandler get /storelink/getResourceSpecs (GetResourceSpecsReq) returns (GetResourceSpecsResp) } @@ -540,35 +540,35 @@ service pcm { @doc "应用列表" @handler AppListHandler get /apps/list (AppListReq) returns (AppListResp) - + @doc "获取应用分发详情" @handler AppDetailHandler get /apps/distribute/:appName (AppDetailReq) returns (AppDetailResp) - + @doc "应用pods列表" @handler AppPodsHandler get /apps/pods/:appName (AppDetailReq) returns (AppDetailResp) - + @doc "获取应用详情" @handler GetAppByAppName get /apps/getAppByAppName/:appName (AppDetailReq) returns (AppTaskResp) - + @doc "删除应用" @handler DeleteAppByAppName delete /apps/deleteApp (DeleteAppReq) returns (DeleteAppResp) - + @doc "更新应用" @handler UpdateAppByAppName put /apps/updateApp (DeleteAppReq) returns (AppTaskResp) - + @doc "重启应用" @handler RestartAppByAppName put /apps/restartApp (DeleteAppReq) returns (AppResp) - + @doc "暂停应用" @handler PauseAppByAppName put /apps/pauseApp (DeleteAppReq) returns (AppResp) - + @doc "启动应用" @handler StartAppByAppName put /apps/startApp (DeleteAppReq) returns (AppResp) @@ -583,31 +583,31 @@ service pcm { service pcm { @handler AdaptersListHandler get /adapter/list (AdapterReq) returns (AdapterListResp) - + @handler CreateAdapterHandler post /adapter/create (AdapterReq) returns (AdapterResp) - + @handler UpdateAdapterHandler put /adapter/update (AdapterReq) returns (AdapterResp) - + @handler DeleteAdapterHandler delete /adapter/delete (AdapterDelReq) returns (AdapterResp) - + @handler GetAdapterHandler get /adapter/get (AdapterDelReq) returns (AdapterResp) - + @handler ClusterListHandler get /adapter/cluster/list (ClusterReq) returns (ClusterListResp) - + @handler CreateClusterHandler post /adapter/cluster/create (ClusterReq) returns (ClusterResp) - + @handler UpdateClusterHandler put /adapter/cluster/update (ClusterReq) returns (ClusterResp) - + @handler DeleteClusterHandler delete /adapter/cluster/delete (ClusterDelReq) returns (ClusterResp) - + @handler GetClusterHandler get /adapter/cluster/get (ClusterDelReq) returns (ClusterResp) } @@ -620,16 +620,16 @@ service pcm { service pcm { @handler ScheduleGetAiResourceTypesHandler get /schedule/ai/getResourceTypes returns (AiResourceTypesResp) - + @handler ScheduleGetAiTaskTypesHandler get /schedule/ai/getTaskTypes returns (AiTaskTypesResp) - + @handler ScheduleGetDatasetsHandler get /schedule/ai/getDatasets returns (AiDatasetsResp) - + @handler ScheduleGetStrategyHandler get /schedule/ai/getStrategies returns (AiStrategyResp) - + @handler ScheduleSubmitHandler post /schedule/submit (ScheduleResp) returns (ScheduleResp) } \ No newline at end of file diff --git a/api/internal/handler/routes.go b/api/internal/handler/routes.go index cc74f23c..89c55ff6 100644 --- a/api/internal/handler/routes.go +++ b/api/internal/handler/routes.go @@ -11,6 +11,7 @@ import ( core "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/handler/core" hpc "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/handler/hpc" image "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/handler/image" + schedule "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/handler/schedule" storage "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/handler/storage" storelink "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/handler/storelink" vm "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/handler/vm" @@ -749,4 +750,35 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { }, rest.WithPrefix("/pcm/v1"), ) + + server.AddRoutes( + []rest.Route{ + { + Method: http.MethodGet, + Path: "/schedule/ai/getResourceTypes", + Handler: schedule.ScheduleGetAiResourceTypesHandler(serverCtx), + }, + { + Method: http.MethodGet, + Path: "/schedule/ai/getTaskTypes", + Handler: schedule.ScheduleGetAiTaskTypesHandler(serverCtx), + }, + { + Method: http.MethodGet, + Path: "/schedule/ai/getDatasets", + Handler: schedule.ScheduleGetDatasetsHandler(serverCtx), + }, + { + Method: http.MethodGet, + Path: "/schedule/ai/getStrategies", + Handler: schedule.ScheduleGetStrategyHandler(serverCtx), + }, + { + Method: http.MethodPost, + Path: "/schedule/submit", + Handler: schedule.ScheduleSubmitHandler(serverCtx), + }, + }, + rest.WithPrefix("/pcm/v1"), + ) } diff --git a/api/internal/handler/schedule/schedulegetairesourcetypeshandler.go b/api/internal/handler/schedule/schedulegetairesourcetypeshandler.go new file mode 100644 index 00000000..e4553fe4 --- /dev/null +++ b/api/internal/handler/schedule/schedulegetairesourcetypeshandler.go @@ -0,0 +1,21 @@ +package schedule + +import ( + "net/http" + + "github.com/zeromicro/go-zero/rest/httpx" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/logic/schedule" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" +) + +func ScheduleGetAiResourceTypesHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := schedule.NewScheduleGetAiResourceTypesLogic(r.Context(), svcCtx) + resp, err := l.ScheduleGetAiResourceTypes() + if err != nil { + httpx.ErrorCtx(r.Context(), w, err) + } else { + httpx.OkJsonCtx(r.Context(), w, resp) + } + } +} diff --git a/api/internal/handler/schedule/schedulegetaitasktypeshandler.go b/api/internal/handler/schedule/schedulegetaitasktypeshandler.go new file mode 100644 index 00000000..da09aa49 --- /dev/null +++ b/api/internal/handler/schedule/schedulegetaitasktypeshandler.go @@ -0,0 +1,21 @@ +package schedule + +import ( + "net/http" + + "github.com/zeromicro/go-zero/rest/httpx" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/logic/schedule" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" +) + +func ScheduleGetAiTaskTypesHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := schedule.NewScheduleGetAiTaskTypesLogic(r.Context(), svcCtx) + resp, err := l.ScheduleGetAiTaskTypes() + if err != nil { + httpx.ErrorCtx(r.Context(), w, err) + } else { + httpx.OkJsonCtx(r.Context(), w, resp) + } + } +} diff --git a/api/internal/handler/schedule/schedulegetdatasetshandler.go b/api/internal/handler/schedule/schedulegetdatasetshandler.go new file mode 100644 index 00000000..a88eed26 --- /dev/null +++ b/api/internal/handler/schedule/schedulegetdatasetshandler.go @@ -0,0 +1,21 @@ +package schedule + +import ( + "net/http" + + "github.com/zeromicro/go-zero/rest/httpx" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/logic/schedule" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" +) + +func ScheduleGetDatasetsHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := schedule.NewScheduleGetDatasetsLogic(r.Context(), svcCtx) + resp, err := l.ScheduleGetDatasets() + if err != nil { + httpx.ErrorCtx(r.Context(), w, err) + } else { + httpx.OkJsonCtx(r.Context(), w, resp) + } + } +} diff --git a/api/internal/handler/schedule/schedulegetstrategyhandler.go b/api/internal/handler/schedule/schedulegetstrategyhandler.go new file mode 100644 index 00000000..2826e4e7 --- /dev/null +++ b/api/internal/handler/schedule/schedulegetstrategyhandler.go @@ -0,0 +1,21 @@ +package schedule + +import ( + "net/http" + + "github.com/zeromicro/go-zero/rest/httpx" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/logic/schedule" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" +) + +func ScheduleGetStrategyHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := schedule.NewScheduleGetStrategyLogic(r.Context(), svcCtx) + resp, err := l.ScheduleGetStrategy() + if err != nil { + httpx.ErrorCtx(r.Context(), w, err) + } else { + httpx.OkJsonCtx(r.Context(), w, resp) + } + } +} diff --git a/api/internal/handler/schedule/schedulesubmithandler.go b/api/internal/handler/schedule/schedulesubmithandler.go new file mode 100644 index 00000000..40df03a8 --- /dev/null +++ b/api/internal/handler/schedule/schedulesubmithandler.go @@ -0,0 +1,28 @@ +package schedule + +import ( + "net/http" + + "github.com/zeromicro/go-zero/rest/httpx" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/logic/schedule" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" +) + +func ScheduleSubmitHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.ScheduleResp + if err := httpx.Parse(r, &req); err != nil { + httpx.ErrorCtx(r.Context(), w, err) + return + } + + l := schedule.NewScheduleSubmitLogic(r.Context(), svcCtx) + resp, err := l.ScheduleSubmit(&req) + if err != nil { + httpx.ErrorCtx(r.Context(), w, err) + } else { + httpx.OkJsonCtx(r.Context(), w, resp) + } + } +} diff --git a/api/internal/logic/schedule/schedulegetairesourcetypeslogic.go b/api/internal/logic/schedule/schedulegetairesourcetypeslogic.go new file mode 100644 index 00000000..8f8348af --- /dev/null +++ b/api/internal/logic/schedule/schedulegetairesourcetypeslogic.go @@ -0,0 +1,30 @@ +package schedule + +import ( + "context" + + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type ScheduleGetAiResourceTypesLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewScheduleGetAiResourceTypesLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ScheduleGetAiResourceTypesLogic { + return &ScheduleGetAiResourceTypesLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *ScheduleGetAiResourceTypesLogic) ScheduleGetAiResourceTypes() (resp *types.AiResourceTypesResp, err error) { + // todo: add your logic here and delete this line + + return +} diff --git a/api/internal/logic/schedule/schedulegetaitasktypeslogic.go b/api/internal/logic/schedule/schedulegetaitasktypeslogic.go new file mode 100644 index 00000000..01f168f9 --- /dev/null +++ b/api/internal/logic/schedule/schedulegetaitasktypeslogic.go @@ -0,0 +1,30 @@ +package schedule + +import ( + "context" + + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type ScheduleGetAiTaskTypesLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewScheduleGetAiTaskTypesLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ScheduleGetAiTaskTypesLogic { + return &ScheduleGetAiTaskTypesLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *ScheduleGetAiTaskTypesLogic) ScheduleGetAiTaskTypes() (resp *types.AiTaskTypesResp, err error) { + // todo: add your logic here and delete this line + + return +} diff --git a/api/internal/logic/schedule/schedulegetdatasetslogic.go b/api/internal/logic/schedule/schedulegetdatasetslogic.go new file mode 100644 index 00000000..54997d6f --- /dev/null +++ b/api/internal/logic/schedule/schedulegetdatasetslogic.go @@ -0,0 +1,30 @@ +package schedule + +import ( + "context" + + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type ScheduleGetDatasetsLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewScheduleGetDatasetsLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ScheduleGetDatasetsLogic { + return &ScheduleGetDatasetsLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *ScheduleGetDatasetsLogic) ScheduleGetDatasets() (resp *types.AiDatasetsResp, err error) { + // todo: add your logic here and delete this line + + return +} diff --git a/api/internal/logic/schedule/schedulegetstrategylogic.go b/api/internal/logic/schedule/schedulegetstrategylogic.go new file mode 100644 index 00000000..102ec4b9 --- /dev/null +++ b/api/internal/logic/schedule/schedulegetstrategylogic.go @@ -0,0 +1,30 @@ +package schedule + +import ( + "context" + + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type ScheduleGetStrategyLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewScheduleGetStrategyLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ScheduleGetStrategyLogic { + return &ScheduleGetStrategyLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *ScheduleGetStrategyLogic) ScheduleGetStrategy() (resp *types.AiStrategyResp, err error) { + // todo: add your logic here and delete this line + + return +} diff --git a/api/internal/logic/schedule/schedulesubmitlogic.go b/api/internal/logic/schedule/schedulesubmitlogic.go new file mode 100644 index 00000000..924aa37b --- /dev/null +++ b/api/internal/logic/schedule/schedulesubmitlogic.go @@ -0,0 +1,30 @@ +package schedule + +import ( + "context" + + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type ScheduleSubmitLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewScheduleSubmitLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ScheduleSubmitLogic { + return &ScheduleSubmitLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *ScheduleSubmitLogic) ScheduleSubmit(req *types.ScheduleResp) (resp *types.ScheduleResp, err error) { + // todo: add your logic here and delete this line + + return +} diff --git a/api/internal/types/types.go b/api/internal/types/types.go index 19e46e4e..2aaa4b95 100644 --- a/api/internal/types/types.go +++ b/api/internal/types/types.go @@ -3736,3 +3736,37 @@ type ParticipantSl struct { ParticipantName string `json:"name"` ParticipantType string `json:"type"` } + +type ScheduleReq struct { + AiOption *AiOption `json:"aiOption,optional"` +} + +type ScheduleResp struct { + Success bool `json:"success"` + TaskId string `json:"taskId"` + ClusterId string `json:"clusterId"` + ErrorMsg string `json:"errorMsg"` +} + +type AiOption struct { + ResourceType string `json:"resourceType"` + TaskType string `json:"taskType"` + Datasets string `json:"datasets"` + Strategy string `json:"strategy"` +} + +type AiResourceTypesResp struct { + ResourceTypes []string `json:"resourceTypes"` +} + +type AiTaskTypesResp struct { + TaskTypes []string `json:"taskTypes"` +} + +type AiDatasetsResp struct { + Datasets []string `json:"datasets"` +} + +type AiStrategyResp struct { + Strategies []string `json:"strategies"` +} From 3102a154c5b8e0132b4e1e3c78213033e8e78c81 Mon Sep 17 00:00:00 2001 From: tzwang Date: Mon, 5 Feb 2024 17:27:33 +0800 Subject: [PATCH 2/3] modified schedule apis Former-commit-id: a30d161c27346206ae0eaeaa2056eb2b1e97f756 --- .../schedule/schedulegetairesourcetypeslogic.go | 7 +++++-- .../logic/schedule/schedulegetaitasktypeslogic.go | 7 +++++-- .../logic/schedule/schedulegetdatasetslogic.go | 13 ++++++++++--- .../logic/schedule/schedulegetstrategylogic.go | 8 +++++--- api/internal/scheduler/schedulers/aiScheduler.go | 2 +- api/internal/scheduler/strategy/strategy.go | 10 +++++----- .../scheduler/strategy/test/strategy_test.go | 4 ++-- 7 files changed, 33 insertions(+), 18 deletions(-) diff --git a/api/internal/logic/schedule/schedulegetairesourcetypeslogic.go b/api/internal/logic/schedule/schedulegetairesourcetypeslogic.go index 8f8348af..025642bf 100644 --- a/api/internal/logic/schedule/schedulegetairesourcetypeslogic.go +++ b/api/internal/logic/schedule/schedulegetairesourcetypeslogic.go @@ -2,6 +2,7 @@ package schedule import ( "context" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/storeLink" "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" @@ -24,7 +25,9 @@ func NewScheduleGetAiResourceTypesLogic(ctx context.Context, svcCtx *svc.Service } func (l *ScheduleGetAiResourceTypesLogic) ScheduleGetAiResourceTypes() (resp *types.AiResourceTypesResp, err error) { - // todo: add your logic here and delete this line + resp = &types.AiResourceTypesResp{} + resourceTypes := storeLink.GetResourceTypes() + resp.ResourceTypes = resourceTypes - return + return resp, nil } diff --git a/api/internal/logic/schedule/schedulegetaitasktypeslogic.go b/api/internal/logic/schedule/schedulegetaitasktypeslogic.go index 01f168f9..926e4dbc 100644 --- a/api/internal/logic/schedule/schedulegetaitasktypeslogic.go +++ b/api/internal/logic/schedule/schedulegetaitasktypeslogic.go @@ -2,6 +2,7 @@ package schedule import ( "context" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/storeLink" "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" @@ -24,7 +25,9 @@ func NewScheduleGetAiTaskTypesLogic(ctx context.Context, svcCtx *svc.ServiceCont } func (l *ScheduleGetAiTaskTypesLogic) ScheduleGetAiTaskTypes() (resp *types.AiTaskTypesResp, err error) { - // todo: add your logic here and delete this line + resp = &types.AiTaskTypesResp{} + taskTypes := storeLink.GetTaskTypes() + resp.TaskTypes = taskTypes - return + return resp, nil } diff --git a/api/internal/logic/schedule/schedulegetdatasetslogic.go b/api/internal/logic/schedule/schedulegetdatasetslogic.go index 54997d6f..99a3dd43 100644 --- a/api/internal/logic/schedule/schedulegetdatasetslogic.go +++ b/api/internal/logic/schedule/schedulegetdatasetslogic.go @@ -2,6 +2,8 @@ package schedule import ( "context" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/scheduler/service" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/storeLink" "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" @@ -24,7 +26,12 @@ func NewScheduleGetDatasetsLogic(ctx context.Context, svcCtx *svc.ServiceContext } func (l *ScheduleGetDatasetsLogic) ScheduleGetDatasets() (resp *types.AiDatasetsResp, err error) { - // todo: add your logic here and delete this line - - return + resp = &types.AiDatasetsResp{} + _, colMap := service.InitAiClusterMap(l.ctx, l.svcCtx) + names, err := storeLink.GetDatasetsNames(colMap) + if err != nil { + return nil, err + } + resp.Datasets = names + return resp, nil } diff --git a/api/internal/logic/schedule/schedulegetstrategylogic.go b/api/internal/logic/schedule/schedulegetstrategylogic.go index 102ec4b9..9b591f83 100644 --- a/api/internal/logic/schedule/schedulegetstrategylogic.go +++ b/api/internal/logic/schedule/schedulegetstrategylogic.go @@ -2,7 +2,7 @@ package schedule import ( "context" - + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/scheduler/strategy" "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" @@ -24,7 +24,9 @@ func NewScheduleGetStrategyLogic(ctx context.Context, svcCtx *svc.ServiceContext } func (l *ScheduleGetStrategyLogic) ScheduleGetStrategy() (resp *types.AiStrategyResp, err error) { - // todo: add your logic here and delete this line + resp = &types.AiStrategyResp{} + names := strategy.GetStrategyNames() + resp.Strategies = names - return + return resp, nil } diff --git a/api/internal/scheduler/schedulers/aiScheduler.go b/api/internal/scheduler/schedulers/aiScheduler.go index 3077b67b..c8b5c72b 100644 --- a/api/internal/scheduler/schedulers/aiScheduler.go +++ b/api/internal/scheduler/schedulers/aiScheduler.go @@ -71,7 +71,7 @@ func (as *AiScheduler) PickOptimalStrategy() (strategy.Strategy, error) { case strategy.REPLICATION: strategy := strategy.NewReplicationStrategy(¶m.ReplicationParams{Params: params, Replicas: 1}) return strategy, nil - case strategy.RESOURCE_PRICING: + case strategy.RESOURCES_PRICING: strategy := strategy.NewPricingStrategy(¶m.ResourcePricingParams{Params: params, Replicas: 1}) return strategy, nil } diff --git a/api/internal/scheduler/strategy/strategy.go b/api/internal/scheduler/strategy/strategy.go index ef9123cc..b73288ef 100644 --- a/api/internal/scheduler/strategy/strategy.go +++ b/api/internal/scheduler/strategy/strategy.go @@ -1,14 +1,14 @@ package strategy const ( - REPLICATION = "replication" - RESOURCE_PRICING = "resourcePricing" - STATIC_WEIGHT = "staticWeight" - DYNAMIC_WEIGHT = "dynamicWeight" + REPLICATION = "replication" + RESOURCES_PRICING = "resourcesPricing" + STATIC_WEIGHT = "staticWeight" + DYNAMIC_RESOURCES = "dynamicResources" ) var ( - strategyNames = []string{REPLICATION, RESOURCE_PRICING} + strategyNames = []string{REPLICATION, RESOURCES_PRICING, STATIC_WEIGHT, DYNAMIC_RESOURCES} ) type Strategy interface { diff --git a/api/internal/scheduler/strategy/test/strategy_test.go b/api/internal/scheduler/strategy/test/strategy_test.go index 6331d136..e92bf482 100644 --- a/api/internal/scheduler/strategy/test/strategy_test.go +++ b/api/internal/scheduler/strategy/test/strategy_test.go @@ -15,7 +15,7 @@ func TestReplication(t *testing.T) { {Name: "test2", Participant_id: 2}, {Name: "test3", Participant_id: 3}, } - rsc := []*collector.ResourceSpecs{ + rsc := []*collector.ResourceStats{ { ParticipantId: 1, Name: "test1", @@ -31,7 +31,7 @@ func TestReplication(t *testing.T) { name string replica int32 ps []entity.Participant - res []*collector.ResourceSpecs + res []*collector.ResourceStats }{ { name: "test1", From c1855def290420a3f045b5e231712429ec8f28eb Mon Sep 17 00:00:00 2001 From: tzwang Date: Wed, 21 Feb 2024 17:04:03 +0800 Subject: [PATCH 3/3] modified aioption and submit func Former-commit-id: c7dce696d7d090075f4395bb6945a8c06bf77236 --- .../logic/storelink/submitlinktasklogic.go | 2 +- .../scheduler/schedulers/option/aiOption.go | 11 +++--- api/internal/scheduler/service/aiService.go | 10 +++--- api/internal/scheduler/strategy/strategy.go | 10 +++--- api/internal/storeLink/modelarts.go | 4 +-- api/internal/storeLink/octopus.go | 34 +++++++++++++++++-- api/internal/storeLink/shuguangHpc.go | 2 +- api/internal/storeLink/shuguangai.go | 4 +-- api/internal/storeLink/storeLink.go | 2 +- 9 files changed, 55 insertions(+), 24 deletions(-) diff --git a/api/internal/logic/storelink/submitlinktasklogic.go b/api/internal/logic/storelink/submitlinktasklogic.go index 45710841..354e5521 100644 --- a/api/internal/logic/storelink/submitlinktasklogic.go +++ b/api/internal/logic/storelink/submitlinktasklogic.go @@ -67,7 +67,7 @@ func (l *SubmitLinkTaskLogic) SubmitLinkTask(req *types.SubmitLinkTaskReq) (resp envs = append(envs, env) } } - task, err := storelink.ILinkage.SubmitTask(req.ImageId, req.Cmd, envs, params, req.ResourceId, "pytorch") + task, err := storelink.ILinkage.SubmitTask(req.ImageId, req.Cmd, envs, params, req.ResourceId, "", "", "pytorch") if err != nil { return nil, err } diff --git a/api/internal/scheduler/schedulers/option/aiOption.go b/api/internal/scheduler/schedulers/option/aiOption.go index 9512b3e9..acff16de 100644 --- a/api/internal/scheduler/schedulers/option/aiOption.go +++ b/api/internal/scheduler/schedulers/option/aiOption.go @@ -9,11 +9,12 @@ type AiOption struct { ClusterToStaticWeight map[string]int32 CodeType string - ImageId string - SpecId string - //DatasetsId string - CodeId string - ResourceId string + ImageId string + SpecId string + DatasetsId string + CodeId string + ResourceId string + AlgorithmId string Cmd string Envs []string diff --git a/api/internal/scheduler/service/aiService.go b/api/internal/scheduler/service/aiService.go index 01e41685..4ac09867 100644 --- a/api/internal/scheduler/service/aiService.go +++ b/api/internal/scheduler/service/aiService.go @@ -16,11 +16,11 @@ const ( var ( AiTypeMap = map[string]string{ - "hanwuji": OCTOPUS, - "suiyan": OCTOPUS, - "sailingsi": OCTOPUS, - "modelarts-CloudBrain2": MODELARTS, - "shuguangAi": SHUGUANGAI, + "hanwuji": OCTOPUS, + //"suiyan": OCTOPUS, + //"sailingsi": OCTOPUS, + //"modelarts-CloudBrain2": MODELARTS, + "shuguangAi": SHUGUANGAI, } ) diff --git a/api/internal/scheduler/strategy/strategy.go b/api/internal/scheduler/strategy/strategy.go index b73288ef..f5c06d64 100644 --- a/api/internal/scheduler/strategy/strategy.go +++ b/api/internal/scheduler/strategy/strategy.go @@ -1,10 +1,12 @@ package strategy const ( - REPLICATION = "replication" - RESOURCES_PRICING = "resourcesPricing" - STATIC_WEIGHT = "staticWeight" - DYNAMIC_RESOURCES = "dynamicResources" + REPLICATION = "replication" + RESOURCES_PRICING = "resourcesPricing" + STATIC_WEIGHT = "staticWeight" + DYNAMIC_RESOURCES = "dynamicResources" + DATA_LOCALITY = "dataLocality" //感知数据位置，数据调度和计算调度协同，近数据调度 + ENERGY_CONSUMPTION = "energyConsumption" //根据各集群总体能耗水平调度作业，优先选择能耗低的集群调度作业 ) var ( diff --git a/api/internal/storeLink/modelarts.go b/api/internal/storeLink/modelarts.go index c3b4cda1..2d65db43 100644 --- a/api/internal/storeLink/modelarts.go +++ b/api/internal/storeLink/modelarts.go @@ -63,7 +63,7 @@ func (m *ModelArtsLink) QueryImageList() (interface{}, error) { return resp, nil } -func (m *ModelArtsLink) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, aiType string) (interface{}, error) { +func (m *ModelArtsLink) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) { // modelArts提交任务 environments := make(map[string]string) parameters := make([]*modelarts.ParametersTrainJob, 0) @@ -162,7 +162,7 @@ func (m *ModelArtsLink) Execute(option *option.AiOption) (interface{}, error) { if err != nil { return nil, err } - task, err := m.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.TaskType) + task, err := m.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.DatasetsId, option.AlgorithmId, option.TaskType) if err != nil { return nil, err } diff --git a/api/internal/storeLink/octopus.go b/api/internal/storeLink/octopus.go index 9448d897..13fe2b07 100644 --- a/api/internal/storeLink/octopus.go +++ b/api/internal/storeLink/octopus.go @@ -108,7 +108,7 @@ func (o *OctopusLink) QueryImageList() (interface{}, error) { return resp, nil } -func (o *OctopusLink) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, aiType string) (interface{}, error) { +func (o *OctopusLink) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) { // octopus提交任务 // python参数 @@ -227,7 +227,7 @@ func (o *OctopusLink) Execute(option *option.AiOption) (interface{}, error) { if err != nil { return nil, err } - task, err := o.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.TaskType) + task, err := o.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.DatasetsId, option.AlgorithmId, option.TaskType) if err != nil { return nil, err } @@ -239,6 +239,10 @@ func (o *OctopusLink) GenerateSubmitParams(option *option.AiOption) error { if err != nil { return err } + err = o.generateDatasetsId(option) + if err != nil { + return err + } err = o.generateImageId(option) if err != nil { return err @@ -259,10 +263,34 @@ func (o *OctopusLink) GenerateSubmitParams(option *option.AiOption) error { } func (o *OctopusLink) generateResourceId(option *option.AiOption) error { - return nil } +func (o *OctopusLink) generateDatasetsId(option *option.AiOption) error { + if option.DatasetsName == "" { + return errors.New("DatasetsName not set") + } + req := &octopus.GetMyDatasetListReq{ + Platform: o.platform, + PageIndex: o.pageIndex, + PageSize: o.pageSize, + } + resp, err := o.svcCtx.OctopusRpc.GetMyDatasetList(o.ctx, req) + if err != nil { + return err + } + if !resp.Success { + return errors.New("failed to get DatasetsId") + } + for _, dataset := range resp.Payload.Datasets { + if dataset.Name == option.DatasetsName { + option.DatasetsId = dataset.Id + return nil + } + } + return errors.New("failed to get DatasetsId") +} + func (o *OctopusLink) generateImageId(option *option.AiOption) error { return nil diff --git a/api/internal/storeLink/shuguangHpc.go b/api/internal/storeLink/shuguangHpc.go index f7f0af82..85b4c32e 100644 --- a/api/internal/storeLink/shuguangHpc.go +++ b/api/internal/storeLink/shuguangHpc.go @@ -144,7 +144,7 @@ func (s ShuguangHpc) QueryImageList() (interface{}, error) { return nil, nil } -func (s ShuguangHpc) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, aiType string) (interface{}, error) { +func (s ShuguangHpc) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) { // shuguangHpc提交任务 //判断是否resourceId匹配自定义资源Id diff --git a/api/internal/storeLink/shuguangai.go b/api/internal/storeLink/shuguangai.go index eef573bf..c2898513 100644 --- a/api/internal/storeLink/shuguangai.go +++ b/api/internal/storeLink/shuguangai.go @@ -141,7 +141,7 @@ func (s *ShuguangAi) SubmitTensorflowTask(imageId string, cmd string, envs []str return nil, nil } -func (s *ShuguangAi) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, aiType string) (interface{}, error) { +func (s *ShuguangAi) SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) { // shuguangAi提交任务 switch aiType { case PYTORCH_TASK: @@ -248,7 +248,7 @@ func (s *ShuguangAi) Execute(option *option.AiOption) (interface{}, error) { if err != nil { return nil, err } - task, err := s.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.TaskType) + task, err := s.SubmitTask(option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.DatasetsId, option.AlgorithmId, option.TaskType) if err != nil { return nil, err } diff --git a/api/internal/storeLink/storeLink.go b/api/internal/storeLink/storeLink.go index 80714797..b1815758 100644 --- a/api/internal/storeLink/storeLink.go +++ b/api/internal/storeLink/storeLink.go @@ -33,7 +33,7 @@ type Linkage interface { UploadImage(path string) (interface{}, error) DeleteImage(imageId string) (interface{}, error) QueryImageList() (interface{}, error) - SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, aiType string) (interface{}, error) + SubmitTask(imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) QueryTask(taskId string) (interface{}, error) QuerySpecs() (interface{}, error) DeleteTask(taskId string) (interface{}, error)