floraachy
/
pcm-coordinator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix:查询获取训练作业支持的公共规格接口 

Former-commit-id: 732d42912260e3e5f21a3792e1dad9037aba04cf
This commit is contained in:
qiwang 2023-05-16 12:48:24 +08:00
parent 256a87f54c
commit f9bf686ede
8 changed files with 3963 additions and 3303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/common/AuthService.go
View File

@ -12,7 +12,7 @@ import (
const (
IAMUser = "pcmmodelarts"
IAMPassword = "!QAZ2wsx3edc4"
IAMDomain = "hid_btkv2tt4_rcdv0q"
IAMDomain = "hw_008618597947378_01"
AuthMethod = "password"
IAMTokenUrl = "https://iam.cn-north-4.myhuaweicloud.com/v3/auth/tokens"
ProjectName = "cn-north-4"

1
adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/config/config.go
View File

@ -9,4 +9,5 @@ type Config struct {
zrpc.RpcServerConf
LogConf logx.LogConf
PcmCoreRpcConf zrpc.RpcClientConf
ModelArtsConfig
}

13
adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/config/modelartsConfig.go Normal file
View File

@ -0,0 +1,13 @@
package config
type ModelArtsConfig struct {
ModelArtsUrl string `json:"ModelArtsUrl"`
IAMUser string `json:"IAMUser"`
NanjingModelArtsUrl string `json:"NanjingModelArtsUrl"`
AK string `json:"AK"`
SK string `json:"SK"`
XProjectId string `json:"X-Project-Id"`
XDomainId string `json:"X-Domain-Id"`
HaweiModelArtsType string `json:"HaweiModelArtsType"`
NanjingModelArtsType string `json:"NanjingModelArtsType"`
}

6
adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/server/modelartsserver.go
View File

@ -118,6 +118,12 @@ func (s *ModelArtsServer) ShowAlgorithmByUuid(ctx context.Context, in *modelarts
return l.ShowAlgorithmByUuid(in)
}
// training-job-flavors 获取训练作业支持的公共规格
func (s *ModelArtsServer) GetTrainingJobFlavors(ctx context.Context, in *modelarts.TrainingJobFlavorsReq) (*modelarts.TrainingJobFlavorsResp, error) {
l := logic.NewGetTrainingJobFlavorsLogic(ctx, s.svcCtx)
return l.GetTrainingJobFlavors(in)
}
// export task
func (s *ModelArtsServer) ExportTask(ctx context.Context, in *modelarts.ExportTaskReq) (*modelarts.ExportTaskDataResp, error) {
l := logic.NewExportTaskLogic(ctx, s.svcCtx)

7090
adaptor/PCM-AI/PCM-MODELARTS/rpc/modelarts/pcm-modelarts.pb.go
View File

File diff suppressed because it is too large Load Diff

122
adaptor/PCM-AI/PCM-MODELARTS/rpc/modelarts/pcm-modelarts_grpc.pb.go
View File

@ -22,64 +22,66 @@ const _ = grpc.SupportPackageIsVersion7
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type ModelArtsClient interface {
// get modelarts Token
//get modelarts Token
GetToken(ctx context.Context, in *TokenReq, opts ...grpc.CallOption) (*TokenResp, error)
// get modelarts Token
//get modelarts Token
GetDatasetList(ctx context.Context, in *DataSetReq, opts ...grpc.CallOption) (*DataSetResp, error)
// create DateSet
//create DateSet
CreateDataSet(ctx context.Context, in *CreateDataSetReq, opts ...grpc.CallOption) (*CreateDataSetResq, error)
// create DateSet
//create DateSet
DeleteDataSet(ctx context.Context, in *DeleteDataSetReq, opts ...grpc.CallOption) (*DeleteDataSetResq, error)
// creat task 创建导入任务
//creat task 创建导入任务
CreateTask(ctx context.Context, in *ImportTaskDataReq, opts ...grpc.CallOption) (*ImportTaskDataResp, error)
// get taskList 查询数据集导入任务列表
//get taskList 查询数据集导入任务列表
GetImportTaskList(ctx context.Context, in *ListImportTasksReq, opts ...grpc.CallOption) (*ListImportTasksResp, error)
// ListTrainingJobs 查询训练作业列表
// ListTrainingJobs 查询训练作业列表
GetListTrainingJobs(ctx context.Context, in *ListTrainingJobsreq, opts ...grpc.CallOption) (*ListTrainingJobsresp, error)
// CreateTrainingJob 创建训练作业
// CreateTrainingJob 创建训练作业
CreateTrainingJob(ctx context.Context, in *CreateTrainingJobReq, opts ...grpc.CallOption) (*CreateTrainingJobResp, error)
// DeleteTrainingJobConfig 删除训练作业
// DeleteTrainingJobConfig 删除训练作业
DeleteTrainingJob(ctx context.Context, in *DeleteTrainingJobReq, opts ...grpc.CallOption) (*DeleteTrainingJobResp, error)
// CreateTrainingJobConfig 创建训练作业参数
// CreateTrainingJobConfig 创建训练作业参数
CreateTrainingJobConfig(ctx context.Context, in *CreateTrainingJobConfigReq, opts ...grpc.CallOption) (*CreateTrainingJobConfigResp, error)
// DeleteTrainingJobConfig 删除训练作业参数
// DeleteTrainingJobConfig 删除训练作业参数
DeleteTrainingJobConfig(ctx context.Context, in *DeleteTrainingJobConfigReq, opts ...grpc.CallOption) (*DeleteTrainingJobConfigResp, error)
// ListTrainingJobConfig 查询训练作业参数
// ListTrainingJobConfig 查询训练作业参数
ListTrainingJobConfig(ctx context.Context, in *ListTrainingJobConfigReq, opts ...grpc.CallOption) (*ListTrainingJobConfigResp, error)
// CreateAlgorithm 创建算法
// CreateAlgorithm 创建算法
CreateAlgorithm(ctx context.Context, in *CreateAlgorithmReq, opts ...grpc.CallOption) (*CreateAlgorithmResp, error)
// ListAlgorithms 查询算法
// ListAlgorithms 查询算法
ListAlgorithms(ctx context.Context, in *ListAlgorithmsReq, opts ...grpc.CallOption) (*ListAlgorithmsResp, error)
// DeleteAlgorithms 删除算法
// DeleteAlgorithms 删除算法
DeleteAlgorithms(ctx context.Context, in *DeleteAlgorithmsReq, opts ...grpc.CallOption) (*DeleteAlgorithmsResp, error)
// ShowAlgorithmByUuid 展示算法详情
// ShowAlgorithmByUuid 展示算法详情
ShowAlgorithmByUuid(ctx context.Context, in *ShowAlgorithmByUuidReq, opts ...grpc.CallOption) (*ShowAlgorithmByUuidResp, error)
// training-job-flavors 获取训练作业支持的公共规格
GetTrainingJobFlavors(ctx context.Context, in *TrainingJobFlavorsReq, opts ...grpc.CallOption) (*TrainingJobFlavorsResp, error)
//export task
ExportTask(ctx context.Context, in *ExportTaskReq, opts ...grpc.CallOption) (*ExportTaskDataResp, error)
GetExportTasksOfDataset(ctx context.Context, in *GetExportTasksOfDatasetReq, opts ...grpc.CallOption) (*GetExportTasksOfDatasetResp, error)
GetExportTaskStatusOfDataset(ctx context.Context, in *GetExportTaskStatusOfDatasetReq, opts ...grpc.CallOption) (*GetExportTaskStatusOfDatasetResp, error)
// processor task
//processor task
CreateProcessorTask(ctx context.Context, in *CreateProcessorTaskReq, opts ...grpc.CallOption) (*CreateProcessorTaskResp, error)
DescribeProcessorTask(ctx context.Context, in *DescribeProcessorTaskReq, opts ...grpc.CallOption) (*DescribeProcessorTaskResp, error)
// model management
//model management
CreateModel(ctx context.Context, in *CreateModelReq, opts ...grpc.CallOption) (*CreateModelResp, error)
DeleteModel(ctx context.Context, in *DeleteModelReq, opts ...grpc.CallOption) (*DeleteModelResp, error)
ListModels(ctx context.Context, in *ListModelReq, opts ...grpc.CallOption) (*ListModelResp, error)
ShowModels(ctx context.Context, in *ShowModelReq, opts ...grpc.CallOption) (*ShowModelResp, error)
// service management
//service management
CreateService(ctx context.Context, in *CreateServiceReq, opts ...grpc.CallOption) (*CreateServiceResp, error)
ListServices(ctx context.Context, in *ListServicesReq, opts ...grpc.CallOption) (*ListServicesResp, error)
ShowService(ctx context.Context, in *ShowServiceReq, opts ...grpc.CallOption) (*ShowServiceResp, error)
DeleteService(ctx context.Context, in *DeleteServiceReq, opts ...grpc.CallOption) (*DeleteServiceResp, error)
ListClusters(ctx context.Context, in *ListClustersReq, opts ...grpc.CallOption) (*ListClustersResp, error)
// notebook task
//notebook task
ListNotebook(ctx context.Context, in *ListNotebookReq, opts ...grpc.CallOption) (*ListNotebookResp, error)
CreateNotebook(ctx context.Context, in *CreateNotebookReq, opts ...grpc.CallOption) (*CreateNotebookResp, error)
StartNotebook(ctx context.Context, in *StartNotebookReq, opts ...grpc.CallOption) (*StartNotebookResp, error)
StopNotebook(ctx context.Context, in *StopNotebookReq, opts ...grpc.CallOption) (*StopNotebookResp, error)
GetNotebookStorage(ctx context.Context, in *GetNotebookStorageReq, opts ...grpc.CallOption) (*GetNotebookStorageResp, error)
MountNotebookStorage(ctx context.Context, in *MountNotebookStorageReq, opts ...grpc.CallOption) (*MountNotebookStorageResp, error)
// visualization-jobs
//visualization-jobs
GetVisualizationJob(ctx context.Context, in *GetVisualizationJobReq, opts ...grpc.CallOption) (*GetVisualizationJobResp, error)
CreateVisualizationJob(ctx context.Context, in *CreateVisualizationJobReq, opts ...grpc.CallOption) (*CreateVisualizationJobResp, error)
}
@ -236,6 +238,15 @@ func (c *modelArtsClient) ShowAlgorithmByUuid(ctx context.Context, in *ShowAlgor
return out, nil
}
func (c *modelArtsClient) GetTrainingJobFlavors(ctx context.Context, in *TrainingJobFlavorsReq, opts ...grpc.CallOption) (*TrainingJobFlavorsResp, error) {
out := new(TrainingJobFlavorsResp)
err := c.cc.Invoke(ctx, "/modelarts.ModelArts/GetTrainingJobFlavors", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *modelArtsClient) ExportTask(ctx context.Context, in *ExportTaskReq, opts ...grpc.CallOption) (*ExportTaskDataResp, error) {
out := new(ExportTaskDataResp)
err := c.cc.Invoke(ctx, "/modelarts.ModelArts/ExportTask", in, out, opts...)
@ -438,64 +449,66 @@ func (c *modelArtsClient) CreateVisualizationJob(ctx context.Context, in *Create
// All implementations must embed UnimplementedModelArtsServer
// for forward compatibility
type ModelArtsServer interface {
// get modelarts Token
//get modelarts Token
GetToken(context.Context, *TokenReq) (*TokenResp, error)
// get modelarts Token
//get modelarts Token
GetDatasetList(context.Context, *DataSetReq) (*DataSetResp, error)
// create DateSet
//create DateSet
CreateDataSet(context.Context, *CreateDataSetReq) (*CreateDataSetResq, error)
// create DateSet
//create DateSet
DeleteDataSet(context.Context, *DeleteDataSetReq) (*DeleteDataSetResq, error)
// creat task 创建导入任务
//creat task 创建导入任务
CreateTask(context.Context, *ImportTaskDataReq) (*ImportTaskDataResp, error)
// get taskList 查询数据集导入任务列表
//get taskList 查询数据集导入任务列表
GetImportTaskList(context.Context, *ListImportTasksReq) (*ListImportTasksResp, error)
// ListTrainingJobs 查询训练作业列表
// ListTrainingJobs 查询训练作业列表
GetListTrainingJobs(context.Context, *ListTrainingJobsreq) (*ListTrainingJobsresp, error)
// CreateTrainingJob 创建训练作业
// CreateTrainingJob 创建训练作业
CreateTrainingJob(context.Context, *CreateTrainingJobReq) (*CreateTrainingJobResp, error)
// DeleteTrainingJobConfig 删除训练作业
// DeleteTrainingJobConfig 删除训练作业
DeleteTrainingJob(context.Context, *DeleteTrainingJobReq) (*DeleteTrainingJobResp, error)
// CreateTrainingJobConfig 创建训练作业参数
// CreateTrainingJobConfig 创建训练作业参数
CreateTrainingJobConfig(context.Context, *CreateTrainingJobConfigReq) (*CreateTrainingJobConfigResp, error)
// DeleteTrainingJobConfig 删除训练作业参数
// DeleteTrainingJobConfig 删除训练作业参数
DeleteTrainingJobConfig(context.Context, *DeleteTrainingJobConfigReq) (*DeleteTrainingJobConfigResp, error)
// ListTrainingJobConfig 查询训练作业参数
// ListTrainingJobConfig 查询训练作业参数
ListTrainingJobConfig(context.Context, *ListTrainingJobConfigReq) (*ListTrainingJobConfigResp, error)
// CreateAlgorithm 创建算法
// CreateAlgorithm 创建算法
CreateAlgorithm(context.Context, *CreateAlgorithmReq) (*CreateAlgorithmResp, error)
// ListAlgorithms 查询算法
// ListAlgorithms 查询算法
ListAlgorithms(context.Context, *ListAlgorithmsReq) (*ListAlgorithmsResp, error)
// DeleteAlgorithms 删除算法
// DeleteAlgorithms 删除算法
DeleteAlgorithms(context.Context, *DeleteAlgorithmsReq) (*DeleteAlgorithmsResp, error)
// ShowAlgorithmByUuid 展示算法详情
// ShowAlgorithmByUuid 展示算法详情
ShowAlgorithmByUuid(context.Context, *ShowAlgorithmByUuidReq) (*ShowAlgorithmByUuidResp, error)
// training-job-flavors 获取训练作业支持的公共规格
GetTrainingJobFlavors(context.Context, *TrainingJobFlavorsReq) (*TrainingJobFlavorsResp, error)
//export task
ExportTask(context.Context, *ExportTaskReq) (*ExportTaskDataResp, error)
GetExportTasksOfDataset(context.Context, *GetExportTasksOfDatasetReq) (*GetExportTasksOfDatasetResp, error)
GetExportTaskStatusOfDataset(context.Context, *GetExportTaskStatusOfDatasetReq) (*GetExportTaskStatusOfDatasetResp, error)
// processor task
//processor task
CreateProcessorTask(context.Context, *CreateProcessorTaskReq) (*CreateProcessorTaskResp, error)
DescribeProcessorTask(context.Context, *DescribeProcessorTaskReq) (*DescribeProcessorTaskResp, error)
// model management
//model management
CreateModel(context.Context, *CreateModelReq) (*CreateModelResp, error)
DeleteModel(context.Context, *DeleteModelReq) (*DeleteModelResp, error)
ListModels(context.Context, *ListModelReq) (*ListModelResp, error)
ShowModels(context.Context, *ShowModelReq) (*ShowModelResp, error)
// service management
//service management
CreateService(context.Context, *CreateServiceReq) (*CreateServiceResp, error)
ListServices(context.Context, *ListServicesReq) (*ListServicesResp, error)
ShowService(context.Context, *ShowServiceReq) (*ShowServiceResp, error)
DeleteService(context.Context, *DeleteServiceReq) (*DeleteServiceResp, error)
ListClusters(context.Context, *ListClustersReq) (*ListClustersResp, error)
// notebook task
//notebook task
ListNotebook(context.Context, *ListNotebookReq) (*ListNotebookResp, error)
CreateNotebook(context.Context, *CreateNotebookReq) (*CreateNotebookResp, error)
StartNotebook(context.Context, *StartNotebookReq) (*StartNotebookResp, error)
StopNotebook(context.Context, *StopNotebookReq) (*StopNotebookResp, error)
GetNotebookStorage(context.Context, *GetNotebookStorageReq) (*GetNotebookStorageResp, error)
MountNotebookStorage(context.Context, *MountNotebookStorageReq) (*MountNotebookStorageResp, error)
// visualization-jobs
//visualization-jobs
GetVisualizationJob(context.Context, *GetVisualizationJobReq) (*GetVisualizationJobResp, error)
CreateVisualizationJob(context.Context, *CreateVisualizationJobReq) (*CreateVisualizationJobResp, error)
mustEmbedUnimplementedModelArtsServer()
@ -553,6 +566,9 @@ func (UnimplementedModelArtsServer) DeleteAlgorithms(context.Context, *DeleteAlg
func (UnimplementedModelArtsServer) ShowAlgorithmByUuid(context.Context, *ShowAlgorithmByUuidReq) (*ShowAlgorithmByUuidResp, error) {
return nil, status.Errorf(codes.Unimplemented, "method ShowAlgorithmByUuid not implemented")
}
func (UnimplementedModelArtsServer) GetTrainingJobFlavors(context.Context, *TrainingJobFlavorsReq) (*TrainingJobFlavorsResp, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetTrainingJobFlavors not implemented")
}
func (UnimplementedModelArtsServer) ExportTask(context.Context, *ExportTaskReq) (*ExportTaskDataResp, error) {
return nil, status.Errorf(codes.Unimplemented, "method ExportTask not implemented")
}
@ -920,6 +936,24 @@ func _ModelArts_ShowAlgorithmByUuid_Handler(srv interface{}, ctx context.Context
return interceptor(ctx, in, info, handler)
}
func _ModelArts_GetTrainingJobFlavors_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(TrainingJobFlavorsReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ModelArtsServer).GetTrainingJobFlavors(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/modelarts.ModelArts/GetTrainingJobFlavors",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ModelArtsServer).GetTrainingJobFlavors(ctx, req.(*TrainingJobFlavorsReq))
}
return interceptor(ctx, in, info, handler)
}
func _ModelArts_ExportTask_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ExportTaskReq)
if err := dec(in); err != nil {
@ -1387,6 +1421,10 @@ var ModelArts_ServiceDesc = grpc.ServiceDesc{
MethodName: "ShowAlgorithmByUuid",
Handler: _ModelArts_ShowAlgorithmByUuid_Handler,
},
{
MethodName: "GetTrainingJobFlavors",
Handler: _ModelArts_GetTrainingJobFlavors_Handler,
},
{
MethodName: "ExportTask",
Handler: _ModelArts_ExportTask_Handler,

10
adaptor/PCM-AI/PCM-MODELARTS/rpc/modelartsclient/modelarts.go
View File

@ -241,6 +241,8 @@ type (
TemplateParam = modelarts.TemplateParam
TokenReq = modelarts.TokenReq
TokenResp = modelarts.TokenResp
TrainingJobFlavorsReq = modelarts.TrainingJobFlavorsReq
TrainingJobFlavorsResp = modelarts.TrainingJobFlavorsResp
User = modelarts.User
UserNotebookDomain = modelarts.UserNotebookDomain
UserNotebookResp = modelarts.UserNotebookResp
@ -283,6 +285,8 @@ type (
DeleteAlgorithms(ctx context.Context, in *DeleteAlgorithmsReq, opts ...grpc.CallOption) (*DeleteAlgorithmsResp, error)
// ShowAlgorithmByUuid 展示算法详情
ShowAlgorithmByUuid(ctx context.Context, in *ShowAlgorithmByUuidReq, opts ...grpc.CallOption) (*ShowAlgorithmByUuidResp, error)
// training-job-flavors 获取训练作业支持的公共规格
GetTrainingJobFlavors(ctx context.Context, in *TrainingJobFlavorsReq, opts ...grpc.CallOption) (*TrainingJobFlavorsResp, error)
// export task
ExportTask(ctx context.Context, in *ExportTaskReq, opts ...grpc.CallOption) (*ExportTaskDataResp, error)
GetExportTasksOfDataset(ctx context.Context, in *GetExportTasksOfDatasetReq, opts ...grpc.CallOption) (*GetExportTasksOfDatasetResp, error)
@ -420,6 +424,12 @@ func (m *defaultModelArts) ShowAlgorithmByUuid(ctx context.Context, in *ShowAlgo
return client.ShowAlgorithmByUuid(ctx, in, opts...)
}
// training-job-flavors 获取训练作业支持的公共规格
func (m *defaultModelArts) GetTrainingJobFlavors(ctx context.Context, in *TrainingJobFlavorsReq, opts ...grpc.CallOption) (*TrainingJobFlavorsResp, error) {
client := modelarts.NewModelArtsClient(m.cli.Conn())
return client.GetTrainingJobFlavors(ctx, in, opts...)
}
// export task
func (m *defaultModelArts) ExportTask(ctx context.Context, in *ExportTaskReq, opts ...grpc.CallOption) (*ExportTaskDataResp, error) {
client := modelarts.NewModelArtsClient(m.cli.Conn())

22
adaptor/PCM-AI/PCM-MODELARTS/rpc/pb/pcm-modelarts.proto
View File

@ -297,7 +297,7 @@ message cpu{
int32 core_num =2;
}
message npu{
string unit_num =1;
int32 unit_num =1;
string product_name =2;
string memory =3;
}
@ -924,6 +924,22 @@ message ShowAlgorithmByUuidResp{
/******************Show Algorithm By Uuid end*************************/
/******************TrainingJobFlavors start*************************/
message TrainingJobFlavorsReq{
string project_id =1; // @gotags: copier:"ProjectId"
string flavor_type =2; // @gotags: copier:"FlavorType"
string modelArtsType =3; // @gotags: copier:"modelArtsType"
}
message TrainingJobFlavorsResp{
int32 total_count = 1; // @gotags: copier:"TotalCount"
repeated FlavorResponse flavors = 2; // @gotags: copier:"FlavorResponse"
string msg =3;// @gotags: copier:"Msg"
int32 code =4;// @gotags: copier:"Code"
}
/******************TrainingJobFlavors end*************************/
/******************Task(export) Start*************************/
message ExportTaskReq{
string annotation_format = 1; // @gotags: copier:"AnnotationFormat"
@ -2060,6 +2076,8 @@ service ModelArts {
rpc DeleteAlgorithms(DeleteAlgorithmsReq) returns (DeleteAlgorithmsResp);
// ShowAlgorithmByUuid 展示算法详情
rpc ShowAlgorithmByUuid(ShowAlgorithmByUuidReq) returns (ShowAlgorithmByUuidResp);
// training-job-flavors 获取训练作业支持的公共规格
rpc GetTrainingJobFlavors(TrainingJobFlavorsReq) returns (TrainingJobFlavorsResp);
//export task
@ -2094,4 +2112,6 @@ service ModelArts {
//visualization-jobs
rpc GetVisualizationJob(GetVisualizationJobReq) returns (GetVisualizationJobResp);
rpc CreateVisualizationJob(CreateVisualizationJobReq) returns (CreateVisualizationJobResp);
}