floraachy
/
pcm-coordinator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix:获取训练作业支持的AI预置框架 

Former-commit-id: 546c6a7db9c2292768b0d586d1c57c70be3d0fba
This commit is contained in:
qiwang 2023-05-16 17:03:50 +08:00
parent 99b841b078
commit ce1952e252
6 changed files with 3200 additions and 2641 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/logic/getaiengineslistlogic.go Normal file
View File

@ -0,0 +1,89 @@
package logic
import (
"APIGW-go-sdk/core"
"PCM/adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/common"
"PCM/common/tool"
"context"
"fmt"
"io/ioutil"
"k8s.io/apimachinery/pkg/util/json"
"net/http"
"PCM/adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/svc"
"PCM/adaptor/PCM-AI/PCM-MODELARTS/rpc/modelarts"
"github.com/zeromicro/go-zero/core/logx"
)
type GetAiEnginesListLogic struct {
ctx context.Context
svcCtx *svc.ServiceContext
logx.Logger
}
func NewGetAiEnginesListLogic(ctx context.Context, svcCtx *svc.ServiceContext) *GetAiEnginesListLogic {
return &GetAiEnginesListLogic{
ctx: ctx,
svcCtx: svcCtx,
Logger: logx.WithContext(ctx),
}
}
// GET ai-engines 查询作业引擎规格
func (l *GetAiEnginesListLogic) GetAiEnginesList(in *modelarts.ListAiEnginesReq) (*modelarts.ListAiEnginesResp, error) {
// todo: add your logic here and delete this line
var resp modelarts.ListAiEnginesResp
//根据智算类型判断走华为智算还是南京智算
modelArtsType := in.ModelArtsType
if modelArtsType == l.svcCtx.Config.HaweiModelArtsType {
modelArtsUrl := l.svcCtx.Config.ModelArtsUrl
url := modelArtsUrl + "v2/" + in.ProjectId + "/training-job-engines"
token := common.GetToken()
statusCode, body, err := tool.HttpClientWithBodyAndCode(tool.GET, url, nil, token)
if err != nil {
return nil, err
}
if statusCode == 200 {
json.Unmarshal(body, &resp)
resp.Code = 200
resp.Msg = "Success"
} else if statusCode != 200 {
json.Unmarshal(body, &resp)
resp.Code = 400
resp.Msg = "Failure"
}
} else if modelArtsType == l.svcCtx.Config.NanjingModelArtsType {
AK := l.svcCtx.Config.AK
SK := l.svcCtx.Config.SK
NanjingModelArtsUrl := l.svcCtx.Config.NanjingModelArtsUrl
XProjectId := l.svcCtx.Config.XProjectId
XDomainId := l.svcCtx.Config.XDomainId
s := core.Signer{
Key: AK,
Secret: SK,
}
r, err := http.NewRequest("GET", NanjingModelArtsUrl+"v2/"+in.ProjectId+"/training-job-engines",
nil)
if err != nil {
fmt.Println(err)
}
r.Header.Add("content-type", "application/json;charset=UTF-8")
r.Header.Add("X-Project-Id", XProjectId)
r.Header.Add("X-Domain-Id", XDomainId)
r.Header.Add("x-stage", "RELEASE")
s.Sign(r)
client := http.DefaultClient
res, err := client.Do(r)
if err != nil {
fmt.Println(err)
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
}
json.Unmarshal(body, &resp)
}
return &resp, nil
}

6
adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/server/modelartsserver.go
View File

@ -124,6 +124,12 @@ func (s *ModelArtsServer) GetTrainingJobFlavors(ctx context.Context, in *modelar
return l.GetTrainingJobFlavors(in)
}
// GET ai-engines 查询作业引擎规格
func (s *ModelArtsServer) GetAiEnginesList(ctx context.Context, in *modelarts.ListAiEnginesReq) (*modelarts.ListAiEnginesResp, error) {
l := logic.NewGetAiEnginesListLogic(ctx, s.svcCtx)
return l.GetAiEnginesList(in)
}
// export task
func (s *ModelArtsServer) ExportTask(ctx context.Context, in *modelarts.ExportTaskReq) (*modelarts.ExportTaskDataResp, error) {
l := logic.NewExportTaskLogic(ctx, s.svcCtx)

5662
adaptor/PCM-AI/PCM-MODELARTS/rpc/modelarts/pcm-modelarts.pb.go
View File

File diff suppressed because it is too large Load Diff

38
adaptor/PCM-AI/PCM-MODELARTS/rpc/modelarts/pcm-modelarts_grpc.pb.go
View File

@ -56,6 +56,8 @@ type ModelArtsClient interface {
ShowAlgorithmByUuid(ctx context.Context, in *ShowAlgorithmByUuidReq, opts ...grpc.CallOption) (*ShowAlgorithmByUuidResp, error)
// training-job-flavors 获取训练作业支持的公共规格
GetTrainingJobFlavors(ctx context.Context, in *TrainingJobFlavorsReq, opts ...grpc.CallOption) (*TrainingJobFlavorsResp, error)
// GET ai-engines 查询作业引擎规格
GetAiEnginesList(ctx context.Context, in *ListAiEnginesReq, opts ...grpc.CallOption) (*ListAiEnginesResp, error)
//export task
ExportTask(ctx context.Context, in *ExportTaskReq, opts ...grpc.CallOption) (*ExportTaskDataResp, error)
GetExportTasksOfDataset(ctx context.Context, in *GetExportTasksOfDatasetReq, opts ...grpc.CallOption) (*GetExportTasksOfDatasetResp, error)
@ -247,6 +249,15 @@ func (c *modelArtsClient) GetTrainingJobFlavors(ctx context.Context, in *Trainin
return out, nil
}
func (c *modelArtsClient) GetAiEnginesList(ctx context.Context, in *ListAiEnginesReq, opts ...grpc.CallOption) (*ListAiEnginesResp, error) {
out := new(ListAiEnginesResp)
err := c.cc.Invoke(ctx, "/modelarts.ModelArts/GetAiEnginesList", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *modelArtsClient) ExportTask(ctx context.Context, in *ExportTaskReq, opts ...grpc.CallOption) (*ExportTaskDataResp, error) {
out := new(ExportTaskDataResp)
err := c.cc.Invoke(ctx, "/modelarts.ModelArts/ExportTask", in, out, opts...)
@ -483,6 +494,8 @@ type ModelArtsServer interface {
ShowAlgorithmByUuid(context.Context, *ShowAlgorithmByUuidReq) (*ShowAlgorithmByUuidResp, error)
// training-job-flavors 获取训练作业支持的公共规格
GetTrainingJobFlavors(context.Context, *TrainingJobFlavorsReq) (*TrainingJobFlavorsResp, error)
// GET ai-engines 查询作业引擎规格
GetAiEnginesList(context.Context, *ListAiEnginesReq) (*ListAiEnginesResp, error)
//export task
ExportTask(context.Context, *ExportTaskReq) (*ExportTaskDataResp, error)
GetExportTasksOfDataset(context.Context, *GetExportTasksOfDatasetReq) (*GetExportTasksOfDatasetResp, error)
@ -569,6 +582,9 @@ func (UnimplementedModelArtsServer) ShowAlgorithmByUuid(context.Context, *ShowAl
func (UnimplementedModelArtsServer) GetTrainingJobFlavors(context.Context, *TrainingJobFlavorsReq) (*TrainingJobFlavorsResp, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetTrainingJobFlavors not implemented")
}
func (UnimplementedModelArtsServer) GetAiEnginesList(context.Context, *ListAiEnginesReq) (*ListAiEnginesResp, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetAiEnginesList not implemented")
}
func (UnimplementedModelArtsServer) ExportTask(context.Context, *ExportTaskReq) (*ExportTaskDataResp, error) {
return nil, status.Errorf(codes.Unimplemented, "method ExportTask not implemented")
}
@ -954,6 +970,24 @@ func _ModelArts_GetTrainingJobFlavors_Handler(srv interface{}, ctx context.Conte
return interceptor(ctx, in, info, handler)
}
func _ModelArts_GetAiEnginesList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListAiEnginesReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ModelArtsServer).GetAiEnginesList(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/modelarts.ModelArts/GetAiEnginesList",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ModelArtsServer).GetAiEnginesList(ctx, req.(*ListAiEnginesReq))
}
return interceptor(ctx, in, info, handler)
}
func _ModelArts_ExportTask_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ExportTaskReq)
if err := dec(in); err != nil {
@ -1425,6 +1459,10 @@ var ModelArts_ServiceDesc = grpc.ServiceDesc{
MethodName: "GetTrainingJobFlavors",
Handler: _ModelArts_GetTrainingJobFlavors_Handler,
},
{
MethodName: "GetAiEnginesList",
Handler: _ModelArts_GetAiEnginesList_Handler,
},
{
MethodName: "ExportTask",
Handler: _ModelArts_ExportTask_Handler,

12
adaptor/PCM-AI/PCM-MODELARTS/rpc/modelartsclient/modelarts.go
View File

@ -128,6 +128,7 @@ type (
Inputs = modelarts.Inputs
InputsAlRp = modelarts.InputsAlRp
InputsAlRq = modelarts.InputsAlRq
Items = modelarts.Items
JobAlgorithmResponse = modelarts.JobAlgorithmResponse
JobConfigAl = modelarts.JobConfigAl
JobConfigAlRq = modelarts.JobConfigAlRq
@ -137,6 +138,8 @@ type (
Jobs = modelarts.Jobs
Lease = modelarts.Lease
LeaseReq = modelarts.LeaseReq
ListAiEnginesReq = modelarts.ListAiEnginesReq
ListAiEnginesResp = modelarts.ListAiEnginesResp
ListAlgorithmsReq = modelarts.ListAlgorithmsReq
ListAlgorithmsResp = modelarts.ListAlgorithmsResp
ListClustersReq = modelarts.ListClustersReq
@ -234,6 +237,7 @@ type (
Status = modelarts.Status
StopNotebookReq = modelarts.StopNotebookReq
StopNotebookResp = modelarts.StopNotebookResp
Tags = modelarts.Tags
TagsAlRp = modelarts.TagsAlRp
TaskResponse = modelarts.TaskResponse
TaskStatuses = modelarts.TaskStatuses
@ -287,6 +291,8 @@ type (
ShowAlgorithmByUuid(ctx context.Context, in *ShowAlgorithmByUuidReq, opts ...grpc.CallOption) (*ShowAlgorithmByUuidResp, error)
// training-job-flavors 获取训练作业支持的公共规格
GetTrainingJobFlavors(ctx context.Context, in *TrainingJobFlavorsReq, opts ...grpc.CallOption) (*TrainingJobFlavorsResp, error)
// GET ai-engines 查询作业引擎规格
GetAiEnginesList(ctx context.Context, in *ListAiEnginesReq, opts ...grpc.CallOption) (*ListAiEnginesResp, error)
// export task
ExportTask(ctx context.Context, in *ExportTaskReq, opts ...grpc.CallOption) (*ExportTaskDataResp, error)
GetExportTasksOfDataset(ctx context.Context, in *GetExportTasksOfDatasetReq, opts ...grpc.CallOption) (*GetExportTasksOfDatasetResp, error)
@ -430,6 +436,12 @@ func (m *defaultModelArts) GetTrainingJobFlavors(ctx context.Context, in *Traini
return client.GetTrainingJobFlavors(ctx, in, opts...)
}
// GET ai-engines 查询作业引擎规格
func (m *defaultModelArts) GetAiEnginesList(ctx context.Context, in *ListAiEnginesReq, opts ...grpc.CallOption) (*ListAiEnginesResp, error) {
client := modelarts.NewModelArtsClient(m.cli.Conn())
return client.GetAiEnginesList(ctx, in, opts...)
}
// export task
func (m *defaultModelArts) ExportTask(ctx context.Context, in *ExportTaskReq, opts ...grpc.CallOption) (*ExportTaskDataResp, error) {
client := modelarts.NewModelArtsClient(m.cli.Conn())

34
adaptor/PCM-AI/PCM-MODELARTS/rpc/pb/pcm-modelarts.proto
View File

@ -940,6 +940,37 @@ message TrainingJobFlavorsResp{
/******************TrainingJobFlavors end*************************/
/******************ListAiEngines start*************************/
message ListAiEnginesReq{
string project_id =1; // @gotags: copier:"ProjectId"
string modelArtsType =2; // @gotags: copier:"modelArtsType"
}
message ListAiEnginesResp{
int32 total = 1; // @gotags: copier:"total"
repeated Items items =2; // @gotags: copier:"items"
string msg =3;// @gotags: copier:"Msg"
int32 code =4;// @gotags: copier:"Code"
}
message Items{
string engine_id =1; // @gotags: copier:"EngineId"
string engine_name =2; // @gotags: copier:"EngineName"
string engine_version =3; // @gotags: copier:"EngineVersion"
bool v1_compatible =4; // @gotags: copier:"V1Compatible"
string run_user =5; // @gotags: copier:"RunUser"
ImageInfo image_info =6; // @gotags: copier:"ImageInfo"
bool image_source =7; // @gotags: copier:"imageSource"
repeated Tags tags =8; // @gotags: copier:"Tags"
}
message Tags{
string key = 1;
string value = 2;
}
/******************ListAiEngines end*************************/
/******************Task(export) Start*************************/
message ExportTaskReq{
string annotation_format = 1; // @gotags: copier:"AnnotationFormat"
@ -2078,7 +2109,8 @@ service ModelArts {
rpc ShowAlgorithmByUuid(ShowAlgorithmByUuidReq) returns (ShowAlgorithmByUuidResp);
// training-job-flavors 获取训练作业支持的公共规格
rpc GetTrainingJobFlavors(TrainingJobFlavorsReq) returns (TrainingJobFlavorsResp);
// GET ai-engines 查询作业引擎规格
rpc GetAiEnginesList(ListAiEnginesReq) returns (ListAiEnginesResp);
//export task
rpc ExportTask(ExportTaskReq) returns (ExportTaskDataResp);