From 4496618d4265b37bafd9394871013cdc6fa95ffd Mon Sep 17 00:00:00 2001 From: zhouqunjie Date: Mon, 12 Jun 2023 19:35:20 +0800 Subject: [PATCH 1/5] listHistoryJob for ac& th job submit fix Former-commit-id: d9bae6d49c1a23bd652abee772df87d62a8378b1 --- adaptor/PCM-HPC/PCM-AC/rpc/etc/hpcac.yaml | 2 +- .../rpc/internal/logic/listhistoryjoblogic.go | 109 ++++++++++++++---- .../PCM-TH/rpc/internal/logic/cronlogic.go | 10 +- 3 files changed, 95 insertions(+), 26 deletions(-) diff --git a/adaptor/PCM-HPC/PCM-AC/rpc/etc/hpcac.yaml b/adaptor/PCM-HPC/PCM-AC/rpc/etc/hpcac.yaml index 68eeba2b..6f28a33b 100644 --- a/adaptor/PCM-HPC/PCM-AC/rpc/etc/hpcac.yaml +++ b/adaptor/PCM-HPC/PCM-AC/rpc/etc/hpcac.yaml @@ -4,7 +4,7 @@ NacosConfig: ServerConfigs: # - IpAddr: 127.0.0.1 # Port: 8848 - - IpAddr: nacos.jcce.dev + - IpAddr: 10.101.15.7 Port: 8848 ClientConfig: NamespaceId: test diff --git a/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go b/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go index 23759cb3..effd8f61 100644 --- a/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go +++ b/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go @@ -3,10 +3,15 @@ package logic import ( "PCM/adaptor/PCM-HPC/PCM-AC/rpc/hpcAC" "PCM/adaptor/PCM-HPC/PCM-AC/rpc/internal/svc" - "PCM/adaptor/PCM-HPC/PCM-AC/rpc/internal/util" "context" + "github.com/bitly/go-simplejson" "github.com/zeromicro/go-zero/core/logx" + "io" + "log" + "net/http" + "net/url" "strconv" + "time" ) type ListHistoryJobLogic struct { @@ -26,31 +31,93 @@ func NewListHistoryJobLogic(ctx context.Context, svcCtx *svc.ServiceContext) *Li // ListHistoryJob list all history jobs func (l *ListHistoryJobLogic) ListHistoryJob(in *hpcAC.ListHistoryJobReq) (*hpcAC.ListHistoryJobResp, error) { + var resp hpcAC.ListHistoryJobResp + historyJobUrl := "hpc/openapi/v2/historyjobs?" + getTokenLogic := NewGetACTokenLogic(l.ctx, l.svcCtx) tokenResp, _ := getTokenLogic.GetACToken(&hpcAC.ACTokenReq{}) token := tokenResp.GetData().Token - url := "hpc/openapi/v2/historyjobs" - resp := hpcAC.ListHistoryJobResp{} - params := map[string]string{ - "strClusterNameList": in.StrClusterNameList, - "startTime": in.StartTime, - "endTime": in.EndTime, - "timeType": in.TimeType, - "queue": in.Queue, - "appType": in.AppType, - "sort": in.Sort, - "orderBy": in.OrderBy, - "jobId": in.JobId, - "jobState": in.JobState, - "hostName": in.HostName, - "strUser": in.StrUser, - "jobName": in.JobName, - "start": strconv.Itoa(int(in.Start)), - "limit": strconv.Itoa(int(in.Limit)), - "isQueryByQueueTime": in.IsQueryByQueueTime, + getACClusterIdLogic := NewGetACClusterIdLogic(l.ctx, l.svcCtx) + clusterIdResp, _ := getACClusterIdLogic.GetACClusterId(&hpcAC.ACClusterReq{Token: token}) + clusterId := clusterIdResp.GetData().Id + + c := http.Client{Timeout: time.Duration(3) * time.Second} + + params := url.Values{} + params.Add("strClusterIDList", strconv.FormatInt(clusterId, 10)) + + reqUrl, err := http.NewRequest("GET", "https://api01.hpccube.com:65106/"+historyJobUrl+params.Encode(), nil) + + if err != nil { + log.Fatal(err) } - _, _ = util.Get(token, l.svcCtx.Config.ClusterUrl, url, nil, ¶ms, &resp) + + reqUrl.Header.Add("token", token) + + respUrl, err := c.Do(reqUrl) + + if err != nil { + log.Fatal(err) + } + + body, err := io.ReadAll(respUrl.Body) + + jsonResult, err := simplejson.NewJson(body) + jsonData := jsonResult.Get("data") + if jsonData.Get("total").MustInt() == 0 { + resp.Code = "200" + resp.Msg = "success" + return &resp, nil + } + historyJobList := jsonResult.Get("data").Get("list") + rows, err := historyJobList.Array() + if err != nil { + log.Fatal(err) + } + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + + } + }(respUrl.Body) + + var Jobs []*hpcAC.HistoryJobList + + for index := range rows { + jobShuguang := historyJobList.GetIndex(index) + var job hpcAC.HistoryJobList + + job.AppType = jobShuguang.Get("appType").MustString() + job.JobId = jobShuguang.Get("jobId").MustString() + job.JobName = jobShuguang.Get("jobName").MustString() + job.JobStartTime = jobShuguang.Get("jobStartTime").MustString() + job.Queue = jobShuguang.Get("queue").MustString() + job.JobState = jobShuguang.Get("jobState").MustString() + job.JobEndTime = jobShuguang.Get("jobEndTime").MustString() + job.JobExecHost = jobShuguang.Get("jobExecHost").MustString() + job.JobWalltimeUsed = jobShuguang.Get("jobWalltimeUsed").MustString() + job.UserName = jobShuguang.Get("userName").MustString() + job.JobExitStatus = int32(jobShuguang.Get("jobExitStatus").MustInt()) + job.AcctTime = jobShuguang.Get("acctTime").MustString() + job.JobProcNum = int32(jobShuguang.Get("jobProcNum").MustInt()) + job.Nodect = int32(jobShuguang.Get("nodect").MustInt()) + job.Workdir = jobShuguang.Get("workdir").MustString() + + startTime, err := time.Parse(l.svcCtx.Config.ShuguangConf.Layout, jobShuguang.Get("jobStartTime").MustString()) + if err == nil { + job.JobStartTime = startTime.String() + } + + Jobs = append(Jobs, &job) + + } + + if jsonResult.Get("code").MustInt() == 0 { + resp.Code = "200" + } + resp.Msg = jsonResult.Get("msg").MustString() + resp.Data.List = Jobs return &resp, nil } diff --git a/adaptor/PCM-HPC/PCM-TH/rpc/internal/logic/cronlogic.go b/adaptor/PCM-HPC/PCM-TH/rpc/internal/logic/cronlogic.go index 0981dfde..5aea002e 100644 --- a/adaptor/PCM-HPC/PCM-TH/rpc/internal/logic/cronlogic.go +++ b/adaptor/PCM-HPC/PCM-TH/rpc/internal/logic/cronlogic.go @@ -66,10 +66,12 @@ func submitJob(infoList *pcmcoreclient.InfoListResp, submitJobLogic *SubmitJobLo for index, _ := range infoList.HpcInfoList { if infoList.HpcInfoList[index].Status == "Saved" { submitReq := hpcTH.SubmitJobReq{ - Account: infoList.HpcInfoList[index].Account, - Name: infoList.HpcInfoList[index].Name, - Script: infoList.HpcInfoList[index].CmdScript, - UserId: 123, + Account: infoList.HpcInfoList[index].Account, + Name: infoList.HpcInfoList[index].Name, + WorkDir: "/root", + Script: infoList.HpcInfoList[index].CmdScript, + UserId: 0, + MinNodes: 1, } jobResult, _ := submitJobLogic.SubmitJob(&submitReq) // 任务提交成功 From 51094ad8db85973bcaa8082777962199d0266086 Mon Sep 17 00:00:00 2001 From: devad Date: Mon, 12 Jun 2023 19:41:27 +0800 Subject: [PATCH 2/5] =?UTF-8?q?fix:=20pcm-ac-service.yaml=20=E6=9B=B4?= =?UTF-8?q?=E6=94=B9=E4=B8=BAnodePort?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: devad Former-commit-id: 44777c38339d8fd03001d841b893bebedfd6a2bf --- deploy/pcm-ac-service.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/pcm-ac-service.yaml b/deploy/pcm-ac-service.yaml index e6c5f1b2..11488f87 100644 --- a/deploy/pcm-ac-service.yaml +++ b/deploy/pcm-ac-service.yaml @@ -13,4 +13,5 @@ spec: protocol: TCP port: 2001 targetPort: 2001 - type: ClusterIP \ No newline at end of file + nodePort: 31617 + type: NodePort \ No newline at end of file From 85024180f916b600a68954e5be7bea20f4b69414 Mon Sep 17 00:00:00 2001 From: zhouqunjie Date: Mon, 12 Jun 2023 19:48:33 +0800 Subject: [PATCH 3/5] start&end time for list history job Former-commit-id: cbaa2409c690bf764fff7056ea35b95706f0acde --- .../PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go b/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go index effd8f61..7f048483 100644 --- a/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go +++ b/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go @@ -46,6 +46,8 @@ func (l *ListHistoryJobLogic) ListHistoryJob(in *hpcAC.ListHistoryJobReq) (*hpcA params := url.Values{} params.Add("strClusterIDList", strconv.FormatInt(clusterId, 10)) + params.Add("startTime", in.StartTime) + params.Add("endTime", in.EndTime) reqUrl, err := http.NewRequest("GET", "https://api01.hpccube.com:65106/"+historyJobUrl+params.Encode(), nil) From 38e1a8e4a76b7353ac5357031705efd677c3f0a4 Mon Sep 17 00:00:00 2001 From: devad Date: Mon, 12 Jun 2023 19:50:19 +0800 Subject: [PATCH 4/5] =?UTF-8?q?fix:=20pcm-ac-service.yaml=20=E6=9B=B4?= =?UTF-8?q?=E6=94=B9=E4=B8=BAnodePort?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: devad Former-commit-id: 8f0ed19d2c634271f7329bc5a419be8a1e912183 --- adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/logic/cronlogic.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/logic/cronlogic.go b/adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/logic/cronlogic.go index 907e22e4..c19ffdbb 100644 --- a/adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/logic/cronlogic.go +++ b/adaptor/PCM-AI/PCM-MODELARTS/rpc/internal/logic/cronlogic.go @@ -65,8 +65,9 @@ func submitJob(infoList *pcmcoreclient.InfoListResp, submitJobLogic *CreateTrain for index, _ := range infoList.AiInfoList { if infoList.AiInfoList[index].Status == "Saved" { submitReq := modelarts.CreateTrainingJobReq{ - Kind: "job", - ProjectId: "0a62ffb0d48026c12fbfc011b8d23f0b", + ModelArtsType: "cn-north-4.myhuawei", + Kind: "job", + ProjectId: "0a62ffb0d48026c12fbfc011b8d23f0b", Metadata: &modelarts.MetadataS{ Name: infoList.AiInfoList[index].Name, WorkspaceId: "0", From 68b68cac8d82f0ecb8384c8a9ef0f5e423f23f58 Mon Sep 17 00:00:00 2001 From: zhouqunjie Date: Mon, 12 Jun 2023 19:55:27 +0800 Subject: [PATCH 5/5] start&end time for list history job Former-commit-id: 36ff5278ce23ec81213a6db1d779765bdc9b7091 --- .../PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go b/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go index 7f048483..5459a4a8 100644 --- a/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go +++ b/adaptor/PCM-HPC/PCM-AC/rpc/internal/logic/listhistoryjoblogic.go @@ -48,6 +48,10 @@ func (l *ListHistoryJobLogic) ListHistoryJob(in *hpcAC.ListHistoryJobReq) (*hpcA params.Add("strClusterIDList", strconv.FormatInt(clusterId, 10)) params.Add("startTime", in.StartTime) params.Add("endTime", in.EndTime) + params.Add("timeType", in.TimeType) + params.Add("start", string(in.Start)) + params.Add("limit", string(in.Limit)) + params.Add("isQueryByQueueTime", in.IsQueryByQueueTime) reqUrl, err := http.NewRequest("GET", "https://api01.hpccube.com:65106/"+historyJobUrl+params.Encode(), nil)