From e8a9843dd0d767030d6464b3ca59000e3f268482 Mon Sep 17 00:00:00 2001 From: zhangwei <894646498@qq.com> Date: Tue, 23 Jan 2024 17:33:15 +0800 Subject: [PATCH 1/2] Resource Monitoring Former-commit-id: 77df78146a1f62018fe384a9ae71711b886b7d00 --- api/desc/core/pcm-core.api | 33 +++++++++++++++++ api/desc/pcm.api | 12 +++++- api/internal/config/config.go | 2 - .../handler/core/resourcecenterinfohandler.go | 25 +++++++++++++ api/internal/handler/core/tasksnumhandler.go | 25 +++++++++++++ api/internal/handler/routes.go | 10 +++++ .../logic/core/resourcecenterinfologic.go | 37 +++++++++++++++++++ api/internal/logic/core/tasksnumlogic.go | 32 ++++++++++++++++ api/internal/types/types.go | 32 ++++++++++++++++ 9 files changed, 204 insertions(+), 4 deletions(-) create mode 100644 api/internal/handler/core/resourcecenterinfohandler.go create mode 100644 api/internal/handler/core/tasksnumhandler.go create mode 100644 api/internal/logic/core/resourcecenterinfologic.go create mode 100644 api/internal/logic/core/tasksnumlogic.go diff --git a/api/desc/core/pcm-core.api b/api/desc/core/pcm-core.api index 0a89111a..128c9c5c 100644 --- a/api/desc/core/pcm-core.api +++ b/api/desc/core/pcm-core.api @@ -6,6 +6,39 @@ info( author: "zhouqj" email: "450705171@qq.com" ) +type ( + resourceCenterInfoReq { + participantId int64 `path:"participantId"` + } + resourceCenterInfoResp { + tasksName []string `json:"tasksName"` + runningTasksNum int64 `json:"runningTasksNum"` + resourceUsage ResourceUsage `json:"resourceUsage"` + resourceTotal ResourceTotal `json:"resourceTotal"` + } + + ResourceUsage { + cpu int64 `json:"cpu"` + memory int64 `json:"memory"` + storage int64 `json:"storage"` + } + + ResourceTotal { + cpu int64 `json:"cpu"` + memory int64 `json:"memory"` + storage int64 `json:"storage"` + } +) +type ( + tasksNumReq { + + } + tasksNumResp { + totalNum int64 `json:"totalNum"` + runningNum int64 `json:"runningNum"` + completedNum int64 `json:"completedNum"` + } +) type ( submitJobReq { diff --git a/api/desc/pcm.api b/api/desc/pcm.api index 69ab4ceb..bee76e1b 100644 --- a/api/desc/pcm.api +++ b/api/desc/pcm.api @@ -42,7 +42,7 @@ service pcm { @doc "查询任务列表" @handler TaskListHandler - get /core/taskList (taskListReq)returns (taskListResp) + get /core/taskList (taskListReq) returns (taskListResp) @doc "查询任务详情" @handler TaskDetailHandler @@ -107,6 +107,14 @@ service pcm { @doc "获取hashcat" @handler getHashcatHandler get /core/getHashcat/:crackTaskId (getHashcatHandlerReq) returns (getHashcatHandlerResp) + + @doc "Task Count Statistics" + @handler tasksNumHandler + get /core/tasks/num (tasksNumReq) returns (tasksNumResp) + + @doc "Resource Center Information" + @handler resourceCenterInfoHandler + get /core/center/resource/:participantId (resourceCenterInfoReq) returns (resourceCenterInfoResp) } //hpc二级接口 @@ -466,7 +474,7 @@ service pcm { @doc "查询节点详情" @handler ShowNodeDetailsHandler - get /vm/showNodeDetails (ShowNodeDetailsReq) returns (ShowNodeDetailsResp) + get /vm/showNodeDetails (ShowNodeDetailsReq) returns (ShowNodeDetailsResp) } //存算联动 接口 diff --git a/api/internal/config/config.go b/api/internal/config/config.go index 7f4b6dee..dee5d7ed 100644 --- a/api/internal/config/config.go +++ b/api/internal/config/config.go @@ -16,7 +16,6 @@ package config import ( "github.com/zeromicro/go-zero/core/logx" - "github.com/zeromicro/go-zero/core/stores/cache" "github.com/zeromicro/go-zero/core/stores/redis" "github.com/zeromicro/go-zero/rest" "github.com/zeromicro/go-zero/zrpc" @@ -28,7 +27,6 @@ type Config struct { DataSource string } Redis redis.RedisConf - Cache cache.CacheConf LogConf logx.LogConf K8sNativeConf zrpc.RpcClientConf ACRpcConf zrpc.RpcClientConf diff --git a/api/internal/handler/core/resourcecenterinfohandler.go b/api/internal/handler/core/resourcecenterinfohandler.go new file mode 100644 index 00000000..f5de45fe --- /dev/null +++ b/api/internal/handler/core/resourcecenterinfohandler.go @@ -0,0 +1,25 @@ +package core + +import ( + "gitlink.org.cn/jcce-pcm/pcm-coordinator/pkg/repository/result" + "net/http" + + "github.com/zeromicro/go-zero/rest/httpx" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/logic/core" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" +) + +func ResourceCenterInfoHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.ResourceCenterInfoReq + if err := httpx.Parse(r, &req); err != nil { + httpx.ErrorCtx(r.Context(), w, err) + return + } + + l := core.NewResourceCenterInfoLogic(r.Context(), svcCtx) + resp, err := l.ResourceCenterInfo(&req) + result.HttpResult(r, w, resp, err) + } +} diff --git a/api/internal/handler/core/tasksnumhandler.go b/api/internal/handler/core/tasksnumhandler.go new file mode 100644 index 00000000..5e0adb28 --- /dev/null +++ b/api/internal/handler/core/tasksnumhandler.go @@ -0,0 +1,25 @@ +package core + +import ( + "gitlink.org.cn/jcce-pcm/pcm-coordinator/pkg/repository/result" + "net/http" + + "github.com/zeromicro/go-zero/rest/httpx" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/logic/core" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" +) + +func TasksNumHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.TasksNumReq + if err := httpx.Parse(r, &req); err != nil { + httpx.ErrorCtx(r.Context(), w, err) + return + } + + l := core.NewTasksNumLogic(r.Context(), svcCtx) + resp, err := l.TasksNum(&req) + result.HttpResult(r, w, resp, err) + } +} diff --git a/api/internal/handler/routes.go b/api/internal/handler/routes.go index 292ed1d1..b4aa1760 100644 --- a/api/internal/handler/routes.go +++ b/api/internal/handler/routes.go @@ -125,6 +125,16 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { Path: "/core/getHashcat/:crackTaskId", Handler: core.GetHashcatHandler(serverCtx), }, + { + Method: http.MethodGet, + Path: "/core/tasks/num", + Handler: core.TasksNumHandler(serverCtx), + }, + { + Method: http.MethodGet, + Path: "/core/center/resource/:participantId", + Handler: core.ResourceCenterInfoHandler(serverCtx), + }, }, rest.WithPrefix("/pcm/v1"), ) diff --git a/api/internal/logic/core/resourcecenterinfologic.go b/api/internal/logic/core/resourcecenterinfologic.go new file mode 100644 index 00000000..ff5682b1 --- /dev/null +++ b/api/internal/logic/core/resourcecenterinfologic.go @@ -0,0 +1,37 @@ +package core + +import ( + "context" + + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type ResourceCenterInfoLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewResourceCenterInfoLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ResourceCenterInfoLogic { + return &ResourceCenterInfoLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *ResourceCenterInfoLogic) ResourceCenterInfo(req *types.ResourceCenterInfoReq) (resp *types.ResourceCenterInfoResp, err error) { + // todo: add your logic here and delete this line + resp = &types.ResourceCenterInfoResp{} + l.svcCtx.DbEngin.Raw("select count(t.id) from task t,cloud c where c.task_id = t.id and c.participant_id = ? and t.status = ?", req.ParticipantId, "Running").Scan(&resp.RunningTasksNum) + + l.svcCtx.DbEngin.Raw("select t.name from task t,cloud c where c.task_id = t.id and c.participant_id = ?", req.ParticipantId).Scan(&resp.TasksName) + + l.svcCtx.DbEngin.Raw("select SUm(cpu_total) as cpu,sum(mem_total) as memory,SUM(disk_total) as storage FROM sc_node_avail_info where participant_id = ? GROUP BY created_time ORDER BY created_time desc LIMIT 1", req.ParticipantId).Scan(&resp.ResourceTotal) + + l.svcCtx.DbEngin.Raw("select IFNULL(SUM(cpu_usable)/COUNT(*),0)as cpu,IFNULL(sum(mem_avail)/SUM(mem_total),0) as memory,IFNULL(sum(disk_avail)/SUM(disk_total),0) as storage FROM sc_node_avail_info where participant_id = ? GROUP BY created_time ORDER BY created_time desc LIMIT 1", req.ParticipantId).Scan(&resp.ResourceUsage) + return +} diff --git a/api/internal/logic/core/tasksnumlogic.go b/api/internal/logic/core/tasksnumlogic.go new file mode 100644 index 00000000..11aba8a7 --- /dev/null +++ b/api/internal/logic/core/tasksnumlogic.go @@ -0,0 +1,32 @@ +package core + +import ( + "context" + + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/svc" + "gitlink.org.cn/jcce-pcm/pcm-coordinator/api/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type TasksNumLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewTasksNumLogic(ctx context.Context, svcCtx *svc.ServiceContext) *TasksNumLogic { + return &TasksNumLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *TasksNumLogic) TasksNum(req *types.TasksNumReq) (resp *types.TasksNumResp, err error) { + resp = &types.TasksNumResp{} + l.svcCtx.DbEngin.Raw("select count(*) from task ").Scan(&resp.TotalNum) + l.svcCtx.DbEngin.Raw("select count(*) from task where status ='Running' ").Scan(&resp.RunningNum) + l.svcCtx.DbEngin.Raw("select count(*) from task where status ='Completed' ").Scan(&resp.CompletedNum) + return resp, nil +} diff --git a/api/internal/types/types.go b/api/internal/types/types.go index b1887a56..d6e4382a 100644 --- a/api/internal/types/types.go +++ b/api/internal/types/types.go @@ -1,6 +1,38 @@ // Code generated by goctl. DO NOT EDIT. package types +type ResourceCenterInfoReq struct { + ParticipantId int64 `path:"participantId"` +} + +type ResourceCenterInfoResp struct { + TasksName []string `json:"tasksName"` + RunningTasksNum int64 `json:"runningTasksNum"` + ResourceUsage ResourceUsage `json:"resourceUsage"` + ResourceTotal ResourceTotal `json:"resourceTotal"` +} + +type ResourceUsage struct { + Cpu int64 `json:"cpu"` + Memory int64 `json:"memory"` + Storage int64 `json:"storage"` +} + +type ResourceTotal struct { + Cpu int64 `json:"cpu"` + Memory int64 `json:"memory"` + Storage int64 `json:"storage"` +} + +type TasksNumReq struct { +} + +type TasksNumResp struct { + TotalNum int64 `json:"totalNum"` + RunningNum int64 `json:"runningNum"` + CompletedNum int64 `json:"completedNum"` +} + type SubmitJobReq struct { SlurmVersion string `json:"slurmVersion"` Apptype string `json:"apptype,optional"` From 373a347ea65fad8abd9e5733ea34201433b31301 Mon Sep 17 00:00:00 2001 From: zhangwei <894646498@qq.com> Date: Wed, 24 Jan 2024 09:53:05 +0800 Subject: [PATCH 2/2] fix bug Former-commit-id: 9c7262a85ad9248a248fd231adcdcb76bd7283e6 --- api/desc/core/pcm-core.api | 6 +++--- api/internal/logic/core/resourcecenterinfologic.go | 2 +- api/internal/types/types.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/desc/core/pcm-core.api b/api/desc/core/pcm-core.api index 128c9c5c..bd1905c9 100644 --- a/api/desc/core/pcm-core.api +++ b/api/desc/core/pcm-core.api @@ -18,9 +18,9 @@ type ( } ResourceUsage { - cpu int64 `json:"cpu"` - memory int64 `json:"memory"` - storage int64 `json:"storage"` + cpu float64 `json:"cpu"` + memory float64 `json:"memory"` + storage float64 `json:"storage"` } ResourceTotal { diff --git a/api/internal/logic/core/resourcecenterinfologic.go b/api/internal/logic/core/resourcecenterinfologic.go index ff5682b1..e37b4b65 100644 --- a/api/internal/logic/core/resourcecenterinfologic.go +++ b/api/internal/logic/core/resourcecenterinfologic.go @@ -32,6 +32,6 @@ func (l *ResourceCenterInfoLogic) ResourceCenterInfo(req *types.ResourceCenterIn l.svcCtx.DbEngin.Raw("select SUm(cpu_total) as cpu,sum(mem_total) as memory,SUM(disk_total) as storage FROM sc_node_avail_info where participant_id = ? GROUP BY created_time ORDER BY created_time desc LIMIT 1", req.ParticipantId).Scan(&resp.ResourceTotal) - l.svcCtx.DbEngin.Raw("select IFNULL(SUM(cpu_usable)/COUNT(*),0)as cpu,IFNULL(sum(mem_avail)/SUM(mem_total),0) as memory,IFNULL(sum(disk_avail)/SUM(disk_total),0) as storage FROM sc_node_avail_info where participant_id = ? GROUP BY created_time ORDER BY created_time desc LIMIT 1", req.ParticipantId).Scan(&resp.ResourceUsage) + l.svcCtx.DbEngin.Raw("select IFNULL(SUM(cpu_usable)/COUNT(*),0)as cpu,IFNULL(sum(mem_avail)/SUM(mem_total),0) as memory,IFNULL(sum(disk_avail)/SUM(disk_total),0) as storage FROM sc_node_avail_info where cpu_total != 0 and participant_id = ? GROUP BY created_time ORDER BY created_time desc LIMIT 1", req.ParticipantId).Scan(&resp.ResourceUsage) return } diff --git a/api/internal/types/types.go b/api/internal/types/types.go index d6e4382a..2e52c925 100644 --- a/api/internal/types/types.go +++ b/api/internal/types/types.go @@ -13,9 +13,9 @@ type ResourceCenterInfoResp struct { } type ResourceUsage struct { - Cpu int64 `json:"cpu"` - Memory int64 `json:"memory"` - Storage int64 `json:"storage"` + Cpu float64 `json:"cpu"` + Memory float64 `json:"memory"` + Storage float64 `json:"storage"` } type ResourceTotal struct {