pcm-client for participant to pull and push task info
Former-commit-id: 9e0057609cdd0c9aa60a1c97d23a8012254fadd1
This commit is contained in:
zhouqunjie
parent
aa6b343f34
commit
cbc4674662
|
|
@ -0,0 +1,13 @@
|
|||
package client
|
||||
|
||||
type Options struct {
|
||||
Url string
|
||||
DataSource string
|
||||
}
|
||||
type Client interface {
|
||||
Task(TaskOptions) (Task, error)
|
||||
}
|
||||
|
||||
func NewClient(options Options) (Client, error) {
|
||||
return newClient(options)
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
package client
|
||||
|
||||
import (
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
"gorm.io/gorm/schema"
|
||||
"time"
|
||||
)
|
||||
|
||||
type client struct {
|
||||
url string
|
||||
dataSource string
|
||||
DbEngin *gorm.DB
|
||||
}
|
||||
|
||||
func (c *client) Task(options TaskOptions) (Task, error) {
|
||||
task, _ := newTask(c, &options)
|
||||
return task, nil
|
||||
}
|
||||
|
||||
func newClient(options Options) (Client, error) {
|
||||
//init dbEngine
|
||||
dbEngin, _ := gorm.Open(mysql.Open(options.DataSource), &gorm.Config{
|
||||
NamingStrategy: schema.NamingStrategy{
|
||||
SingularTable: true,
|
||||
},
|
||||
Logger: logger.Default.LogMode(logger.Warn),
|
||||
})
|
||||
sqlDB, _ := dbEngin.DB()
|
||||
sqlDB.SetMaxIdleConns(10)
|
||||
sqlDB.SetMaxOpenConns(50)
|
||||
sqlDB.SetConnMaxLifetime(time.Hour)
|
||||
c := &client{
|
||||
url: options.Url,
|
||||
dataSource: options.DataSource,
|
||||
DbEngin: dbEngin,
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
package client
|
||||
|
||||
type TaskOptions struct {
|
||||
pullTaskInfoReq PullTaskInfoReq
|
||||
pushTaskInfoReq PushTaskInfoReq
|
||||
pushResourceInfoReq PushResourceInfoReq
|
||||
}
|
||||
|
||||
type PullTaskInfoReq struct {
|
||||
AdapterId int64 `json:"adapterId"`
|
||||
}
|
||||
|
||||
type PullTaskInfoResp struct {
|
||||
HpcInfoList []*HpcInfo `json:"HpcInfoList,omitempty"`
|
||||
CloudInfoList []*CloudInfo `json:"CloudInfoList,omitempty"`
|
||||
AiInfoList []*AiInfo `json:"AiInfoList,omitempty"`
|
||||
VmInfoList []*VmInfo `json:"VmInfoList,omitempty"`
|
||||
}
|
||||
|
||||
type PushTaskInfoReq struct {
|
||||
AdapterId int64 `json:"adapterId"`
|
||||
HpcInfoList []*HpcInfo
|
||||
CloudInfoList []*CloudInfo
|
||||
AiInfoList []*AiInfo
|
||||
VmInfoList []*VmInfo
|
||||
}
|
||||
|
||||
type PushTaskInfoResp struct {
|
||||
Code int64
|
||||
Msg string
|
||||
}
|
||||
|
||||
type PushResourceInfoReq struct {
|
||||
AdapterId int64 `json:"adapterId"`
|
||||
}
|
||||
|
||||
type Task interface {
|
||||
PullTaskInfo(pullTaskInfoReq PullTaskInfoReq) (*PullTaskInfoResp, error)
|
||||
PushTaskInfo(pushTaskInfoReq PushTaskInfoReq) (*PushTaskInfoResp, error)
|
||||
PushResourceInfo(pushResourceInfoReq PushResourceInfoReq)
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
package client
|
||||
|
||||
import (
|
||||
"github.com/zeromicro/go-zero/core/logx"
|
||||
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/constants"
|
||||
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/models"
|
||||
"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/utils"
|
||||
"gorm.io/gorm"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type task struct {
|
||||
sync.RWMutex
|
||||
client *client
|
||||
options *TaskOptions
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func newTask(client *client, options *TaskOptions) (*task, error) {
|
||||
task := &task{
|
||||
RWMutex: sync.RWMutex{},
|
||||
client: client,
|
||||
options: options,
|
||||
log: log.Logger{},
|
||||
}
|
||||
return task, nil
|
||||
}
|
||||
|
||||
func (t task) PullTaskInfo(pullTaskInfoReq PullTaskInfoReq) (*PullTaskInfoResp, error) {
|
||||
result := PullTaskInfoResp{}
|
||||
// 查询p端类型
|
||||
var kind int32
|
||||
t.client.DbEngin.Raw("select type as kind from `t_adapter` where id = ?", pullTaskInfoReq.AdapterId).Scan(&kind)
|
||||
// 查询云智超中的数据列表
|
||||
switch kind {
|
||||
case 2:
|
||||
var hpcModelList []models.TaskHpc
|
||||
findModelList(pullTaskInfoReq.AdapterId, t.client.DbEngin, &hpcModelList)
|
||||
utils.Convert(hpcModelList, &result.HpcInfoList)
|
||||
case 0:
|
||||
var cloudModelList []models.Cloud
|
||||
findModelList(pullTaskInfoReq.AdapterId, t.client.DbEngin, &cloudModelList)
|
||||
utils.Convert(cloudModelList, &result.CloudInfoList)
|
||||
case 1:
|
||||
var aiModelList []models.Ai
|
||||
findModelList(pullTaskInfoReq.AdapterId, t.client.DbEngin, &aiModelList)
|
||||
utils.Convert(aiModelList, &result.AiInfoList)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (t task) PushTaskInfo(pushTaskInfoReq PushTaskInfoReq) (*PushTaskInfoResp, error) {
|
||||
// 查询p端类型
|
||||
var kind int32
|
||||
t.client.DbEngin.Raw("select type as kind from t_adapter where id = ?", pushTaskInfoReq.AdapterId).Scan(&kind)
|
||||
switch kind {
|
||||
case 0:
|
||||
for _, cloudInfo := range pushTaskInfoReq.CloudInfoList {
|
||||
t.client.DbEngin.Exec("update cloud set status = ?,start_time = ?,result = ? where participant_id = ? and id = ?",
|
||||
cloudInfo.Status, cloudInfo.StartTime, cloudInfo.Result, pushTaskInfoReq.AdapterId, cloudInfo.Id)
|
||||
syncTask(t.client.DbEngin, cloudInfo.TaskId)
|
||||
}
|
||||
case 2:
|
||||
for _, hpcInfo := range pushTaskInfoReq.HpcInfoList {
|
||||
t.client.DbEngin.Exec("update task_hpc set status = ?,start_time = ?,job_id = ? where cluster_id = ? and task_id = ? and name = ?",
|
||||
hpcInfo.Status, hpcInfo.StartTime, hpcInfo.RunningTime, hpcInfo.JobId, pushTaskInfoReq.AdapterId, hpcInfo.TaskId, hpcInfo.Name)
|
||||
syncTask(t.client.DbEngin, hpcInfo.TaskId)
|
||||
}
|
||||
case 1:
|
||||
for _, aiInfo := range pushTaskInfoReq.AiInfoList {
|
||||
t.client.DbEngin.Exec("update ai set status = ?,start_time = ?,project_id = ?,job_id = ? where participant_id = ? and task_id = ? and name = ?",
|
||||
aiInfo.Status, aiInfo.StartTime, aiInfo.ProjectId, aiInfo.JobId, pushTaskInfoReq.AdapterId, aiInfo.TaskId, aiInfo.Name)
|
||||
syncTask(t.client.DbEngin, aiInfo.TaskId)
|
||||
}
|
||||
}
|
||||
|
||||
return &PushTaskInfoResp{}, nil
|
||||
}
|
||||
|
||||
func (t task) PushResourceInfo(pushResourceInfoReq PushResourceInfoReq) {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func findModelList(participantId int64, dbEngin *gorm.DB, data interface{}) error {
|
||||
tx := dbEngin.Where("cluster_id = (select id from t_cluster where adapter_id = ?) AND status NOT IN ?", participantId, []string{"Deleted", "Succeeded", "Completed", "Failed"}).Find(data)
|
||||
if tx.Error != nil {
|
||||
return tx.Error
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func syncTask(gorm *gorm.DB, taskId int64) {
|
||||
|
||||
var allStatus string
|
||||
tx := gorm.Raw("SELECT CONCAT_WS(',',GROUP_CONCAT(DISTINCT h.status) ,GROUP_CONCAT(DISTINCT a.status) ,GROUP_CONCAT(DISTINCT c.status))as status from task t left join hpc h on t.id = h.task_id left join cloud c on t.id = c.task_id left join ai a on t.id = a.task_id where t.id = ?", taskId).Scan(&allStatus)
|
||||
if tx.Error != nil {
|
||||
logx.Error(tx.Error)
|
||||
}
|
||||
// 子状态统一则修改主任务状态
|
||||
statusArray := strings.Split(allStatus, ",")
|
||||
if len(removeRepeatedElement(statusArray)) == 1 {
|
||||
updateTask(gorm, taskId, statusArray[0])
|
||||
|
||||
}
|
||||
// 子任务包含失败状态 主任务则失败
|
||||
if strings.Contains(allStatus, constants.Failed) {
|
||||
updateTask(gorm, taskId, constants.Failed)
|
||||
|
||||
}
|
||||
if strings.Contains(allStatus, constants.Running) {
|
||||
updateTask(gorm, taskId, constants.Running)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func updateTask(gorm *gorm.DB, taskId int64, status string) {
|
||||
var task models.Task
|
||||
gorm.Where("id = ? ", taskId).Find(&task)
|
||||
if task.Status != status {
|
||||
task.Status = status
|
||||
gorm.Updates(&task)
|
||||
}
|
||||
}
|
||||
|
||||
func removeRepeatedElement(arr []string) (newArr []string) {
|
||||
newArr = make([]string, 0)
|
||||
for i := 0; i < len(arr); i++ {
|
||||
repeat := false
|
||||
for j := i + 1; j < len(arr); j++ {
|
||||
if arr[i] == arr[j] {
|
||||
repeat = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !repeat {
|
||||
newArr = append(newArr, arr[i])
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
package client
|
||||
|
||||
type HpcInfo struct {
|
||||
ParticipantId int64 `json:"participantId,omitempty"`
|
||||
TaskId int64 `json:"taskId,omitempty"`
|
||||
JobId string `json:"jobId,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
StartTime string `json:"startTime,omitempty"`
|
||||
RunningTime int64 `json:"runningTime,omitempty"`
|
||||
Result string `json:"result,omitempty"`
|
||||
WorkDir string `json:"workDir,omitempty"`
|
||||
WallTime string `json:"wallTime,omitempty"`
|
||||
CmdScript string `json:"cmdScript,omitempty"`
|
||||
DerivedEs string `json:"derivedEs,omitempty"`
|
||||
Cluster string `json:"cluster,omitempty"`
|
||||
BlockId string `json:"blockId,omitempty"`
|
||||
AllocNodes uint32 `json:"allocNodes,omitempty"`
|
||||
AllocCpu uint32 `json:"allocCpu,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Account string `json:"account,omitempty"`
|
||||
ExitCode uint32 `json:"exitCode,omitempty"`
|
||||
AssocId uint32 `json:"assocId,omitempty"`
|
||||
AppType string `json:"appType,omitempty"`
|
||||
AppName string `json:"appName,omitempty"`
|
||||
Queue string `json:"queue,omitempty"`
|
||||
SubmitType string `json:"submitType,omitempty"`
|
||||
NNode string `json:"nNode,omitempty"`
|
||||
StdOutFile string `json:"stdOutFile,omitempty"`
|
||||
StdErrFile string `json:"stdErrFile,omitempty"`
|
||||
StdInput string `json:"stdInput,omitempty"`
|
||||
Environment string `json:"environment,omitempty"`
|
||||
}
|
||||
|
||||
type CloudInfo struct {
|
||||
Participant int64 `json:"participant,omitempty"`
|
||||
Id int64 `json:"id,omitempty"`
|
||||
TaskId int64 `json:"taskId,omitempty"`
|
||||
ApiVersion string `json:"apiVersion,omitempty"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
StartTime string `json:"startTime,omitempty"`
|
||||
RunningTime int64 `json:"runningTime,omitempty"`
|
||||
Result string `json:"result,omitempty"`
|
||||
YamlString string `json:"yamlString,omitempty"`
|
||||
}
|
||||
|
||||
type AiInfo struct {
|
||||
ParticipantId int64 `json:"participantId,omitempty"`
|
||||
TaskId int64 `json:"taskId,omitempty"`
|
||||
ProjectId string `json:"project_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
StartTime string `json:"startTime,omitempty"`
|
||||
RunningTime int64 `json:"runningTime,omitempty"`
|
||||
Result string `json:"result,omitempty"`
|
||||
JobId string `json:"jobId,omitempty"`
|
||||
CreateTime string `json:"createTime,omitempty"`
|
||||
ImageUrl string `json:"imageUrl,omitempty"`
|
||||
Command string `json:"command,omitempty"`
|
||||
FlavorId string `json:"flavorId,omitempty"`
|
||||
SubscriptionId string `json:"subscriptionId,omitempty"`
|
||||
ItemVersionId string `json:"itemVersionId,omitempty"`
|
||||
}
|
||||
|
||||
type VmInfo struct {
|
||||
ParticipantId int64 `json:"participantId,omitempty"`
|
||||
TaskId int64 `json:"taskId,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
FlavorRef string `json:"flavor_ref,omitempty"`
|
||||
ImageRef string `json:"image_ref,omitempty"`
|
||||
NetworkUuid string `json:"network_uuid,omitempty"`
|
||||
BlockUuid string `json:"block_uuid,omitempty"`
|
||||
SourceType string `json:"source_type,omitempty"`
|
||||
DeleteOnTermination bool `json:"delete_on_termination,omitempty"`
|
||||
State string `json:"state,omitempty"`
|
||||
}
|
||||
Loading…
Reference in New Issue