From b3a0c91d5e83b5d05f788558dbebc56616660f26 Mon Sep 17 00:00:00 2001 From: zhangwei <894646498@qq.com> Date: Wed, 24 May 2023 16:18:40 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=86=E7=89=87=E4=B8=8A=E4=BC=A0=E5=BC=80?= =?UTF-8?q?=E5=8F=91=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Former-commit-id: fb304e3e7d3b5321b18197be1fba6d2a36acdbd2 --- adaptor/PCM-CORE/api/desc/pcm.api | 4 +- .../handler/image/chunkimagehandler.go | 104 ++++++++++++++++++ .../handler/image/uploadimagehandler.go | 102 +++++++++++------ .../PCM-CORE/api/internal/handler/routes.go | 6 + .../internal/logic/image/chunkimagelogic.go | 28 +++++ adaptor/PCM-CORE/api/pcm.go | 2 +- 6 files changed, 211 insertions(+), 35 deletions(-) create mode 100644 adaptor/PCM-CORE/api/internal/handler/image/chunkimagehandler.go create mode 100644 adaptor/PCM-CORE/api/internal/logic/image/chunkimagelogic.go diff --git a/adaptor/PCM-CORE/api/desc/pcm.api b/adaptor/PCM-CORE/api/desc/pcm.api index ce3f80bd..9f1b671d 100644 --- a/adaptor/PCM-CORE/api/desc/pcm.api +++ b/adaptor/PCM-CORE/api/desc/pcm.api @@ -181,7 +181,9 @@ service pcm { @handler uploadImageHandler post /image/upload () returns () + @handler chunkImageHandler + post /image/chunk () returns () + @handler imageListHandler get /image/list () returns (imageListResp) - } \ No newline at end of file diff --git a/adaptor/PCM-CORE/api/internal/handler/image/chunkimagehandler.go b/adaptor/PCM-CORE/api/internal/handler/image/chunkimagehandler.go new file mode 100644 index 00000000..60003800 --- /dev/null +++ b/adaptor/PCM-CORE/api/internal/handler/image/chunkimagehandler.go @@ -0,0 +1,104 @@ +package image + +import ( + result2 "PCM/common/result" + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + + "PCM/adaptor/PCM-CORE/api/internal/logic/image" + "PCM/adaptor/PCM-CORE/api/internal/svc" +) + +var dir, _ = os.Getwd() +var windowsUploadPath = strings.ReplaceAll(path.Join(dir, "uploads"), "/", "\\") +var linuxUploadPath = path.Join(dir, "uploads") + +func ChunkImageHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + size, _ := strconv.ParseInt(r.PostFormValue("size"), 10, 64) + hash := r.PostFormValue("hash") + name := r.PostFormValue("name") + + toSize, _ := getDirSize(path.Join(linuxUploadTempPath, hash)) + if size != toSize { + fmt.Fprintf(w, "文件上传错误") + } + chunksPath := path.Join(linuxUploadTempPath, hash) + files, _ := ioutil.ReadDir(chunksPath) + // 排序 + filesSort := make(map[string]string) + for _, f := range files { + nameArr := strings.Split(f.Name(), "-") + filesSort[nameArr[1]] = f.Name() + } + saveFile := path.Join(linuxUploadPath, name) + if exists, _ := PathExists(saveFile); exists { + os.Remove(saveFile) + } + fs, _ := os.OpenFile(saveFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, os.ModeAppend|os.ModePerm) + var wg sync.WaitGroup + filesCount := len(files) + if filesCount != len(filesSort) { + fmt.Fprintf(w, "文件上传错误2") + } + wg.Add(filesCount) + for i := 0; i < filesCount; i++ { + // 这里一定要注意按顺序读取不然文件就会损坏 + fileName := path.Join(chunksPath, "\\"+filesSort[strconv.Itoa(i)]) + data, err := ioutil.ReadFile(fileName) + fmt.Println(err) + fs.Write(data) + + wg.Done() + } + wg.Wait() + os.RemoveAll(path.Join(chunksPath, "\\")) + defer fs.Close() + + //// 加载镜像文件到docker + //body, err := svcCtx.DockerClient.ImageLoad(context.Background(), multipartFile, false) + //if err != nil { + // httpx.ErrorCtx(r.Context(), w, err) + // return + //} + //bytes, err := ioutil.ReadAll(body.Body) + //loadBody := LoadBody{} + //err = json.Unmarshal(bytes, &loadBody) + //if err != nil { + // httpx.ErrorCtx(r.Context(), w, err) + // return + //} + //imageName := strings.TrimSpace(loadBody.Stream[13:]) + //req.Name = "hub.jcce.dev:18443/repository/docker-hub/jcce/" + imageName + //// 给镜像打上私有仓库的tag + //err = svcCtx.DockerClient.ImageTag(context.Background(), imageName, req.Name) + //if err != nil { + // httpx.ErrorCtx(r.Context(), w, err) + // return + //} + //l := image.NewUploadImageLogic(r.Context(), svcCtx) + //err = l.UploadImage(&req) + l := image.NewChunkImageLogic(r.Context(), svcCtx) + err := l.ChunkImage() + result2.HttpResult(r, w, nil, err) + } +} + +// DirSize 获取整体文件夹大小 +func getDirSize(path string) (int64, error) { + var size int64 + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if !info.IsDir() { + size += info.Size() + } + return err + }) + return size, err +} diff --git a/adaptor/PCM-CORE/api/internal/handler/image/uploadimagehandler.go b/adaptor/PCM-CORE/api/internal/handler/image/uploadimagehandler.go index 3a306d55..15a3e2c2 100644 --- a/adaptor/PCM-CORE/api/internal/handler/image/uploadimagehandler.go +++ b/adaptor/PCM-CORE/api/internal/handler/image/uploadimagehandler.go @@ -1,15 +1,18 @@ package image import ( - "PCM/adaptor/PCM-CORE/api/internal/logic/image" - types2 "PCM/adaptor/PCM-CORE/api/internal/types" - "PCM/common/result" - "context" - "github.com/zeromicro/go-zero/rest/httpx" + result2 "PCM/common/result" + "bufio" + "encoding/json" + "fmt" + "io" "io/ioutil" - "k8s.io/apimachinery/pkg/util/json" + "log" "net/http" + "os" + "path" "strings" + "syscall" "PCM/adaptor/PCM-CORE/api/internal/svc" ) @@ -18,38 +21,71 @@ type LoadBody struct { Stream string `json:"stream"` } +var windowsUploadTempPath = strings.ReplaceAll(path.Join(windowsUploadPath, "temp"), "/", "\\") +var linuxUploadTempPath = path.Join(windowsUploadPath) + func UploadImageHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - var req types2.UploadImageReq - // 解析yaml文件 - multipartFile, _, err := r.FormFile("file") - if err != nil { - result.HttpResult(r, w, nil, err) - return + file, _, err := r.FormFile("file") + index := r.PostFormValue("index") + hash := r.PostFormValue("hash") + // 获取uploads下所有的文件夹 + nameList, err := ioutil.ReadDir(linuxUploadPath) + m := map[string]interface{}{ + "code": 46900, + "msg": "文件已上传", } - // 加载镜像文件到docker - body, err := svcCtx.DockerClient.ImageLoad(context.Background(), multipartFile, false) - if err != nil { - httpx.ErrorCtx(r.Context(), w, err) - return + result, _ := json.MarshalIndent(m, "", " ") + // 循环判断hash是否在文件里如果有就返回上传已完成 + for _, name := range nameList { + tmpName := strings.Split(name.Name(), "_")[0] + if tmpName == hash { + fmt.Fprintf(w, string(result)) + return + } } - bytes, err := ioutil.ReadAll(body.Body) - loadBody := LoadBody{} - err = json.Unmarshal(bytes, &loadBody) - if err != nil { - httpx.ErrorCtx(r.Context(), w, err) - return + + chunksPath := path.Join(linuxUploadTempPath, hash) + + isPathExists, err := PathExists(chunksPath) + if !isPathExists { + err = os.MkdirAll(chunksPath, os.ModePerm) } - imageName := strings.TrimSpace(loadBody.Stream[13:]) - req.Name = "hub.jcce.dev:18443/repository/docker-hub/jcce/" + imageName - // 给镜像打上私有仓库的tag - err = svcCtx.DockerClient.ImageTag(context.Background(), imageName, req.Name) - if err != nil { - httpx.ErrorCtx(r.Context(), w, err) - return + destFile, err := os.OpenFile(path.Join(chunksPath+"\\"+hash+"-"+index), syscall.O_CREAT|syscall.O_WRONLY, 0777) + reader := bufio.NewReader(file) + writer := bufio.NewWriter(destFile) + buf := make([]byte, 1024*1024) // 1M buf + for { + n, err := reader.Read(buf) + if err == io.EOF { + writer.Flush() + break + } else if err != nil { + return + } else { + writer.Write(buf[:n]) + } } - l := image.NewUploadImageLogic(r.Context(), svcCtx) - err = l.UploadImage(&req) - result.HttpResult(r, w, nil, err) + + defer file.Close() + defer destFile.Close() + if err != nil { + log.Fatal("%v", err) + } + fmt.Printf("第%s:%s块上传完成\n", index, destFile.Name()) + + result2.HttpResult(r, w, nil, err) } } + +// PathExists 判断文件夹是否存在 +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} diff --git a/adaptor/PCM-CORE/api/internal/handler/routes.go b/adaptor/PCM-CORE/api/internal/handler/routes.go index 734d774d..c1ba6219 100644 --- a/adaptor/PCM-CORE/api/internal/handler/routes.go +++ b/adaptor/PCM-CORE/api/internal/handler/routes.go @@ -266,12 +266,18 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { Path: "/image/upload", Handler: image.UploadImageHandler(serverCtx), }, + { + Method: http.MethodPost, + Path: "/image/chunk", + Handler: image.ChunkImageHandler(serverCtx), + }, { Method: http.MethodGet, Path: "/image/list", Handler: image.ImageListHandler(serverCtx), }, }, + rest.WithMaxBytes(624288000), rest.WithPrefix("/pcm/v1"), ) } diff --git a/adaptor/PCM-CORE/api/internal/logic/image/chunkimagelogic.go b/adaptor/PCM-CORE/api/internal/logic/image/chunkimagelogic.go new file mode 100644 index 00000000..c03f4866 --- /dev/null +++ b/adaptor/PCM-CORE/api/internal/logic/image/chunkimagelogic.go @@ -0,0 +1,28 @@ +package image + +import ( + "context" + + "PCM/adaptor/PCM-CORE/api/internal/svc" + "github.com/zeromicro/go-zero/core/logx" +) + +type ChunkImageLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewChunkImageLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ChunkImageLogic { + return &ChunkImageLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *ChunkImageLogic) ChunkImage() error { + // todo: add your logic here and delete this line + + return nil +} diff --git a/adaptor/PCM-CORE/api/pcm.go b/adaptor/PCM-CORE/api/pcm.go index 15627376..e74d166a 100644 --- a/adaptor/PCM-CORE/api/pcm.go +++ b/adaptor/PCM-CORE/api/pcm.go @@ -44,7 +44,7 @@ func main() { serviceGroup := service.NewServiceGroup() defer serviceGroup.Stop() - server := rest.MustNewServer(c.RestConf) + server := rest.MustNewServer(c.RestConf, rest.WithCors()) ctx := svc.NewServiceContext(c) // start log component