From f60e5f99f0816fc2d9ecb338008ea420248d2943 Mon Sep 17 00:00:00 2001 From: Heschi Kreinick Date: Tue, 23 Oct 2018 15:13:28 -0400 Subject: [PATCH] go/packages/packagestest: use a module proxy Instead of writing replace directives to link the test's modules together, fill a GOPROXY dir and use `go mod download` to fill a real GOCACHE with them. This makes the tests more realistic, since replace directives will be somewhat unusual. It also gives the name= query something to search. Actually doing this in a way that's compatible with packagestest's abstraction is a little tricky, since it wants to know where the files will be. The actual files will be created by go mod download, so we have to get Export to put them there to begin with, then move them out of the way. Since the GOPROXY zip format doesn't support symlinks, those will only work in the primary module. Change-Id: I6bc1d368f1c950d789e409213107d60bb1389802 Reviewed-on: https://go-review.googlesource.com/c/144498 Run-TryBot: Heschi Kreinick TryBot-Result: Gobot Gobot Reviewed-by: Ian Cottrell --- go/packages/packagestest/export.go | 12 +- go/packages/packagestest/gopath.go | 4 - go/packages/packagestest/modules.go | 158 ++++++++++++++++++++--- go/packages/packagestest/modules_test.go | 12 +- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/go/packages/packagestest/export.go b/go/packages/packagestest/export.go index 1181ec24..a48da618 100644 --- a/go/packages/packagestest/export.go +++ b/go/packages/packagestest/export.go @@ -24,14 +24,6 @@ import ( "golang.org/x/tools/go/packages" ) -const ( - // gorootModule is a special module name that indicates it contains source files - // that should replace the normal GOROOT - // in general you should not use this, it only exists for some very specialized - // tests. - gorootModule = "GOROOT" -) - var ( skipCleanup = flag.Bool("skip-cleanup", false, "Do not delete the temporary export folders") // for debugging ) @@ -112,6 +104,7 @@ func Export(t *testing.T, exporter Exporter, modules []Module) *Exported { Env: append(os.Environ(), "GOPACKAGESDRIVER=off"), }, temp: temp, + primary: modules[0].Name, written: map[string]map[string]string{}, } defer func() { @@ -120,9 +113,6 @@ func Export(t *testing.T, exporter Exporter, modules []Module) *Exported { } }() for _, module := range modules { - if exported.primary == "" && module.Name != gorootModule { - exported.primary = module.Name - } for fragment, value := range module.Files { fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment)) written, ok := exported.written[module.Name] diff --git a/go/packages/packagestest/gopath.go b/go/packages/packagestest/gopath.go index 090f49a2..4a086c5a 100644 --- a/go/packages/packagestest/gopath.go +++ b/go/packages/packagestest/gopath.go @@ -56,10 +56,6 @@ func (gopath) Finalize(exported *Exported) error { gopath += string(filepath.ListSeparator) } dir := gopathDir(exported, module) - if module == gorootModule { - exported.Config.Env = append(exported.Config.Env, "GOROOT="+dir) - continue - } gopath += dir if module == exported.primary { exported.Config.Dir = filepath.Join(dir, "src") diff --git a/go/packages/packagestest/modules.go b/go/packages/packagestest/modules.go index 01004c05..8897a531 100644 --- a/go/packages/packagestest/modules.go +++ b/go/packages/packagestest/modules.go @@ -5,9 +5,13 @@ package packagestest import ( + "archive/zip" "bytes" "fmt" + "golang.org/x/tools/go/packages" "io/ioutil" + "os" + "os/exec" "path" "path/filepath" ) @@ -32,6 +36,8 @@ import ( // /sometemporarydirectory/repoa var Modules = modules{} +const theVersion = "v1.0.0" + type modules struct{} func (modules) Name() string { @@ -39,38 +45,156 @@ func (modules) Name() string { } func (modules) Filename(exported *Exported, module, fragment string) string { + if module == exported.primary { + return filepath.Join(primaryDir(exported), fragment) + } return filepath.Join(moduleDir(exported, module), fragment) } func (modules) Finalize(exported *Exported) error { - exported.Config.Env = append(exported.Config.Env, "GO111MODULE=on") - for module, files := range exported.written { - dir := gopathDir(exported, module) - if module == gorootModule { - exported.Config.Env = append(exported.Config.Env, "GOROOT="+dir) + // Write out the primary module. This module can use symlinks and + // other weird stuff, and will be the working dir for the go command. + // It depends on all the other modules. + primaryDir := primaryDir(exported) + exported.Config.Dir = primaryDir + exported.written[exported.primary]["go.mod"] = filepath.Join(primaryDir, "go.mod") + primaryGomod := "module " + exported.primary + "\nrequire (\n" + for other := range exported.written { + if other == exported.primary { continue } - buf := &bytes.Buffer{} - fmt.Fprintf(buf, "module %v\n", module) - // add replace directives to the paths of all other modules written - for other := range exported.written { - if other == gorootModule || other == module { - continue - } - fmt.Fprintf(buf, "replace %v => %v\n", other, moduleDir(exported, other)) + primaryGomod += fmt.Sprintf("\t%v %v\n", other, theVersion) + } + primaryGomod += ")\n" + if err := ioutil.WriteFile(filepath.Join(primaryDir, "go.mod"), []byte(primaryGomod), 0644); err != nil { + return err + } + + // Create the mod cache so we can rename it later, even if we don't need it. + if err := os.MkdirAll(modCache(exported), 0755); err != nil { + return err + } + + // Write out the go.mod files for the other modules. + for module, files := range exported.written { + if module == exported.primary { + continue } + dir := moduleDir(exported, module) + modfile := filepath.Join(dir, "go.mod") - if err := ioutil.WriteFile(modfile, buf.Bytes(), 0644); err != nil { + if err := ioutil.WriteFile(modfile, []byte("module "+module+"\n"), 0644); err != nil { return err } files["go.mod"] = modfile + } + + // Zip up all the secondary modules into the proxy dir. + proxyDir := filepath.Join(exported.temp, "modproxy") + for module, files := range exported.written { if module == exported.primary { - exported.Config.Dir = dir + continue } + dir := filepath.Join(proxyDir, module, "@v") + + if err := writeModuleProxy(dir, module, files); err != nil { + return fmt.Errorf("creating module proxy dir for %v: %v", module, err) + } + } + + // Discard the original mod cache dir, which contained the files written + // for us by Export. + if err := os.Rename(modCache(exported), modCache(exported)+".orig"); err != nil { + return err + } + exported.Config.Env = append(exported.Config.Env, + "GO111MODULE=on", + "GOPATH="+filepath.Join(exported.temp, "modcache"), + "GOPROXY=file://"+filepath.ToSlash(proxyDir)) + + // Run go mod download to recreate the mod cache dir with all the extra + // stuff in cache. All the files created by Export should be recreated. + if err := invokeGo(exported.Config, "mod", "download"); err != nil { + return err + } + + return nil +} + +// writeModuleProxy creates a directory in the proxy dir for a module. +func writeModuleProxy(dir, module string, files map[string]string) error { + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + + // list file. Just the single version. + if err := ioutil.WriteFile(filepath.Join(dir, "list"), []byte(theVersion+"\n"), 0644); err != nil { + return err + } + + // go.mod, copied from the file written in Finalize. + modContents, err := ioutil.ReadFile(files["go.mod"]) + if err != nil { + return err + } + if err := ioutil.WriteFile(filepath.Join(dir, theVersion+".mod"), modContents, 0644); err != nil { + return err + } + + // info file, just the bare bones. + infoContents := []byte(fmt.Sprintf(`{"Version": "%v", "Time":"2017-12-14T13:08:43Z"}`, theVersion)) + if err := ioutil.WriteFile(filepath.Join(dir, theVersion+".info"), infoContents, 0644); err != nil { + return err + } + + // zip of all the source files. + f, err := os.OpenFile(filepath.Join(dir, theVersion+".zip"), os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + z := zip.NewWriter(f) + for name, path := range files { + zf, err := z.Create(module + "@" + theVersion + "/" + name) + if err != nil { + return err + } + contents, err := ioutil.ReadFile(path) + if err != nil { + return err + } + if _, err := zf.Write(contents); err != nil { + return err + } + } + if err := z.Close(); err != nil { + return err + } + + return nil +} + +func invokeGo(cfg *packages.Config, args ...string) error { + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + cmd := exec.Command("go", args...) + cmd.Env = append(append([]string{}, cfg.Env...), "PWD="+cfg.Dir) + cmd.Dir = cfg.Dir + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("go %v: %s: %s", args, err, stderr) } return nil } -func moduleDir(exported *Exported, module string) string { - return filepath.Join(exported.temp, path.Base(module)) +func modCache(exported *Exported) string { + return filepath.Join(exported.temp, "modcache/pkg/mod") +} + +func primaryDir(exported *Exported) string { + return filepath.Join(exported.temp, "primarymod", path.Base(exported.primary)) +} + +func moduleDir(exported *Exported, module string) string { + return filepath.Join(modCache(exported), path.Dir(module), path.Base(module)+"@"+theVersion) } diff --git a/go/packages/packagestest/modules_test.go b/go/packages/packagestest/modules_test.go index 3530f079..c4868a55 100644 --- a/go/packages/packagestest/modules_test.go +++ b/go/packages/packagestest/modules_test.go @@ -17,15 +17,15 @@ func TestModulesExport(t *testing.T) { exported := packagestest.Export(t, packagestest.Modules, testdata) defer exported.Cleanup() // Check that the cfg contains all the right bits - var expectDir = filepath.Join(exported.Temp(), "fake1") + var expectDir = filepath.Join(exported.Temp(), "primarymod/fake1") if exported.Config.Dir != expectDir { t.Errorf("Got working directory %v expected %v", exported.Config.Dir, expectDir) } checkFiles(t, exported, []fileTest{ - {"golang.org/fake1", "go.mod", "fake1/go.mod", nil}, - {"golang.org/fake1", "a.go", "fake1/a.go", checkLink("testdata/a.go")}, - {"golang.org/fake1", "b.go", "fake1/b.go", checkContent("package fake1")}, - {"golang.org/fake2", "go.mod", "fake2/go.mod", nil}, - {"golang.org/fake2", "other/a.go", "fake2/other/a.go", checkContent("package fake2")}, + {"golang.org/fake1", "go.mod", "primarymod/fake1/go.mod", nil}, + {"golang.org/fake1", "a.go", "primarymod/fake1/a.go", checkLink("testdata/a.go")}, + {"golang.org/fake1", "b.go", "primarymod/fake1/b.go", checkContent("package fake1")}, + {"golang.org/fake2", "go.mod", "modcache/pkg/mod/golang.org/fake2@v1.0.0/go.mod", nil}, + {"golang.org/fake2", "other/a.go", "modcache/pkg/mod/golang.org/fake2@v1.0.0/other/a.go", checkContent("package fake2")}, }) }