go/packages/packagestest: use a module proxy

Instead of writing replace directives to link the test's modules
together, fill a GOPROXY dir and use `go mod download` to fill a real
GOCACHE with them. This makes the tests more realistic, since replace
directives will be somewhat unusual. It also gives the name= query
something to search.

Actually doing this in a way that's compatible with packagestest's
abstraction is a little tricky, since it wants to know where the files
will be. The actual files will be created by go mod download, so we have
to get Export to put them there to begin with, then move them out of the
way.

Since the GOPROXY zip format doesn't support symlinks, those will only
work in the primary module.

Change-Id: I6bc1d368f1c950d789e409213107d60bb1389802
Reviewed-on: https://go-review.googlesource.com/c/144498
Run-TryBot: Heschi Kreinick <heschi@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Cottrell <iancottrell@google.com>

go/packages/packagestest/export.go

@@ -24,14 +24,6 @@ import (
 	"golang.org/x/tools/go/packages"
 )
 
-const (
-	// gorootModule is a special module name that indicates it contains source files
-	// that should replace the normal GOROOT
-	// in general you should not use this, it only exists for some very specialized
-	// tests.
-	gorootModule = "GOROOT"
-)
-
 var (
 	skipCleanup = flag.Bool("skip-cleanup", false, "Do not delete the temporary export folders") // for debugging
 )
@@ -112,6 +104,7 @@ func Export(t *testing.T, exporter Exporter, modules []Module) *Exported {
 			Env: append(os.Environ(), "GOPACKAGESDRIVER=off"),
 		},
 		temp:    temp,
+		primary: modules[0].Name,
 		written: map[string]map[string]string{},
 	}
 	defer func() {
@@ -120,9 +113,6 @@ func Export(t *testing.T, exporter Exporter, modules []Module) *Exported {
 		}
 	}()
 	for _, module := range modules {
-		if exported.primary == "" && module.Name != gorootModule {
-			exported.primary = module.Name
-		}
 		for fragment, value := range module.Files {
 			fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment))
 			written, ok := exported.written[module.Name]

go/packages/packagestest/gopath.go

@@ -56,10 +56,6 @@ func (gopath) Finalize(exported *Exported) error {
 			gopath += string(filepath.ListSeparator)
 		}
 		dir := gopathDir(exported, module)
-		if module == gorootModule {
-			exported.Config.Env = append(exported.Config.Env, "GOROOT="+dir)
-			continue
-		}
 		gopath += dir
 		if module == exported.primary {
 			exported.Config.Dir = filepath.Join(dir, "src")

go/packages/packagestest/modules.go

@@ -5,9 +5,13 @@
 package packagestest
 
 import (
+	"archive/zip"
 	"bytes"
 	"fmt"
+	"golang.org/x/tools/go/packages"
 	"io/ioutil"
+	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 )
@@ -32,6 +36,8 @@ import (
 //     /sometemporarydirectory/repoa
 var Modules = modules{}
 
+const theVersion = "v1.0.0"
+
 type modules struct{}
 
 func (modules) Name() string {
@@ -39,38 +45,156 @@ func (modules) Name() string {
 }
 
 func (modules) Filename(exported *Exported, module, fragment string) string {
+	if module == exported.primary {
+		return filepath.Join(primaryDir(exported), fragment)
+	}
 	return filepath.Join(moduleDir(exported, module), fragment)
 }
 
 func (modules) Finalize(exported *Exported) error {
-	exported.Config.Env = append(exported.Config.Env, "GO111MODULE=on")
+	// Write out the primary module. This module can use symlinks and
-	for module, files := range exported.written {
+	// other weird stuff, and will be the working dir for the go command.
-		dir := gopathDir(exported, module)
+	// It depends on all the other modules.
-		if module == gorootModule {
+	primaryDir := primaryDir(exported)
-			exported.Config.Env = append(exported.Config.Env, "GOROOT="+dir)
+	exported.Config.Dir = primaryDir
-			continue
+	exported.written[exported.primary]["go.mod"] = filepath.Join(primaryDir, "go.mod")
-		}
+	primaryGomod := "module " + exported.primary + "\nrequire (\n"
-		buf := &bytes.Buffer{}
-		fmt.Fprintf(buf, "module %v\n", module)
-		// add replace directives to the paths of all other modules written
 	for other := range exported.written {
-			if other == gorootModule || other == module {
+		if other == exported.primary {
 			continue
 		}
-			fmt.Fprintf(buf, "replace %v => %v\n", other, moduleDir(exported, other))
+		primaryGomod += fmt.Sprintf("\t%v %v\n", other, theVersion)
 	}
+	primaryGomod += ")\n"
+	if err := ioutil.WriteFile(filepath.Join(primaryDir, "go.mod"), []byte(primaryGomod), 0644); err != nil {
+		return err
+	}
+
+	// Create the mod cache so we can rename it later, even if we don't need it.
+	if err := os.MkdirAll(modCache(exported), 0755); err != nil {
+		return err
+	}
+
+	// Write out the go.mod files for the other modules.
+	for module, files := range exported.written {
+		if module == exported.primary {
+			continue
+		}
+		dir := moduleDir(exported, module)
+
 		modfile := filepath.Join(dir, "go.mod")
-		if err := ioutil.WriteFile(modfile, buf.Bytes(), 0644); err != nil {
+		if err := ioutil.WriteFile(modfile, []byte("module "+module+"\n"), 0644); err != nil {
 			return err
 		}
 		files["go.mod"] = modfile
-		if module == exported.primary {
-			exported.Config.Dir = dir
 	}
+
+	// Zip up all the secondary modules into the proxy dir.
+	proxyDir := filepath.Join(exported.temp, "modproxy")
+	for module, files := range exported.written {
+		if module == exported.primary {
+			continue
+		}
+		dir := filepath.Join(proxyDir, module, "@v")
+
+		if err := writeModuleProxy(dir, module, files); err != nil {
+			return fmt.Errorf("creating module proxy dir for %v: %v", module, err)
+		}
+	}
+
+	// Discard the original mod cache dir, which contained the files written
+	// for us by Export.
+	if err := os.Rename(modCache(exported), modCache(exported)+".orig"); err != nil {
+		return err
+	}
+	exported.Config.Env = append(exported.Config.Env,
+		"GO111MODULE=on",
+		"GOPATH="+filepath.Join(exported.temp, "modcache"),
+		"GOPROXY=file://"+filepath.ToSlash(proxyDir))
+
+	// Run go mod download to recreate the mod cache dir with all the extra
+	// stuff in cache. All the files created by Export should be recreated.
+	if err := invokeGo(exported.Config, "mod", "download"); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// writeModuleProxy creates a directory in the proxy dir for a module.
+func writeModuleProxy(dir, module string, files map[string]string) error {
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+
+	// list file. Just the single version.
+	if err := ioutil.WriteFile(filepath.Join(dir, "list"), []byte(theVersion+"\n"), 0644); err != nil {
+		return err
+	}
+
+	// go.mod, copied from the file written in Finalize.
+	modContents, err := ioutil.ReadFile(files["go.mod"])
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(filepath.Join(dir, theVersion+".mod"), modContents, 0644); err != nil {
+		return err
+	}
+
+	// info file, just the bare bones.
+	infoContents := []byte(fmt.Sprintf(`{"Version": "%v", "Time":"2017-12-14T13:08:43Z"}`, theVersion))
+	if err := ioutil.WriteFile(filepath.Join(dir, theVersion+".info"), infoContents, 0644); err != nil {
+		return err
+	}
+
+	// zip of all the source files.
+	f, err := os.OpenFile(filepath.Join(dir, theVersion+".zip"), os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	z := zip.NewWriter(f)
+	for name, path := range files {
+		zf, err := z.Create(module + "@" + theVersion + "/" + name)
+		if err != nil {
+			return err
+		}
+		contents, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		if _, err := zf.Write(contents); err != nil {
+			return err
+		}
+	}
+	if err := z.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func invokeGo(cfg *packages.Config, args ...string) error {
+	stdout := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
+	cmd := exec.Command("go", args...)
+	cmd.Env = append(append([]string{}, cfg.Env...), "PWD="+cfg.Dir)
+	cmd.Dir = cfg.Dir
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("go %v: %s: %s", args, err, stderr)
 	}
 	return nil
 }
 
-func moduleDir(exported *Exported, module string) string {
+func modCache(exported *Exported) string {
-	return filepath.Join(exported.temp, path.Base(module))
+	return filepath.Join(exported.temp, "modcache/pkg/mod")
+}
+
+func primaryDir(exported *Exported) string {
+	return filepath.Join(exported.temp, "primarymod", path.Base(exported.primary))
+}
+
+func moduleDir(exported *Exported, module string) string {
+	return filepath.Join(modCache(exported), path.Dir(module), path.Base(module)+"@"+theVersion)
 }

go/packages/packagestest/modules_test.go

@@ -17,15 +17,15 @@ func TestModulesExport(t *testing.T) {
 	exported := packagestest.Export(t, packagestest.Modules, testdata)
 	defer exported.Cleanup()
 	// Check that the cfg contains all the right bits
-	var expectDir = filepath.Join(exported.Temp(), "fake1")
+	var expectDir = filepath.Join(exported.Temp(), "primarymod/fake1")
 	if exported.Config.Dir != expectDir {
 		t.Errorf("Got working directory %v expected %v", exported.Config.Dir, expectDir)
 	}
 	checkFiles(t, exported, []fileTest{
-		{"golang.org/fake1", "go.mod", "fake1/go.mod", nil},
+		{"golang.org/fake1", "go.mod", "primarymod/fake1/go.mod", nil},
-		{"golang.org/fake1", "a.go", "fake1/a.go", checkLink("testdata/a.go")},
+		{"golang.org/fake1", "a.go", "primarymod/fake1/a.go", checkLink("testdata/a.go")},
-		{"golang.org/fake1", "b.go", "fake1/b.go", checkContent("package fake1")},
+		{"golang.org/fake1", "b.go", "primarymod/fake1/b.go", checkContent("package fake1")},
-		{"golang.org/fake2", "go.mod", "fake2/go.mod", nil},
+		{"golang.org/fake2", "go.mod", "modcache/pkg/mod/golang.org/fake2@v1.0.0/go.mod", nil},
-		{"golang.org/fake2", "other/a.go", "fake2/other/a.go", checkContent("package fake2")},
+		{"golang.org/fake2", "other/a.go", "modcache/pkg/mod/golang.org/fake2@v1.0.0/other/a.go", checkContent("package fake2")},
 	})
 }