go/packages: add basic support for overlays

This allows users of go/packages to replace the contents of already
existing files, to support use-cases such as unsaved files in editors.

BREAKING CHANGE: This CL changes the signature of the function provided
to Config.ParseFile.

Change-Id: I6ce50336060832679e9f64f8d201b44651772e0b
Reviewed-on: https://go-review.googlesource.com/c/139798
Reviewed-by: Alan Donovan <adonovan@google.com>

go/packages/packages.go

@@ -20,6 +20,8 @@ import (
 	"sync"
 
 	"golang.org/x/tools/go/gcexportdata"
+	"io/ioutil"
+	"path/filepath"
 )
 
 // A LoadMode specifies the amount of detail to return when loading.
@@ -96,12 +98,14 @@ type Config struct {
 	// It must be safe to call ParseFile simultaneously from multiple goroutines.
 	// If ParseFile is nil, the loader will uses parser.ParseFile.
 	//
-	// Setting ParseFile to a custom implementation can allow
+	// ParseFile should parse the source from src and use filename only for
-	// providing alternate file content in order to type-check
+	// recording position information.
-	// unsaved text editor buffers, or to selectively eliminate
+	//
-	// unwanted function bodies to reduce the amount of work
+	// An application may supply a custom implementation of ParseFile
-	// done by the type checker.
+	// to change the effective file contents or the behavior of the parser,
-	ParseFile func(fset *token.FileSet, filename string) (*ast.File, error)
+	// or to modify the syntax tree. For example, selectively eliminating
+	// unwanted function bodies can significantly accelerate type checking.
+	ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error)
 
 	// If Tests is set, the loader includes not just the packages
 	// matching a particular pattern but also any related test packages,
@@ -116,6 +120,15 @@ type Config struct {
 	// In build systems with explicit names for tests,
 	// setting Tests may have no effect.
 	Tests bool
+
+	// Overlay provides a mapping of absolute file paths to file contents.
+	// If the file  with the given path already exists, the parser will use the
+	// alternative file contents provided by the map.
+	//
+	// The Package.Imports map may not include packages that are imported only
+	// by the alternative file contents provided by Overlay. This may cause
+	// type-checking to fail.
+	Overlay map[string][]byte
 }
 
 // driver is the type for functions that query the build system for the
@@ -380,9 +393,13 @@ func newLoader(cfg *Config) *loader {
 		// ParseFile is required even in LoadTypes mode
 		// because we load source if export data is missing.
 		if ld.ParseFile == nil {
-			ld.ParseFile = func(fset *token.FileSet, filename string) (*ast.File, error) {
+			ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) {
+				var isrc interface{}
+				if src != nil {
+					isrc = src
+				}
 				const mode = parser.AllErrors | parser.ParseComments
-				return parser.ParseFile(fset, filename, nil, mode)
+				return parser.ParseFile(fset, filename, isrc, mode)
 			}
 		}
 	}
@@ -743,7 +760,21 @@ func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) {
 		go func(i int, filename string) {
 			ioLimit <- true // wait
 			// ParseFile may return both an AST and an error.
-			parsed[i], errors[i] = ld.ParseFile(ld.Fset, filename)
+			var src []byte
+			for f, contents := range ld.Config.Overlay {
+				if sameFile(f, filename) {
+					src = contents
+				}
+			}
+			var err error
+			if src == nil {
+				src, err = ioutil.ReadFile(filename)
+			}
+			if err != nil {
+				parsed[i], errors[i] = nil, err
+			} else {
+				parsed[i], errors[i] = ld.ParseFile(ld.Fset, filename, src)
+			}
 			<-ioLimit // signal
 			wg.Done()
 		}(i, file)
@@ -772,6 +803,20 @@ func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) {
 	return parsed, errors
 }
 
+// sameFile returns true if x and y have the same basename and denote
+// the same file.
+//
+func sameFile(x, y string) bool {
+	if filepath.Base(x) == filepath.Base(y) { // (optimisation)
+		if xi, err := os.Stat(x); err == nil {
+			if yi, err := os.Stat(y); err == nil {
+				return os.SameFile(xi, yi)
+			}
+		}
+	}
+	return false
+}
+
 // loadFromExportData returns type information for the specified
 // package, loading it from an export data file on the first request.
 func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error) {

go/packages/packages_test.go

@@ -763,12 +763,45 @@ func TestLoadSyntaxError(t *testing.T) {
 	}
 }
 
-// This function tests use of the ParseFile hook to supply
+// This function tests use of the ParseFile hook to modify
-// alternative file contents to the parser and type-checker.
+// the AST after parsing.
-func TestLoadAllSyntaxOverlay(t *testing.T) {
+func TestParseFileModifyAST(t *testing.T) {
 	type M = map[string]string
 
 	tmp, cleanup := makeTree(t, M{
+		"src/a/a.go": `package a; const A = "a" `,
+	})
+	defer cleanup()
+
+	parseFile := func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) {
+		const mode = parser.AllErrors | parser.ParseComments
+		f, err := parser.ParseFile(fset, filename, src, mode)
+		// modify AST to change `const A = "a"` to `const A = "b"`
+		spec := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.ValueSpec)
+		spec.Values[0].(*ast.BasicLit).Value = `"b"`
+		return f, err
+	}
+	cfg := &packages.Config{
+		Mode:      packages.LoadAllSyntax,
+		Env:       append(os.Environ(), "GOPATH="+tmp, "GO111MODULE=off"),
+		ParseFile: parseFile,
+	}
+	initial, err := packages.Load(cfg, "a")
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Check value of a.A has been set to "b"
+	a := initial[0]
+	got := constant(a, "A").Val().String()
+	if got != `"b"` {
+		t.Errorf("a.A: got %s, want %s", got, `"b"`)
+	}
+}
+
+// This function tests config.Overlay functionality.
+func TestOverlay(t *testing.T) {
+	tmp, cleanup := makeTree(t, map[string]string{
 		"src/a/a.go": `package a; import "b"; const A = "a" + b.B`,
 		"src/b/b.go": `package b; import "c"; const B = "b" + c.C`,
 		"src/c/c.go": `package c; const C = "c"`,
@@ -777,32 +810,23 @@ func TestLoadAllSyntaxOverlay(t *testing.T) {
 	defer cleanup()
 
 	for i, test := range []struct {
-		overlay  M
+		overlay  map[string][]byte
 		want     string // expected value of a.A
 		wantErrs []string
 	}{
-		{nil, `"abc"`, nil}, // default
+		{nil, `"abc"`, nil},                 // default
-		{M{}, `"abc"`, nil}, // empty overlay
+		{map[string][]byte{}, `"abc"`, nil}, // empty overlay
-		{M{filepath.Join(tmp, "src/c/c.go"): `package c; const C = "C"`}, `"abC"`, nil},
+		{map[string][]byte{filepath.Join(tmp, "src/c/c.go"): []byte(`package c; const C = "C"`)}, `"abC"`, nil},
-		{M{filepath.Join(tmp, "src/b/b.go"): `package b; import "c"; const B = "B" + c.C`}, `"aBc"`, nil},
+		{map[string][]byte{filepath.Join(tmp, "src/b/b.go"): []byte(`package b; import "c"; const B = "B" + c.C`)}, `"aBc"`, nil},
-		{M{filepath.Join(tmp, "src/b/b.go"): `package b; import "d"; const B = "B" + d.D`}, `unknown`,
+		{map[string][]byte{filepath.Join(tmp, "src/b/b.go"): []byte(`package b; import "d"; const B = "B" + d.D`)}, `unknown`,
 			[]string{`could not import d (no metadata for d)`}},
 	} {
-		var parseFile func(fset *token.FileSet, filename string) (*ast.File, error)
+		var parseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error)
-		if test.overlay != nil {
-			parseFile = func(fset *token.FileSet, filename string) (*ast.File, error) {
-				var src interface{}
-				if content, ok := test.overlay[filename]; ok {
-					src = content
-				}
-				const mode = parser.AllErrors | parser.ParseComments
-				return parser.ParseFile(fset, filename, src, mode)
-			}
-		}
 		cfg := &packages.Config{
 			Mode:      packages.LoadAllSyntax,
 			Env:       append(os.Environ(), "GOPATH="+tmp, "GO111MODULE=off"),
 			ParseFile: parseFile,
+			Overlay:   test.overlay,
 		}
 		initial, err := packages.Load(cfg, "a")
 		if err != nil {