go/packages: add support to overlays for unwritten files to existing packages

This support is not perfect, but should produce more accurate results than
the previous behavior, which is to silently drop those files.

Change-Id: Ia00c042bf303e9ec0fb1cbd579c0fccb29073de0
Reviewed-on: https://go-review.googlesource.com/c/151999
Run-TryBot: Michael Matloob <matloob@golang.org>
Reviewed-by: Ian Cottrell <iancottrell@google.com>

go/packages/golist.go

@@ -118,10 +118,6 @@ extractQueries:
 	// types.SizesFor always returns nil or a *types.StdSizes
 	response.Sizes, _ = sizes.(*types.StdSizes)
 
-	if len(containFiles) == 0 && len(packagesNamed) == 0 {
-		return response, nil
-	}
-
 	seenPkgs := make(map[string]*Package) // for deduplication. different containing queries could produce same packages
 	for _, pkg := range response.Packages {
 		seenPkgs[pkg.ID] = pkg
@@ -134,20 +130,44 @@ extractQueries:
 		response.Packages = append(response.Packages, p)
 	}
 
+	if len(containFiles) != 0 {
 		containsResults, err := runContainsQueries(cfg, listfunc, isFallback, addPkg, containFiles)
 		if err != nil {
 			return nil, err
 		}
 		response.Roots = append(response.Roots, containsResults...)
+	}
 
+	if len(packagesNamed) != 0 {
 		namedResults, err := runNamedQueries(cfg, listfunc, addPkg, packagesNamed)
 		if err != nil {
 			return nil, err
 		}
 		response.Roots = append(response.Roots, namedResults...)
+	}
+
+	needPkgs, err := processGolistOverlay(cfg, response)
+	if err != nil {
+		return nil, err
+	}
+	if len(needPkgs) > 0 {
+		addNeededOverlayPackages(cfg, listfunc, addPkg, needPkgs)
+	}
+
 	return response, nil
 }
 
+func addNeededOverlayPackages(cfg *Config, driver driver, addPkg func(*Package), pkgs []string) error {
+	response, err := driver(cfg, pkgs...)
+	if err != nil {
+		return err
+	}
+	for _, pkg := range response.Packages {
+		addPkg(pkg)
+	}
+	return nil
+}
+
 func runContainsQueries(cfg *Config, driver driver, isFallback bool, addPkg func(*Package), queries []string) ([]string, error) {
 	var results []string
 	for _, query := range queries {

go/packages/golist_overlay.go

@@ -0,0 +1,98 @@
+package packages
+
+import (
+	"go/parser"
+	"go/token"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+// processGolistOverlay provides rudimentary support for adding
+// files that don't exist on disk to an overlay. The results can be
+// sometimes incorrect.
+// TODO(matloob): Handle unsupported cases, including the following:
+// - test files
+// - adding test and non-test files to test variants of packages
+// - determining the correct package to add given a new import path
+// - creating packages that don't exist
+func processGolistOverlay(cfg *Config, response *driverResponse) (needPkgs []string, err error) {
+	havePkgs := make(map[string]string) // importPath -> non-test package ID
+	needPkgsSet := make(map[string]bool)
+
+	for _, pkg := range response.Packages {
+		// This is an approximation of import path to id. This can be
+		// wrong for tests, vendored packages, and a number of other cases.
+		havePkgs[pkg.PkgPath] = pkg.ID
+	}
+
+outer:
+	for path, contents := range cfg.Overlay {
+		base := filepath.Base(path)
+		if strings.HasSuffix(path, "_test.go") {
+			// Overlays don't support adding new test files yet.
+			// TODO(matloob): support adding new test files.
+			continue
+		}
+		dir := filepath.Dir(path)
+		for _, pkg := range response.Packages {
+			var dirContains, fileExists bool
+			for _, f := range pkg.GoFiles {
+				if sameFile(filepath.Dir(f), dir) {
+					dirContains = true
+				}
+				if filepath.Base(f) == base {
+					fileExists = true
+				}
+			}
+			if dirContains {
+				if !fileExists {
+					pkg.GoFiles = append(pkg.GoFiles, path) // TODO(matloob): should the file just be added to GoFiles?
+					pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, path)
+				}
+				imports, err := extractImports(path, contents)
+				if err != nil {
+					// Let the parser or type checker report errors later.
+					continue outer
+				}
+				for _, imp := range imports {
+					_, found := pkg.Imports[imp]
+					if !found {
+						needPkgsSet[imp] = true
+						// TODO(matloob): Handle cases when the following block isn't correct.
+						// These include imports of test variants, imports of vendored packages, etc.
+						id, ok := havePkgs[imp]
+						if !ok {
+							id = imp
+						}
+						pkg.Imports[imp] = &Package{ID: id}
+					}
+				}
+				continue outer
+			}
+		}
+	}
+
+	needPkgs = make([]string, 0, len(needPkgsSet))
+	for pkg := range needPkgsSet {
+		needPkgs = append(needPkgs, pkg)
+	}
+	return needPkgs, err
+}
+
+func extractImports(filename string, contents []byte) ([]string, error) {
+	f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.ImportsOnly) // TODO(matloob): reuse fileset?
+	if err != nil {
+		return nil, err
+	}
+	var res []string
+	for _, imp := range f.Imports {
+		quotedPath := imp.Path.Value
+		path, err := strconv.Unquote(quotedPath)
+		if err != nil {
+			return nil, err
+		}
+		res = append(res, path)
+	}
+	return res, nil
+}

go/packages/packages.go

@@ -432,6 +432,7 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 				ld.Mode >= LoadTypes && rootIndex >= 0,
 			needsrc: ld.Mode >= LoadAllSyntax ||
 				ld.Mode >= LoadSyntax && rootIndex >= 0 ||
+				ld.Overlay != nil || // Overlays can invalidate export data. TODO(matloob): make this check fine-grained based on dependencies on overlaid files
 				pkg.ExportFile == "" && pkg.PkgPath != "unsafe",
 		}
 		ld.pkgs[lpkg.ID] = lpkg
@@ -819,6 +820,15 @@ func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) {
 // the same file.
 //
 func sameFile(x, y string) bool {
+	if x == y {
+		// It could be the case that y doesn't exist.
+		// For instance, it may be an overlay file that
+		// hasn't been written to disk. To handle that case
+		// let x == y through. (We added the exact absolute path
+		// string to the CompiledGoFiles list, so the unwritten
+		// overlay case implies x==y.)
+		return true
+	}
 	if filepath.Base(x) == filepath.Base(y) { // (optimisation)
 		if xi, err := os.Stat(x); err == nil {
 			if yi, err := os.Stat(y); err == nil {

go/packages/packages_test.go

@@ -824,8 +824,18 @@ func testOverlay(t *testing.T, exporter packagestest.Exporter) {
 		{map[string][]byte{}, `"abc"`, nil}, // empty overlay
 		{map[string][]byte{exported.File("golang.org/fake", "c/c.go"): []byte(`package c; const C = "C"`)}, `"abC"`, nil},
 		{map[string][]byte{exported.File("golang.org/fake", "b/b.go"): []byte(`package b; import "golang.org/fake/c"; const B = "B" + c.C`)}, `"aBc"`, nil},
-		{map[string][]byte{exported.File("golang.org/fake", "b/b.go"): []byte(`package b; import "d"; const B = "B" + d.D`)}, `unknown`,
-			[]string{`could not import d (no metadata for d)`}},
+		// Overlay with an existing file in an existing package adding a new import.
+		{map[string][]byte{exported.File("golang.org/fake", "b/b.go"): []byte(`package b; import "golang.org/fake/d"; const B = "B" + d.D`)}, `"aBd"`, nil},
+		// Overlay with a new file in an existing package.
+		{map[string][]byte{
+			exported.File("golang.org/fake", "c/c.go"):                                               []byte(`package c;`),
+			filepath.Join(filepath.Dir(exported.File("golang.org/fake", "c/c.go")), "c_new_file.go"): []byte(`package c; const C = "Ç"`)},
+			`"abÇ"`, nil},
+		// Overlay with a new file in an existing package, adding a new dependency to that package.
+		{map[string][]byte{
+			exported.File("golang.org/fake", "c/c.go"):                                               []byte(`package c;`),
+			filepath.Join(filepath.Dir(exported.File("golang.org/fake", "c/c.go")), "c_new_file.go"): []byte(`package c; import "golang.org/fake/d"; const C = "c" + d.D`)},
+			`"abcd"`, nil},
 	} {
 		exported.Config.Overlay = test.overlay
 		exported.Config.Mode = packages.LoadAllSyntax