diff --git a/go/packages/golist.go b/go/packages/golist.go index 6be07b9a..5181f6d1 100644 --- a/go/packages/golist.go +++ b/go/packages/golist.go @@ -118,10 +118,6 @@ extractQueries: // types.SizesFor always returns nil or a *types.StdSizes response.Sizes, _ = sizes.(*types.StdSizes) - if len(containFiles) == 0 && len(packagesNamed) == 0 { - return response, nil - } - seenPkgs := make(map[string]*Package) // for deduplication. different containing queries could produce same packages for _, pkg := range response.Packages { seenPkgs[pkg.ID] = pkg @@ -134,20 +130,44 @@ extractQueries: response.Packages = append(response.Packages, p) } - containsResults, err := runContainsQueries(cfg, listfunc, isFallback, addPkg, containFiles) - if err != nil { - return nil, err + if len(containFiles) != 0 { + containsResults, err := runContainsQueries(cfg, listfunc, isFallback, addPkg, containFiles) + if err != nil { + return nil, err + } + response.Roots = append(response.Roots, containsResults...) } - response.Roots = append(response.Roots, containsResults...) - namedResults, err := runNamedQueries(cfg, listfunc, addPkg, packagesNamed) + if len(packagesNamed) != 0 { + namedResults, err := runNamedQueries(cfg, listfunc, addPkg, packagesNamed) + if err != nil { + return nil, err + } + response.Roots = append(response.Roots, namedResults...) + } + + needPkgs, err := processGolistOverlay(cfg, response) if err != nil { return nil, err } - response.Roots = append(response.Roots, namedResults...) + if len(needPkgs) > 0 { + addNeededOverlayPackages(cfg, listfunc, addPkg, needPkgs) + } + return response, nil } +func addNeededOverlayPackages(cfg *Config, driver driver, addPkg func(*Package), pkgs []string) error { + response, err := driver(cfg, pkgs...) + if err != nil { + return err + } + for _, pkg := range response.Packages { + addPkg(pkg) + } + return nil +} + func runContainsQueries(cfg *Config, driver driver, isFallback bool, addPkg func(*Package), queries []string) ([]string, error) { var results []string for _, query := range queries { diff --git a/go/packages/golist_overlay.go b/go/packages/golist_overlay.go new file mode 100644 index 00000000..60438a1a --- /dev/null +++ b/go/packages/golist_overlay.go @@ -0,0 +1,98 @@ +package packages + +import ( + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" +) + +// processGolistOverlay provides rudimentary support for adding +// files that don't exist on disk to an overlay. The results can be +// sometimes incorrect. +// TODO(matloob): Handle unsupported cases, including the following: +// - test files +// - adding test and non-test files to test variants of packages +// - determining the correct package to add given a new import path +// - creating packages that don't exist +func processGolistOverlay(cfg *Config, response *driverResponse) (needPkgs []string, err error) { + havePkgs := make(map[string]string) // importPath -> non-test package ID + needPkgsSet := make(map[string]bool) + + for _, pkg := range response.Packages { + // This is an approximation of import path to id. This can be + // wrong for tests, vendored packages, and a number of other cases. + havePkgs[pkg.PkgPath] = pkg.ID + } + +outer: + for path, contents := range cfg.Overlay { + base := filepath.Base(path) + if strings.HasSuffix(path, "_test.go") { + // Overlays don't support adding new test files yet. + // TODO(matloob): support adding new test files. + continue + } + dir := filepath.Dir(path) + for _, pkg := range response.Packages { + var dirContains, fileExists bool + for _, f := range pkg.GoFiles { + if sameFile(filepath.Dir(f), dir) { + dirContains = true + } + if filepath.Base(f) == base { + fileExists = true + } + } + if dirContains { + if !fileExists { + pkg.GoFiles = append(pkg.GoFiles, path) // TODO(matloob): should the file just be added to GoFiles? + pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, path) + } + imports, err := extractImports(path, contents) + if err != nil { + // Let the parser or type checker report errors later. + continue outer + } + for _, imp := range imports { + _, found := pkg.Imports[imp] + if !found { + needPkgsSet[imp] = true + // TODO(matloob): Handle cases when the following block isn't correct. + // These include imports of test variants, imports of vendored packages, etc. + id, ok := havePkgs[imp] + if !ok { + id = imp + } + pkg.Imports[imp] = &Package{ID: id} + } + } + continue outer + } + } + } + + needPkgs = make([]string, 0, len(needPkgsSet)) + for pkg := range needPkgsSet { + needPkgs = append(needPkgs, pkg) + } + return needPkgs, err +} + +func extractImports(filename string, contents []byte) ([]string, error) { + f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.ImportsOnly) // TODO(matloob): reuse fileset? + if err != nil { + return nil, err + } + var res []string + for _, imp := range f.Imports { + quotedPath := imp.Path.Value + path, err := strconv.Unquote(quotedPath) + if err != nil { + return nil, err + } + res = append(res, path) + } + return res, nil +} diff --git a/go/packages/packages.go b/go/packages/packages.go index c1110f3c..81e5b82e 100644 --- a/go/packages/packages.go +++ b/go/packages/packages.go @@ -432,6 +432,7 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) { ld.Mode >= LoadTypes && rootIndex >= 0, needsrc: ld.Mode >= LoadAllSyntax || ld.Mode >= LoadSyntax && rootIndex >= 0 || + ld.Overlay != nil || // Overlays can invalidate export data. TODO(matloob): make this check fine-grained based on dependencies on overlaid files pkg.ExportFile == "" && pkg.PkgPath != "unsafe", } ld.pkgs[lpkg.ID] = lpkg @@ -819,6 +820,15 @@ func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) { // the same file. // func sameFile(x, y string) bool { + if x == y { + // It could be the case that y doesn't exist. + // For instance, it may be an overlay file that + // hasn't been written to disk. To handle that case + // let x == y through. (We added the exact absolute path + // string to the CompiledGoFiles list, so the unwritten + // overlay case implies x==y.) + return true + } if filepath.Base(x) == filepath.Base(y) { // (optimisation) if xi, err := os.Stat(x); err == nil { if yi, err := os.Stat(y); err == nil { diff --git a/go/packages/packages_test.go b/go/packages/packages_test.go index 50e6344f..2548e571 100644 --- a/go/packages/packages_test.go +++ b/go/packages/packages_test.go @@ -824,8 +824,18 @@ func testOverlay(t *testing.T, exporter packagestest.Exporter) { {map[string][]byte{}, `"abc"`, nil}, // empty overlay {map[string][]byte{exported.File("golang.org/fake", "c/c.go"): []byte(`package c; const C = "C"`)}, `"abC"`, nil}, {map[string][]byte{exported.File("golang.org/fake", "b/b.go"): []byte(`package b; import "golang.org/fake/c"; const B = "B" + c.C`)}, `"aBc"`, nil}, - {map[string][]byte{exported.File("golang.org/fake", "b/b.go"): []byte(`package b; import "d"; const B = "B" + d.D`)}, `unknown`, - []string{`could not import d (no metadata for d)`}}, + // Overlay with an existing file in an existing package adding a new import. + {map[string][]byte{exported.File("golang.org/fake", "b/b.go"): []byte(`package b; import "golang.org/fake/d"; const B = "B" + d.D`)}, `"aBd"`, nil}, + // Overlay with a new file in an existing package. + {map[string][]byte{ + exported.File("golang.org/fake", "c/c.go"): []byte(`package c;`), + filepath.Join(filepath.Dir(exported.File("golang.org/fake", "c/c.go")), "c_new_file.go"): []byte(`package c; const C = "Ç"`)}, + `"abÇ"`, nil}, + // Overlay with a new file in an existing package, adding a new dependency to that package. + {map[string][]byte{ + exported.File("golang.org/fake", "c/c.go"): []byte(`package c;`), + filepath.Join(filepath.Dir(exported.File("golang.org/fake", "c/c.go")), "c_new_file.go"): []byte(`package c; import "golang.org/fake/d"; const C = "c" + d.D`)}, + `"abcd"`, nil}, } { exported.Config.Overlay = test.overlay exported.Config.Mode = packages.LoadAllSyntax