diff --git a/go/packages/golist.go b/go/packages/golist/golist.go similarity index 64% rename from go/packages/golist.go rename to go/packages/golist/golist.go index ff3d66bf..845fb5b8 100644 --- a/go/packages/golist.go +++ b/go/packages/golist/golist.go @@ -2,9 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package packages - -// This file defines the "go list" implementation of the Packages metadata query. +// Package golist defines the "go list" implementation of the Packages metadata query. +package golist import ( "bytes" @@ -16,18 +15,95 @@ import ( "os/exec" "path/filepath" "strings" + + "golang.org/x/tools/go/packages/raw" ) -// A GoTooOldError reports that the go command -// found by exec.LookPath does not contain the necessary -// support to be used with go/packages. -// Currently, go/packages requires Go 1.11 or later. -// (We intend to issue a point release for Go 1.10 -// so that go/packages can be used with updated Go 1.10 systems too.) -type GoTooOldError struct { +// A goTooOldError reports that the go command +// found by exec.LookPath is too old to use the new go list behavior. +type goTooOldError struct { error } +// LoadRaw and returns the raw Go packages named by the given patterns. +// This is a low level API, in general you should be using the packages.Load +// unless you have a very strong need for the raw data, and you know that you +// are using conventional go layout as supported by `go list` +// It returns the packages identifiers that directly matched the patterns, the +// full set of packages requested (which may include the dependencies) and +// an error if the operation failed. +func LoadRaw(ctx context.Context, cfg *raw.Config, patterns ...string) ([]string, []*raw.Package, error) { + if len(patterns) == 0 { + return nil, nil, fmt.Errorf("no packages to load") + } + if cfg == nil { + return nil, nil, fmt.Errorf("Load must be passed a valid Config") + } + if cfg.Dir == "" { + return nil, nil, fmt.Errorf("Config does not have a working directory") + } + // Determine files requested in contains patterns + var containFiles []string + { + restPatterns := patterns[:0] + for _, pattern := range patterns { + if containFile := strings.TrimPrefix(pattern, "contains:"); containFile != pattern { + containFiles = append(containFiles, containFile) + } else { + restPatterns = append(restPatterns, pattern) + } + } + containFiles = absJoin(cfg.Dir, containFiles) + patterns = restPatterns + } + + listfunc := golistPackages + // TODO(matloob): Patterns may now be empty, if it was solely comprised of contains: patterns. + // See if the extra process invocation can be avoided. + roots, pkgs, err := listfunc(ctx, cfg, patterns...) + if _, ok := err.(goTooOldError); ok { + listfunc = golistPackagesFallback + roots, pkgs, err = listfunc(ctx, cfg, patterns...) + } + if err != nil { + return nil, nil, err + } + + // Run go list for contains: patterns. + seenPkgs := make(map[string]bool) // for deduplication. different containing queries could produce same packages + if len(containFiles) > 0 { + for _, pkg := range pkgs { + seenPkgs[pkg.ID] = true + } + } + for _, f := range containFiles { + // TODO(matloob): Do only one query per directory. + fdir := filepath.Dir(f) + cfg.Dir = fdir + _, cList, err := listfunc(ctx, cfg, ".") + if err != nil { + return nil, nil, err + } + // Deduplicate and set deplist to set of packages requested files. + dedupedList := cList[:0] // invariant: only packages that haven't been seen before + for _, pkg := range cList { + if seenPkgs[pkg.ID] { + continue + } + seenPkgs[pkg.ID] = true + dedupedList = append(dedupedList, pkg) + for _, pkgFile := range pkg.GoFiles { + if filepath.Base(f) == filepath.Base(pkgFile) { + roots = append(roots, pkg.ID) + break + } + } + } + pkgs = append(pkgs, dedupedList...) + } + return roots, pkgs, nil +} + // Fields must match go list; // see $GOROOT/src/cmd/go/internal/load/pkg.go. type jsonPackage struct { @@ -53,7 +129,7 @@ type jsonPackage struct { // golistPackages uses the "go list" command to expand the // pattern words and return metadata for the specified packages. // dir may be "" and env may be nil, as per os/exec.Command. -func golistPackages(ctx context.Context, cfg *rawConfig, words ...string) ([]string, []*rawPackage, error) { +func golistPackages(ctx context.Context, cfg *raw.Config, words ...string) ([]string, []*raw.Package, error) { // go list uses the following identifiers in ImportPath and Imports: // // "p" -- importable package or main (command) @@ -72,9 +148,9 @@ func golistPackages(ctx context.Context, cfg *rawConfig, words ...string) ([]str if err != nil { return nil, nil, err } - // Decode the JSON and convert it to rawPackage form. + // Decode the JSON and convert it to Package form. var roots []string - var result []*rawPackage + var result []*raw.Package for dec := json.NewDecoder(buf); dec.More(); { p := new(jsonPackage) if err := dec.Decode(p); err != nil { @@ -153,7 +229,7 @@ func golistPackages(ctx context.Context, cfg *rawConfig, words ...string) ([]str imports[id] = id // identity import } - pkg := &rawPackage{ + pkg := &raw.Package{ ID: id, Name: p.Name, GoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles), @@ -184,7 +260,7 @@ func absJoin(dir string, fileses ...[]string) (res []string) { return res } -func golistargs(cfg *rawConfig, words []string) []string { +func golistargs(cfg *raw.Config, words []string) []string { fullargs := []string{ "list", "-e", "-json", "-cgo=true", fmt.Sprintf("-test=%t", cfg.Tests), @@ -198,7 +274,7 @@ func golistargs(cfg *rawConfig, words []string) []string { } // golist returns the JSON-encoded result of a "go list args..." query. -func golist(ctx context.Context, cfg *rawConfig, args []string) (*bytes.Buffer, error) { +func golist(ctx context.Context, cfg *raw.Config, args []string) (*bytes.Buffer, error) { out := new(bytes.Buffer) cmd := exec.CommandContext(ctx, "go", args...) cmd.Env = cfg.Env @@ -216,7 +292,7 @@ func golist(ctx context.Context, cfg *rawConfig, args []string) (*bytes.Buffer, // Old go list? if strings.Contains(fmt.Sprint(cmd.Stderr), "flag provided but not defined") { - return nil, GoTooOldError{fmt.Errorf("unsupported version of go list: %s: %s", exitErr, cmd.Stderr)} + return nil, goTooOldError{fmt.Errorf("unsupported version of go list: %s: %s", exitErr, cmd.Stderr)} } // Export mode entails a build. diff --git a/go/packages/golist_fallback.go b/go/packages/golist/golist_fallback.go similarity index 83% rename from go/packages/golist_fallback.go rename to go/packages/golist/golist_fallback.go index a06aab21..dda2b7b7 100644 --- a/go/packages/golist_fallback.go +++ b/go/packages/golist/golist_fallback.go @@ -1,10 +1,15 @@ -package packages +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package golist import ( "context" "encoding/json" "fmt" + "golang.org/x/tools/go/packages/raw" "golang.org/x/tools/imports" ) @@ -19,13 +24,13 @@ import ( // TODO(matloob): Support cgo. Copy code from the loader that runs cgo. -func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string) ([]string, []*rawPackage, error) { +func golistPackagesFallback(ctx context.Context, cfg *raw.Config, words ...string) ([]string, []*raw.Package, error) { original, deps, err := getDeps(ctx, cfg, words...) if err != nil { return nil, nil, err } - var result []*rawPackage + var result []*raw.Package var roots []string addPackage := func(p *jsonPackage) { if p.Name == "" { @@ -59,7 +64,7 @@ func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string if isRoot { roots = append(roots, id) } - result = append(result, &rawPackage{ + result = append(result, &raw.Package{ ID: id, Name: p.Name, GoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles), @@ -73,7 +78,7 @@ func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string if isRoot { roots = append(roots, testID) } - result = append(result, &rawPackage{ + result = append(result, &raw.Package{ ID: testID, Name: p.Name, GoFiles: absJoin(p.Dir, p.GoFiles, p.TestGoFiles, p.CgoFiles), @@ -87,7 +92,7 @@ func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string if isRoot { roots = append(roots, xtestID) } - result = append(result, &rawPackage{ + result = append(result, &raw.Package{ ID: xtestID, Name: p.Name + "_test", GoFiles: absJoin(p.Dir, p.XTestGoFiles), @@ -110,7 +115,7 @@ func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string return nil, nil, err } - // Decode the JSON and convert it to rawPackage form. + // Decode the JSON and convert it to Package form. for dec := json.NewDecoder(buf); dec.More(); { p := new(jsonPackage) if err := dec.Decode(p); err != nil { @@ -124,7 +129,7 @@ func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string } // getDeps runs an initial go list to determine all the dependency packages. -func getDeps(ctx context.Context, cfg *rawConfig, words ...string) (originalSet map[string]*jsonPackage, deps []string, err error) { +func getDeps(ctx context.Context, cfg *raw.Config, words ...string) (originalSet map[string]*jsonPackage, deps []string, err error) { buf, err := golist(ctx, cfg, golistargs_fallback(cfg, words)) if err != nil { return nil, nil, err @@ -156,7 +161,7 @@ func getDeps(ctx context.Context, cfg *rawConfig, words ...string) (originalSet return originalSet, deps, nil } -func golistargs_fallback(cfg *rawConfig, words []string) []string { +func golistargs_fallback(cfg *raw.Config, words []string) []string { fullargs := []string{"list", "-e", "-json"} fullargs = append(fullargs, cfg.Flags...) fullargs = append(fullargs, "--") diff --git a/go/packages/packages.go b/go/packages/packages.go index 858e554a..96641ee3 100644 --- a/go/packages/packages.go +++ b/go/packages/packages.go @@ -17,10 +17,9 @@ import ( "os" "sync" - "path/filepath" - "strings" - "golang.org/x/tools/go/gcexportdata" + "golang.org/x/tools/go/packages/golist" + "golang.org/x/tools/go/packages/raw" ) // A LoadMode specifies the amount of detail to return when loading packages. @@ -143,7 +142,23 @@ type Config struct { // Load and returns the Go packages named by the given patterns. func Load(cfg *Config, patterns ...string) ([]*Package, error) { l := newLoader(cfg) - return l.load(patterns...) + rawCfg := newRawConfig(&l.Config) + roots, pkgs, err := loadRaw(l.Context, rawCfg, patterns...) + if err != nil { + return nil, err + } + return l.loadFrom(roots, pkgs...) +} + +// loadRaw returns the raw Go packages named by the given patterns. +// This is a low level API, in general you should be using the Load function +// unless you have a very strong need for the raw data. +// It returns the packages identifiers that directly matched the patterns, the +// full set of packages requested (which may include the dependencies) and +// an error if the operation failed. +func loadRaw(ctx context.Context, cfg *raw.Config, patterns ...string) ([]string, []*raw.Package, error) { + //TODO: this is the seam at which we enable alternate build systems + return golist.LoadRaw(ctx, cfg, patterns...) } // A Package describes a single loaded Go package. @@ -213,7 +228,7 @@ type Package struct { // loaderPackage augments Package with state used during the loading phase type loaderPackage struct { - raw *rawPackage + raw *raw.Package *Package importErrors map[string]error // maps each bad import to its error loadOnce sync.Once @@ -265,82 +280,22 @@ func newLoader(cfg *Config) *loader { return ld } -func (ld *loader) load(patterns ...string) ([]*Package, error) { - if len(patterns) == 0 { - return nil, fmt.Errorf("no packages to load") +func newRawConfig(cfg *Config) *raw.Config { + rawCfg := &raw.Config{ + Dir: cfg.Dir, + Env: cfg.Env, + Flags: cfg.Flags, + Export: cfg.Mode > LoadImports && cfg.Mode < LoadAllSyntax, + Tests: cfg.Tests, + Deps: cfg.Mode >= LoadImports, } - - if ld.Dir == "" { - return nil, fmt.Errorf("failed to get working directory") + if rawCfg.Env == nil { + rawCfg.Env = os.Environ() } - - // Determine files requested in contains patterns - var containFiles []string - { - restPatterns := patterns[:0] - for _, pattern := range patterns { - if containFile := strings.TrimPrefix(pattern, "contains:"); containFile != pattern { - containFiles = append(containFiles, containFile) - } else { - restPatterns = append(restPatterns, pattern) - } - } - containFiles = absJoin(ld.Dir, containFiles) - patterns = restPatterns - } - - // Do the metadata query and partial build. - // TODO(adonovan): support alternative build systems at this seam. - rawCfg := newRawConfig(&ld.Config) - listfunc := golistPackages - // TODO(matloob): Patterns may now be empty, if it was solely comprised of contains: patterns. - // See if the extra process invocation can be avoided. - roots, list, err := listfunc(ld.Context, rawCfg, patterns...) - if _, ok := err.(GoTooOldError); ok { - listfunc = golistPackagesFallback - roots, list, err = listfunc(ld.Context, rawCfg, patterns...) - } - if err != nil { - return nil, err - } - - // Run go list for contains: patterns. - seenPkgs := make(map[string]bool) // for deduplication. different containing queries could produce same packages - if len(containFiles) > 0 { - for _, pkg := range list { - seenPkgs[pkg.ID] = true - } - } - for _, f := range containFiles { - // TODO(matloob): Do only one query per directory. - fdir := filepath.Dir(f) - rawCfg.Dir = fdir - _, cList, err := listfunc(ld.Context, rawCfg, ".") - if err != nil { - return nil, err - } - // Deduplicate and set deplist to set of packages requested files. - dedupedList := cList[:0] // invariant: only packages that haven't been seen before - for _, pkg := range cList { - if seenPkgs[pkg.ID] { - continue - } - seenPkgs[pkg.ID] = true - dedupedList = append(dedupedList, pkg) - for _, pkgFile := range pkg.GoFiles { - if filepath.Base(f) == filepath.Base(pkgFile) { - roots = append(roots, pkg.ID) - break - } - } - } - list = append(list, dedupedList...) - } - - return ld.loadFrom(roots, list...) + return rawCfg } -func (ld *loader) loadFrom(roots []string, list ...*rawPackage) ([]*Package, error) { +func (ld *loader) loadFrom(roots []string, list ...*raw.Package) ([]*Package, error) { if len(list) == 0 { return nil, fmt.Errorf("packages not found") } diff --git a/go/packages/raw.go b/go/packages/raw.go deleted file mode 100644 index f13ecae6..00000000 --- a/go/packages/raw.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package packages - -import ( - "os" -) - -// This file contains the structs needed at the seam between the packages -// loader and the underlying build tool - -// rawPackage is the serialized form of a package -type rawPackage struct { - // ID is a unique identifier for a package. - // This is the same as Package.ID - ID string - // Name is the package name as it appears in the package source code. - // This is the same as Package.name - Name string `json:",omitempty"` - // This is the package path as used in the export data. - // This is used to map entries in the export data back to the package they - // come from. - // This is not currently exposed in Package. - PkgPath string `json:",omitempty"` - // Imports maps import paths appearing in the package's Go source files - // to corresponding package identifiers. - // This is similar to Package.Imports, but maps to the ID rather than the - // package itself. - Imports map[string]string `json:",omitempty"` - // Export is the absolute path to a file containing the export data for the - // package. - // This is not currently exposed in Package. - Export string `json:",omitempty"` - // GoFiles lists the absolute file paths of the package's Go source files. - // This is the same as Package.GoFiles - GoFiles []string `json:",omitempty"` - // OtherFiles lists the absolute file paths of the package's non-Go source - // files. - // This is the same as Package.OtherFiles - OtherFiles []string `json:",omitempty"` - // DepOnly marks a package that is in a list because it was a dependency. - // It is used to find the roots when constructing a graph from a package list. - // This is not exposed in Package. - DepOnly bool `json:",omitempty"` -} - -// rawConfig specifies details about what raw package information is needed -// and how the underlying build tool should load package data. -type rawConfig struct { - Dir string - Env []string - Flags []string - Export bool - Tests bool - Deps bool -} - -func newRawConfig(cfg *Config) *rawConfig { - rawCfg := &rawConfig{ - Dir: cfg.Dir, - Env: cfg.Env, - Flags: cfg.Flags, - Export: cfg.Mode > LoadImports && cfg.Mode < LoadAllSyntax, - Tests: cfg.Tests, - Deps: cfg.Mode >= LoadImports, - } - if rawCfg.Env == nil { - rawCfg.Env = os.Environ() - } - return rawCfg -} diff --git a/go/packages/raw/raw.go b/go/packages/raw/raw.go new file mode 100644 index 00000000..413ee866 --- /dev/null +++ b/go/packages/raw/raw.go @@ -0,0 +1,104 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package raw is the experimental API to raw package information. + +NOTE: THIS PACKAGE IS NOT YET READY FOR WIDESPREAD USE: + - The interface is not yet stable. + - We reserve the right to add and remove fields and change the command line args. + - This may remain unstable even after the higher level x/tools/go/packages API is stable. + +This package is used by x/tools/go/packages to provide the low level raw +information about file layout. +It should not be needed unless you are attempting to implement a new source of +data for the packages API, all tools should interact only with the packages API. +*/ +package raw + +// Package is the raw serialized form of a packages.Package +type Package struct { + // ID is a unique identifier for a package, + // in a syntax provided by the underlying build system. + // + // Because the syntax varies based on the build system, + // clients should treat IDs as opaque and not attempt to + // interpret them. + ID string + + // Name is the package name as it appears in the package source code. + Name string `json:",omitempty"` + + // This is the package path as used by the types package. + // This is used to map entries in the export data back to the package they + // come from. + PkgPath string `json:",omitempty"` + + // Imports maps import paths appearing in the package's Go source files + // to corresponding package identifiers. + Imports map[string]string `json:",omitempty"` + + // Export is the absolute path to a file containing the export data for the + // package. + Export string `json:",omitempty"` + + // GoFiles lists the absolute file paths of the package's Go source files. + GoFiles []string `json:",omitempty"` + + // CompiledGoFiles lists the absolute file paths of the package's source + // files that were handed to the compiler. + // This is allowed to be different to GoFiles in the presence of files that + // were automatically modified or processed before compilation. + CompiledGoFiles []string `json:",omitempty"` + + // OtherFiles lists the absolute file paths of the package's non-Go source + // files, including assembly, C, C++, Fortran, Objective-C, SWIG, and so on. + OtherFiles []string `json:",omitempty"` +} + +// Config specifies details about what raw package information is needed +// and how the underlying build tool should load package data. +type Config struct { + // Dir is the directory in which to run the build system tool + // that provides information about the packages. + // If Dir is empty, the tool is run in the current directory. + Dir string + + // Env is the environment to use when invoking the build system tool. + // If Env is nil, the current environment is used. + // Like in os/exec's Cmd, only the last value in the slice for + // each environment key is used. To specify the setting of only + // a few variables, append to the current environment, as in: + // + // opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386") + // + Env []string + + // Flags is a list of command-line flags to be passed through to + // the underlying query tool. + Flags []string + + // Export controls whether the raw packages must contain the export + // data file. + Export bool + + // If Tests is set, the loader includes not just the packages + // matching a particular pattern but also any related test packages, + // including test-only variants of the package and the test executable. + // + // For example, when using the go command, loading "fmt" with Tests=true + // returns four packages, with IDs "fmt" (the standard package), + // "fmt [fmt.test]" (the package as compiled for the test), + // "fmt_test" (the test functions from source files in package fmt_test), + // and "fmt.test" (the test binary). + // + // In build systems with explicit names for tests, + // setting Tests may have no effect. + Tests bool + + // If Deps is set, the loader will include the full dependency graph. + // Packages that are only in the results because of Deps will have DepOnly + // set on them. + Deps bool +}