close_issue_1
/
tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

go/packages: extract the raw API to its own package 

We want to make it public to enable external sources of information
(other than go list), but we don't want to clutter the main
go/packages API with things that are intended for implementers only.
We also want to move the two API's from experimental to stable at
different times.
We also need to split the API because the golist implementation has to depend on
the raw structs, and then the main packages.Load has to depend on the golist
implementation.

The golist functionality is now in it's own package, and provides a Load method
to return the raw structures. This can be re-used by more complex raw providers.

Change-Id: I767af216b0d1ad9391e613cbf24eb2db5f503369
Reviewed-on: https://go-review.googlesource.com/126718
Run-TryBot: Ian Cottrell <iancottrell@google.com>
Reviewed-by: Michael Matloob <matloob@golang.org>
This commit is contained in:
Ian Cottrell 2018-07-30 14:10:18 -04:00 committed by Michael Matloob
parent 7ebc57ba19
commit 8149a73649
5 changed files with 243 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
go/packages/golist.go → go/packages/golist/golist.go
View File

@ -2,9 +2,8 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package packages
// This file defines the "go list" implementation of the Packages metadata query.
// Package golist defines the "go list" implementation of the Packages metadata query.
package golist
import (
"bytes"
@ -16,18 +15,95 @@ import (
"os/exec"
"path/filepath"
"strings"
"golang.org/x/tools/go/packages/raw"
)
// A GoTooOldError reports that the go command
// found by exec.LookPath does not contain the necessary
// support to be used with go/packages.
// Currently, go/packages requires Go 1.11 or later.
// (We intend to issue a point release for Go 1.10
// so that go/packages can be used with updated Go 1.10 systems too.)
type GoTooOldError struct {
// A goTooOldError reports that the go command
// found by exec.LookPath is too old to use the new go list behavior.
type goTooOldError struct {
error
}
// LoadRaw and returns the raw Go packages named by the given patterns.
// This is a low level API, in general you should be using the packages.Load
// unless you have a very strong need for the raw data, and you know that you
// are using conventional go layout as supported by `go list`
// It returns the packages identifiers that directly matched the patterns, the
// full set of packages requested (which may include the dependencies) and
// an error if the operation failed.
func LoadRaw(ctx context.Context, cfg *raw.Config, patterns ...string) ([]string, []*raw.Package, error) {
if len(patterns) == 0 {
return nil, nil, fmt.Errorf("no packages to load")
}
if cfg == nil {
return nil, nil, fmt.Errorf("Load must be passed a valid Config")
}
if cfg.Dir == "" {
return nil, nil, fmt.Errorf("Config does not have a working directory")
}
// Determine files requested in contains patterns
var containFiles []string
{
restPatterns := patterns[:0]
for _, pattern := range patterns {
if containFile := strings.TrimPrefix(pattern, "contains:"); containFile != pattern {
containFiles = append(containFiles, containFile)
} else {
restPatterns = append(restPatterns, pattern)
}
}
containFiles = absJoin(cfg.Dir, containFiles)
patterns = restPatterns
}
listfunc := golistPackages
// TODO(matloob): Patterns may now be empty, if it was solely comprised of contains: patterns.
// See if the extra process invocation can be avoided.
roots, pkgs, err := listfunc(ctx, cfg, patterns...)
if _, ok := err.(goTooOldError); ok {
listfunc = golistPackagesFallback
roots, pkgs, err = listfunc(ctx, cfg, patterns...)
}
if err != nil {
return nil, nil, err
}
// Run go list for contains: patterns.
seenPkgs := make(map[string]bool) // for deduplication. different containing queries could produce same packages
if len(containFiles) > 0 {
for _, pkg := range pkgs {
seenPkgs[pkg.ID] = true
}
}
for _, f := range containFiles {
// TODO(matloob): Do only one query per directory.
fdir := filepath.Dir(f)
cfg.Dir = fdir
_, cList, err := listfunc(ctx, cfg, ".")
if err != nil {
return nil, nil, err
}
// Deduplicate and set deplist to set of packages requested files.
dedupedList := cList[:0] // invariant: only packages that haven't been seen before
for _, pkg := range cList {
if seenPkgs[pkg.ID] {
continue
}
seenPkgs[pkg.ID] = true
dedupedList = append(dedupedList, pkg)
for _, pkgFile := range pkg.GoFiles {
if filepath.Base(f) == filepath.Base(pkgFile) {
roots = append(roots, pkg.ID)
break
}
}
}
pkgs = append(pkgs, dedupedList...)
}
return roots, pkgs, nil
}
// Fields must match go list;
// see $GOROOT/src/cmd/go/internal/load/pkg.go.
type jsonPackage struct {
@ -53,7 +129,7 @@ type jsonPackage struct {
// golistPackages uses the "go list" command to expand the
// pattern words and return metadata for the specified packages.
// dir may be "" and env may be nil, as per os/exec.Command.
func golistPackages(ctx context.Context, cfg *rawConfig, words ...string) ([]string, []*rawPackage, error) {
func golistPackages(ctx context.Context, cfg *raw.Config, words ...string) ([]string, []*raw.Package, error) {
// go list uses the following identifiers in ImportPath and Imports:
//
// "p" -- importable package or main (command)
@ -72,9 +148,9 @@ func golistPackages(ctx context.Context, cfg *rawConfig, words ...string) ([]str
if err != nil {
return nil, nil, err
}
// Decode the JSON and convert it to rawPackage form.
// Decode the JSON and convert it to Package form.
var roots []string
var result []*rawPackage
var result []*raw.Package
for dec := json.NewDecoder(buf); dec.More(); {
p := new(jsonPackage)
if err := dec.Decode(p); err != nil {
@ -153,7 +229,7 @@ func golistPackages(ctx context.Context, cfg *rawConfig, words ...string) ([]str
imports[id] = id // identity import
}
pkg := &rawPackage{
pkg := &raw.Package{
ID: id,
Name: p.Name,
GoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles),
@ -184,7 +260,7 @@ func absJoin(dir string, fileses ...[]string) (res []string) {
return res
}
func golistargs(cfg *rawConfig, words []string) []string {
func golistargs(cfg *raw.Config, words []string) []string {
fullargs := []string{
"list", "-e", "-json", "-cgo=true",
fmt.Sprintf("-test=%t", cfg.Tests),
@ -198,7 +274,7 @@ func golistargs(cfg *rawConfig, words []string) []string {
}
// golist returns the JSON-encoded result of a "go list args..." query.
func golist(ctx context.Context, cfg *rawConfig, args []string) (*bytes.Buffer, error) {
func golist(ctx context.Context, cfg *raw.Config, args []string) (*bytes.Buffer, error) {
out := new(bytes.Buffer)
cmd := exec.CommandContext(ctx, "go", args...)
cmd.Env = cfg.Env
@ -216,7 +292,7 @@ func golist(ctx context.Context, cfg *rawConfig, args []string) (*bytes.Buffer,
// Old go list?
if strings.Contains(fmt.Sprint(cmd.Stderr), "flag provided but not defined") {
return nil, GoTooOldError{fmt.Errorf("unsupported version of go list: %s: %s", exitErr, cmd.Stderr)}
return nil, goTooOldError{fmt.Errorf("unsupported version of go list: %s: %s", exitErr, cmd.Stderr)}
}
// Export mode entails a build.

23
go/packages/golist_fallback.go → go/packages/golist/golist_fallback.go
View File

@ -1,10 +1,15 @@
package packages
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package golist
import (
"context"
"encoding/json"
"fmt"
"golang.org/x/tools/go/packages/raw"
"golang.org/x/tools/imports"
)
@ -19,13 +24,13 @@ import (
// TODO(matloob): Support cgo. Copy code from the loader that runs cgo.
func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string) ([]string, []*rawPackage, error) {
func golistPackagesFallback(ctx context.Context, cfg *raw.Config, words ...string) ([]string, []*raw.Package, error) {
original, deps, err := getDeps(ctx, cfg, words...)
if err != nil {
return nil, nil, err
}
var result []*rawPackage
var result []*raw.Package
var roots []string
addPackage := func(p *jsonPackage) {
if p.Name == "" {
@ -59,7 +64,7 @@ func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string
if isRoot {
roots = append(roots, id)
}
result = append(result, &rawPackage{
result = append(result, &raw.Package{
ID: id,
Name: p.Name,
GoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles),
@ -73,7 +78,7 @@ func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string
if isRoot {
roots = append(roots, testID)
}
result = append(result, &rawPackage{
result = append(result, &raw.Package{
ID: testID,
Name: p.Name,
GoFiles: absJoin(p.Dir, p.GoFiles, p.TestGoFiles, p.CgoFiles),
@ -87,7 +92,7 @@ func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string
if isRoot {
roots = append(roots, xtestID)
}
result = append(result, &rawPackage{
result = append(result, &raw.Package{
ID: xtestID,
Name: p.Name + "_test",
GoFiles: absJoin(p.Dir, p.XTestGoFiles),
@ -110,7 +115,7 @@ func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string
return nil, nil, err
}
// Decode the JSON and convert it to rawPackage form.
// Decode the JSON and convert it to Package form.
for dec := json.NewDecoder(buf); dec.More(); {
p := new(jsonPackage)
if err := dec.Decode(p); err != nil {
@ -124,7 +129,7 @@ func golistPackagesFallback(ctx context.Context, cfg *rawConfig, words ...string
}
// getDeps runs an initial go list to determine all the dependency packages.
func getDeps(ctx context.Context, cfg *rawConfig, words ...string) (originalSet map[string]*jsonPackage, deps []string, err error) {
func getDeps(ctx context.Context, cfg *raw.Config, words ...string) (originalSet map[string]*jsonPackage, deps []string, err error) {
buf, err := golist(ctx, cfg, golistargs_fallback(cfg, words))
if err != nil {
return nil, nil, err
@ -156,7 +161,7 @@ func getDeps(ctx context.Context, cfg *rawConfig, words ...string) (originalSet
return originalSet, deps, nil
}
func golistargs_fallback(cfg *rawConfig, words []string) []string {
func golistargs_fallback(cfg *raw.Config, words []string) []string {
fullargs := []string{"list", "-e", "-json"}
fullargs = append(fullargs, cfg.Flags...)
fullargs = append(fullargs, "--")

113
go/packages/packages.go
View File

@ -17,10 +17,9 @@ import (
"os"
"sync"
"path/filepath"
"strings"
"golang.org/x/tools/go/gcexportdata"
"golang.org/x/tools/go/packages/golist"
"golang.org/x/tools/go/packages/raw"
)
// A LoadMode specifies the amount of detail to return when loading packages.
@ -143,7 +142,23 @@ type Config struct {
// Load and returns the Go packages named by the given patterns.
func Load(cfg *Config, patterns ...string) ([]*Package, error) {
l := newLoader(cfg)
return l.load(patterns...)
rawCfg := newRawConfig(&l.Config)
roots, pkgs, err := loadRaw(l.Context, rawCfg, patterns...)
if err != nil {
return nil, err
}
return l.loadFrom(roots, pkgs...)
}
// loadRaw returns the raw Go packages named by the given patterns.
// This is a low level API, in general you should be using the Load function
// unless you have a very strong need for the raw data.
// It returns the packages identifiers that directly matched the patterns, the
// full set of packages requested (which may include the dependencies) and
// an error if the operation failed.
func loadRaw(ctx context.Context, cfg *raw.Config, patterns ...string) ([]string, []*raw.Package, error) {
//TODO: this is the seam at which we enable alternate build systems
return golist.LoadRaw(ctx, cfg, patterns...)
}
// A Package describes a single loaded Go package.
@ -213,7 +228,7 @@ type Package struct {
// loaderPackage augments Package with state used during the loading phase
type loaderPackage struct {
raw *rawPackage
raw *raw.Package
*Package
importErrors map[string]error // maps each bad import to its error
loadOnce sync.Once
@ -265,82 +280,22 @@ func newLoader(cfg *Config) *loader {
return ld
}
func (ld *loader) load(patterns ...string) ([]*Package, error) {
if len(patterns) == 0 {
return nil, fmt.Errorf("no packages to load")
func newRawConfig(cfg *Config) *raw.Config {
rawCfg := &raw.Config{
Dir: cfg.Dir,
Env: cfg.Env,
Flags: cfg.Flags,
Export: cfg.Mode > LoadImports && cfg.Mode < LoadAllSyntax,
Tests: cfg.Tests,
Deps: cfg.Mode >= LoadImports,
}
if rawCfg.Env == nil {
rawCfg.Env = os.Environ()
}
return rawCfg
}
if ld.Dir == "" {
return nil, fmt.Errorf("failed to get working directory")
}
// Determine files requested in contains patterns
var containFiles []string
{
restPatterns := patterns[:0]
for _, pattern := range patterns {
if containFile := strings.TrimPrefix(pattern, "contains:"); containFile != pattern {
containFiles = append(containFiles, containFile)
} else {
restPatterns = append(restPatterns, pattern)
}
}
containFiles = absJoin(ld.Dir, containFiles)
patterns = restPatterns
}
// Do the metadata query and partial build.
// TODO(adonovan): support alternative build systems at this seam.
rawCfg := newRawConfig(&ld.Config)
listfunc := golistPackages
// TODO(matloob): Patterns may now be empty, if it was solely comprised of contains: patterns.
// See if the extra process invocation can be avoided.
roots, list, err := listfunc(ld.Context, rawCfg, patterns...)
if _, ok := err.(GoTooOldError); ok {
listfunc = golistPackagesFallback
roots, list, err = listfunc(ld.Context, rawCfg, patterns...)
}
if err != nil {
return nil, err
}
// Run go list for contains: patterns.
seenPkgs := make(map[string]bool) // for deduplication. different containing queries could produce same packages
if len(containFiles) > 0 {
for _, pkg := range list {
seenPkgs[pkg.ID] = true
}
}
for _, f := range containFiles {
// TODO(matloob): Do only one query per directory.
fdir := filepath.Dir(f)
rawCfg.Dir = fdir
_, cList, err := listfunc(ld.Context, rawCfg, ".")
if err != nil {
return nil, err
}
// Deduplicate and set deplist to set of packages requested files.
dedupedList := cList[:0] // invariant: only packages that haven't been seen before
for _, pkg := range cList {
if seenPkgs[pkg.ID] {
continue
}
seenPkgs[pkg.ID] = true
dedupedList = append(dedupedList, pkg)
for _, pkgFile := range pkg.GoFiles {
if filepath.Base(f) == filepath.Base(pkgFile) {
roots = append(roots, pkg.ID)
break
}
}
}
list = append(list, dedupedList...)
}
return ld.loadFrom(roots, list...)
}
func (ld *loader) loadFrom(roots []string, list ...*rawPackage) ([]*Package, error) {
func (ld *loader) loadFrom(roots []string, list ...*raw.Package) ([]*Package, error) {
if len(list) == 0 {
return nil, fmt.Errorf("packages not found")
}

73
go/packages/raw.go
View File

@ -1,73 +0,0 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package packages
import (
"os"
)
// This file contains the structs needed at the seam between the packages
// loader and the underlying build tool
// rawPackage is the serialized form of a package
type rawPackage struct {
// ID is a unique identifier for a package.
// This is the same as Package.ID
ID string
// Name is the package name as it appears in the package source code.
// This is the same as Package.name
Name string `json:",omitempty"`
// This is the package path as used in the export data.
// This is used to map entries in the export data back to the package they
// come from.
// This is not currently exposed in Package.
PkgPath string `json:",omitempty"`
// Imports maps import paths appearing in the package's Go source files
// to corresponding package identifiers.
// This is similar to Package.Imports, but maps to the ID rather than the
// package itself.
Imports map[string]string `json:",omitempty"`
// Export is the absolute path to a file containing the export data for the
// package.
// This is not currently exposed in Package.
Export string `json:",omitempty"`
// GoFiles lists the absolute file paths of the package's Go source files.
// This is the same as Package.GoFiles
GoFiles []string `json:",omitempty"`
// OtherFiles lists the absolute file paths of the package's non-Go source
// files.
// This is the same as Package.OtherFiles
OtherFiles []string `json:",omitempty"`
// DepOnly marks a package that is in a list because it was a dependency.
// It is used to find the roots when constructing a graph from a package list.
// This is not exposed in Package.
DepOnly bool `json:",omitempty"`
}
// rawConfig specifies details about what raw package information is needed
// and how the underlying build tool should load package data.
type rawConfig struct {
Dir string
Env []string
Flags []string
Export bool
Tests bool
Deps bool
}
func newRawConfig(cfg *Config) *rawConfig {
rawCfg := &rawConfig{
Dir: cfg.Dir,
Env: cfg.Env,
Flags: cfg.Flags,
Export: cfg.Mode > LoadImports && cfg.Mode < LoadAllSyntax,
Tests: cfg.Tests,
Deps: cfg.Mode >= LoadImports,
}
if rawCfg.Env == nil {
rawCfg.Env = os.Environ()
}
return rawCfg
}

104
go/packages/raw/raw.go Normal file
View File

@ -0,0 +1,104 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package raw is the experimental API to raw package information.
NOTE: THIS PACKAGE IS NOT YET READY FOR WIDESPREAD USE:
- The interface is not yet stable.
- We reserve the right to add and remove fields and change the command line args.
- This may remain unstable even after the higher level x/tools/go/packages API is stable.
This package is used by x/tools/go/packages to provide the low level raw
information about file layout.
It should not be needed unless you are attempting to implement a new source of
data for the packages API, all tools should interact only with the packages API.
*/
package raw
// Package is the raw serialized form of a packages.Package
type Package struct {
// ID is a unique identifier for a package,
// in a syntax provided by the underlying build system.
//
// Because the syntax varies based on the build system,
// clients should treat IDs as opaque and not attempt to
// interpret them.
ID string
// Name is the package name as it appears in the package source code.
Name string `json:",omitempty"`
// This is the package path as used by the types package.
// This is used to map entries in the export data back to the package they
// come from.
PkgPath string `json:",omitempty"`
// Imports maps import paths appearing in the package's Go source files
// to corresponding package identifiers.
Imports map[string]string `json:",omitempty"`
// Export is the absolute path to a file containing the export data for the
// package.
Export string `json:",omitempty"`
// GoFiles lists the absolute file paths of the package's Go source files.
GoFiles []string `json:",omitempty"`
// CompiledGoFiles lists the absolute file paths of the package's source
// files that were handed to the compiler.
// This is allowed to be different to GoFiles in the presence of files that
// were automatically modified or processed before compilation.
CompiledGoFiles []string `json:",omitempty"`
// OtherFiles lists the absolute file paths of the package's non-Go source
// files, including assembly, C, C++, Fortran, Objective-C, SWIG, and so on.
OtherFiles []string `json:",omitempty"`
}
// Config specifies details about what raw package information is needed
// and how the underlying build tool should load package data.
type Config struct {
// Dir is the directory in which to run the build system tool
// that provides information about the packages.
// If Dir is empty, the tool is run in the current directory.
Dir string
// Env is the environment to use when invoking the build system tool.
// If Env is nil, the current environment is used.
// Like in os/exec's Cmd, only the last value in the slice for
// each environment key is used. To specify the setting of only
// a few variables, append to the current environment, as in:
//
// opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386")
//
Env []string
// Flags is a list of command-line flags to be passed through to
// the underlying query tool.
Flags []string
// Export controls whether the raw packages must contain the export
// data file.
Export bool
// If Tests is set, the loader includes not just the packages
// matching a particular pattern but also any related test packages,
// including test-only variants of the package and the test executable.
//
// For example, when using the go command, loading "fmt" with Tests=true
// returns four packages, with IDs "fmt" (the standard package),
// "fmt [fmt.test]" (the package as compiled for the test),
// "fmt_test" (the test functions from source files in package fmt_test),
// and "fmt.test" (the test binary).
//
// In build systems with explicit names for tests,
// setting Tests may have no effect.
Tests bool
// If Deps is set, the loader will include the full dependency graph.
// Packages that are only in the results because of Deps will have DepOnly
// set on them.
Deps bool
}