[release-branch.go1.9] godoc: don't try to follow all symlinks

Revert https://golang.org/cl/45096.

Original change description:
    godoc: follow symbolic links to folders in GOROOT

    Directory walking in godoc relies on ReadDir which returns the result
    of os.Lstat.

    Instead make the the OS VFS's ReadDir use os.Stat on symlinks before
    returning.

Updates golang/go#15049
Fixes golang/go#21061

Change-Id: Ieaa7923d85842f3da5696a7f46134d16407dae66
Reviewed-on: https://go-review.googlesource.com/53634
Run-TryBot: Ian Lance Taylor <iant@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Reviewed-on: https://go-review.googlesource.com/57870
Reviewed-by: Russ Cox <rsc@golang.org>

CONTRIBUTING.md

@@ -4,15 +4,16 @@ Go is an open source project.
 
 It is the work of hundreds of contributors. We appreciate your help!
 
+
 ## Filing issues
 
 When [filing an issue](https://golang.org/issue/new), make sure to answer these five questions:
 
-1.  What version of Go are you using (`go version`)?
+1. What version of Go are you using (`go version`)?
-2.  What operating system and processor architecture are you using?
+2. What operating system and processor architecture are you using?
-3.  What did you do?
+3. What did you do?
-4.  What did you expect to see?
+4. What did you expect to see?
-5.  What did you see instead?
+5. What did you see instead?
 
 General questions should go to the [golang-nuts mailing list](https://groups.google.com/group/golang-nuts) instead of the issue tracker.
 The gophers there will answer or ask you to file an issue if you've tripped over a bug.
@@ -22,5 +23,9 @@ The gophers there will answer or ask you to file an issue if you've tripped over
 Please read the [Contribution Guidelines](https://golang.org/doc/contribute.html)
 before sending patches.
 
+**We do not accept GitHub pull requests**
+(we use [Gerrit](https://code.google.com/p/gerrit/) instead for code review).
+
 Unless otherwise noted, the Go source files are distributed under
 the BSD-style license found in the LICENSE file.
+

README

@@ -0,0 +1,10 @@
+This subrepository holds the source for various packages and tools that support
+the Go programming language.
+
+Some of the tools, godoc and vet for example, are included in binary Go distributions.
+Others, including the Go guru and the test coverage tool, can be fetched with "go get".
+
+Packages include a type-checker for Go and an implementation of the
+Static Single Assignment form (SSA) representation for Go programs.
+
+To submit changes to this repository, see http://golang.org/doc/contribute.html.

README.md

@@ -1,27 +0,0 @@
-# Go Tools
-
-This subrepository holds the source for various packages and tools that support
-the Go programming language.
-
-Some of the tools, `godoc` and `vet` for example, are included in binary Go
-distributions.
-
-Others, including the Go `guru` and the test coverage tool, can be fetched with
-`go get`.
-
-Packages include a type-checker for Go and an implementation of the
-Static Single Assignment form (SSA) representation for Go programs.
-
-## Download/Install
-
-The easiest way to install is to run `go get -u golang.org/x/tools/...`. You can
-also manually git clone the repository to `$GOPATH/src/golang.org/x/tools`.
-
-## Report Issues / Send Patches
-
-This repository uses Gerrit for code changes. To learn how to submit changes to
-this repository, see https://golang.org/doc/contribute.html.
-
-The main issue tracker for the tools repository is located at
-https://github.com/golang/go/issues. Prefix your issue with "x/tools/(your
-subdir):" in the subject line, so it is easy to find.

blog/blog.go

@@ -24,14 +24,7 @@ import (
 	"golang.org/x/tools/present"
 )
 
-var (
+var validJSONPFunc = regexp.MustCompile(`(?i)^[a-z_][a-z0-9_.]*$`)
-	validJSONPFunc = regexp.MustCompile(`(?i)^[a-z_][a-z0-9_.]*$`)
-	// used to serve relative paths when ServeLocalLinks is enabled.
-	golangOrgAbsLinkReplacer = strings.NewReplacer(
-		`href="https://golang.org/pkg`, `href="/pkg`,
-		`href="https://golang.org/cmd`, `href="/cmd`,
-	)
-)
 
 // Config specifies Server configuration values.
 type Config struct {
@@ -47,8 +40,7 @@ type Config struct {
 	FeedArticles int    // Articles to include in Atom and JSON feeds.
 	FeedTitle    string // The title of the Atom XML feed
 
-	PlayEnabled     bool
+	PlayEnabled bool
-	ServeLocalLinks bool // rewrite golang.org/{pkg,cmd} links to host-less, relative paths.
 }
 
 // Doc represents an article adorned with presentation data.
@@ -81,17 +73,10 @@ type Server struct {
 func NewServer(cfg Config) (*Server, error) {
 	present.PlayEnabled = cfg.PlayEnabled
 
-	if notExist(cfg.TemplatePath) {
-		return nil, fmt.Errorf("template directory not found: %s", cfg.TemplatePath)
-	}
 	root := filepath.Join(cfg.TemplatePath, "root.tmpl")
 	parse := func(name string) (*template.Template, error) {
-		path := filepath.Join(cfg.TemplatePath, name)
-		if notExist(path) {
-			return nil, fmt.Errorf("template %s was not found in %s", name, cfg.TemplatePath)
-		}
 		t := template.New("").Funcs(funcMap)
-		return t.ParseFiles(root, path)
+		return t.ParseFiles(root, filepath.Join(cfg.TemplatePath, name))
 	}
 
 	s := &Server{cfg: cfg}
@@ -199,8 +184,8 @@ func (s *Server) loadDocs(root string) error {
 		if err != nil {
 			return err
 		}
-		var html bytes.Buffer
+		html := new(bytes.Buffer)
-		err = d.Render(&html, s.template.doc)
+		err = d.Render(html, s.template.doc)
 		if err != nil {
 			return err
 		}
@@ -425,18 +410,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		d.Doc = doc
 		t = s.template.article
 	}
-	var err error
+	err := t.ExecuteTemplate(w, "root", d)
-	if s.cfg.ServeLocalLinks {
-		var buf bytes.Buffer
-		err = t.ExecuteTemplate(&buf, "root", d)
-		if err != nil {
-			log.Println(err)
-			return
-		}
-		_, err = golangOrgAbsLinkReplacer.WriteString(w, buf.String())
-	} else {
-		err = t.ExecuteTemplate(w, "root", d)
-	}
 	if err != nil {
 		log.Println(err)
 	}
@@ -448,9 +422,3 @@ type docsByTime []*Doc
 func (s docsByTime) Len() int           { return len(s) }
 func (s docsByTime) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 func (s docsByTime) Less(i, j int) bool { return s[i].Time.After(s[j].Time) }
-
-// notExist reports whether the path exists or not.
-func notExist(path string) bool {
-	_, err := os.Stat(path)
-	return os.IsNotExist(err)
-}

blog/blog_test.go

@@ -1,44 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package blog
-
-import (
-	"bytes"
-	"testing"
-)
-
-func TestLinkRewrite(t *testing.T) {
-	tests := []struct {
-		input  string
-		output string
-	}{
-		{
-			`For instance, the <a href="https://golang.org/pkg/bytes/" target="_blank">bytes package</a> from the standard library exports the <code>Buffer</code> type.`,
-			`For instance, the <a href="/pkg/bytes/" target="_blank">bytes package</a> from the standard library exports the <code>Buffer</code> type.`},
-		{
-			`(The <a href="https://golang.org/cmd/gofmt/" target="_blank">gofmt command</a> has a <code>-r</code> flag that provides a syntax-aware search and replace, making large-scale refactoring easier.)`,
-			`(The <a href="/cmd/gofmt/" target="_blank">gofmt command</a> has a <code>-r</code> flag that provides a syntax-aware search and replace, making large-scale refactoring easier.)`,
-		},
-		{
-			`<a href="//golang.org/LICENSE">BSD license</a>.<br> <a href="//golang.org/doc/tos.html">Terms of Service</a> `,
-			`<a href="//golang.org/LICENSE">BSD license</a>.<br> <a href="//golang.org/doc/tos.html">Terms of Service</a> `,
-		},
-		{
-			`For instance, the <code>websocket</code> package from the <code>go.net</code> sub-repository has an import path of <code>&#34;golang.org/x/net/websocket&#34;</code>.`,
-			`For instance, the <code>websocket</code> package from the <code>go.net</code> sub-repository has an import path of <code>&#34;golang.org/x/net/websocket&#34;</code>.`,
-		},
-	}
-	for _, test := range tests {
-		var buf bytes.Buffer
-		_, err := golangOrgAbsLinkReplacer.WriteString(&buf, test.input)
-		if err != nil {
-			t.Errorf("unexpected error during replacing links. Got: %#v, Want: nil.\n", err)
-			continue
-		}
-		if got, want := buf.String(), test.output; got != want {
-			t.Errorf("WriteString(%q) = %q. Expected: %q", test.input, got, want)
-		}
-	}
-}

cmd/auth/authtest/authtest.go

@@ -1,231 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// authtest is a diagnostic tool for implementations of the GOAUTH protocol
-// described in https://golang.org/issue/26232.
-//
-// It accepts a single URL as an argument, and executes the GOAUTH protocol to
-// fetch and display the headers for that URL.
-//
-// CAUTION: authtest logs the GOAUTH responses, which may include user
-// credentials, to stderr. Do not post its output unless you are certain that
-// all of the credentials involved are fake!
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"net/textproto"
-	"net/url"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-)
-
-var v = flag.Bool("v", false, "if true, log GOAUTH responses to stderr")
-
-func main() {
-	log.SetFlags(log.LstdFlags | log.Lshortfile)
-	flag.Parse()
-	args := flag.Args()
-	if len(args) != 1 {
-		log.Fatalf("usage: [GOAUTH=CMD...] %s URL", filepath.Base(os.Args[0]))
-	}
-
-	resp := try(args[0], nil)
-	if resp.StatusCode == http.StatusOK {
-		return
-	}
-
-	resp = try(args[0], resp)
-	if resp.StatusCode != http.StatusOK {
-		os.Exit(1)
-	}
-}
-
-func try(url string, prev *http.Response) *http.Response {
-	req := new(http.Request)
-	if prev != nil {
-		*req = *prev.Request
-	} else {
-		var err error
-		req, err = http.NewRequest("HEAD", os.Args[1], nil)
-		if err != nil {
-			log.Fatal(err)
-		}
-	}
-
-goauth:
-	for _, argList := range strings.Split(os.Getenv("GOAUTH"), ";") {
-		// TODO(golang.org/issue/26849): If we escape quoted strings in GOFLAGS, use
-		// the same quoting here.
-		args := strings.Split(argList, " ")
-		if len(args) == 0 || args[0] == "" {
-			log.Fatalf("invalid or empty command in GOAUTH")
-		}
-
-		creds, err := getCreds(args, prev)
-		if err != nil {
-			log.Fatal(err)
-		}
-		for _, c := range creds {
-			if c.Apply(req) {
-				fmt.Fprintf(os.Stderr, "# request to %s\n", req.URL)
-				fmt.Fprintf(os.Stderr, "%s %s %s\n", req.Method, req.URL, req.Proto)
-				req.Header.Write(os.Stderr)
-				fmt.Fprintln(os.Stderr)
-				break goauth
-			}
-		}
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK && resp.StatusCode < 400 || resp.StatusCode > 500 {
-		log.Fatalf("unexpected status: %v", resp.Status)
-	}
-
-	fmt.Fprintf(os.Stderr, "# response from %s\n", resp.Request.URL)
-	formatHead(os.Stderr, resp)
-	return resp
-}
-
-func formatHead(out io.Writer, resp *http.Response) {
-	fmt.Fprintf(out, "%s %s\n", resp.Proto, resp.Status)
-	if err := resp.Header.Write(out); err != nil {
-		log.Fatal(err)
-	}
-	fmt.Fprintln(out)
-}
-
-type Cred struct {
-	URLPrefixes []*url.URL
-	Header      http.Header
-}
-
-func (c Cred) Apply(req *http.Request) bool {
-	if req.URL == nil {
-		return false
-	}
-	ok := false
-	for _, prefix := range c.URLPrefixes {
-		if prefix.Host == req.URL.Host &&
-			(req.URL.Path == prefix.Path ||
-				(strings.HasPrefix(req.URL.Path, prefix.Path) &&
-					(strings.HasSuffix(prefix.Path, "/") ||
-						req.URL.Path[len(prefix.Path)] == '/'))) {
-			ok = true
-			break
-		}
-	}
-	if !ok {
-		return false
-	}
-
-	for k, vs := range c.Header {
-		req.Header.Del(k)
-		for _, v := range vs {
-			req.Header.Add(k, v)
-		}
-	}
-	return true
-}
-
-func (c Cred) String() string {
-	var buf strings.Builder
-	for _, u := range c.URLPrefixes {
-		fmt.Fprintln(&buf, u)
-	}
-	buf.WriteString("\n")
-	c.Header.Write(&buf)
-	buf.WriteString("\n")
-	return buf.String()
-}
-
-func getCreds(args []string, resp *http.Response) ([]Cred, error) {
-	cmd := exec.Command(args[0], args[1:]...)
-	cmd.Stderr = os.Stderr
-
-	if resp != nil {
-		u := *resp.Request.URL
-		u.RawQuery = ""
-		cmd.Args = append(cmd.Args, u.String())
-	}
-
-	var head strings.Builder
-	if resp != nil {
-		formatHead(&head, resp)
-	}
-	cmd.Stdin = strings.NewReader(head.String())
-
-	fmt.Fprintf(os.Stderr, "# %s\n", strings.Join(cmd.Args, " "))
-	out, err := cmd.Output()
-	if err != nil {
-		return nil, fmt.Errorf("%s: %v", strings.Join(cmd.Args, " "), err)
-	}
-	os.Stderr.Write(out)
-	os.Stderr.WriteString("\n")
-
-	var creds []Cred
-	r := textproto.NewReader(bufio.NewReader(bytes.NewReader(out)))
-	line := 0
-readLoop:
-	for {
-		var prefixes []*url.URL
-		for {
-			prefix, err := r.ReadLine()
-			if err == io.EOF {
-				if len(prefixes) > 0 {
-					return nil, fmt.Errorf("line %d: %v", line, io.ErrUnexpectedEOF)
-				}
-				break readLoop
-			}
-			line++
-
-			if prefix == "" {
-				if len(prefixes) == 0 {
-					return nil, fmt.Errorf("line %d: unexpected newline", line)
-				}
-				break
-			}
-			u, err := url.Parse(prefix)
-			if err != nil {
-				return nil, fmt.Errorf("line %d: malformed URL: %v", line, err)
-			}
-			if u.Scheme != "https" {
-				return nil, fmt.Errorf("line %d: non-HTTPS URL %q", line, prefix)
-			}
-			if len(u.RawQuery) > 0 {
-				return nil, fmt.Errorf("line %d: unexpected query string in URL %q", line, prefix)
-			}
-			if len(u.Fragment) > 0 {
-				return nil, fmt.Errorf("line %d: unexpected fragment in URL %q", line, prefix)
-			}
-			prefixes = append(prefixes, u)
-		}
-
-		header, err := r.ReadMIMEHeader()
-		if err != nil {
-			return nil, fmt.Errorf("headers at line %d: %v", line, err)
-		}
-		if len(header) > 0 {
-			creds = append(creds, Cred{
-				URLPrefixes: prefixes,
-				Header:      http.Header(header),
-			})
-		}
-	}
-
-	return creds, nil
-}

cmd/auth/cookieauth/cookieauth.go

@@ -1,166 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// cookieauth uses a “Netscape cookie file” to implement the GOAUTH protocol
-// described in https://golang.org/issue/26232.
-// It expects the location of the file as the first command-line argument.
-//
-// Example GOAUTH usage:
-// 	export GOAUTH="cookieauth $(git config --get http.cookieFile)"
-//
-// See http://www.cookiecentral.com/faq/#3.5 for a description of the Netscape
-// cookie file format.
-package main
-
-import (
-	"bufio"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"net/http/cookiejar"
-	"net/url"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-	"unicode"
-)
-
-func main() {
-	if len(os.Args) < 2 {
-		fmt.Fprintf(os.Stderr, "usage: %s COOKIEFILE [URL]\n", os.Args[0])
-		os.Exit(2)
-	}
-
-	log.SetPrefix("cookieauth: ")
-
-	f, err := os.Open(os.Args[1])
-	if err != nil {
-		log.Fatalf("failed to read cookie file: %v\n", os.Args[1])
-		os.Exit(1)
-	}
-	defer f.Close()
-
-	var (
-		targetURL  *url.URL
-		targetURLs = map[string]*url.URL{}
-	)
-	if len(os.Args) == 3 {
-		targetURL, err = url.ParseRequestURI(os.Args[2])
-		if err != nil {
-			log.Fatalf("invalid request URI (%v): %q\n", err, os.Args[2])
-		}
-		targetURLs[targetURL.String()] = targetURL
-	} else if len(os.Args) > 3 {
-		// Extra arguments were passed: maybe the protocol was expanded?
-		// We don't know how to interpret the request, so ignore it.
-		return
-	}
-
-	entries, err := parseCookieFile(f.Name(), f)
-	if err != nil {
-		log.Fatalf("error reading cookie file: %v\n", f.Name())
-	}
-
-	jar, err := cookiejar.New(nil)
-	if err != nil {
-		log.Fatalf("failed to initialize cookie jar: %v\n", err)
-	}
-
-	for _, e := range entries {
-		u := &url.URL{
-			Scheme: "https",
-			Host:   e.Host,
-			Path:   e.Cookie.Path,
-		}
-
-		if targetURL == nil {
-			targetURLs[u.String()] = u
-		}
-
-		jar.SetCookies(u, []*http.Cookie{&e.Cookie})
-	}
-
-	for _, u := range targetURLs {
-		req := &http.Request{URL: u, Header: make(http.Header)}
-		for _, c := range jar.Cookies(req.URL) {
-			req.AddCookie(c)
-		}
-		fmt.Printf("%s\n\n", u)
-		req.Header.Write(os.Stdout)
-		fmt.Println()
-	}
-}
-
-type Entry struct {
-	Host   string
-	Cookie http.Cookie
-}
-
-// parseCookieFile parses a Netscape cookie file as described in
-// http://www.cookiecentral.com/faq/#3.5.
-func parseCookieFile(name string, r io.Reader) ([]*Entry, error) {
-	var entries []*Entry
-	s := bufio.NewScanner(r)
-	line := 0
-	for s.Scan() {
-		line++
-		text := strings.TrimSpace(s.Text())
-		if len(text) < 2 || (text[0] == '#' && unicode.IsSpace(rune(text[1]))) {
-			continue
-		}
-
-		e, err := parseCookieLine(text)
-		if err != nil {
-			log.Printf("%s:%d: %v\n", name, line, err)
-			continue
-		}
-		entries = append(entries, e)
-	}
-	return entries, s.Err()
-}
-
-func parseCookieLine(line string) (*Entry, error) {
-	f := strings.Fields(line)
-	if len(f) < 7 {
-		return nil, fmt.Errorf("found %d columns; want 7", len(f))
-	}
-
-	e := new(Entry)
-	c := &e.Cookie
-
-	if domain := f[0]; strings.HasPrefix(domain, "#HttpOnly_") {
-		c.HttpOnly = true
-		e.Host = strings.TrimPrefix(domain[10:], ".")
-	} else {
-		e.Host = strings.TrimPrefix(domain, ".")
-	}
-
-	isDomain, err := strconv.ParseBool(f[1])
-	if err != nil {
-		return nil, fmt.Errorf("non-boolean domain flag: %v", err)
-	}
-	if isDomain {
-		c.Domain = e.Host
-	}
-
-	c.Path = f[2]
-
-	c.Secure, err = strconv.ParseBool(f[3])
-	if err != nil {
-		return nil, fmt.Errorf("non-boolean secure flag: %v", err)
-	}
-
-	expiration, err := strconv.ParseInt(f[4], 10, 64)
-	if err != nil {
-		return nil, fmt.Errorf("malformed expiration: %v", err)
-	}
-	c.Expires = time.Unix(expiration, 0)
-
-	c.Name = f[5]
-	c.Value = f[6]
-
-	return e, nil
-}

cmd/auth/gitauth/gitauth.go

@@ -1,149 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// gitauth uses 'git credential' to implement the GOAUTH protocol described in
-// https://golang.org/issue/26232. It expects an absolute path to the working
-// directory for the 'git' command as the first command-line argument.
-//
-// Example GOAUTH usage:
-// 	export GOAUTH="gitauth $HOME"
-//
-// See https://git-scm.com/docs/gitcredentials or run 'man gitcredentials' for
-// information on how to configure 'git credential'.
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-)
-
-func main() {
-	if len(os.Args) < 2 || !filepath.IsAbs(os.Args[1]) {
-		fmt.Fprintf(os.Stderr, "usage: %s WORKDIR [URL]", os.Args[0])
-		os.Exit(2)
-	}
-
-	log.SetPrefix("gitauth: ")
-
-	if len(os.Args) != 3 {
-		// No explicit URL was passed on the command line, but 'git credential'
-		// provides no way to enumerate existing credentials.
-		// Wait for a request for a specific URL.
-		return
-	}
-
-	u, err := url.ParseRequestURI(os.Args[2])
-	if err != nil {
-		log.Fatalf("invalid request URI (%v): %q\n", err, os.Args[1])
-	}
-
-	var (
-		prefix     *url.URL
-		lastHeader http.Header
-		lastStatus = http.StatusUnauthorized
-	)
-	for lastStatus == http.StatusUnauthorized {
-		cmd := exec.Command("git", "credential", "fill")
-
-		// We don't want to execute a 'git' command in an arbitrary directory, since
-		// that opens up a number of config-injection attacks (for example,
-		// https://golang.org/issue/29230). Instead, we have the user configure a
-		// directory explicitly on the command line.
-		cmd.Dir = os.Args[1]
-
-		cmd.Stdin = strings.NewReader(fmt.Sprintf("url=%s\n", u))
-		cmd.Stderr = os.Stderr
-		out, err := cmd.Output()
-		if err != nil {
-			log.Fatalf("'git credential fill' failed: %v\n", err)
-		}
-
-		prefix = new(url.URL)
-		var username, password string
-		lines := strings.Split(string(out), "\n")
-		for _, line := range lines {
-			frags := strings.SplitN(line, "=", 2)
-			if len(frags) != 2 {
-				continue // Ignore unrecognized response lines.
-			}
-			switch strings.TrimSpace(frags[0]) {
-			case "protocol":
-				prefix.Scheme = frags[1]
-			case "host":
-				prefix.Host = frags[1]
-			case "path":
-				prefix.Path = frags[1]
-			case "username":
-				username = frags[1]
-			case "password":
-				password = frags[1]
-			case "url":
-				// Write to a local variable instead of updating prefix directly:
-				// if the url field is malformed, we don't want to invalidate
-				// information parsed from the protocol, host, and path fields.
-				u, err := url.ParseRequestURI(frags[1])
-				if err == nil {
-					prefix = u
-				} else {
-					log.Printf("malformed URL from 'git credential fill' (%v): %q\n", err, frags[1])
-					// Proceed anyway: we might be able to parse the prefix from other fields of the response.
-				}
-			}
-		}
-
-		// Double-check that the URL Git gave us is a prefix of the one we requested.
-		if !strings.HasPrefix(u.String(), prefix.String()) {
-			log.Fatalf("requested a credential for %q, but 'git credential fill' provided one for %q\n", u, prefix)
-		}
-
-		// Send a HEAD request to try to detect whether the credential is valid.
-		// If the user just typed in a correct password and has caching enabled,
-		// we don't want to nag them for it again the next time they run a 'go' command.
-		req, err := http.NewRequest("HEAD", u.String(), nil)
-		if err != nil {
-			log.Fatalf("internal error constructing HTTP HEAD request: %v\n", err)
-		}
-		req.SetBasicAuth(username, password)
-		lastHeader = req.Header
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			log.Printf("HTTPS HEAD request failed to connect: %v\n", err)
-			// Couldn't verify the credential, but we have no evidence that it is invalid either.
-			// Proceed, but don't update git's credential cache.
-			break
-		}
-		lastStatus = resp.StatusCode
-
-		if resp.StatusCode != http.StatusOK {
-			log.Printf("%s: %v %s\n", u, resp.StatusCode, http.StatusText(resp.StatusCode))
-		}
-
-		if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusUnauthorized {
-			// We learned something about the credential: it either worked or it was invalid.
-			// Approve or reject the credential (on a best-effort basis)
-			// so that the git credential helper can update its cache as appropriate.
-			action := "approve"
-			if resp.StatusCode != http.StatusOK {
-				action = "reject"
-			}
-			cmd = exec.Command("git", "credential", action)
-			cmd.Stderr = os.Stderr
-			cmd.Stdout = os.Stderr
-			cmd.Stdin = bytes.NewReader(out)
-			_ = cmd.Run()
-		}
-	}
-
-	// Write out the credential in the format expected by the 'go' command.
-	fmt.Printf("%s\n\n", prefix)
-	lastHeader.Write(os.Stdout)
-	fmt.Println()
-}

cmd/auth/netrcauth/netrcauth.go

@@ -1,123 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// netrcauth uses a .netrc file (or _netrc file on Windows) to implement the
-// GOAUTH protocol described in https://golang.org/issue/26232.
-// It expects the location of the file as the first command-line argument.
-//
-// Example GOAUTH usage:
-// 	export GOAUTH="netrcauth $HOME/.netrc"
-//
-// See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
-// or run 'man 5 netrc' for a description of the .netrc file format.
-package main
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"strings"
-)
-
-func main() {
-	if len(os.Args) < 2 {
-		fmt.Fprintf(os.Stderr, "usage: %s NETRCFILE [URL]", os.Args[0])
-		os.Exit(2)
-	}
-
-	log.SetPrefix("netrcauth: ")
-
-	if len(os.Args) != 2 {
-		// An explicit URL was passed on the command line, but netrcauth does not
-		// have any URL-specific output: it dumps the entire .netrc file at the
-		// first call.
-		return
-	}
-
-	path := os.Args[1]
-
-	data, err := ioutil.ReadFile(path)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return
-		}
-		log.Fatalf("failed to read %s: %v\n", path, err)
-	}
-
-	u := &url.URL{Scheme: "https"}
-	lines := parseNetrc(string(data))
-	for _, l := range lines {
-		u.Host = l.machine
-		fmt.Printf("%s\n\n", u)
-
-		req := &http.Request{Header: make(http.Header)}
-		req.SetBasicAuth(l.login, l.password)
-		req.Header.Write(os.Stdout)
-		fmt.Println()
-	}
-}
-
-// The following functions were extracted from src/cmd/go/internal/web2/web.go
-// as of https://golang.org/cl/161698.
-
-type netrcLine struct {
-	machine  string
-	login    string
-	password string
-}
-
-func parseNetrc(data string) []netrcLine {
-	// See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
-	// for documentation on the .netrc format.
-	var nrc []netrcLine
-	var l netrcLine
-	inMacro := false
-	for _, line := range strings.Split(data, "\n") {
-		if inMacro {
-			if line == "" {
-				inMacro = false
-			}
-			continue
-		}
-
-		f := strings.Fields(line)
-		i := 0
-		for ; i < len(f)-1; i += 2 {
-			// Reset at each "machine" token.
-			// “The auto-login process searches the .netrc file for a machine token
-			// that matches […]. Once a match is made, the subsequent .netrc tokens
-			// are processed, stopping when the end of file is reached or another
-			// machine or a default token is encountered.”
-			switch f[i] {
-			case "machine":
-				l = netrcLine{machine: f[i+1]}
-			case "default":
-				break
-			case "login":
-				l.login = f[i+1]
-			case "password":
-				l.password = f[i+1]
-			case "macdef":
-				// “A macro is defined with the specified name; its contents begin with
-				// the next .netrc line and continue until a null line (consecutive
-				// new-line characters) is encountered.”
-				inMacro = true
-			}
-			if l.machine != "" && l.login != "" && l.password != "" {
-				nrc = append(nrc, l)
-				l = netrcLine{}
-			}
-		}
-
-		if i < len(f) && f[i] == "default" {
-			// “There can be only one default token, and it must be after all machine tokens.”
-			break
-		}
-	}
-
-	return nrc
-}

cmd/bundle/main.go

@@ -105,8 +105,8 @@ var (
 	outputFile = flag.String("o", "", "write output to `file` (default standard output)")
 	dstPath    = flag.String("dst", "", "set destination import `path` (default taken from current directory)")
 	pkgName    = flag.String("pkg", "", "set destination package `name` (default taken from current directory)")
-	prefix     = flag.String("prefix", "&_", "set bundled identifier prefix to `p` (default is \"&_\", where & stands for the original name)")
+	prefix     = flag.String("prefix", "", "set bundled identifier prefix to `p` (default source package name + \"_\")")
-	underscore = flag.Bool("underscore", false, "rewrite golang.org/x/* to internal/x/* imports; temporary workaround for golang.org/issue/16333")
+	underscore = flag.Bool("underscore", false, "rewrite golang.org to golang_org in imports; temporary workaround for golang.org/issue/16333")
 
 	importMap = map[string]string{}
 )
@@ -203,8 +203,9 @@ func bundle(src, dst, dstpkg, prefix string) ([]byte, error) {
 	// Because there was a single Import call and Load succeeded,
 	// InitialPackages is guaranteed to hold the sole requested package.
 	info := lprog.InitialPackages()[0]
-	if strings.Contains(prefix, "&") {
+	if prefix == "" {
-		prefix = strings.Replace(prefix, "&", info.Files[0].Name.Name, -1)
+		pkgName := info.Files[0].Name.Name
+		prefix = pkgName + "_"
 	}
 
 	objsToUpdate := make(map[types.Object]bool)
@@ -298,7 +299,7 @@ func bundle(src, dst, dstpkg, prefix string) ([]byte, error) {
 				pkgStd[spec] = true
 			} else {
 				if *underscore {
-					spec = strings.Replace(spec, "golang.org/x/", "internal/x/", 1)
+					spec = strings.Replace(spec, "golang.org/", "golang_org/", 1)
 				}
 				pkgExt[spec] = true
 			}

cmd/callgraph/main.go

@@ -37,7 +37,7 @@ import (
 	"golang.org/x/tools/go/callgraph/cha"
 	"golang.org/x/tools/go/callgraph/rta"
 	"golang.org/x/tools/go/callgraph/static"
-	"golang.org/x/tools/go/packages"
+	"golang.org/x/tools/go/loader"
 	"golang.org/x/tools/go/pointer"
 	"golang.org/x/tools/go/ssa"
 	"golang.org/x/tools/go/ssa/ssautil"
@@ -67,7 +67,7 @@ const Usage = `callgraph: display the the call graph of a Go program.
 
 Usage:
 
-  callgraph [-algo=static|cha|rta|pta] [-test] [-format=...] package...
+  callgraph [-algo=static|cha|rta|pta] [-test] [-format=...] <args>...
 
 Flags:
 
@@ -114,10 +114,12 @@ Flags:
 
            Caller and Callee are *ssa.Function values, which print as
            "(*sync/atomic.Mutex).Lock", but other attributes may be
-           derived from them, e.g. Caller.Pkg.Pkg.Path yields the
+           derived from them, e.g. Caller.Pkg.Object.Path yields the
            import path of the enclosing package.  Consult the go/ssa
            API documentation for details.
 
+` + loader.FromArgsUsage + `
+
 Examples:
 
   Show the call graph of the trivial web server application:
@@ -126,7 +128,7 @@ Examples:
 
   Same, but show only the packages of each function:
 
-    callgraph -format '{{.Caller.Pkg.Pkg.Path}} -> {{.Callee.Pkg.Pkg.Path}}' \
+    callgraph -format '{{.Caller.Pkg.Object.Path}} -> {{.Callee.Pkg.Object.Path}}' \
       $GOROOT/src/net/http/triv.go | sort | uniq
 
   Show functions that make dynamic calls into the 'fmt' test package,
@@ -156,7 +158,7 @@ func init() {
 
 func main() {
 	flag.Parse()
-	if err := doCallgraph("", "", *algoFlag, *formatFlag, *testFlag, flag.Args()); err != nil {
+	if err := doCallgraph(&build.Default, *algoFlag, *formatFlag, *testFlag, flag.Args()); err != nil {
 		fmt.Fprintf(os.Stderr, "callgraph: %s\n", err)
 		os.Exit(1)
 	}
@@ -164,30 +166,28 @@ func main() {
 
 var stdout io.Writer = os.Stdout
 
-func doCallgraph(dir, gopath, algo, format string, tests bool, args []string) error {
+func doCallgraph(ctxt *build.Context, algo, format string, tests bool, args []string) error {
+	conf := loader.Config{Build: ctxt}
+
 	if len(args) == 0 {
 		fmt.Fprintln(os.Stderr, Usage)
 		return nil
 	}
 
-	cfg := &packages.Config{
+	// Use the initial packages from the command line.
-		Mode:  packages.LoadAllSyntax,
+	_, err := conf.FromArgs(args, tests)
-		Tests: tests,
-		Dir:   dir,
-	}
-	if gopath != "" {
-		cfg.Env = append(os.Environ(), "GOPATH="+gopath) // to enable testing
-	}
-	initial, err := packages.Load(cfg, args...)
 	if err != nil {
 		return err
 	}
-	if packages.PrintErrors(initial) > 0 {
+
-		return fmt.Errorf("packages contain errors")
+	// Load, parse and type-check the whole program.
+	iprog, err := conf.Load()
+	if err != nil {
+		return err
 	}
 
 	// Create and build SSA-form program representation.
-	prog, pkgs := ssautil.AllPackages(initial, 0)
+	prog := ssautil.CreateProgram(iprog, 0)
 	prog.Build()
 
 	// -- call graph construction ------------------------------------------
@@ -221,7 +221,7 @@ func doCallgraph(dir, gopath, algo, format string, tests bool, args []string) er
 			}
 		}
 
-		mains, err := mainPackages(pkgs)
+		mains, err := mainPackages(prog, tests)
 		if err != nil {
 			return err
 		}
@@ -237,7 +237,7 @@ func doCallgraph(dir, gopath, algo, format string, tests bool, args []string) er
 		cg = ptares.CallGraph
 
 	case "rta":
-		mains, err := mainPackages(pkgs)
+		mains, err := mainPackages(prog, tests)
 		if err != nil {
 			return err
 		}
@@ -305,13 +305,25 @@ func doCallgraph(dir, gopath, algo, format string, tests bool, args []string) er
 
 // mainPackages returns the main packages to analyze.
 // Each resulting package is named "main" and has a main function.
-func mainPackages(pkgs []*ssa.Package) ([]*ssa.Package, error) {
+func mainPackages(prog *ssa.Program, tests bool) ([]*ssa.Package, error) {
+	pkgs := prog.AllPackages() // TODO(adonovan): use only initial packages
+
+	// If tests, create a "testmain" package for each test.
 	var mains []*ssa.Package
-	for _, p := range pkgs {
+	if tests {
-		if p != nil && p.Pkg.Name() == "main" && p.Func("main") != nil {
+		for _, pkg := range pkgs {
-			mains = append(mains, p)
+			if main := prog.CreateTestMainPackage(pkg); main != nil {
+				mains = append(mains, main)
+			}
 		}
+		if mains == nil {
+			return nil, fmt.Errorf("no tests")
+		}
+		return mains, nil
 	}
+
+	// Otherwise, use the main packages.
+	mains = append(mains, ssautil.MainPackages(pkgs)...)
 	if len(mains) == 0 {
 		return nil, fmt.Errorf("no main packages")
 	}

cmd/callgraph/main_test.go

@@ -5,44 +5,31 @@
 // No testdata on Android.
 
 // +build !android
-// +build go1.11
 
 package main
 
 import (
 	"bytes"
 	"fmt"
-	"log"
+	"go/build"
-	"os"
+	"reflect"
-	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 )
 
-func init() {
-	// This test currently requires GOPATH mode.
-	// Explicitly disabling module mode should suffix, but
-	// we'll also turn off GOPROXY just for good measure.
-	if err := os.Setenv("GO111MODULE", "off"); err != nil {
-		log.Fatal(err)
-	}
-	if err := os.Setenv("GOPROXY", "off"); err != nil {
-		log.Fatal(err)
-	}
-}
-
 func TestCallgraph(t *testing.T) {
-	gopath, err := filepath.Abs("testdata")
+	ctxt := build.Default // copy
-	if err != nil {
+	ctxt.GOPATH = "testdata"
-		t.Fatal(err)
+
-	}
+	const format = "{{.Caller}} --> {{.Callee}}"
 
 	for _, test := range []struct {
-		algo  string
+		algo, format string
-		tests bool
+		tests        bool
-		want  []string
+		want         []string
 	}{
-		{"rta", false, []string{
+		{"rta", format, false, []string{
 			// rta imprecisely shows cross product of {main,main2} x {C,D}
 			`pkg.main --> (pkg.C).f`,
 			`pkg.main --> (pkg.D).f`,
@@ -50,7 +37,7 @@ func TestCallgraph(t *testing.T) {
 			`pkg.main2 --> (pkg.C).f`,
 			`pkg.main2 --> (pkg.D).f`,
 		}},
-		{"pta", false, []string{
+		{"pta", format, false, []string{
 			// pta distinguishes main->C, main2->D.  Also has a root node.
 			`<root> --> pkg.init`,
 			`<root> --> pkg.main`,
@@ -58,42 +45,37 @@ func TestCallgraph(t *testing.T) {
 			`pkg.main --> pkg.main2`,
 			`pkg.main2 --> (pkg.D).f`,
 		}},
-		// tests: both the package's main and the test's main are called.
+		// tests: main is not called.
-		// The callgraph includes all the guts of the "testing" package.
+		{"rta", format, true, []string{
-		{"rta", true, []string{
+			`pkg$testmain.init --> pkg.init`,
-			`pkg.test.main --> testing.MainStart`,
-			`testing.runExample --> pkg.Example`,
 			`pkg.Example --> (pkg.C).f`,
-			`pkg.main --> (pkg.C).f`,
 		}},
-		{"pta", true, []string{
+		{"pta", format, true, []string{
-			`<root> --> pkg.test.main`,
+			`<root> --> pkg$testmain.init`,
-			`<root> --> pkg.main`,
+			`<root> --> pkg.Example`,
-			`pkg.test.main --> testing.MainStart`,
+			`pkg$testmain.init --> pkg.init`,
-			`testing.runExample --> pkg.Example`,
 			`pkg.Example --> (pkg.C).f`,
-			`pkg.main --> (pkg.C).f`,
 		}},
 	} {
-		const format = "{{.Caller}} --> {{.Callee}}"
 		stdout = new(bytes.Buffer)
-		if err := doCallgraph("testdata/src", gopath, test.algo, format, test.tests, []string{"pkg"}); err != nil {
+		if err := doCallgraph(&ctxt, test.algo, test.format, test.tests, []string{"pkg"}); err != nil {
 			t.Error(err)
 			continue
 		}
 
-		edges := make(map[string]bool)
+		got := sortedLines(fmt.Sprint(stdout))
-		for _, line := range strings.Split(fmt.Sprint(stdout), "\n") {
+		if !reflect.DeepEqual(got, test.want) {
-			edges[line] = true
+			t.Errorf("callgraph(%q, %q, %t):\ngot:\n%s\nwant:\n%s",
-		}
+				test.algo, test.format, test.tests,
-		for _, edge := range test.want {
+				strings.Join(got, "\n"),
-			if !edges[edge] {
+				strings.Join(test.want, "\n"))
-				t.Errorf("callgraph(%q, %t): missing edge: %s",
-					test.algo, test.tests, edge)
-			}
-		}
-		if t.Failed() {
-			t.Log("got:\n", stdout)
 		}
 	}
 }
+
+func sortedLines(s string) []string {
+	s = strings.TrimSpace(s)
+	lines := strings.Split(s, "\n")
+	sort.Strings(lines)
+	return lines
+}

cmd/callgraph/testdata/src/pkg/pkg_test.go

@@ -1,10 +1,7 @@
 package main
 
-// An Example function must have an "Output:" comment for the go build
+// Don't import "testing", it adds a lot of callgraph edges.
-// system to generate a call to it from the test main package.
 
 func Example() {
 	C(0).f()
-
-	// Output:
 }

cmd/compilebench/main.go

@@ -22,21 +22,12 @@
 //	-compileflags 'list'
 //		Pass the space-separated list of flags to the compilation.
 //
-//	-link exe
-//		Use exe as the path to the cmd/link binary.
-//
-//	-linkflags 'list'
-//		Pass the space-separated list of flags to the linker.
-//
 //	-count n
 //		Run each benchmark n times (default 1).
 //
 //	-cpuprofile file
 //		Write a CPU profile of the compiler to file.
 //
-//	-go path
-//		Path to "go" command (default "go").
-//
 //	-memprofile file
 //		Write a memory profile of the compiler to file.
 //
@@ -46,15 +37,12 @@
 //	-obj
 //		Report object file statistics.
 //
-//	-pkg pkg
+//  -pkg
 //		Benchmark compiling a single package.
 //
 //	-run regexp
 //		Only run benchmarks with names matching regexp.
 //
-//	-short
-//		Skip long-running benchmarks.
-//
 // Although -cpuprofile and -memprofile are intended to write a
 // combined profile for all the executed benchmarks to file,
 // today they write only the profile for the last benchmark executed.
@@ -79,9 +67,9 @@ package main
 
 import (
 	"bytes"
-	"encoding/json"
 	"flag"
 	"fmt"
+	"go/build"
 	"io/ioutil"
 	"log"
 	"os"
@@ -96,7 +84,6 @@ import (
 var (
 	goroot   string
 	compiler string
-	linker   string
 	runRE    *regexp.Regexp
 	is6g     bool
 )
@@ -107,8 +94,6 @@ var (
 	flagObj            = flag.Bool("obj", false, "report object file stats")
 	flagCompiler       = flag.String("compile", "", "use `exe` as the cmd/compile binary")
 	flagCompilerFlags  = flag.String("compileflags", "", "additional `flags` to pass to compile")
-	flagLinker         = flag.String("link", "", "use `exe` as the cmd/link binary")
-	flagLinkerFlags    = flag.String("linkflags", "", "additional `flags` to pass to link")
 	flagRun            = flag.String("run", "", "run benchmarks matching `regexp`")
 	flagCount          = flag.Int("count", 1, "run benchmarks `n` times")
 	flagCpuprofile     = flag.String("cpuprofile", "", "write CPU profile to `file`")
@@ -118,31 +103,24 @@ var (
 	flagShort          = flag.Bool("short", false, "skip long-running benchmarks")
 )
 
-type test struct {
+var tests = []struct {
 	name string
-	r    runner
+	dir  string
-}
+	long bool
-
+}{
-type runner interface {
+	{"BenchmarkTemplate", "html/template", false},
-	long() bool
+	{"BenchmarkUnicode", "unicode", false},
-	run(name string, count int) error
+	{"BenchmarkGoTypes", "go/types", false},
-}
+	{"BenchmarkCompiler", "cmd/compile/internal/gc", false},
-
+	{"BenchmarkSSA", "cmd/compile/internal/ssa", false},
-var tests = []test{
+	{"BenchmarkFlate", "compress/flate", false},
-	{"BenchmarkTemplate", compile{"html/template"}},
+	{"BenchmarkGoParser", "go/parser", false},
-	{"BenchmarkUnicode", compile{"unicode"}},
+	{"BenchmarkReflect", "reflect", false},
-	{"BenchmarkGoTypes", compile{"go/types"}},
+	{"BenchmarkTar", "archive/tar", false},
-	{"BenchmarkCompiler", compile{"cmd/compile/internal/gc"}},
+	{"BenchmarkXML", "encoding/xml", false},
-	{"BenchmarkSSA", compile{"cmd/compile/internal/ssa"}},
+	{"BenchmarkStdCmd", "", true},
-	{"BenchmarkFlate", compile{"compress/flate"}},
+	{"BenchmarkHelloSize", "", false},
-	{"BenchmarkGoParser", compile{"go/parser"}},
+	{"BenchmarkCmdGoSize", "", true},
-	{"BenchmarkReflect", compile{"reflect"}},
-	{"BenchmarkTar", compile{"archive/tar"}},
-	{"BenchmarkXML", compile{"encoding/xml"}},
-	{"BenchmarkLinkCompiler", link{"cmd/compile"}},
-	{"BenchmarkStdCmd", goBuild{[]string{"std", "cmd"}}},
-	{"BenchmarkHelloSize", size{"$GOROOT/test/helloworld.go", false}},
-	{"BenchmarkCmdGoSize", size{"cmd/go", true}},
 }
 
 func usage() {
@@ -166,27 +144,19 @@ func main() {
 		log.Fatalf("%s env GOROOT: %v", *flagGoCmd, err)
 	}
 	goroot = strings.TrimSpace(string(s))
-	os.Setenv("GOROOT", goroot) // for any subcommands
 
 	compiler = *flagCompiler
 	if compiler == "" {
-		var foundTool string
+		out, err := exec.Command(*flagGoCmd, "tool", "-n", "compile").CombinedOutput()
-		foundTool, compiler = toolPath("compile", "6g")
+		if err != nil {
-		if foundTool == "6g" {
+			out, err = exec.Command(*flagGoCmd, "tool", "-n", "6g").CombinedOutput()
 			is6g = true
+			if err != nil {
+				out, err = exec.Command(*flagGoCmd, "tool", "-n", "compile").CombinedOutput()
+				log.Fatalf("go tool -n compiler: %v\n%s", err, out)
+			}
 		}
-	}
+		compiler = strings.TrimSpace(string(out))
-
-	linker = *flagLinker
-	if linker == "" && !is6g { // TODO: Support 6l
-		_, linker = toolPath("link")
-	}
-
-	if is6g {
-		*flagMemprofilerate = -1
-		*flagAlloc = false
-		*flagCpuprofile = ""
-		*flagMemprofile = ""
 	}
 
 	if *flagRun != "" {
@@ -197,117 +167,67 @@ func main() {
 		runRE = r
 	}
 
-	if *flagPackage != "" {
-		tests = []test{
-			{"BenchmarkPkg", compile{*flagPackage}},
-			{"BenchmarkPkgLink", link{*flagPackage}},
-		}
-		runRE = nil
-	}
-
 	for i := 0; i < *flagCount; i++ {
+		if *flagPackage != "" {
+			runBuild("BenchmarkPkg", *flagPackage, i)
+			continue
+		}
 		for _, tt := range tests {
-			if tt.r.long() && *flagShort {
+			if tt.long && *flagShort {
 				continue
 			}
 			if runRE == nil || runRE.MatchString(tt.name) {
-				if err := tt.r.run(tt.name, i); err != nil {
+				runBuild(tt.name, tt.dir, i)
-					log.Printf("%s: %v", tt.name, err)
-				}
 			}
 		}
 	}
 }
 
-func toolPath(names ...string) (found, path string) {
+func runCmd(name string, cmd *exec.Cmd) {
-	var out1 []byte
-	var err1 error
-	for i, name := range names {
-		out, err := exec.Command(*flagGoCmd, "tool", "-n", name).CombinedOutput()
-		if err == nil {
-			return name, strings.TrimSpace(string(out))
-		}
-		if i == 0 {
-			out1, err1 = out, err
-		}
-	}
-	log.Fatalf("go tool -n %s: %v\n%s", names[0], err1, out1)
-	return "", ""
-}
-
-type Pkg struct {
-	Dir     string
-	GoFiles []string
-}
-
-func goList(dir string) (*Pkg, error) {
-	var pkg Pkg
-	out, err := exec.Command(*flagGoCmd, "list", "-json", dir).Output()
-	if err != nil {
-		return nil, fmt.Errorf("go list -json %s: %v\n", dir, err)
-	}
-	if err := json.Unmarshal(out, &pkg); err != nil {
-		return nil, fmt.Errorf("go list -json %s: unmarshal: %v", dir, err)
-	}
-	return &pkg, nil
-}
-
-func runCmd(name string, cmd *exec.Cmd) error {
 	start := time.Now()
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("%v\n%s", err, out)
+		log.Printf("%v: %v\n%s", name, err, out)
+		return
 	}
 	fmt.Printf("%s 1 %d ns/op\n", name, time.Since(start).Nanoseconds())
-	return nil
 }
 
-type goBuild struct{ pkgs []string }
+func runStdCmd() {
-
-func (goBuild) long() bool { return true }
-
-func (r goBuild) run(name string, count int) error {
 	args := []string{"build", "-a"}
 	if *flagCompilerFlags != "" {
 		args = append(args, "-gcflags", *flagCompilerFlags)
 	}
-	args = append(args, r.pkgs...)
+	args = append(args, "std", "cmd")
 	cmd := exec.Command(*flagGoCmd, args...)
 	cmd.Dir = filepath.Join(goroot, "src")
-	return runCmd(name, cmd)
+	runCmd("BenchmarkStdCmd", cmd)
 }
 
-type size struct {
+// path is either a path to a file ("$GOROOT/test/helloworld.go") or a package path ("cmd/go").
-	// path is either a path to a file ("$GOROOT/test/helloworld.go") or a package path ("cmd/go").
+func runSize(name, path string) {
-	path   string
+	cmd := exec.Command(*flagGoCmd, "build", "-o", "_compilebenchout_", path)
-	isLong bool
-}
-
-func (r size) long() bool { return r.isLong }
-
-func (r size) run(name string, count int) error {
-	if strings.HasPrefix(r.path, "$GOROOT/") {
-		r.path = goroot + "/" + r.path[len("$GOROOT/"):]
-	}
-
-	cmd := exec.Command(*flagGoCmd, "build", "-o", "_compilebenchout_", r.path)
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
-		return err
+		log.Print(err)
+		return
 	}
 	defer os.Remove("_compilebenchout_")
 	info, err := os.Stat("_compilebenchout_")
 	if err != nil {
-		return err
+		log.Print(err)
+		return
 	}
 	out, err := exec.Command("size", "_compilebenchout_").CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("size: %v\n%s", err, out)
+		log.Printf("size: %v\n%s", err, out)
+		return
 	}
 	lines := strings.Split(string(out), "\n")
 	if len(lines) < 2 {
-		return fmt.Errorf("not enough output from size: %s", out)
+		log.Printf("not enough output from size: %s", out)
+		return
 	}
 	f := strings.Fields(lines[1])
 	if strings.HasPrefix(lines[0], "__TEXT") && len(f) >= 2 { // OS X
@@ -315,31 +235,110 @@ func (r size) run(name string, count int) error {
 	} else if strings.Contains(lines[0], "bss") && len(f) >= 3 {
 		fmt.Printf("%s 1 %s text-bytes %s data-bytes %s bss-bytes %v exe-bytes\n", name, f[0], f[1], f[2], info.Size())
 	}
-	return nil
 }
 
-type compile struct{ dir string }
+func runBuild(name, dir string, count int) {
-
+	switch name {
-func (compile) long() bool { return false }
+	case "BenchmarkStdCmd":
-
+		runStdCmd()
-func (c compile) run(name string, count int) error {
+		return
-	// Make sure dependencies needed by go tool compile are installed to GOROOT/pkg.
+	case "BenchmarkCmdGoSize":
-	out, err := exec.Command(*flagGoCmd, "build", "-i", c.dir).CombinedOutput()
+		runSize("BenchmarkCmdGoSize", "cmd/go")
-	if err != nil {
+		return
-		return fmt.Errorf("go build -i %s: %v\n%s", c.dir, err, out)
+	case "BenchmarkHelloSize":
+		runSize("BenchmarkHelloSize", filepath.Join(goroot, "test/helloworld.go"))
+		return
 	}
 
-	// Find dir and source file list.
+	pkg, err := build.Import(dir, ".", 0)
-	pkg, err := goList(c.dir)
 	if err != nil {
-		return err
+		log.Print(err)
+		return
 	}
-
 	args := []string{"-o", "_compilebench_.o"}
+	if is6g {
+		*flagMemprofilerate = -1
+		*flagAlloc = false
+		*flagCpuprofile = ""
+		*flagMemprofile = ""
+	}
+	if *flagMemprofilerate >= 0 {
+		args = append(args, "-memprofilerate", fmt.Sprint(*flagMemprofilerate))
+	}
 	args = append(args, strings.Fields(*flagCompilerFlags)...)
+	if *flagAlloc || *flagCpuprofile != "" || *flagMemprofile != "" {
+		if *flagAlloc || *flagMemprofile != "" {
+			args = append(args, "-memprofile", "_compilebench_.memprof")
+		}
+		if *flagCpuprofile != "" {
+			args = append(args, "-cpuprofile", "_compilebench_.cpuprof")
+		}
+	}
 	args = append(args, pkg.GoFiles...)
-	if err := runBuildCmd(name, count, pkg.Dir, compiler, args); err != nil {
+	cmd := exec.Command(compiler, args...)
-		return err
+	cmd.Dir = pkg.Dir
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	start := time.Now()
+	err = cmd.Run()
+	if err != nil {
+		log.Printf("%v: %v", name, err)
+		return
+	}
+	end := time.Now()
+
+	var allocs, allocbytes int64
+	if *flagAlloc || *flagMemprofile != "" {
+		out, err := ioutil.ReadFile(pkg.Dir + "/_compilebench_.memprof")
+		if err != nil {
+			log.Print("cannot find memory profile after compilation")
+		}
+		for _, line := range strings.Split(string(out), "\n") {
+			f := strings.Fields(line)
+			if len(f) < 4 || f[0] != "#" || f[2] != "=" {
+				continue
+			}
+			val, err := strconv.ParseInt(f[3], 0, 64)
+			if err != nil {
+				continue
+			}
+			switch f[1] {
+			case "TotalAlloc":
+				allocbytes = val
+			case "Mallocs":
+				allocs = val
+			}
+		}
+
+		if *flagMemprofile != "" {
+			if err := ioutil.WriteFile(*flagMemprofile, out, 0666); err != nil {
+				log.Print(err)
+			}
+		}
+		os.Remove(pkg.Dir + "/_compilebench_.memprof")
+	}
+
+	if *flagCpuprofile != "" {
+		out, err := ioutil.ReadFile(pkg.Dir + "/_compilebench_.cpuprof")
+		if err != nil {
+			log.Print(err)
+		}
+		outpath := *flagCpuprofile
+		if *flagCount != 1 {
+			outpath = fmt.Sprintf("%s_%d", outpath, count)
+		}
+		if err := ioutil.WriteFile(outpath, out, 0666); err != nil {
+			log.Print(err)
+		}
+		os.Remove(pkg.Dir + "/_compilebench_.cpuprof")
+	}
+
+	wallns := end.Sub(start).Nanoseconds()
+	userns := cmd.ProcessState.UserTime().Nanoseconds()
+
+	fmt.Printf("%s 1 %d ns/op %d user-ns/op", name, wallns, userns)
+	if *flagAlloc {
+		fmt.Printf(" %d B/op %d allocs/op", allocbytes, allocs)
 	}
 
 	opath := pkg.Dir + "/_compilebench_.o"
@@ -358,147 +357,4 @@ func (c compile) run(name string, count int) error {
 	fmt.Println()
 
 	os.Remove(opath)
-	return nil
-}
-
-type link struct{ dir string }
-
-func (link) long() bool { return false }
-
-func (r link) run(name string, count int) error {
-	if linker == "" {
-		// No linker. Skip the test.
-		return nil
-	}
-
-	// Build dependencies.
-	out, err := exec.Command(*flagGoCmd, "build", "-i", "-o", "/dev/null", r.dir).CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("go build -i %s: %v\n%s", r.dir, err, out)
-	}
-
-	// Build the main package.
-	pkg, err := goList(r.dir)
-	if err != nil {
-		return err
-	}
-	args := []string{"-o", "_compilebench_.o"}
-	args = append(args, pkg.GoFiles...)
-	cmd := exec.Command(compiler, args...)
-	cmd.Dir = pkg.Dir
-	cmd.Stdout = os.Stderr
-	cmd.Stderr = os.Stderr
-	err = cmd.Run()
-	if err != nil {
-		return fmt.Errorf("compiling: %v", err)
-	}
-	defer os.Remove(pkg.Dir + "/_compilebench_.o")
-
-	// Link the main package.
-	args = []string{"-o", "_compilebench_.exe"}
-	args = append(args, strings.Fields(*flagLinkerFlags)...)
-	args = append(args, "_compilebench_.o")
-	if err := runBuildCmd(name, count, pkg.Dir, linker, args); err != nil {
-		return err
-	}
-	fmt.Println()
-	defer os.Remove(pkg.Dir + "/_compilebench_.exe")
-
-	return err
-}
-
-// runBuildCmd runs "tool args..." in dir, measures standard build
-// tool metrics, and prints a benchmark line. The caller may print
-// additional metrics and then must print a newline.
-//
-// This assumes tool accepts standard build tool flags like
-// -memprofilerate, -memprofile, and -cpuprofile.
-func runBuildCmd(name string, count int, dir, tool string, args []string) error {
-	var preArgs []string
-	if *flagMemprofilerate >= 0 {
-		preArgs = append(preArgs, "-memprofilerate", fmt.Sprint(*flagMemprofilerate))
-	}
-	if *flagAlloc || *flagCpuprofile != "" || *flagMemprofile != "" {
-		if *flagAlloc || *flagMemprofile != "" {
-			preArgs = append(preArgs, "-memprofile", "_compilebench_.memprof")
-		}
-		if *flagCpuprofile != "" {
-			preArgs = append(preArgs, "-cpuprofile", "_compilebench_.cpuprof")
-		}
-	}
-	cmd := exec.Command(tool, append(preArgs, args...)...)
-	cmd.Dir = dir
-	cmd.Stdout = os.Stderr
-	cmd.Stderr = os.Stderr
-	start := time.Now()
-	err := cmd.Run()
-	if err != nil {
-		return err
-	}
-	end := time.Now()
-
-	haveAllocs := false
-	var allocs, allocbytes int64
-	if *flagAlloc || *flagMemprofile != "" {
-		out, err := ioutil.ReadFile(dir + "/_compilebench_.memprof")
-		if err != nil {
-			log.Print("cannot find memory profile after compilation")
-		}
-		for _, line := range strings.Split(string(out), "\n") {
-			f := strings.Fields(line)
-			if len(f) < 4 || f[0] != "#" || f[2] != "=" {
-				continue
-			}
-			val, err := strconv.ParseInt(f[3], 0, 64)
-			if err != nil {
-				continue
-			}
-			haveAllocs = true
-			switch f[1] {
-			case "TotalAlloc":
-				allocbytes = val
-			case "Mallocs":
-				allocs = val
-			}
-		}
-		if !haveAllocs {
-			log.Println("missing stats in memprof (golang.org/issue/18641)")
-		}
-
-		if *flagMemprofile != "" {
-			outpath := *flagMemprofile
-			if *flagCount != 1 {
-				outpath = fmt.Sprintf("%s_%d", outpath, count)
-			}
-			if err := ioutil.WriteFile(outpath, out, 0666); err != nil {
-				log.Print(err)
-			}
-		}
-		os.Remove(dir + "/_compilebench_.memprof")
-	}
-
-	if *flagCpuprofile != "" {
-		out, err := ioutil.ReadFile(dir + "/_compilebench_.cpuprof")
-		if err != nil {
-			log.Print(err)
-		}
-		outpath := *flagCpuprofile
-		if *flagCount != 1 {
-			outpath = fmt.Sprintf("%s_%d", outpath, count)
-		}
-		if err := ioutil.WriteFile(outpath, out, 0666); err != nil {
-			log.Print(err)
-		}
-		os.Remove(dir + "/_compilebench_.cpuprof")
-	}
-
-	wallns := end.Sub(start).Nanoseconds()
-	userns := cmd.ProcessState.UserTime().Nanoseconds()
-
-	fmt.Printf("%s 1 %d ns/op %d user-ns/op", name, wallns, userns)
-	if haveAllocs {
-		fmt.Printf(" %d B/op %d allocs/op", allocbytes, allocs)
-	}
-
-	return nil
 }

cmd/digraph/digraph.go

@@ -1,83 +1,18 @@
-// Copyright 2019 The Go Authors. All rights reserved.
+// The digraph command performs queries over unlabelled directed graphs
-// Use of this source code is governed by a BSD-style
+// represented in text form.  It is intended to integrate nicely with
-// license that can be found in the LICENSE file.
+// typical UNIX command pipelines.
-
+//
-/*
+// Since directed graphs (import graphs, reference graphs, call graphs,
-The digraph command performs queries over unlabelled directed graphs
+// etc) often arise during software tool development and debugging, this
-represented in text form.  It is intended to integrate nicely with
+// command is included in the go.tools repository.
-typical UNIX command pipelines.
+//
-
-Usage:
-
-	your-application | digraph [command]
-
-The support commands are:
-
-	nodes
-		the set of all nodes
-	degree
-		the in-degree and out-degree of each node
-	preds <node> ...
-		the set of immediate predecessors of the specified nodes
-	succs <node> ...
-		the set of immediate successors of the specified nodes
-	forward <node> ...
-		the set of nodes transitively reachable from the specified nodes
-	reverse <node> ...
-		the set of nodes that transitively reach the specified nodes
-	somepath <node> <node>
-		the list of nodes on some arbitrary path from the first node to the second
-	allpaths <node> <node>
-		the set of nodes on all paths from the first node to the second
-	sccs
-		all strongly connected components (one per line)
-	scc <node>
-		the set of nodes nodes strongly connected to the specified one
-
-Input format:
-
-Each line contains zero or more words. Words are separated by unquoted
-whitespace; words may contain Go-style double-quoted portions, allowing spaces
-and other characters to be expressed.
-
-Each word declares a node, and if there are more than one, an edge from the
-first to each subsequent one. The graph is provided on the standard input.
-
-For instance, the following (acyclic) graph specifies a partial order among the
-subtasks of getting dressed:
-
-	$ cat clothes.txt
-	socks shoes
-	"boxer shorts" pants
-	pants belt shoes
-	shirt tie sweater
-	sweater jacket
-	hat
-
-The line "shirt tie sweater" indicates the two edges shirt -> tie and
-shirt -> sweater, not shirt -> tie -> sweater.
-
-Example usage:
-
-Using digraph with existing Go tools:
-
-	$ go mod graph | digraph nodes # Operate on the Go module graph.
-	$ go list -m all | digraph nodes # Operate on the Go package graph.
-
-Show the transitive closure of imports of the digraph tool itself:
-	$ go list -f '{{.ImportPath}} {{join .Imports " "}}' ... | digraph forward golang.org/x/tools/cmd/digraph
-
-Show which clothes (see above) must be donned before a jacket:
-	$ digraph reverse jacket
-
-*/
-package main // import "golang.org/x/tools/cmd/digraph"
-
 // TODO(adonovan):
 // - support input files other than stdin
-// - support alternative formats (AT&T GraphViz, CSV, etc),
+// - suport alternative formats (AT&T GraphViz, CSV, etc),
 //   a comment syntax, etc.
 // - allow queries to nest, like Blaze query language.
+//
+package main // import "golang.org/x/tools/cmd/digraph"
 
 import (
 	"bufio"
@@ -93,41 +28,73 @@ import (
 	"unicode/utf8"
 )
 
-func usage() {
+const Usage = `digraph: queries over directed graphs in text form.
-	fmt.Fprintf(os.Stderr, `Usage: your-application | digraph [command]
 
-The support commands are:
+Graph format:
-	nodes
+
-		the set of all nodes
+  Each line contains zero or more words.  Words are separated by
-	degree
+  unquoted whitespace; words may contain Go-style double-quoted portions,
-		the in-degree and out-degree of each node
+  allowing spaces and other characters to be expressed.
-	preds <node> ...
+
-		the set of immediate predecessors of the specified nodes
+  Each field declares a node, and if there are more than one,
-	succs <node> ...
+  an edge from the first to each subsequent one.
-		the set of immediate successors of the specified nodes
+  The graph is provided on the standard input.
-	forward <node> ...
+
-		the set of nodes transitively reachable from the specified nodes
+  For instance, the following (acyclic) graph specifies a partial order
-	reverse <node> ...
+  among the subtasks of getting dressed:
-		the set of nodes that transitively reach the specified nodes
+
-	somepath <node> <node>
+	% cat clothes.txt
-		the list of nodes on some arbitrary path from the first node to the second
+	socks shoes
-	allpaths <node> <node>
+	"boxer shorts" pants
-		the set of nodes on all paths from the first node to the second
+	pants belt shoes
-	sccs
+	shirt tie sweater
-		all strongly connected components (one per line)
+	sweater jacket
-	scc <node>
+	hat
-		the set of nodes nodes strongly connected to the specified one
+
-`)
+  The line "shirt tie sweater" indicates the two edges shirt -> tie and
-	os.Exit(2)
+  shirt -> sweater, not shirt -> tie -> sweater.
-}
+
+Supported queries:
+
+  nodes
+	the set of all nodes
+  degree
+	the in-degree and out-degree of each node.
+  preds <label> ...
+	the set of immediate predecessors of the specified nodes
+  succs <label> ...
+	the set of immediate successors of the specified nodes
+  forward <label> ...
+	the set of nodes transitively reachable from the specified nodes
+  reverse <label> ...
+	the set of nodes that transitively reach the specified nodes
+  somepath <label> <label>
+	the list of nodes on some arbitrary path from the first node to the second
+  allpaths <label> <label>
+	the set of nodes on all paths from the first node to the second
+  sccs
+	all strongly connected components (one per line)
+  scc <label>
+	the set of nodes nodes strongly connected to the specified one
+
+Example usage:
+
+   Show the transitive closure of imports of the digraph tool itself:
+   % go list -f '{{.ImportPath}}{{.Imports}}' ... | tr '[]' '  ' |
+         digraph forward golang.org/x/tools/cmd/digraph
+
+   Show which clothes (see above) must be donned before a jacket:
+   %  digraph reverse jacket <clothes.txt
+
+`
 
 func main() {
-	flag.Usage = usage
 	flag.Parse()
 
 	args := flag.Args()
 	if len(args) == 0 {
-		usage()
+		fmt.Println(Usage)
+		return
 	}
 
 	if err := digraph(args[0], args[1:]); err != nil {
@@ -262,47 +229,6 @@ func (g graph) sccs() []nodeset {
 	return sccs
 }
 
-func (g graph) allpaths(from, to string) error {
-	// Mark all nodes to "to".
-	seen := make(nodeset) // value of seen[x] indicates whether x is on some path to "to"
-	var visit func(node string) bool
-	visit = func(node string) bool {
-		reachesTo, ok := seen[node]
-		if !ok {
-			reachesTo = node == to
-			seen[node] = reachesTo
-			for e := range g[node] {
-				if visit(e) {
-					reachesTo = true
-				}
-			}
-			if reachesTo && node != to {
-				seen[node] = true
-			}
-		}
-		return reachesTo
-	}
-	visit(from)
-
-	// For each marked node, collect its marked successors.
-	var edges []string
-	for n := range seen {
-		for succ := range g[n] {
-			if seen[succ] {
-				edges = append(edges, n+" "+succ)
-			}
-		}
-	}
-
-	// Sort (so that this method is deterministic) and print edges.
-	sort.Strings(edges)
-	for _, e := range edges {
-		fmt.Fprintln(stdout, e)
-	}
-
-	return nil
-}
-
 func parse(rd io.Reader) (graph, error) {
 	g := make(graph)
 
@@ -325,7 +251,6 @@ func parse(rd io.Reader) (graph, error) {
 	return g, nil
 }
 
-// Overridable for testing purposes.
 var stdin io.Reader = os.Stdin
 var stdout io.Writer = os.Stdout
 
@@ -440,7 +365,33 @@ func digraph(cmd string, args []string) error {
 		if g[to] == nil {
 			return fmt.Errorf("no such 'to' node %q", to)
 		}
-		g.allpaths(from, to)
+
+		seen := make(nodeset) // value of seen[x] indicates whether x is on some path to 'to'
+		var visit func(label string) bool
+		visit = func(label string) bool {
+			reachesTo, ok := seen[label]
+			if !ok {
+				reachesTo = label == to
+
+				seen[label] = reachesTo
+				for e := range g[label] {
+					if visit(e) {
+						reachesTo = true
+					}
+				}
+				seen[label] = reachesTo
+			}
+			return reachesTo
+		}
+		if !visit(from) {
+			return fmt.Errorf("no path from %q to %q", from, to)
+		}
+		for label, reachesTo := range seen {
+			if !reachesTo {
+				delete(seen, label)
+			}
+		}
+		seen.sort().println("\n")
 
 	case "sccs":
 		if len(args) != 0 {

cmd/digraph/digraph_test.go

@@ -1,6 +1,3 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
 package main
 
 import (
@@ -29,34 +26,35 @@ d c
 `
 
 	for _, test := range []struct {
-		name  string
 		input string
 		cmd   string
 		args  []string
 		want  string
 	}{
-		{"nodes", g1, "nodes", nil, "belt\nhat\njacket\npants\nshirt\nshoes\nshorts\nsocks\nsweater\ntie\n"},
+		{g1, "nodes", nil, "belt\nhat\njacket\npants\nshirt\nshoes\nshorts\nsocks\nsweater\ntie\n"},
-		{"reverse", g1, "reverse", []string{"jacket"}, "jacket\nshirt\nsweater\n"},
+		{g1, "reverse", []string{"jacket"}, "jacket\nshirt\nsweater\n"},
-		{"forward", g1, "forward", []string{"socks"}, "shoes\nsocks\n"},
+		{g1, "forward", []string{"socks"}, "shoes\nsocks\n"},
-		{"forward multiple args", g1, "forward", []string{"socks", "sweater"}, "jacket\nshoes\nsocks\nsweater\n"},
+		{g1, "forward", []string{"socks", "sweater"}, "jacket\nshoes\nsocks\nsweater\n"},
-		{"scss", g2, "sccs", nil, "a\nb\nc d\n"},
-		{"scc", g2, "scc", []string{"d"}, "c\nd\n"},
-		{"succs", g2, "succs", []string{"a"}, "b\nc\n"},
-		{"preds", g2, "preds", []string{"c"}, "a\nd\n"},
-		{"preds multiple args", g2, "preds", []string{"c", "d"}, "a\nb\nc\nd\n"},
-	} {
-		t.Run(test.name, func(t *testing.T) {
-			stdin = strings.NewReader(test.input)
-			stdout = new(bytes.Buffer)
-			if err := digraph(test.cmd, test.args); err != nil {
-				t.Fatal(err)
-			}
 
-			got := stdout.(fmt.Stringer).String()
+		{g2, "allpaths", []string{"a", "d"}, "a\nb\nc\nd\n"},
-			if got != test.want {
+
-				t.Errorf("digraph(%s, %s) = got %q, want %q", test.cmd, test.args, got, test.want)
+		{g2, "sccs", nil, "a\nb\nc d\n"},
-			}
+		{g2, "scc", []string{"d"}, "c\nd\n"},
-		})
+		{g2, "succs", []string{"a"}, "b\nc\n"},
+		{g2, "preds", []string{"c"}, "a\nd\n"},
+		{g2, "preds", []string{"c", "d"}, "a\nb\nc\nd\n"},
+	} {
+		stdin = strings.NewReader(test.input)
+		stdout = new(bytes.Buffer)
+		if err := digraph(test.cmd, test.args); err != nil {
+			t.Error(err)
+			continue
+		}
+
+		got := stdout.(fmt.Stringer).String()
+		if got != test.want {
+			t.Errorf("digraph(%s, %s) = %q, want %q", test.cmd, test.args, got, test.want)
+		}
 	}
 
 	// TODO(adonovan):
@@ -64,110 +62,6 @@ d c
 	// - test errors
 }
 
-func TestAllpaths(t *testing.T) {
-	for _, test := range []struct {
-		name string
-		in   string
-		to   string // from is always "A"
-		want string
-	}{
-		{
-			name: "Basic",
-			in:   "A B\nB C",
-			to:   "B",
-			want: "A B\n",
-		},
-		{
-			name: "Long",
-			in:   "A B\nB C\n",
-			to:   "C",
-			want: "A B\nB C\n",
-		},
-		{
-			name: "Cycle Basic",
-			in:   "A B\nB A",
-			to:   "B",
-			want: "A B\nB A\n",
-		},
-		{
-			name: "Cycle Path Out",
-			// A <-> B -> C -> D
-			in:   "A B\nB A\nB C\nC D",
-			to:   "C",
-			want: "A B\nB A\nB C\n",
-		},
-		{
-			name: "Cycle Path Out Further Out",
-			// A -> B <-> C -> D -> E
-			in:   "A B\nB C\nC D\nC B\nD E",
-			to:   "D",
-			want: "A B\nB C\nC B\nC D\n",
-		},
-		{
-			name: "Two Paths Basic",
-			//           /-> C --\
-			// A -> B --          -> E -> F
-			//           \-> D --/
-			in:   "A B\nB C\nC E\nB D\nD E\nE F",
-			to:   "E",
-			want: "A B\nB C\nB D\nC E\nD E\n",
-		},
-		{
-			name: "Two Paths With One Immediately From Start",
-			//      /-> B -+ -> D
-			// A --        |
-			//      \-> C <+
-			in:   "A B\nA C\nB C\nB D",
-			to:   "C",
-			want: "A B\nA C\nB C\n",
-		},
-		{
-			name: "Two Paths Further Up",
-			//      /-> B --\
-			// A --          -> D -> E -> F
-			//      \-> C --/
-			in:   "A B\nA C\nB D\nC D\nD E\nE F",
-			to:   "E",
-			want: "A B\nA C\nB D\nC D\nD E\n",
-		},
-		{
-			// We should include A - C  - D even though it's further up the
-			// second path than D (which would already be in the graph by
-			// the time we get around to integrating the second path).
-			name: "Two Splits",
-			//      /-> B --\         /-> E --\
-			// A --           -> D --          -> G -> H
-			//      \-> C --/         \-> F --/
-			in:   "A B\nA C\nB D\nC D\nD E\nD F\nE G\nF G\nG H",
-			to:   "G",
-			want: "A B\nA C\nB D\nC D\nD E\nD F\nE G\nF G\n",
-		},
-		{
-			// D - E should not be duplicated.
-			name: "Two Paths - Two Splits With Gap",
-			//      /-> B --\              /-> F --\
-			// A --           -> D -> E --          -> H -> I
-			//      \-> C --/              \-> G --/
-			in:   "A B\nA C\nB D\nC D\nD E\nE F\nE G\nF H\nG H\nH I",
-			to:   "H",
-			want: "A B\nA C\nB D\nC D\nD E\nE F\nE G\nF H\nG H\n",
-		},
-	} {
-		t.Run(test.name, func(t *testing.T) {
-			stdin = strings.NewReader(test.in)
-			stdout = new(bytes.Buffer)
-			if err := digraph("allpaths", []string{"A", test.to}); err != nil {
-				t.Fatal(err)
-			}
-
-			got := stdout.(fmt.Stringer).String()
-			if got != test.want {
-				t.Errorf("digraph(allpaths, A, %s) = got %q, want %q", test.to, got, test.want)
-			}
-		})
-	}
-}
-
 func TestSplit(t *testing.T) {
 	for _, test := range []struct {
 		line string

cmd/fiximports/main.go

@@ -491,18 +491,15 @@ func list(args ...string) ([]*listPackage, error) {
 	return pkgs, nil
 }
 
-// cwd contains the current working directory of the tool.
+var cwd string
-//
+
-// It is initialized directly so that its value will be set for any other
+func init() {
-// package variables or init functions that depend on it, such as the gopath
+	var err error
-// variable in main_test.go.
+	cwd, err = os.Getwd()
-var cwd string = func() string {
-	cwd, err := os.Getwd()
 	if err != nil {
 		log.Fatalf("os.Getwd: %v", err)
 	}
-	return cwd
+}
-}()
 
 // shortPath returns an absolute or relative name for path, whatever is shorter.
 // Plundered from $GOROOT/src/cmd/go/build.go.

cmd/fiximports/main_test.go

@@ -10,7 +10,6 @@ package main
 
 import (
 	"bytes"
-	"log"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -33,25 +32,11 @@ import (
 // titanic.biz/bar	-- domain is sinking; package has jumped ship to new.com/bar
 // titanic.biz/foo	-- domain is sinking but package has no import comment yet
 
-var gopath = filepath.Join(cwd, "testdata")
-
-func init() {
-	if err := os.Setenv("GOPATH", gopath); err != nil {
-		log.Fatal(err)
-	}
-
-	// This test currently requires GOPATH mode.
-	// Explicitly disabling module mode should suffix, but
-	// we'll also turn off GOPROXY just for good measure.
-	if err := os.Setenv("GO111MODULE", "off"); err != nil {
-		log.Fatal(err)
-	}
-	if err := os.Setenv("GOPROXY", "off"); err != nil {
-		log.Fatal(err)
-	}
-}
-
 func TestFixImports(t *testing.T) {
+	gopath := filepath.Join(cwd, "testdata")
+	if err := os.Setenv("GOPATH", gopath); err != nil {
+		t.Fatalf("os.Setenv: %v", err)
+	}
 	defer func() {
 		stderr = os.Stderr
 		*badDomains = "code.google.com"
@@ -239,6 +224,11 @@ import (
 
 // TestDryRun tests that the -n flag suppresses calls to writeFile.
 func TestDryRun(t *testing.T) {
+	gopath := filepath.Join(cwd, "testdata")
+	if err := os.Setenv("GOPATH", gopath); err != nil {
+		t.Fatalf("os.Setenv: %v", err)
+	}
+
 	*dryrun = true
 	defer func() { *dryrun = false }() // restore
 	stderr = new(bytes.Buffer)

cmd/getgo/download.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !plan9
-
 package main
 
 import (
@@ -22,7 +20,7 @@ import (
 
 const (
 	currentVersionURL = "https://golang.org/VERSION?m=text"
-	downloadURLPrefix = "https://dl.google.com/go"
+	downloadURLPrefix = "https://storage.googleapis.com/golang"
 )
 
 // downloadGoVersion downloads and upacks the specific go version to dest/go.
@@ -46,7 +44,7 @@ func downloadGoVersion(version, ops, arch, dest string) error {
 		return fmt.Errorf("Downloading Go from %s failed: %v", uri, err)
 	}
 	if resp.StatusCode > 299 {
-		return fmt.Errorf("Downloading Go from %s failed with HTTP status %s", uri, resp.Status)
+		return fmt.Errorf("Downloading Go from %s failed with HTTP status %s", resp.Status)
 	}
 	defer resp.Body.Close()
 
@@ -71,7 +69,7 @@ func downloadGoVersion(version, ops, arch, dest string) error {
 	}
 	defer sresp.Body.Close()
 	if sresp.StatusCode > 299 {
-		return fmt.Errorf("Downloading Go sha256 from %s.sha256 failed with HTTP status %s", uri, sresp.Status)
+		return fmt.Errorf("Downloading Go sha256 from %s.sha256 failed with HTTP status %s", sresp.Status)
 	}
 
 	shasum, err := ioutil.ReadAll(sresp.Body)

cmd/getgo/download_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !plan9
-
 package main
 
 import (

cmd/getgo/main.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !plan9
-
 // The getgo command installs Go to the user's system.
 package main
 

cmd/getgo/main_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !plan9
-
 package main
 
 import (

cmd/getgo/path.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !plan9
-
 package main
 
 import (

cmd/getgo/path_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !plan9
-
 package main
 
 import (

cmd/getgo/server/main.go

@@ -14,7 +14,7 @@ import (
 )
 
 const (
-	base             = "https://dl.google.com/go/getgo/"
+	base             = "https://storage.googleapis.com/golang/getgo/"
 	windowsInstaller = base + "installer.exe"
 	linuxInstaller   = base + "installer_linux"
 	macInstaller     = base + "installer_darwin"

cmd/getgo/steps.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !plan9
-
 package main
 
 import (

cmd/getgo/system.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !plan9
-
 package main
 
 import (

cmd/getgo/system_unix.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build aix darwin dragonfly freebsd linux nacl netbsd openbsd solaris
+// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris
 
 package main
 

cmd/godex/print.go

@@ -7,7 +7,7 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"go/constant"
+	exact "go/constant"
 	"go/token"
 	"go/types"
 	"io"
@@ -213,9 +213,9 @@ func (p *printer) printDecl(keyword string, n int, printGroup func()) {
 
 // absInt returns the absolute value of v as a *big.Int.
 // v must be a numeric value.
-func absInt(v constant.Value) *big.Int {
+func absInt(v exact.Value) *big.Int {
 	// compute big-endian representation of v
-	b := constant.Bytes(v) // little-endian
+	b := exact.Bytes(v) // little-endian
 	for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 {
 		b[i], b[j] = b[j], b[i]
 	}
@@ -229,14 +229,14 @@ var (
 
 // floatString returns the string representation for a
 // numeric value v in normalized floating-point format.
-func floatString(v constant.Value) string {
+func floatString(v exact.Value) string {
-	if constant.Sign(v) == 0 {
+	if exact.Sign(v) == 0 {
 		return "0.0"
 	}
 	// x != 0
 
 	// convert |v| into a big.Rat x
-	x := new(big.Rat).SetFrac(absInt(constant.Num(v)), absInt(constant.Denom(v)))
+	x := new(big.Rat).SetFrac(absInt(exact.Num(v)), absInt(exact.Denom(v)))
 
 	// normalize x and determine exponent e
 	// (This is not very efficient, but also not speed-critical.)
@@ -272,7 +272,7 @@ func floatString(v constant.Value) string {
 	if e != 0 {
 		s += fmt.Sprintf("e%+d", e)
 	}
-	if constant.Sign(v) < 0 {
+	if exact.Sign(v) < 0 {
 		s = "-" + s
 	}
 
@@ -286,29 +286,29 @@ func floatString(v constant.Value) string {
 // valString returns the string representation for the value v.
 // Setting floatFmt forces an integer value to be formatted in
 // normalized floating-point format.
-// TODO(gri) Move this code into package constant.
+// TODO(gri) Move this code into package exact.
-func valString(v constant.Value, floatFmt bool) string {
+func valString(v exact.Value, floatFmt bool) string {
 	switch v.Kind() {
-	case constant.Int:
+	case exact.Int:
 		if floatFmt {
 			return floatString(v)
 		}
-	case constant.Float:
+	case exact.Float:
 		return floatString(v)
-	case constant.Complex:
+	case exact.Complex:
-		re := constant.Real(v)
+		re := exact.Real(v)
-		im := constant.Imag(v)
+		im := exact.Imag(v)
 		var s string
-		if constant.Sign(re) != 0 {
+		if exact.Sign(re) != 0 {
 			s = floatString(re)
-			if constant.Sign(im) >= 0 {
+			if exact.Sign(im) >= 0 {
 				s += " + "
 			} else {
 				s += " - "
-				im = constant.UnaryOp(token.SUB, im, 0) // negate im
+				im = exact.UnaryOp(token.SUB, im, 0) // negate im
 			}
 		}
-		// im != 0, otherwise v would be constant.Int or constant.Float
+		// im != 0, otherwise v would be exact.Int or exact.Float
 		return s + floatString(im) + "i"
 	}
 	return v.String()

cmd/godoc/README.godoc-app

@@ -0,0 +1,56 @@
+godoc on appengine
+------------------
+
+Prerequisites
+-------------
+
+* Go appengine SDK
+  https://developers.google.com/appengine/downloads#Google_App_Engine_SDK_for_Go
+
+* Go sources at tip under $GOROOT
+
+* Godoc sources at tip inside $GOPATH
+  (go get -d golang.org/x/tools/cmd/godoc)
+
+
+Directory structure
+-------------------
+
+* Let $APPDIR be the directory containing the app engine files.
+  (e.g., $APPDIR=$HOME/godoc-app)
+
+* $APPDIR contains the following entries (this may change depending on
+  app-engine release and version of godoc):
+
+	app.yaml
+	golang.org/x/tools/cmd/godoc
+	godoc.zip
+	index.split.*
+
+* The app.yaml file is set up per app engine documentation.
+  For instance:
+
+	application: godoc-app
+	version: 1
+	runtime: go
+	api_version: go1
+
+	handlers:
+	- url: /.*
+	  script: _go_app
+
+
+Configuring and running godoc
+-----------------------------
+
+To configure godoc, run
+
+	bash setup-godoc-app.bash
+
+to prepare an $APPDIR as described above. See the script for details on usage.
+
+To run godoc locally, using the App Engine development server, run
+
+	<path to go_appengine>/dev_appserver.py $APPDIR
+
+godoc should come up at http://localhost:8080 .

cmd/godoc/appinit.go

@@ -0,0 +1,82 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine
+
+package main
+
+// This file replaces main.go when running godoc under app-engine.
+// See README.godoc-app for details.
+
+import (
+	"archive/zip"
+	"log"
+	"net/http"
+	"path"
+	"regexp"
+
+	"golang.org/x/tools/godoc"
+	"golang.org/x/tools/godoc/dl"
+	"golang.org/x/tools/godoc/proxy"
+	"golang.org/x/tools/godoc/short"
+	"golang.org/x/tools/godoc/static"
+	"golang.org/x/tools/godoc/vfs"
+	"golang.org/x/tools/godoc/vfs/mapfs"
+	"golang.org/x/tools/godoc/vfs/zipfs"
+
+	"google.golang.org/appengine"
+)
+
+func init() {
+	enforceHosts = !appengine.IsDevAppServer()
+	playEnabled = true
+
+	log.Println("initializing godoc ...")
+	log.Printf(".zip file   = %s", zipFilename)
+	log.Printf(".zip GOROOT = %s", zipGoroot)
+	log.Printf("index files = %s", indexFilenames)
+
+	goroot := path.Join("/", zipGoroot) // fsHttp paths are relative to '/'
+
+	// read .zip file and set up file systems
+	const zipfile = zipFilename
+	rc, err := zip.OpenReader(zipfile)
+	if err != nil {
+		log.Fatalf("%s: %s\n", zipfile, err)
+	}
+	// rc is never closed (app running forever)
+	fs.Bind("/", zipfs.New(rc, zipFilename), goroot, vfs.BindReplace)
+	fs.Bind("/lib/godoc", mapfs.New(static.Files), "/", vfs.BindReplace)
+
+	corpus := godoc.NewCorpus(fs)
+	corpus.Verbose = false
+	corpus.MaxResults = 10000 // matches flag default in main.go
+	corpus.IndexEnabled = true
+	corpus.IndexFiles = indexFilenames
+	if err := corpus.Init(); err != nil {
+		log.Fatal(err)
+	}
+	corpus.IndexDirectory = indexDirectoryDefault
+	go corpus.RunIndexer()
+
+	pres = godoc.NewPresentation(corpus)
+	pres.TabWidth = 8
+	pres.ShowPlayground = true
+	pres.ShowExamples = true
+	pres.DeclLinks = true
+	pres.NotesRx = regexp.MustCompile("BUG")
+
+	readTemplates(pres, true)
+
+	mux := registerHandlers(pres)
+	dl.RegisterHandlers(mux)
+	short.RegisterHandlers(mux)
+
+	// Register /compile and /share handlers against the default serve mux
+	// so that other app modules can make plain HTTP requests to those
+	// hosts. (For reasons, HTTPS communication between modules is broken.)
+	proxy.RegisterHandlers(http.DefaultServeMux)
+
+	log.Println("godoc initialization complete")
+}

cmd/godoc/autocert.go

@@ -0,0 +1,77 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build autocert
+
+// This file adds automatic TLS certificate support (using
+// golang.org/x/crypto/acme/autocert), conditional on the use of the
+// autocert build tag. It sets the serveAutoCertHook func variable
+// non-nil. It is used by main.go.
+//
+// TODO: make this the default? We're in the Go 1.8 freeze now, so
+// this is too invasive to be default, but we want it for
+// https://beta.golang.org/
+
+package main
+
+import (
+	"crypto/tls"
+	"flag"
+	"net"
+	"net/http"
+	"time"
+
+	"golang.org/x/crypto/acme/autocert"
+	"golang.org/x/net/http2"
+)
+
+var (
+	autoCertDirFlag  = flag.String("autocert_cache_dir", "/var/cache/autocert", "Directory to cache TLS certs")
+	autoCertHostFlag = flag.String("autocert_hostname", "", "optional hostname to require in autocert SNI requests")
+)
+
+func init() {
+	serveAutoCertHook = serveAutoCert
+}
+
+func serveAutoCert(h http.Handler) error {
+	m := autocert.Manager{
+		Cache:  autocert.DirCache(*autoCertDirFlag),
+		Prompt: autocert.AcceptTOS,
+	}
+	if *autoCertHostFlag != "" {
+		m.HostPolicy = autocert.HostWhitelist(*autoCertHostFlag)
+	}
+	srv := &http.Server{
+		Handler: h,
+		TLSConfig: &tls.Config{
+			GetCertificate: m.GetCertificate,
+		},
+		IdleTimeout: 60 * time.Second,
+	}
+	http2.ConfigureServer(srv, &http2.Server{})
+	ln, err := net.Listen("tcp", ":443")
+	if err != nil {
+		return err
+	}
+	return srv.Serve(tls.NewListener(tcpKeepAliveListener{ln.(*net.TCPListener)}, srv.TLSConfig))
+}
+
+// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
+// connections. It's used by ListenAndServe and ListenAndServeTLS so
+// dead TCP connections (e.g. closing laptop mid-download) eventually
+// go away.
+type tcpKeepAliveListener struct {
+	*net.TCPListener
+}
+
+func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
+	tc, err := ln.AcceptTCP()
+	if err != nil {
+		return
+	}
+	tc.SetKeepAlive(true)
+	tc.SetKeepAlivePeriod(3 * time.Minute)
+	return tc, nil
+}

cmd/godoc/blog.go

@@ -21,7 +21,7 @@ import (
 
 const (
 	blogRepo = "golang.org/x/blog"
-	blogURL  = "https://blog.golang.org/"
+	blogURL  = "http://blog.golang.org/"
 	blogPath = "/blog/"
 )
 
@@ -34,19 +34,16 @@ var (
 func init() {
 	// Initialize blog only when first accessed.
 	http.HandleFunc(blogPath, func(w http.ResponseWriter, r *http.Request) {
-		blogInitOnce.Do(func() {
+		blogInitOnce.Do(blogInit)
-			blogInit(r.Host)
-		})
 		blogServer.ServeHTTP(w, r)
 	})
 }
 
-func blogInit(host string) {
+func blogInit() {
-	// Binary distributions included the blog content in "/blog".
+	// Binary distributions will include the blog content in "/blog".
-	// We stopped including this in Go 1.11.
 	root := filepath.Join(runtime.GOROOT(), "blog")
 
-	// Prefer content from the golang.org/x/blog repository if present.
+	// Prefer content from go.blog repository if present.
 	if pkg, err := build.Import(blogRepo, "", build.FindOnly); err == nil {
 		root = pkg.Dir
 	}
@@ -60,13 +57,12 @@ func blogInit(host string) {
 	}
 
 	s, err := blog.NewServer(blog.Config{
-		BaseURL:         blogPath,
+		BaseURL:      blogPath,
-		BasePath:        strings.TrimSuffix(blogPath, "/"),
+		BasePath:     strings.TrimSuffix(blogPath, "/"),
-		ContentPath:     filepath.Join(root, "content"),
+		ContentPath:  filepath.Join(root, "content"),
-		TemplatePath:    filepath.Join(root, "template"),
+		TemplatePath: filepath.Join(root, "template"),
-		HomeArticles:    5,
+		HomeArticles: 5,
-		PlayEnabled:     playEnabled,
+		PlayEnabled:  playEnabled,
-		ServeLocalLinks: strings.HasPrefix(host, "localhost"),
 	})
 	if err != nil {
 		log.Fatal(err)

cmd/godoc/dl.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build !appengine
+
 package main
 
 import "net/http"

cmd/godoc/doc.go

@@ -6,19 +6,50 @@
 
 Godoc extracts and generates documentation for Go programs.
 
-It runs as a web server and presents the documentation as a
+It has two modes.
+
+Without the -http flag, it runs in command-line mode and prints plain text
+documentation to standard output and exits. If both a library package and
+a command with the same name exists, using the prefix cmd/ will force
+documentation on the command rather than the library package. If the -src
+flag is specified, godoc prints the exported interface of a package in Go
+source form, or the implementation of a specific exported language entity:
+
+	godoc fmt                # documentation for package fmt
+	godoc fmt Printf         # documentation for fmt.Printf
+	godoc cmd/go             # force documentation for the go command
+	godoc -src fmt           # fmt package interface in Go source form
+	godoc -src fmt Printf    # implementation of fmt.Printf
+
+In command-line mode, the -q flag enables search queries against a godoc running
+as a webserver. If no explicit server address is specified with the -server flag,
+godoc first tries localhost:6060 and then http://golang.org.
+
+	godoc -q Reader
+	godoc -q math.Sin
+	godoc -server=:6060 -q sin
+
+With the -http flag, it runs as a web server and presents the documentation as a
 web page.
 
 	godoc -http=:6060
 
 Usage:
 
-	godoc [flag]
+	godoc [flag] package [name ...]
 
 The flags are:
 
 	-v
 		verbose mode
+	-q
+		arguments are considered search queries: a legal query is a
+		single identifier (such as ToLower) or a qualified identifier
+		(such as math.Sin)
+	-src
+		print (exported) source in command-line mode
+	-tabwidth=4
+		width of tabs in units of spaces
 	-timestamps=true
 		show timestamps with directory listings
 	-index
@@ -32,12 +63,7 @@ The flags are:
 		to the indexer (the indexer will never finish), a value of 1.0
 		means that index creation is running at full throttle (other
 		goroutines may get no time while the index is built)
-	-index_interval=0
+	-links=true:
-		interval of indexing; a value of 0 sets it to 5 minutes, a
-		negative value indexes only once at startup
-	-play=false
-		enable playground
-	-links=true
 		link identifiers to their declarations
 	-write_index=false
 		write index to a file; the file name must be specified with
@@ -48,17 +74,21 @@ The flags are:
 	-notes="BUG"
 		regular expression matching note markers to show
 		(e.g., "BUG|TODO", ".*")
+	-html
+		print HTML in command-line mode
 	-goroot=$GOROOT
 		Go root directory
 	-http=addr
 		HTTP service address (e.g., '127.0.0.1:6060' or just ':6060')
+	-server=addr
+		webserver address for command line searches
 	-analysis=type,pointer
 		comma-separated list of analyses to perform
 		"type": display identifier resolution, type info, method sets,
 			'implements', and static callees
 		"pointer": display channel peers, callers and dynamic callees
 			(significantly slower)
-		See https://golang.org/lib/godoc/analysis/help.html for details.
+		See http://golang.org/lib/godoc/analysis/help.html for details.
 	-templates=""
 		directory containing alternate template files; if set,
 		the directory may provide alternative template files
@@ -73,7 +103,7 @@ By default, godoc looks at the packages it finds via $GOROOT and $GOPATH (if set
 This behavior can be altered by providing an alternative $GOROOT with the -goroot
 flag.
 
-When the -index flag is set, a search index is maintained.
+When godoc runs as a web server and -index is set, a search index is maintained.
 The index is created at startup.
 
 The index contains both identifier and full text search information (searchable
@@ -81,19 +111,23 @@ via regular expressions). The maximum number of full text search results shown
 can be set with the -maxresults flag; if set to 0, no full text results are
 shown, and only an identifier index but no full text search index is created.
 
-By default, godoc uses the system's GOOS/GOARCH. You can provide the URL parameters
+By default, godoc uses the system's GOOS/GOARCH; in command-line mode you can
-"GOOS" and "GOARCH" to set the output on the web page for the target system.
+set the GOOS/GOARCH environment variables to get output for the system specified.
+If -http was specified you can provide the URL parameters "GOOS" and "GOARCH"
+to set the output on the web page.
 
 The presentation mode of web pages served by godoc can be controlled with the
 "m" URL parameter; it accepts a comma-separated list of flag names as value:
 
 	all	show documentation for all declarations, not just the exported ones
 	methods	show all embedded methods, not just those of unexported anonymous fields
-	src	show the original source code rather than the extracted documentation
+	src	show the original source code rather then the extracted documentation
+	text	present the page in textual (command-line) form rather than HTML
 	flat	present flat (not indented) directory listings using full paths
 
-For instance, https://golang.org/pkg/math/big/?m=all shows the documentation
+For instance, http://golang.org/pkg/math/big/?m=all,text shows the documentation
-for all (not just the exported) declarations of package big.
+for all (not just the exported) declarations of package big, in textual form (as
+it would appear when using godoc from the command line: "godoc -src math/big .*").
 
 By default, godoc serves files from the file system of the underlying OS.
 Instead, a .zip file may be provided via the -zip flag, which contains
@@ -109,11 +143,11 @@ one may run godoc as follows:
 	godoc -http=:6060 -zip=go.zip -goroot=$HOME/go
 
 Godoc documentation is converted to HTML or to text using the go/doc package;
-see https://golang.org/pkg/go/doc/#ToHTML for the exact rules.
+see http://golang.org/pkg/go/doc/#ToHTML for the exact rules.
 Godoc also shows example code that is runnable by the testing package;
-see https://golang.org/pkg/testing/#hdr-Examples for the conventions.
+see http://golang.org/pkg/testing/#hdr-Examples for the conventions.
 See "Godoc: documenting Go code" for how to write good comments for godoc:
-https://golang.org/doc/articles/godoc_documenting_go_code.html
+http://golang.org/doc/articles/godoc_documenting_go_code.html
 
 */
 package main // import "golang.org/x/tools/cmd/godoc"

cmd/godoc/godoc19_test.go

@@ -0,0 +1,9 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.9
+
+package main_test
+
+func init() { isGo19 = true }

cmd/godoc/godoc_test.go

@@ -8,7 +8,6 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
-	"go/build"
 	"io"
 	"io/ioutil"
 	"net"
@@ -32,10 +31,6 @@ func buildGodoc(t *testing.T) (bin string, cleanup func()) {
 	if runtime.GOARCH == "arm" {
 		t.Skip("skipping test on arm platforms; too slow")
 	}
-	if runtime.GOOS == "android" {
-		t.Skipf("the dependencies are not available on android")
-	}
-
 	tmp, err := ioutil.TempDir("", "godoc-regtest-")
 	if err != nil {
 		t.Fatal(err)
@@ -58,6 +53,91 @@ func buildGodoc(t *testing.T) (bin string, cleanup func()) {
 	return bin, func() { os.RemoveAll(tmp) }
 }
 
+var isGo19 bool // godoc19_test.go sets it to true.
+
+// Basic regression test for godoc command-line tool.
+func TestCLI(t *testing.T) {
+	bin, cleanup := buildGodoc(t)
+	defer cleanup()
+
+	// condStr returns s if cond is true, otherwise empty string.
+	condStr := func(cond bool, s string) string {
+		if !cond {
+			return ""
+		}
+		return s
+	}
+
+	tests := []struct {
+		args      []string
+		matches   []string // regular expressions
+		dontmatch []string // regular expressions
+	}{
+		{
+			args: []string{"fmt"},
+			matches: []string{
+				`import "fmt"`,
+				`Package fmt implements formatted I/O`,
+			},
+		},
+		{
+			args: []string{"io", "WriteString"},
+			matches: []string{
+				`func WriteString\(`,
+				`WriteString writes the contents of the string s to w`,
+			},
+		},
+		{
+			args: []string{"nonexistingpkg"},
+			matches: []string{
+				`cannot find package` +
+					// TODO: Remove this when support for Go 1.8 is dropped.
+					condStr(!isGo19,
+						// For Go 1.8 and older, because it doesn't have CL 33158 change applied to go/build.
+						// The last pattern (does not e) is for plan9:
+						// http://build.golang.org/log/2d8e5e14ed365bfa434b37ec0338cd9e6f8dd9bf
+						`|no such file or directory|does not exist|cannot find the file|(?:' does not e)`),
+			},
+		},
+		{
+			args: []string{"fmt", "NonexistentSymbol"},
+			matches: []string{
+				`No match found\.`,
+			},
+		},
+		{
+			args: []string{"-src", "syscall", "Open"},
+			matches: []string{
+				`func Open\(`,
+			},
+			dontmatch: []string{
+				`No match found\.`,
+			},
+		},
+	}
+	for _, test := range tests {
+		cmd := exec.Command(bin, test.args...)
+		cmd.Args[0] = "godoc"
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Errorf("Running with args %#v: %v", test.args, err)
+			continue
+		}
+		for _, pat := range test.matches {
+			re := regexp.MustCompile(pat)
+			if !re.Match(out) {
+				t.Errorf("godoc %v =\n%s\nwanted /%v/", strings.Join(test.args, " "), out, pat)
+			}
+		}
+		for _, pat := range test.dontmatch {
+			re := regexp.MustCompile(pat)
+			if re.Match(out) {
+				t.Errorf("godoc %v =\n%s\ndid not want /%v/", strings.Join(test.args, " "), out, pat)
+			}
+		}
+	}
+}
+
 func serverAddress(t *testing.T) string {
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -74,32 +154,19 @@ func waitForServerReady(t *testing.T, addr string) {
 	waitForServer(t,
 		fmt.Sprintf("http://%v/", addr),
 		"The Go Programming Language",
-		15*time.Second,
+		15*time.Second)
-		false)
 }
 
 func waitForSearchReady(t *testing.T, addr string) {
 	waitForServer(t,
 		fmt.Sprintf("http://%v/search?q=FALLTHROUGH", addr),
 		"The list of tokens.",
-		2*time.Minute,
+		2*time.Minute)
-		false)
-}
-
-func waitUntilScanComplete(t *testing.T, addr string) {
-	waitForServer(t,
-		fmt.Sprintf("http://%v/pkg", addr),
-		"Scan is not yet complete",
-		2*time.Minute,
-		true,
-	)
-	// setting reverse as true, which means this waits
-	// until the string is not returned in the response anymore
 }
 
 const pollInterval = 200 * time.Millisecond
 
-func waitForServer(t *testing.T, url, match string, timeout time.Duration, reverse bool) {
+func waitForServer(t *testing.T, url, match string, timeout time.Duration) {
 	// "health check" duplicated from x/tools/cmd/tipgodoc/tip.go
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
@@ -110,74 +177,19 @@ func waitForServer(t *testing.T, url, match string, timeout time.Duration, rever
 		}
 		rbody, err := ioutil.ReadAll(res.Body)
 		res.Body.Close()
-		if err == nil && res.StatusCode == http.StatusOK {
+		if err == nil && res.StatusCode == http.StatusOK &&
-			if bytes.Contains(rbody, []byte(match)) && !reverse {
+			bytes.Contains(rbody, []byte(match)) {
-				return
+			return
-			}
-			if !bytes.Contains(rbody, []byte(match)) && reverse {
-				return
-			}
 		}
 	}
 	t.Fatalf("Server failed to respond in %v", timeout)
 }
 
-// hasTag checks whether a given release tag is contained in the current version
-// of the go binary.
-func hasTag(t string) bool {
-	for _, v := range build.Default.ReleaseTags {
-		if t == v {
-			return true
-		}
-	}
-	return false
-}
-
 func killAndWait(cmd *exec.Cmd) {
 	cmd.Process.Kill()
 	cmd.Wait()
 }
 
-func TestURL(t *testing.T) {
-	if runtime.GOOS == "plan9" {
-		t.Skip("skipping on plan9; fails to start up quickly enough")
-	}
-	bin, cleanup := buildGodoc(t)
-	defer cleanup()
-
-	testcase := func(url string, contents string) func(t *testing.T) {
-		return func(t *testing.T) {
-			stdout, stderr := new(bytes.Buffer), new(bytes.Buffer)
-
-			args := []string{fmt.Sprintf("-url=%s", url)}
-			cmd := exec.Command(bin, args...)
-			cmd.Stdout = stdout
-			cmd.Stderr = stderr
-			cmd.Args[0] = "godoc"
-
-			// Set GOPATH variable to non-existing path
-			// and GOPROXY=off to disable module fetches.
-			// We cannot just unset GOPATH variable because godoc would default it to ~/go.
-			// (We don't want the indexer looking at the local workspace during tests.)
-			cmd.Env = append(os.Environ(),
-				"GOPATH=does_not_exist",
-				"GOPROXY=off",
-				"GO111MODULE=off")
-
-			if err := cmd.Run(); err != nil {
-				t.Fatalf("failed to run godoc -url=%q: %s\nstderr:\n%s", url, err, stderr)
-			}
-
-			if !strings.Contains(stdout.String(), contents) {
-				t.Errorf("did not find substring %q in output of godoc -url=%q:\n%s", contents, url, stdout)
-			}
-		}
-	}
-
-	t.Run("index", testcase("/", "Go is an open source programming language"))
-	t.Run("fmt", testcase("/pkg/fmt", "Package fmt implements formatted I/O"))
-}
-
 // Basic integration test for godoc HTTP interface.
 func TestWeb(t *testing.T) {
 	testWeb(t, false)
@@ -193,9 +205,6 @@ func TestWebIndex(t *testing.T) {
 
 // Basic integration test for godoc HTTP interface.
 func testWeb(t *testing.T, withIndex bool) {
-	if runtime.GOOS == "plan9" {
-		t.Skip("skipping on plan9; fails to start up quickly enough")
-	}
 	bin, cleanup := buildGodoc(t)
 	defer cleanup()
 	addr := serverAddress(t)
@@ -207,16 +216,7 @@ func testWeb(t *testing.T, withIndex bool) {
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 	cmd.Args[0] = "godoc"
-
+	cmd.Env = godocEnv()
-	// Set GOPATH variable to non-existing path
-	// and GOPROXY=off to disable module fetches.
-	// We cannot just unset GOPATH variable because godoc would default it to ~/go.
-	// (We don't want the indexer looking at the local workspace during tests.)
-	cmd.Env = append(os.Environ(),
-		"GOPATH=does_not_exist",
-		"GOPROXY=off",
-		"GO111MODULE=off")
-
 	if err := cmd.Start(); err != nil {
 		t.Fatalf("failed to start godoc: %s", err)
 	}
@@ -226,113 +226,74 @@ func testWeb(t *testing.T, withIndex bool) {
 		waitForSearchReady(t, addr)
 	} else {
 		waitForServerReady(t, addr)
-		waitUntilScanComplete(t, addr)
 	}
 
 	tests := []struct {
-		path        string
+		path      string
-		contains    []string // substring
+		match     []string
-		match       []string // regexp
+		dontmatch []string
-		notContains []string
+		needIndex bool
-		needIndex   bool
-		releaseTag  string // optional release tag that must be in go/build.ReleaseTags
 	}{
 		{
-			path:     "/",
+			path:  "/",
-			contains: []string{"Go is an open source programming language"},
+			match: []string{"Go is an open source programming language"},
 		},
 		{
-			path:     "/pkg/fmt/",
+			path:  "/pkg/fmt/",
-			contains: []string{"Package fmt implements formatted I/O"},
+			match: []string{"Package fmt implements formatted I/O"},
 		},
 		{
-			path:     "/src/fmt/",
+			path:  "/src/fmt/",
-			contains: []string{"scan_test.go"},
+			match: []string{"scan_test.go"},
 		},
 		{
-			path:     "/src/fmt/print.go",
+			path:  "/src/fmt/print.go",
-			contains: []string{"// Println formats using"},
+			match: []string{"// Println formats using"},
 		},
 		{
 			path: "/pkg",
-			contains: []string{
+			match: []string{
 				"Standard library",
 				"Package fmt implements formatted I/O",
 			},
-			notContains: []string{
+			dontmatch: []string{
 				"internal/syscall",
 				"cmd/gc",
 			},
 		},
 		{
 			path: "/pkg/?m=all",
-			contains: []string{
+			match: []string{
 				"Standard library",
 				"Package fmt implements formatted I/O",
 				"internal/syscall/?m=all",
 			},
-			notContains: []string{
+			dontmatch: []string{
 				"cmd/gc",
 			},
 		},
 		{
 			path: "/search?q=ListenAndServe",
-			contains: []string{
+			match: []string{
 				"/src",
 			},
-			notContains: []string{
+			dontmatch: []string{
 				"/pkg/bootstrap",
 			},
 			needIndex: true,
 		},
 		{
 			path: "/pkg/strings/",
-			contains: []string{
+			match: []string{
 				`href="/src/strings/strings.go"`,
 			},
 		},
 		{
 			path: "/cmd/compile/internal/amd64/",
-			contains: []string{
+			match: []string{
 				`href="/src/cmd/compile/internal/amd64/ssa.go"`,
 			},
 		},
-		{
-			path: "/pkg/math/bits/",
-			contains: []string{
-				`Added in Go 1.9`,
-			},
-		},
-		{
-			path: "/pkg/net/",
-			contains: []string{
-				`// IPv6 scoped addressing zone; added in Go 1.1`,
-			},
-		},
-		{
-			path: "/pkg/net/http/httptrace/",
-			match: []string{
-				`Got1xxResponse.*// Go 1\.11`,
-			},
-			releaseTag: "go1.11",
-		},
-		// Verify we don't add version info to a struct field added the same time
-		// as the struct itself:
-		{
-			path: "/pkg/net/http/httptrace/",
-			match: []string{
-				`(?m)GotFirstResponseByte func\(\)\s*$`,
-			},
-		},
-		// Remove trailing periods before adding semicolons:
-		{
-			path: "/pkg/database/sql/",
-			contains: []string{
-				"The number of connections currently in use; added in Go 1.11",
-				"The number of idle connections; added in Go 1.11",
-			},
-			releaseTag: "go1.11",
-		},
 	}
 	for _, test := range tests {
 		if test.needIndex && !withIndex {
@@ -345,34 +306,18 @@ func testWeb(t *testing.T, withIndex bool) {
 			continue
 		}
 		body, err := ioutil.ReadAll(resp.Body)
-		strBody := string(body)
 		resp.Body.Close()
 		if err != nil {
 			t.Errorf("GET %s: failed to read body: %s (response: %v)", url, err, resp)
 		}
 		isErr := false
-		for _, substr := range test.contains {
+		for _, substr := range test.match {
-			if test.releaseTag != "" && !hasTag(test.releaseTag) {
-				continue
-			}
 			if !bytes.Contains(body, []byte(substr)) {
 				t.Errorf("GET %s: wanted substring %q in body", url, substr)
 				isErr = true
 			}
 		}
-		for _, re := range test.match {
+		for _, substr := range test.dontmatch {
-			if test.releaseTag != "" && !hasTag(test.releaseTag) {
-				continue
-			}
-			if ok, err := regexp.MatchString(re, strBody); !ok || err != nil {
-				if err != nil {
-					t.Fatalf("Bad regexp %q: %v", re, err)
-				}
-				t.Errorf("GET %s: wanted to match %s in body", url, re)
-				isErr = true
-			}
-		}
-		for _, substr := range test.notContains {
 			if bytes.Contains(body, []byte(substr)) {
 				t.Errorf("GET %s: didn't want substring %q in body", url, substr)
 				isErr = true
@@ -424,11 +369,14 @@ func main() { print(lib.V) }
 	defer cleanup()
 	addr := serverAddress(t)
 	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr), "-analysis=type")
-	cmd.Env = os.Environ()
 	cmd.Env = append(cmd.Env, fmt.Sprintf("GOROOT=%s", filepath.Join(tmpdir, "goroot")))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("GOPATH=%s", filepath.Join(tmpdir, "gopath")))
-	cmd.Env = append(cmd.Env, "GO111MODULE=off")
+	for _, e := range os.Environ() {
-	cmd.Env = append(cmd.Env, "GOPROXY=off")
+		if strings.HasPrefix(e, "GOROOT=") || strings.HasPrefix(e, "GOPATH=") {
+			continue
+		}
+		cmd.Env = append(cmd.Env, e)
+	}
 	cmd.Stdout = os.Stderr
 	stderr, err := cmd.StderrPipe()
 	if err != nil {
@@ -506,3 +454,15 @@ tryagain:
 		}
 	}
 }
+
+// godocEnv returns the process environment without the GOPATH variable.
+// (We don't want the indexer looking at the local workspace during tests.)
+func godocEnv() (env []string) {
+	for _, v := range os.Environ() {
+		if strings.HasPrefix(v, "GOPATH=") {
+			continue
+		}
+		env = append(env, v)
+	}
+	return
+}

cmd/godoc/goroot.go

@@ -1,74 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"os"
-	"path/filepath"
-	"runtime"
-)
-
-// Copies of functions from src/cmd/go/internal/cfg/cfg.go for
-// finding the GOROOT.
-// Keep them in sync until support is moved to a common place, if ever.
-
-func findGOROOT() string {
-	if env := os.Getenv("GOROOT"); env != "" {
-		return filepath.Clean(env)
-	}
-	def := filepath.Clean(runtime.GOROOT())
-	if runtime.Compiler == "gccgo" {
-		// gccgo has no real GOROOT, and it certainly doesn't
-		// depend on the executable's location.
-		return def
-	}
-	exe, err := os.Executable()
-	if err == nil {
-		exe, err = filepath.Abs(exe)
-		if err == nil {
-			if dir := filepath.Join(exe, "../.."); isGOROOT(dir) {
-				// If def (runtime.GOROOT()) and dir are the same
-				// directory, prefer the spelling used in def.
-				if isSameDir(def, dir) {
-					return def
-				}
-				return dir
-			}
-			exe, err = filepath.EvalSymlinks(exe)
-			if err == nil {
-				if dir := filepath.Join(exe, "../.."); isGOROOT(dir) {
-					if isSameDir(def, dir) {
-						return def
-					}
-					return dir
-				}
-			}
-		}
-	}
-	return def
-}
-
-// isGOROOT reports whether path looks like a GOROOT.
-//
-// It does this by looking for the path/pkg/tool directory,
-// which is necessary for useful operation of the cmd/go tool,
-// and is not typically present in a GOPATH.
-func isGOROOT(path string) bool {
-	stat, err := os.Stat(filepath.Join(path, "pkg", "tool"))
-	if err != nil {
-		return false
-	}
-	return stat.IsDir()
-}
-
-// isSameDir reports whether dir1 and dir2 are the same directory.
-func isSameDir(dir1, dir2 string) bool {
-	if dir1 == dir2 {
-		return true
-	}
-	info1, err1 := os.Stat(dir1)
-	info2, err2 := os.Stat(dir2)
-	return err1 == nil && err2 == nil && os.SameFile(info1, info2)
-}

cmd/godoc/handlers.go

@@ -21,7 +21,6 @@ import (
 	"text/template"
 
 	"golang.org/x/tools/godoc"
-	"golang.org/x/tools/godoc/golangorgenv"
 	"golang.org/x/tools/godoc/redirect"
 	"golang.org/x/tools/godoc/vfs"
 )
@@ -31,20 +30,21 @@ var (
 	fs   = vfs.NameSpace{}
 )
 
+var enforceHosts = false // set true in production on app engine
+
 // hostEnforcerHandler redirects requests to "http://foo.golang.org/bar"
 // to "https://golang.org/bar".
-// It permits requests to the host "godoc-test.golang.org" for testing and
+// It permits requests to the host "godoc-test.golang.org" for testing.
-// golang.google.cn for Chinese users.
 type hostEnforcerHandler struct {
 	h http.Handler
 }
 
 func (h hostEnforcerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if !golangorgenv.EnforceHosts() {
+	if !enforceHosts {
 		h.h.ServeHTTP(w, r)
 		return
 	}
-	if !h.isHTTPS(r) || !h.validHost(r.Host) {
+	if r.TLS == nil || !h.validHost(r.Host) {
 		r.URL.Scheme = "https"
 		if h.validHost(r.Host) {
 			r.URL.Host = r.Host
@@ -54,21 +54,13 @@ func (h hostEnforcerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Redirect(w, r, r.URL.String(), http.StatusFound)
 		return
 	}
-	w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")
+	w.Header().Set("Strict-Transport-Security", "max-age=31536000; preload")
 	h.h.ServeHTTP(w, r)
 }
 
-func (h hostEnforcerHandler) isHTTPS(r *http.Request) bool {
-	return r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https"
-}
-
 func (h hostEnforcerHandler) validHost(host string) bool {
 	switch strings.ToLower(host) {
-	case "golang.org", "golang.google.cn":
+	case "golang.org", "godoc-test.golang.org":
-		return true
-	}
-	if strings.HasSuffix(host, "-dot-golang-org.appspot.com") {
-		// staging/test
 		return true
 	}
 	return false
@@ -112,23 +104,27 @@ func readTemplate(name string) *template.Template {
 	return t
 }
 
-func readTemplates(p *godoc.Presentation) {
+func readTemplates(p *godoc.Presentation, html bool) {
-	codewalkHTML = readTemplate("codewalk.html")
+	p.PackageText = readTemplate("package.txt")
-	codewalkdirHTML = readTemplate("codewalkdir.html")
+	p.SearchText = readTemplate("search.txt")
-	p.CallGraphHTML = readTemplate("callgraph.html")
+
-	p.DirlistHTML = readTemplate("dirlist.html")
+	if html || p.HTMLMode {
-	p.ErrorHTML = readTemplate("error.html")
+		codewalkHTML = readTemplate("codewalk.html")
-	p.ExampleHTML = readTemplate("example.html")
+		codewalkdirHTML = readTemplate("codewalkdir.html")
-	p.GodocHTML = readTemplate("godoc.html")
+		p.CallGraphHTML = readTemplate("callgraph.html")
-	p.ImplementsHTML = readTemplate("implements.html")
+		p.DirlistHTML = readTemplate("dirlist.html")
-	p.MethodSetHTML = readTemplate("methodset.html")
+		p.ErrorHTML = readTemplate("error.html")
-	p.PackageHTML = readTemplate("package.html")
+		p.ExampleHTML = readTemplate("example.html")
-	p.PackageRootHTML = readTemplate("packageroot.html")
+		p.GodocHTML = readTemplate("godoc.html")
-	p.SearchHTML = readTemplate("search.html")
+		p.ImplementsHTML = readTemplate("implements.html")
-	p.SearchDocHTML = readTemplate("searchdoc.html")
+		p.MethodSetHTML = readTemplate("methodset.html")
-	p.SearchCodeHTML = readTemplate("searchcode.html")
+		p.PackageHTML = readTemplate("package.html")
-	p.SearchTxtHTML = readTemplate("searchtxt.html")
+		p.SearchHTML = readTemplate("search.html")
-	p.SearchDescXML = readTemplate("opensearch.xml")
+		p.SearchDocHTML = readTemplate("searchdoc.html")
+		p.SearchCodeHTML = readTemplate("searchcode.html")
+		p.SearchTxtHTML = readTemplate("searchtxt.html")
+		p.SearchDescXML = readTemplate("opensearch.xml")
+	}
 }
 
 type fmtResponse struct {

cmd/godoc/main.go

@@ -14,18 +14,28 @@
 //				(idea is if you say import "compress/zlib", you go to
 //				http://godoc/pkg/compress/zlib)
 //
+// Command-line interface:
+//
+//	godoc packagepath [name ...]
+//
+//	godoc compress/zlib
+//		- prints doc for package compress/zlib
+//	godoc crypto/block Cipher NewCMAC
+//		- prints doc for Cipher and NewCMAC in package crypto/block
+
+// +build !appengine
 
 package main
 
 import (
 	"archive/zip"
-	"bytes"
 	_ "expvar" // to serve /debug/vars
 	"flag"
 	"fmt"
 	"go/build"
 	"log"
 	"net/http"
+	"net/http/httptest"
 	_ "net/http/pprof" // to serve /debug/pprof/*
 	"net/url"
 	"os"
@@ -43,7 +53,7 @@ import (
 	"golang.org/x/tools/godoc/vfs/zipfs"
 )
 
-const defaultAddr = "localhost:6060" // default webserver address
+const defaultAddr = ":6060" // default webserver address
 
 var (
 	// file system to serve
@@ -56,21 +66,29 @@ var (
 	analysisFlag = flag.String("analysis", "", `comma-separated list of analyses to perform (supported: type, pointer). See http://golang.org/lib/godoc/analysis/help.html`)
 
 	// network
-	httpAddr = flag.String("http", defaultAddr, "HTTP service address")
+	httpAddr   = flag.String("http", "", "HTTP service address (e.g., '"+defaultAddr+"')")
+	serverAddr = flag.String("server", "", "webserver address for command line searches")
 
 	// layout control
+	html    = flag.Bool("html", false, "print HTML in command-line mode")
+	srcMode = flag.Bool("src", false, "print (exported) source in command-line mode")
 	urlFlag = flag.String("url", "", "print HTML for named URL")
 
+	// command-line searches
+	query = flag.Bool("q", false, "arguments are considered search queries")
+
 	verbose = flag.Bool("v", false, "verbose mode")
 
 	// file system roots
 	// TODO(gri) consider the invariant that goroot always end in '/'
-	goroot = flag.String("goroot", findGOROOT(), "Go root directory")
+	goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory")
 
 	// layout control
+	tabWidth       = flag.Int("tabwidth", 4, "tab width")
 	showTimestamps = flag.Bool("timestamps", false, "show timestamps with directory listings")
 	templateDir    = flag.String("templates", "", "load templates/JS/CSS from disk in this directory")
-	showPlayground = flag.Bool("play", false, "enable playground")
+	showPlayground = flag.Bool("play", false, "enable playground in web interface")
+	showExamples   = flag.Bool("ex", false, "show examples in command line mode")
 	declLinks      = flag.Bool("links", true, "link identifiers to their declarations")
 
 	// search index
@@ -84,19 +102,10 @@ var (
 	notesRx = flag.String("notes", "BUG", "regular expression matching note markers to show")
 )
 
-// An httpResponseRecorder is an http.ResponseWriter
-type httpResponseRecorder struct {
-	body   *bytes.Buffer
-	header http.Header
-	code   int
-}
-
-func (w *httpResponseRecorder) Header() http.Header         { return w.header }
-func (w *httpResponseRecorder) Write(b []byte) (int, error) { return w.body.Write(b) }
-func (w *httpResponseRecorder) WriteHeader(code int)        { w.code = code }
-
 func usage() {
-	fmt.Fprintf(os.Stderr, "usage: godoc -http="+defaultAddr+"\n")
+	fmt.Fprintf(os.Stderr,
+		"usage: godoc package [name ...]\n"+
+			"	godoc -http="+defaultAddr+"\n")
 	flag.PrintDefaults()
 	os.Exit(2)
 }
@@ -123,58 +132,47 @@ func handleURLFlag() {
 
 		// Invoke default HTTP handler to serve request
 		// to our buffering httpWriter.
-		w := &httpResponseRecorder{code: 200, header: make(http.Header), body: new(bytes.Buffer)}
+		w := httptest.NewRecorder()
 		http.DefaultServeMux.ServeHTTP(w, req)
 
 		// Return data, error, or follow redirect.
-		switch w.code {
+		switch w.Code {
 		case 200: // ok
-			os.Stdout.Write(w.body.Bytes())
+			os.Stdout.Write(w.Body.Bytes())
 			return
 		case 301, 302, 303, 307: // redirect
-			redirect := w.header.Get("Location")
+			redirect := w.HeaderMap.Get("Location")
 			if redirect == "" {
-				log.Fatalf("HTTP %d without Location header", w.code)
+				log.Fatalf("HTTP %d without Location header", w.Code)
 			}
 			urlstr = redirect
 		default:
-			log.Fatalf("HTTP error %d", w.code)
+			log.Fatalf("HTTP error %d", w.Code)
 		}
 	}
 	log.Fatalf("too many redirects")
 }
 
-func initCorpus(corpus *godoc.Corpus) {
-	err := corpus.Init()
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
 func main() {
 	flag.Usage = usage
 	flag.Parse()
 
-	if certInit != nil {
-		certInit()
-	}
-
 	playEnabled = *showPlayground
 
-	// Check usage.
+	// Check usage: server and no args.
-	if flag.NArg() > 0 {
+	if (*httpAddr != "" || *urlFlag != "") && (flag.NArg() > 0) {
-		fmt.Fprintln(os.Stderr, `Unexpected arguments. Use "go doc" for command-line help output instead. For example, "go doc fmt.Printf".`)
+		fmt.Fprintln(os.Stderr, "can't use -http with args.")
-		usage()
-	}
-	if *httpAddr == "" && *urlFlag == "" && !*writeIndex {
-		fmt.Fprintln(os.Stderr, "At least one of -http, -url, or -write_index must be set to a non-zero value.")
 		usage()
 	}
 
-	// Set the resolved goroot.
+	// Check usage: command line args or index creation mode.
-	vfs.GOROOT = *goroot
+	if (*httpAddr != "" || *urlFlag != "") != (flag.NArg() == 0) && !*writeIndex {
+		fmt.Fprintln(os.Stderr, "missing args.")
+		usage()
+	}
 
-	fsGate := make(chan bool, 20)
+	var fsGate chan bool
+	fsGate = make(chan bool, 20)
 
 	// Determine file system to use.
 	if *zipfile == "" {
@@ -201,6 +199,8 @@ func main() {
 		fs.Bind("/src", gatefs.New(vfs.OS(p), fsGate), "/src", vfs.BindAfter)
 	}
 
+	httpMode := *httpAddr != ""
+
 	var typeAnalysis, pointerAnalysis bool
 	if *analysisFlag != "" {
 		for _, a := range strings.Split(*analysisFlag, ",") {
@@ -218,7 +218,7 @@ func main() {
 	corpus := godoc.NewCorpus(fs)
 	corpus.Verbose = *verbose
 	corpus.MaxResults = *maxResults
-	corpus.IndexEnabled = *indexEnabled
+	corpus.IndexEnabled = *indexEnabled && httpMode
 	if *maxResults == 0 {
 		corpus.IndexFullText = false
 	}
@@ -226,27 +226,29 @@ func main() {
 	corpus.IndexDirectory = indexDirectoryDefault
 	corpus.IndexThrottle = *indexThrottle
 	corpus.IndexInterval = *indexInterval
-	if *writeIndex || *urlFlag != "" {
+	if *writeIndex {
 		corpus.IndexThrottle = 1.0
 		corpus.IndexEnabled = true
-		initCorpus(corpus)
+	}
-	} else {
+	if *writeIndex || httpMode || *urlFlag != "" {
-		go initCorpus(corpus)
+		if err := corpus.Init(); err != nil {
+			log.Fatal(err)
+		}
 	}
 
-	// Initialize the version info before readTemplates, which saves
-	// the map value in a method value.
-	corpus.InitVersionInfo()
-
 	pres = godoc.NewPresentation(corpus)
+	pres.TabWidth = *tabWidth
 	pres.ShowTimestamps = *showTimestamps
 	pres.ShowPlayground = *showPlayground
+	pres.ShowExamples = *showExamples
 	pres.DeclLinks = *declLinks
+	pres.SrcMode = *srcMode
+	pres.HTMLMode = *html
 	if *notesRx != "" {
 		pres.NotesRx = regexp.MustCompile(*notesRx)
 	}
 
-	readTemplates(pres)
+	readTemplates(pres, httpMode || *urlFlag != "")
 	registerHandlers(pres)
 
 	if *writeIndex {
@@ -281,58 +283,63 @@ func main() {
 		return
 	}
 
-	var handler http.Handler = http.DefaultServeMux
+	if httpMode {
-	if *verbose {
+		// HTTP server mode.
-		log.Printf("Go Documentation Server")
+		var handler http.Handler = http.DefaultServeMux
-		log.Printf("version = %s", runtime.Version())
+		if *verbose {
-		log.Printf("address = %s", *httpAddr)
+			log.Printf("Go Documentation Server")
-		log.Printf("goroot = %s", *goroot)
+			log.Printf("version = %s", runtime.Version())
-		switch {
+			log.Printf("address = %s", *httpAddr)
-		case !*indexEnabled:
+			log.Printf("goroot = %s", *goroot)
-			log.Print("search index disabled")
+			log.Printf("tabwidth = %d", *tabWidth)
-		case *maxResults > 0:
+			switch {
-			log.Printf("full text index enabled (maxresults = %d)", *maxResults)
+			case !*indexEnabled:
-		default:
+				log.Print("search index disabled")
-			log.Print("identifier search index enabled")
+			case *maxResults > 0:
-		}
+				log.Printf("full text index enabled (maxresults = %d)", *maxResults)
-		fs.Fprint(os.Stderr)
+			default:
-		handler = loggingHandler(handler)
+				log.Print("identifier search index enabled")
-	}
-
-	// Initialize search index.
-	if *indexEnabled {
-		go corpus.RunIndexer()
-	}
-
-	// Start type/pointer analysis.
-	if typeAnalysis || pointerAnalysis {
-		go analysis.Run(pointerAnalysis, &corpus.Analysis)
-	}
-
-	if runHTTPS != nil {
-		go func() {
-			if err := runHTTPS(handler); err != nil {
-				log.Fatalf("ListenAndServe TLS: %v", err)
 			}
-		}()
+			fs.Fprint(os.Stderr)
+			handler = loggingHandler(handler)
+		}
+
+		// Initialize search index.
+		if *indexEnabled {
+			go corpus.RunIndexer()
+		}
+
+		// Start type/pointer analysis.
+		if typeAnalysis || pointerAnalysis {
+			go analysis.Run(pointerAnalysis, &corpus.Analysis)
+		}
+
+		if serveAutoCertHook != nil {
+			go func() {
+				if err := serveAutoCertHook(handler); err != nil {
+					log.Fatalf("ListenAndServe TLS: %v", err)
+				}
+			}()
+		}
+
+		// Start http server.
+		if err := http.ListenAndServe(*httpAddr, handler); err != nil {
+			log.Fatalf("ListenAndServe %s: %v", *httpAddr, err)
+		}
+
+		return
 	}
 
-	// Start http server.
+	if *query {
-	if *verbose {
+		handleRemoteSearch()
-		log.Println("starting HTTP server")
+		return
 	}
-	if wrapHTTPMux != nil {
+
-		handler = wrapHTTPMux(handler)
+	if err := godoc.CommandLine(os.Stdout, fs, pres, flag.Args()); err != nil {
-	}
+		log.Print(err)
-	if err := http.ListenAndServe(*httpAddr, handler); err != nil {
-		log.Fatalf("ListenAndServe %s: %v", *httpAddr, err)
 	}
 }
 
-// Hooks that are set non-nil in autocert.go if the "autocert" build tag
+// serveAutoCertHook if non-nil specifies a function to listen on port 443.
-// is used.
+// See autocert.go.
-var (
+var serveAutoCertHook func(http.Handler) error
-	certInit    func()
-	runHTTPS    func(http.Handler) error
-	wrapHTTPMux func(http.Handler) http.Handler
-)

cmd/godoc/play.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build !appengine
+
 package main
 
 // This package registers "/compile" and "/share" handlers

cmd/godoc/remotesearch.go

@@ -0,0 +1,72 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !appengine
+
+package main
+
+import (
+	"errors"
+	"flag"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+)
+
+func handleRemoteSearch() {
+	// Command-line queries.
+	for i := 0; i < flag.NArg(); i++ {
+		res, err := remoteSearch(flag.Arg(i))
+		if err != nil {
+			log.Fatalf("remoteSearch: %s", err)
+		}
+		io.Copy(os.Stdout, res.Body)
+	}
+	return
+}
+
+// remoteSearchURL returns the search URL for a given query as needed by
+// remoteSearch. If html is set, an html result is requested; otherwise
+// the result is in textual form.
+// Adjust this function as necessary if modeNames or FormValue parameters
+// change.
+func remoteSearchURL(query string, html bool) string {
+	s := "/search?m=text&q="
+	if html {
+		s = "/search?q="
+	}
+	return s + url.QueryEscape(query)
+}
+
+func remoteSearch(query string) (res *http.Response, err error) {
+	// list of addresses to try
+	var addrs []string
+	if *serverAddr != "" {
+		// explicit server address - only try this one
+		addrs = []string{*serverAddr}
+	} else {
+		addrs = []string{
+			defaultAddr,
+			"golang.org",
+		}
+	}
+
+	// remote search
+	search := remoteSearchURL(query, *html)
+	for _, addr := range addrs {
+		url := "http://" + addr + search
+		res, err = http.Get(url)
+		if err == nil && res.StatusCode == http.StatusOK {
+			break
+		}
+	}
+
+	if err == nil && res.StatusCode != http.StatusOK {
+		err = errors.New(res.Status)
+	}
+
+	return
+}

cmd/godoc/setup-godoc-app.bash

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+
+# Copyright 2011 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This script creates a complete godoc app in $APPDIR.
+# It copies the cmd/godoc and src/go/... sources from GOROOT,
+# synthesizes an app.yaml file, and creates the .zip, index, and
+# configuration files.
+#
+# If an argument is provided it is assumed to be the app-engine godoc directory.
+# Without an argument, $APPDIR is used instead. If GOROOT is not set, "go env"
+# is consulted to find the $GOROOT.
+#
+# The script creates a .zip file representing the $GOROOT file system
+# and computes the correspondig search index files. These files are then
+# copied to $APPDIR. A corresponding godoc configuration file is created
+# in $APPDIR/appconfig.go.
+
+ZIPFILE=godoc.zip
+INDEXFILE=godoc.index
+SPLITFILES=index.split.
+GODOC=golang.org/x/tools/cmd/godoc
+CONFIGFILE=$GODOC/appconfig.go
+
+error() {
+	echo "error: $1"
+	exit 2
+}
+
+getArgs() {
+	if [ -z $APPENGINE_SDK ]; then
+		error "APPENGINE_SDK environment variable not set"
+	fi
+	if [ ! -x $APPENGINE_SDK/goapp ]; then
+		error "couldn't find goapp command in $APPENGINE_SDK"
+	fi
+	if [ -z $GOROOT ]; then
+		GOROOT=$(go env GOROOT)
+		echo "GOROOT not set explicitly, using go env value instead"
+	fi
+	if [ -z $APPDIR ]; then
+		if [ $# == 0 ]; then
+			error "APPDIR not set, and no argument provided"
+		fi
+		APPDIR=$1
+		echo "APPDIR not set, using argument instead"
+	fi
+
+	# safety checks
+	if [ ! -d $GOROOT ]; then
+		error "$GOROOT is not a directory"
+	fi
+	if [ -e $APPDIR ]; then
+		error "$APPDIR exists; check and remove it before trying again"
+	fi
+
+	# reporting
+	echo "GOROOT = $GOROOT"
+	echo "APPDIR = $APPDIR"
+}
+
+fetchGodoc() {
+	echo "*** Fetching godoc (if not already in GOPATH)"
+	unset GOBIN
+	go=$APPENGINE_SDK/goapp
+	$go get -d -tags appengine $GODOC
+	mkdir -p $APPDIR/$GODOC
+	cp $(find $($go list -f '{{.Dir}}' $GODOC) -mindepth 1 -maxdepth 1 -type f) $APPDIR/$GODOC/
+}
+
+makeAppYaml() {
+	echo "*** make $APPDIR/app.yaml"
+	cat > $APPDIR/app.yaml <<EOF
+application: godoc
+version: 1
+runtime: go
+api_version: go1.4beta
+
+handlers:
+- url: /.*
+  script: _go_app
+EOF
+}
+
+makeZipfile() {
+	echo "*** make $APPDIR/$ZIPFILE"
+	zip -q -r $APPDIR/$ZIPFILE $GOROOT/*
+}
+
+makeIndexfile() {
+	echo "*** make $APPDIR/$INDEXFILE"
+	GOPATH= godoc -write_index -index_files=$APPDIR/$INDEXFILE -zip=$APPDIR/$ZIPFILE
+}
+
+splitIndexfile() {
+	echo "*** split $APPDIR/$INDEXFILE"
+	split -b8m $APPDIR/$INDEXFILE $APPDIR/$SPLITFILES
+}
+
+makeConfigfile() {
+	echo "*** make $APPDIR/$CONFIGFILE"
+	cat > $APPDIR/$CONFIGFILE <<EOF
+package main
+
+// GENERATED FILE - DO NOT MODIFY BY HAND.
+// (generated by golang.org/x/tools/cmd/godoc/setup-godoc-app.bash)
+
+const (
+	// .zip filename
+	zipFilename = "$ZIPFILE"
+
+	// goroot directory in .zip file
+	zipGoroot = "$GOROOT"
+
+	// glob pattern describing search index files
+	// (if empty, the index is built at run-time)
+	indexFilenames = "$SPLITFILES*"
+)
+EOF
+}
+
+getArgs "$@"
+set -e
+mkdir $APPDIR
+fetchGodoc
+makeAppYaml
+makeZipfile
+makeIndexfile
+splitIndexfile
+makeConfigfile
+
+echo "*** setup complete"

cmd/godoc/x.go

@@ -0,0 +1,90 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file contains the handlers that serve go-import redirects for Go
+// sub-repositories. It specifies the mapping from import paths like
+// "golang.org/x/tools" to the actual repository locations.
+
+package main
+
+import (
+	"html/template"
+	"log"
+	"net/http"
+	"strings"
+)
+
+const xPrefix = "/x/"
+
+type xRepo struct {
+	URL, VCS string
+}
+
+var xMap = map[string]xRepo{
+	"codereview": {"https://code.google.com/p/go.codereview", "hg"},
+
+	"arch":       {"https://go.googlesource.com/arch", "git"},
+	"benchmarks": {"https://go.googlesource.com/benchmarks", "git"},
+	"blog":       {"https://go.googlesource.com/blog", "git"},
+	"build":      {"https://go.googlesource.com/build", "git"},
+	"crypto":     {"https://go.googlesource.com/crypto", "git"},
+	"debug":      {"https://go.googlesource.com/debug", "git"},
+	"exp":        {"https://go.googlesource.com/exp", "git"},
+	"image":      {"https://go.googlesource.com/image", "git"},
+	"mobile":     {"https://go.googlesource.com/mobile", "git"},
+	"net":        {"https://go.googlesource.com/net", "git"},
+	"oauth2":     {"https://go.googlesource.com/oauth2", "git"},
+	"perf":       {"https://go.googlesource.com/perf", "git"},
+	"playground": {"https://go.googlesource.com/playground", "git"},
+	"review":     {"https://go.googlesource.com/review", "git"},
+	"sync":       {"https://go.googlesource.com/sync", "git"},
+	"sys":        {"https://go.googlesource.com/sys", "git"},
+	"talks":      {"https://go.googlesource.com/talks", "git"},
+	"term":       {"https://go.googlesource.com/term", "git"},
+	"text":       {"https://go.googlesource.com/text", "git"},
+	"time":       {"https://go.googlesource.com/time", "git"},
+	"tools":      {"https://go.googlesource.com/tools", "git"},
+	"tour":       {"https://go.googlesource.com/tour", "git"},
+}
+
+func init() {
+	http.HandleFunc(xPrefix, xHandler)
+}
+
+func xHandler(w http.ResponseWriter, r *http.Request) {
+	head, tail := strings.TrimPrefix(r.URL.Path, xPrefix), ""
+	if i := strings.Index(head, "/"); i != -1 {
+		head, tail = head[:i], head[i:]
+	}
+	if head == "" {
+		http.Redirect(w, r, "https://godoc.org/-/subrepo", http.StatusTemporaryRedirect)
+		return
+	}
+	repo, ok := xMap[head]
+	if !ok {
+		http.NotFound(w, r)
+		return
+	}
+	data := struct {
+		Prefix, Head, Tail string
+		Repo               xRepo
+	}{xPrefix, head, tail, repo}
+	if err := xTemplate.Execute(w, data); err != nil {
+		log.Println("xHandler:", err)
+	}
+}
+
+var xTemplate = template.Must(template.New("x").Parse(`<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+<meta name="go-import" content="golang.org{{.Prefix}}{{.Head}} {{.Repo.VCS}} {{.Repo.URL}}">
+<meta name="go-source" content="golang.org{{.Prefix}}{{.Head}} https://github.com/golang/{{.Head}}/ https://github.com/golang/{{.Head}}/tree/master{/dir} https://github.com/golang/{{.Head}}/blob/master{/dir}/{file}#L{line}">
+<meta http-equiv="refresh" content="0; url=https://godoc.org/golang.org{{.Prefix}}{{.Head}}{{.Tail}}">
+</head>
+<body>
+Nothing to see here; <a href="https://godoc.org/golang.org{{.Prefix}}{{.Head}}{{.Tail}}">move along</a>.
+</body>
+</html>
+`))

cmd/goimports/doc.go

@@ -13,6 +13,8 @@ For emacs, make sure you have the latest go-mode.el:
    https://github.com/dominikh/go-mode.el
 Then in your .emacs file:
    (setq gofmt-command "goimports")
+   (add-to-list 'load-path "/home/you/somewhere/emacs/")
+   (require 'go-mode-autoloads)
    (add-hook 'before-save-hook 'gofmt-before-save)
 
 For vim, set "gofmt_command" to "goimports":

cmd/goimports/goimports.go

@@ -10,7 +10,6 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"go/build"
 	"go/scanner"
 	"io"
 	"io/ioutil"
@@ -22,16 +21,15 @@ import (
 	"runtime/pprof"
 	"strings"
 
-	"golang.org/x/tools/internal/imports"
+	"golang.org/x/tools/imports"
 )
 
 var (
 	// main operation modes
-	list   = flag.Bool("l", false, "list files whose formatting differs from goimport's")
+	list    = flag.Bool("l", false, "list files whose formatting differs from goimport's")
-	write  = flag.Bool("w", false, "write result to (source) file instead of stdout")
+	write   = flag.Bool("w", false, "write result to (source) file instead of stdout")
-	doDiff = flag.Bool("d", false, "display diffs instead of rewriting files")
+	doDiff  = flag.Bool("d", false, "display diffs instead of rewriting files")
-	srcdir = flag.String("srcdir", "", "choose imports as if source code is from `dir`. When operating on a single file, dir may instead be the complete file name.")
+	srcdir  = flag.String("srcdir", "", "choose imports as if source code is from `dir`. When operating on a single file, dir may instead be the complete file name.")
-
 	verbose bool // verbose logging
 
 	cpuProfile     = flag.String("cpuprofile", "", "CPU profile output")
@@ -43,19 +41,13 @@ var (
 		TabIndent: true,
 		Comments:  true,
 		Fragment:  true,
-		// This environment, and its caches, will be reused for the whole run.
-		Env: &imports.ProcessEnv{
-			GOPATH: build.Default.GOPATH,
-			GOROOT: build.Default.GOROOT,
-		},
 	}
 	exitCode = 0
 )
 
 func init() {
 	flag.BoolVar(&options.AllErrors, "e", false, "report all errors (not just the first 10 on different lines)")
-	flag.StringVar(&options.Env.LocalPrefix, "local", "", "put imports beginning with this string after 3rd-party packages; comma-separated list")
+	flag.StringVar(&imports.LocalPrefix, "local", "", "put imports beginning with this string after 3rd-party packages")
-	flag.BoolVar(&options.FormatOnly, "format-only", false, "if true, don't fix imports and only format. In this mode, goimports is effectively gofmt, with the addition that imports are grouped into sections.")
 }
 
 func report(err error) {
@@ -258,7 +250,7 @@ func gofmtMain() {
 
 	if verbose {
 		log.SetFlags(log.LstdFlags | log.Lmicroseconds)
-		options.Env.Debug = true
+		imports.Debug = true
 	}
 	if options.TabWidth < 0 {
 		fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", options.TabWidth)

cmd/gomvpkg/main.go

@@ -19,7 +19,7 @@ import (
 var (
 	fromFlag     = flag.String("from", "", "Import path of package to be moved")
 	toFlag       = flag.String("to", "", "Destination import path for package")
-	vcsMvCmdFlag = flag.String("vcs_mv_cmd", "", `A template for the version control system's "move directory" command, e.g. "git mv {{.Src}} {{.Dst}}"`)
+	vcsMvCmdFlag = flag.String("vcs_mv_cmd", "", `A template for the version control system's "move directory" command, e.g. "git mv {{.Src}} {{.Dst}}`)
 	helpFlag     = flag.Bool("help", false, "show usage message")
 )
 

cmd/gopls/integration/vscode/.gitignore

@@ -1,3 +0,0 @@
-gopls*.vsix
-out
-node_modules

cmd/gopls/integration/vscode/.vscode/launch.json

@@ -1,20 +0,0 @@
-{
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Launch Extension",
-            "type": "extensionHost",
-            "request": "launch",
-            "runtimeExecutable": "${execPath}",
-            "args": [
-                "--extensionDevelopmentPath=${workspaceFolder}"
-            ],
-            "stopOnEntry": false,
-            "sourceMaps": true,
-            "outFiles": [
-                "${workspaceFolder}/out/**/*.js"
-            ],
-            "preLaunchTask": "npm"
-        },
-    ]
-}

cmd/gopls/integration/vscode/.vscode/tasks.json

@@ -1,29 +0,0 @@
-// Available variables which can be used inside of strings.
-// ${workspaceFolder}: the root folder of the team
-// ${file}: the current opened file
-// ${fileBasename}: the current opened file's basename
-// ${fileDirname}: the current opened file's dirname
-// ${fileExtname}: the current opened file's extension
-// ${cwd}: the current working directory of the spawned process
-// A task runner that calls the Typescript compiler (tsc) and
-// compiles the extension.
-{
-    "version": "2.0.0",
-    // we want to run npm
-    "command": "npm",
-    // the command is a shell script
-    "type": "shell",
-    // show the output window only if unrecognized errors occur.
-    "presentation": {
-        "reveal": "silent"
-    },
-    // we run the custom script "compile" as defined in package.json
-    "args": [
-        "run",
-        "compile"
-    ],
-    // The tsc compiler is started in watching mode
-    "isBackground": true,
-    // use the standard tsc in watch mode problem matcher to find compile problems in the output.
-    "problemMatcher": "$tsc-watch"
-}

cmd/gopls/integration/vscode/README.md

@@ -1,21 +0,0 @@
-# gopls testing extension
-
-An extension for debugging the Go Language Server provided by 
-https://golang.org/x/tools/cmd/gopls. The code for this extension comes from
-a combination of 
-https://github.com/Microsoft/vscode-extension-samples/blob/master/lsp-sample
-and https://github.com/Microsoft/vscode-go. 
-
-## Features
-
-* Diagnostics (on file change)
-* Completion (Ctrl + Space)
-* Jump to definition (F12 or right-click -> Go to Definition)
-* Signature help (Ctrl + Shift + Space)
-
-## Installation
-
-To package the extension, run `vsce package` from this directory. To install
-the extension, navigate to the "Extensions" panel in VSCode, and select
-"Install from VSIX..." from the menu in the top right corner. Choose the 
-`gopls-1.0.0.vsix file` and reload VSCode.

cmd/gopls/integration/vscode/package.json

@@ -1,58 +0,0 @@
-{
-    "name": "gopls",
-    "description": "Go Language Server Client for testing",
-    "author": "The Go authors",
-    "license": "SEE LICENSE IN ../../../../LICENSE",
-    "version": "1.0.0",
-    "repository": {
-        "type": "git",
-        "url": "https://golang.org/x/tools"
-    },
-    "publisher": "golang",
-    "engines": {
-        "vscode": "^1.23.0"
-    },
-    "activationEvents": [
-        "onLanguage:go"
-    ],
-    "main": "./out/extension",
-    "scripts": {
-        "vscode:prepublish": "tsc -p ./",
-        "compile": "tsc -watch -p ./",
-        "postinstall": "node ./node_modules/vscode/bin/install",
-        "lint": "node ./node_modules/tslint/bin/tslint ./src/*.ts"
-    },
-    "extensionDependencies": [],
-    "dependencies": {
-        "vscode-languageclient": "~4.3.0"
-    },
-    "devDependencies": {
-        "@types/mocha": "^5.2.5",
-        "@types/node": "^8.10.39",
-        "tslint": "^5.11.0",
-        "typescript": "^3.1.3",
-        "vscode": "^1.1.24"
-    },
-    "contributes": {
-        "configuration": {
-            "title": "gopls",
-            "properties": {
-                "gopls.flags": {
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    },
-                    "default": [],
-                    "description": "Flags to pass to gopls",
-                    "scope": "resource"
-                },
-                "gopls.command": {
-                    "type": "string",
-                    "default": "gopls",
-                    "description": "Name of the gopls binary",
-                    "scope": "resource"
-                }
-            }
-        }
-    }
-}

cmd/gopls/integration/vscode/src/extension.ts

@@ -1,96 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-'use strict';
-
-import fs = require('fs');
-import lsp = require('vscode-languageclient');
-import vscode = require('vscode');
-import path = require('path');
-
-export function activate(ctx: vscode.ExtensionContext): void {
-  // The handleDiagnostics middleware writes to the diagnostics log, in order
-  // to confirm the order in which the extension received diagnostics.
-  let r = Math.floor(Math.random() * 100000000);
-  let diagnosticsLog = fs.openSync('/tmp/diagnostics' + r + '.log', 'w');
-
-  let document = vscode.window.activeTextEditor.document;
-  let config = vscode.workspace.getConfiguration('gopls', document.uri);
-  let goplsCommand: string = config['command'];
-  let goplsFlags: string[] = config['flags'];
-  let serverOptions:
-      lsp.ServerOptions = {command: getBinPath(goplsCommand), args: goplsFlags};
-  let clientOptions: lsp.LanguageClientOptions = {
-    initializationOptions: {},
-    documentSelector: ['go'],
-    uriConverters: {
-      code2Protocol: (uri: vscode.Uri): string =>
-          (uri.scheme ? uri : uri.with({scheme: 'file'})).toString(),
-      protocol2Code: (uri: string) => vscode.Uri.parse(uri),
-    },
-    middleware: {
-      handleDiagnostics: (uri: vscode.Uri, diagnostics: vscode.Diagnostic[], next: lsp.HandleDiagnosticsSignature) => {
-        let diagString = "-------------------------------------------\n";
-        diagString += uri.toString(); + ": " + diagnostics.length + "\n";
-        if (diagnostics.length > 0) {
-          diagString += "\n";
-          for (const diag of diagnostics) {
-            diagString += diag.message + "\n";
-          }
-        }
-        fs.writeSync(diagnosticsLog, diagString);
-        return next(uri, diagnostics);
-      }
-    },
-    revealOutputChannelOn: lsp.RevealOutputChannelOn.Never,
-  };
-  const c = new lsp.LanguageClient('gopls', serverOptions, clientOptions);
-  c.onReady().then(() => {
-    const capabilities = c.initializeResult && c.initializeResult.capabilities;
-    if (!capabilities) {
-      return vscode.window.showErrorMessage(
-          'The language server is not able to serve any features at the moment.');
-    }
-  });
-  ctx.subscriptions.push(c.start());
-}
-
-function getBinPath(toolName: string): string {
-  toolName = correctBinname(toolName);
-  let tool = findToolIn(toolName, 'PATH', false);
-  if (tool) {
-    return tool;
-  }
-  return findToolIn(toolName, 'GOPATH', true);
-}
-
-function findToolIn(
-    toolName: string, envVar: string, appendBinToPath: boolean): string {
-  let value = process.env[envVar];
-  if (value) {
-    let paths = value.split(path.delimiter);
-    for (let i = 0; i < paths.length; i++) {
-      let binpath = path.join(paths[i], appendBinToPath ? 'bin' : '', toolName);
-      if (fileExists(binpath)) {
-        return binpath;
-      }
-    }
-  }
-  return null;
-}
-
-function fileExists(filePath: string): boolean {
-  try {
-    return fs.statSync(filePath).isFile();
-  } catch (e) {
-    return false;
-  }
-}
-
-function correctBinname(toolName: string) {
-  if (process.platform === 'win32')
-    return toolName + '.exe';
-  else
-    return toolName;
-}

cmd/gopls/integration/vscode/tsconfig.json

@@ -1,15 +0,0 @@
-{
-    "compilerOptions": {
-        "module": "commonjs",
-        "target": "es6",
-        "outDir": "out",
-        "rootDir": "src",
-        "lib": [
-            "es6"
-        ],
-        "sourceMap": true
-    },
-    "exclude": [
-        "node_modules"
-    ]
-}

cmd/gopls/integration/vscode/tslint.json

@@ -1,12 +0,0 @@
-{
-    "rules": {
-        "indent": [
-            true,
-            "tabs"
-        ],
-        "semicolon": [
-            true,
-            "always"
-        ]
-    }
-}

cmd/gopls/main.go

@@ -1,23 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// The gopls command is an LSP server for Go.
-// The Language Server Protocol allows any text editor
-// to be extended with IDE-like features;
-// see https://langserver.org/ for details.
-package main // import "golang.org/x/tools/cmd/gopls"
-
-import (
-	"context"
-	"os"
-
-	"golang.org/x/tools/internal/lsp/cmd"
-	"golang.org/x/tools/internal/lsp/debug"
-	"golang.org/x/tools/internal/tool"
-)
-
-func main() {
-	debug.Version += "-cmd.gopls"
-	tool.Main(context.Background(), cmd.New("gopls-legacy", "", nil), os.Args[1:])
-}

cmd/gorename/gorename_test.go

@@ -48,8 +48,6 @@ func TestGeneratedFiles(t *testing.T) {
 			env = append(env, envVar)
 		}
 	}
-	// gorename currently requires GOPATH mode.
-	env = append(env, "GO111MODULE=off")
 
 	// Testing renaming in packages that include cgo files:
 	for iter, renameTest := range []test{
@@ -312,9 +310,6 @@ func g() { fmt.Println(test.Foo(3)) }
 // buildGorename builds the gorename executable.
 // It returns its path, and a cleanup function.
 func buildGorename(t *testing.T) (tmp, bin string, cleanup func()) {
-	if runtime.GOOS == "android" {
-		t.Skipf("the dependencies are not available on android")
-	}
 
 	tmp, err := ioutil.TempDir("", "gorename-regtest-")
 	if err != nil {
@@ -375,7 +370,7 @@ func modifiedFiles(t *testing.T, dir string, packages map[string][]string) (resu
 			file := filepath.Join(pkgDir, strconv.Itoa(i)+".go")
 			// read file contents and compare to val
 			if contents, err := ioutil.ReadFile(file); err != nil {
-				t.Fatalf("File missing: %s", err)
+				t.Fatal("File missing: %s", err)
 			} else if string(contents) != val {
 				results = append(results, strings.TrimPrefix(dir, file))
 			}

cmd/goyacc/yacc.go

@@ -387,8 +387,6 @@ func setup() {
 	yaccpar = strings.Replace(yaccpartext, "$$", prefix, -1)
 	openup()
 
-	fmt.Fprintf(ftable, "// Code generated by goyacc %s. DO NOT EDIT.\n", strings.Join(os.Args[1:], " "))
-
 	defin(0, "$end")
 	extval = PRIVATE // tokens start in unicode 'private use'
 	defin(0, "error")
@@ -1272,7 +1270,7 @@ l1:
 func cpyact(curprod []int, max int) {
 
 	if !lflag {
-		fmt.Fprintf(fcode, "\n//line %v:%v", infile, lineno)
+		fmt.Fprintf(fcode, "\n\t\t//line %v:%v", infile, lineno)
 	}
 	fmt.Fprint(fcode, "\n\t\t")
 
@@ -1414,7 +1412,8 @@ loop:
 					if nnc == '/' {
 						fcode.WriteRune('*')
 						fcode.WriteRune('/')
-						continue loop
+						c = getrune(finput)
+						break swt
 					}
 					ungetrune(finput, nnc)
 				}

cmd/guru/callees.go

@@ -18,7 +18,7 @@ import (
 	"golang.org/x/tools/go/ssa/ssautil"
 )
 
-// The callees function reports the possible callees of the function call site
+// Callees reports the possible callees of the function call site
 // identified by the specified source location.
 func callees(q *Query) error {
 	lconf := loader.Config{Build: q.Build}

cmd/guru/callers.go

@@ -16,7 +16,7 @@ import (
 	"golang.org/x/tools/go/ssa/ssautil"
 )
 
-// The callers function reports the possible callers of the function
+// Callers reports the possible callers of the function
 // immediately enclosing the specified source location.
 //
 func callers(q *Query) error {

cmd/guru/callstack.go

@@ -16,7 +16,7 @@ import (
 	"golang.org/x/tools/go/ssa/ssautil"
 )
 
-// The callstack function displays an arbitrary path from a root of the callgraph
+// Callstack displays an arbitrary path from a root of the callgraph
 // to the function at the current position.
 //
 // The information may be misleading in a context-insensitive

cmd/guru/describe.go

@@ -8,7 +8,7 @@ import (
 	"bytes"
 	"fmt"
 	"go/ast"
-	"go/constant"
+	exact "go/constant"
 	"go/token"
 	"go/types"
 	"os"
@@ -162,9 +162,6 @@ func findInterestingNode(pkginfo *loader.PackageInfo, path []ast.Node) ([]ast.No
 			path = append([]ast.Node{n.Name}, path...)
 			continue
 
-		case *ast.Comment, *ast.CommentGroup, *ast.File, *ast.KeyValueExpr, *ast.CommClause:
-			return path, actionUnknown // uninteresting
-
 		case ast.Stmt:
 			return path, actionStmt
 
@@ -176,6 +173,9 @@ func findInterestingNode(pkginfo *loader.PackageInfo, path []ast.Node) ([]ast.No
 			*ast.ChanType:
 			return path, actionType
 
+		case *ast.Comment, *ast.CommentGroup, *ast.File, *ast.KeyValueExpr, *ast.CommClause:
+			return path, actionUnknown // uninteresting
+
 		case *ast.Ellipsis:
 			// Continue to enclosing node.
 			// e.g. [...]T in ArrayType
@@ -340,7 +340,6 @@ func describeValue(qpos *queryPos, path []ast.Node) (*describeValueResult, error
 		qpos:     qpos,
 		expr:     expr,
 		typ:      typ,
-		names:    appendNames(nil, typ),
 		constVal: constVal,
 		obj:      obj,
 		methods:  accessibleMethods(typ, qpos.info.Pkg),
@@ -348,36 +347,12 @@ func describeValue(qpos *queryPos, path []ast.Node) (*describeValueResult, error
 	}, nil
 }
 
-// appendNames returns named types found within the Type by
-// removing map, pointer, channel, slice, and array constructors.
-// It does not descend into structs or interfaces.
-func appendNames(names []*types.Named, typ types.Type) []*types.Named {
-	// elemType specifies type that has some element in it
-	// such as array, slice, chan, pointer
-	type elemType interface {
-		Elem() types.Type
-	}
-
-	switch t := typ.(type) {
-	case *types.Named:
-		names = append(names, t)
-	case *types.Map:
-		names = appendNames(names, t.Key())
-		names = appendNames(names, t.Elem())
-	case elemType:
-		names = appendNames(names, t.Elem())
-	}
-
-	return names
-}
-
 type describeValueResult struct {
 	qpos     *queryPos
-	expr     ast.Expr       // query node
+	expr     ast.Expr     // query node
-	typ      types.Type     // type of expression
+	typ      types.Type   // type of expression
-	names    []*types.Named // named types within typ
+	constVal exact.Value  // value of expression, if constant
-	constVal constant.Value // value of expression, if constant
+	obj      types.Object // var/func/const object, if expr was Ident
-	obj      types.Object   // var/func/const object, if expr was Ident
 	methods  []*types.Selection
 	fields   []describeField
 }
@@ -423,7 +398,6 @@ func (r *describeValueResult) PrintPlain(printf printfFunc) {
 
 	printMethods(printf, r.expr, r.methods)
 	printFields(printf, r.expr, r.fields)
-	printNamedTypes(printf, r.expr, r.names)
 }
 
 func (r *describeValueResult) JSON(fset *token.FileSet) []byte {
@@ -435,23 +409,14 @@ func (r *describeValueResult) JSON(fset *token.FileSet) []byte {
 		objpos = fset.Position(r.obj.Pos()).String()
 	}
 
-	typesPos := make([]serial.Definition, len(r.names))
-	for i, t := range r.names {
-		typesPos[i] = serial.Definition{
-			ObjPos: fset.Position(t.Obj().Pos()).String(),
-			Desc:   r.qpos.typeString(t),
-		}
-	}
-
 	return toJSON(&serial.Describe{
 		Desc:   astutil.NodeDescription(r.expr),
 		Pos:    fset.Position(r.expr.Pos()).String(),
 		Detail: "value",
 		Value: &serial.DescribeValue{
-			Type:     r.qpos.typeString(r.typ),
+			Type:   r.qpos.typeString(r.typ),
-			TypesPos: typesPos,
+			Value:  value,
-			Value:    value,
+			ObjPos: objpos,
-			ObjPos:   objpos,
 		},
 	})
 }
@@ -559,19 +524,6 @@ func printFields(printf printfFunc, node ast.Node, fields []describeField) {
 	}
 }
 
-func printNamedTypes(printf printfFunc, node ast.Node, names []*types.Named) {
-	if len(names) > 0 {
-		printf(node, "Named types:")
-	}
-
-	for _, t := range names {
-		// Print the type relative to the package
-		// in which it was defined, not the query package,
-		printf(t.Obj(), "\ttype %s defined here",
-			types.TypeString(t.Obj().Type(), types.RelativeTo(t.Obj().Pkg())))
-	}
-}
-
 func (r *describeTypeResult) PrintPlain(printf printfFunc) {
 	printf(r.node, "%s", r.description)
 

cmd/guru/guru.go

@@ -75,7 +75,7 @@ type Query struct {
 	PTALog     io.Writer // (optional) pointer-analysis log file
 	Reflection bool      // model reflection soundly (currently slow).
 
-	// result-printing function, safe for concurrent use
+	// result-printing function
 	Output func(*token.FileSet, QueryResult)
 }
 

cmd/guru/guru_test.go

@@ -6,7 +6,7 @@ package main_test
 
 // This file defines a test framework for guru queries.
 //
-// The files beneath testdata/src contain Go programs containing
+// The files beneath testdata/src/main contain Go programs containing
 // query annotations of the form:
 //
 //   @verb id "select"
@@ -35,7 +35,6 @@ import (
 	"go/token"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -50,18 +49,6 @@ import (
 	guru "golang.org/x/tools/cmd/guru"
 )
 
-func init() {
-	// This test currently requires GOPATH mode.
-	// Explicitly disabling module mode should suffix, but
-	// we'll also turn off GOPROXY just for good measure.
-	if err := os.Setenv("GO111MODULE", "off"); err != nil {
-		log.Fatal(err)
-	}
-	if err := os.Setenv("GOPROXY", "off"); err != nil {
-		log.Fatal(err)
-	}
-}
-
 var updateFlag = flag.Bool("update", false, "Update the golden files.")
 
 type query struct {
@@ -223,11 +210,6 @@ func doQuery(out io.Writer, q *query, json bool) {
 }
 
 func TestGuru(t *testing.T) {
-	if testing.Short() {
-		// These tests are super slow.
-		// TODO: make a lighter version of the tests for short mode?
-		t.Skipf("skipping in short mode")
-	}
 	switch runtime.GOOS {
 	case "android":
 		t.Skipf("skipping test on %q (no testdata dir)", runtime.GOOS)
@@ -236,9 +218,10 @@ func TestGuru(t *testing.T) {
 	}
 
 	for _, filename := range []string{
-		"testdata/src/alias/alias.go",
+		"testdata/src/alias/alias.go", // iff guru.HasAlias (go1.9)
 		"testdata/src/calls/main.go",
 		"testdata/src/describe/main.go",
+		"testdata/src/describe/main19.go", // iff go1.9
 		"testdata/src/freevars/main.go",
 		"testdata/src/implements/main.go",
 		"testdata/src/implements-methods/main.go",
@@ -255,6 +238,7 @@ func TestGuru(t *testing.T) {
 		"testdata/src/calls-json/main.go",
 		"testdata/src/peers-json/main.go",
 		"testdata/src/definition-json/main.go",
+		"testdata/src/definition-json/main19.go",
 		"testdata/src/describe-json/main.go",
 		"testdata/src/implements-json/main.go",
 		"testdata/src/implements-methods-json/main.go",
@@ -262,58 +246,72 @@ func TestGuru(t *testing.T) {
 		"testdata/src/referrers-json/main.go",
 		"testdata/src/what-json/main.go",
 	} {
-		filename := filename
+		if filename == "testdata/src/referrers/main.go" && runtime.GOOS == "plan9" {
-		name := strings.Split(filename, "/")[2]
+			// Disable this test on plan9 since it expects a particular
-		t.Run(name, func(t *testing.T) {
+			// wording for a "no such file or directory" error.
-			t.Parallel()
+			continue
-			if filename == "testdata/src/referrers/main.go" && runtime.GOOS == "plan9" {
+		}
-				// Disable this test on plan9 since it expects a particular
+		if filename == "testdata/src/alias/alias.go" && !guru.HasAlias {
-				// wording for a "no such file or directory" error.
+			continue
-				t.Skip()
+		}
-			}
+		if strings.HasSuffix(filename, "19.go") && !contains(build.Default.ReleaseTags, "go1.9") {
-			json := strings.Contains(filename, "-json/")
+			// TODO(adonovan): recombine the 'describe' and 'definition'
-			queries := parseQueries(t, filename)
+			// tests once we drop support for go1.8.
-			golden := filename + "lden"
+			continue
-			got := filename + "t"
+		}
-			gotfh, err := os.Create(got)
-			if err != nil {
-				t.Fatalf("Create(%s) failed: %s", got, err)
-			}
-			defer os.Remove(got)
-			defer gotfh.Close()
 
-			// Run the guru on each query, redirecting its output
+		json := strings.Contains(filename, "-json/")
-			// and error (if any) to the foo.got file.
+		queries := parseQueries(t, filename)
-			for _, q := range queries {
+		golden := filename + "lden"
-				doQuery(gotfh, q, json)
+		got := filename + "t"
-			}
+		gotfh, err := os.Create(got)
+		if err != nil {
+			t.Errorf("Create(%s) failed: %s", got, err)
+			continue
+		}
+		defer os.Remove(got)
+		defer gotfh.Close()
 
-			// Compare foo.got with foo.golden.
+		// Run the guru on each query, redirecting its output
-			var cmd *exec.Cmd
+		// and error (if any) to the foo.got file.
-			switch runtime.GOOS {
+		for _, q := range queries {
-			case "plan9":
+			doQuery(gotfh, q, json)
-				cmd = exec.Command("/bin/diff", "-c", golden, got)
+		}
-			default:
-				cmd = exec.Command("/usr/bin/diff", "-u", golden, got)
-			}
-			buf := new(bytes.Buffer)
-			cmd.Stdout = buf
-			cmd.Stderr = os.Stderr
-			if err := cmd.Run(); err != nil {
-				t.Errorf("Guru tests for %s failed: %s.\n%s\n",
-					filename, err, buf)
 
-				if *updateFlag {
+		// Compare foo.got with foo.golden.
-					t.Logf("Updating %s...", golden)
+		var cmd *exec.Cmd
-					if err := exec.Command("/bin/cp", got, golden).Run(); err != nil {
+		switch runtime.GOOS {
-						t.Errorf("Update failed: %s", err)
+		case "plan9":
-					}
+			cmd = exec.Command("/bin/diff", "-c", golden, got)
+		default:
+			cmd = exec.Command("/usr/bin/diff", "-u", golden, got)
+		}
+		buf := new(bytes.Buffer)
+		cmd.Stdout = buf
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			t.Errorf("Guru tests for %s failed: %s.\n%s\n",
+				filename, err, buf)
+
+			if *updateFlag {
+				t.Logf("Updating %s...", golden)
+				if err := exec.Command("/bin/cp", got, golden).Run(); err != nil {
+					t.Errorf("Update failed: %s", err)
 				}
 			}
-		})
+		}
 	}
 }
 
+func contains(haystack []string, needle string) bool {
+	for _, x := range haystack {
+		if needle == x {
+			return true
+		}
+	}
+	return false
+}
+
 func TestIssue14684(t *testing.T) {
 	var buildContext = build.Default
 	buildContext.GOPATH = "testdata"

cmd/guru/implements.go

@@ -19,7 +19,7 @@ import (
 	"golang.org/x/tools/refactor/importgraph"
 )
 
-// The implements function displays the "implements" relation as it pertains to the
+// Implements displays the "implements" relation as it pertains to the
 // selected type.
 // If the selection is a method, 'implements' displays
 // the corresponding methods of the types that would have been reported

cmd/guru/main.go

@@ -19,8 +19,6 @@ import (
 	"io"
 	"log"
 	"os"
-	"path/filepath"
-	"runtime"
 	"runtime/pprof"
 	"strings"
 	"sync"
@@ -40,14 +38,6 @@ var (
 
 func init() {
 	flag.Var((*buildutil.TagsFlag)(&build.Default.BuildTags), "tags", buildutil.TagsFlagDoc)
-
-	// gccgo does not provide a GOROOT with standard library sources.
-	// If we have one in the environment, force gc mode.
-	if build.Default.Compiler == "gccgo" {
-		if _, err := os.Stat(filepath.Join(runtime.GOROOT(), "src", "runtime", "runtime.go")); err == nil {
-			build.Default.Compiler = "gc"
-		}
-	}
 }
 
 const useHelp = "Run 'guru -help' for more information.\n"

cmd/guru/referrers.go

@@ -9,25 +9,21 @@ import (
 	"fmt"
 	"go/ast"
 	"go/build"
-	"go/parser"
 	"go/token"
 	"go/types"
 	"io"
 	"log"
-	"os"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 
 	"golang.org/x/tools/cmd/guru/serial"
 	"golang.org/x/tools/go/buildutil"
 	"golang.org/x/tools/go/loader"
-	"golang.org/x/tools/imports"
 	"golang.org/x/tools/refactor/importgraph"
 )
 
-// The referrers function reports all identifiers that resolve to the same object
+// Referrers reports all identifiers that resolve to the same object
 // as the queried identifier, within any package in the workspace.
 func referrers(q *Query) error {
 	fset := token.NewFileSet()
@@ -38,12 +34,6 @@ func referrers(q *Query) error {
 		return err
 	}
 
-	// Load tests of the query package
-	// even if the query location is not in the tests.
-	for path := range lconf.ImportPkgs {
-		lconf.ImportPkgs[path] = true
-	}
-
 	// Load/parse/type-check the query package.
 	lprog, err := lconf.Load()
 	if err != nil {
@@ -80,27 +70,23 @@ func referrers(q *Query) error {
 		return fmt.Errorf("references to predeclared %q are everywhere!", obj.Name())
 	}
 
+	// For a globally accessible object defined in package P, we
+	// must load packages that depend on P.  Specifically, for a
+	// package-level object, we need load only direct importers
+	// of P, but for a field or interface method, we must load
+	// any package that transitively imports P.
+	if global, pkglevel := classify(obj); global {
+		// We'll use the the object's position to identify it in the larger program.
+		objposn := fset.Position(obj.Pos())
+		defpkg := obj.Pkg().Path() // defining package
+		return globalReferrers(q, qpos.info.Pkg.Path(), defpkg, objposn, pkglevel)
+	}
+
 	q.Output(fset, &referrersInitialResult{
 		qinfo: qpos.info,
 		obj:   obj,
 	})
 
-	// For a globally accessible object defined in package P, we
-	// must load packages that depend on P.  Specifically, for a
-	// package-level object, we need load only direct importers
-	// of P, but for a field or method, we must load
-	// any package that transitively imports P.
-
-	if global, pkglevel := classify(obj); global {
-		if pkglevel {
-			return globalReferrersPkgLevel(q, obj, fset)
-		}
-		// We'll use the the object's position to identify it in the larger program.
-		objposn := fset.Position(obj.Pos())
-		defpkg := obj.Pkg().Path() // defining package
-		return globalReferrers(q, qpos.info.Pkg.Path(), defpkg, objposn)
-	}
-
 	outputUses(q, fset, usesOf(obj, qpos.info), obj.Pkg())
 
 	return nil // success
@@ -224,16 +210,26 @@ func outputUses(q *Query, fset *token.FileSet, refs []*ast.Ident, pkg *types.Pac
 }
 
 // globalReferrers reports references throughout the entire workspace to the
-// object (a field or method) at the specified source position.
+// object at the specified source position.  Its defining package is defpkg,
-// Its defining package is defpkg, and the query package is qpkg.
+// and the query package is qpkg.  isPkgLevel indicates whether the object
-func globalReferrers(q *Query, qpkg, defpkg string, objposn token.Position) error {
+// is defined at package-level.
+func globalReferrers(q *Query, qpkg, defpkg string, objposn token.Position, isPkgLevel bool) error {
 	// Scan the workspace and build the import graph.
 	// Ignore broken packages.
 	_, rev, _ := importgraph.Build(q.Build)
 
 	// Find the set of packages that depend on defpkg.
 	// Only function bodies in those packages need type-checking.
-	users := rev.Search(defpkg) // transitive importers
+	var users map[string]bool
+	if isPkgLevel {
+		users = rev[defpkg] // direct importers
+		if users == nil {
+			users = make(map[string]bool)
+		}
+		users[defpkg] = true // plus the defining package itself
+	} else {
+		users = rev.Search(defpkg) // transitive importers
+	}
 
 	// Prepare to load the larger program.
 	fset := token.NewFileSet()
@@ -265,8 +261,9 @@ func globalReferrers(q *Query, qpkg, defpkg string, objposn token.Position) erro
 	// to completion.
 
 	var (
-		mu   sync.Mutex
+		mu    sync.Mutex
-		qobj types.Object
+		qobj  types.Object
+		qinfo *loader.PackageInfo // info for qpkg
 	)
 
 	// For efficiency, we scan each package for references
@@ -290,6 +287,13 @@ func globalReferrers(q *Query, qpkg, defpkg string, objposn token.Position) erro
 					log.Fatalf("object at %s not found in package %s",
 						objposn, defpkg)
 				}
+
+				// Object found.
+				qinfo = info
+				q.Output(fset, &referrersInitialResult{
+					qinfo: qinfo,
+					obj:   qobj,
+				})
 			}
 			obj := qobj
 			mu.Unlock()
@@ -312,287 +316,6 @@ func globalReferrers(q *Query, qpkg, defpkg string, objposn token.Position) erro
 	return nil // success
 }
 
-// globalReferrersPkgLevel reports references throughout the entire workspace to the package-level object obj.
-// It assumes that the query object itself has already been reported.
-func globalReferrersPkgLevel(q *Query, obj types.Object, fset *token.FileSet) error {
-	// globalReferrersPkgLevel uses go/ast and friends instead of go/types.
-	// This affords a considerable performance benefit.
-	// It comes at the cost of some code complexity.
-	//
-	// Here's a high level summary.
-	//
-	// The goal is to find references to the query object p.Q.
-	// There are several possible scenarios, each handled differently.
-	//
-	// 1. We are looking in a package other than p, and p is not dot-imported.
-	//    This is the simplest case. Q must be referred to as n.Q,
-	//    where n is the name under which p is imported.
-	//    We look at all imports of p to gather all names under which it is imported.
-	//    (In the typical case, it is imported only once, under its default name.)
-	//    Then we look at all selector expressions and report any matches.
-	//
-	// 2. We are looking in a package other than p, and p is dot-imported.
-	//    In this case, Q will be referred to just as Q.
-	//    Furthermore, go/ast's object resolution will not be able to resolve
-	//    Q to any other object, unlike any local (file- or function- or block-scoped) object.
-	//    So we look at all matching identifiers and report all unresolvable ones.
-	//
-	// 3. We are looking in package p.
-	//    (Care must be taken to separate p and p_test (an xtest package),
-	//    and make sure that they are treated as separate packages.)
-	//    In this case, we give go/ast the entire package for object resolution,
-	//    instead of going file by file.
-	//    We then iterate over all identifiers that resolve to the query object.
-	//    (The query object itself has already been reported, so we don't re-report it.)
-	//
-	// We always skip all files that don't contain the string Q, as they cannot be
-	// relevant to finding references to Q.
-	//
-	// We parse all files leniently. In the presence of parsing errors, results are best-effort.
-
-	// Scan the workspace and build the import graph.
-	// Ignore broken packages.
-	_, rev, _ := importgraph.Build(q.Build)
-
-	// Find the set of packages that directly import defpkg.
-	defpkg := obj.Pkg().Path()
-	defpkg = strings.TrimSuffix(defpkg, "_test") // package x_test actually has package name x
-	defpkg = imports.VendorlessPath(defpkg)      // remove vendor goop
-
-	users := rev[defpkg]
-	if len(users) == 0 {
-		users = make(map[string]bool)
-	}
-	// We also need to check defpkg itself, and its xtests.
-	// For the reverse graph packages, we process xtests with the main package.
-	// defpkg gets special handling; we must distinguish between in-package vs out-of-package.
-	// To make the control flow below simpler, add defpkg and defpkg xtest placeholders.
-	// Use "!test" instead of "_test" because "!" is not a valid character in an import path.
-	// (More precisely, it is not guaranteed to be a valid character in an import path,
-	// so it is unlikely that it will be in use. See https://golang.org/ref/spec#Import_declarations.)
-	users[defpkg] = true
-	users[defpkg+"!test"] = true
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		return err
-	}
-
-	defname := obj.Pkg().Name()                    // name of defining package, used for imports using import path only
-	isxtest := strings.HasSuffix(defname, "_test") // indicates whether the query object is defined in an xtest package
-
-	name := obj.Name()
-	namebytes := []byte(name)          // byte slice version of query object name, for early filtering
-	objpos := fset.Position(obj.Pos()) // position of query object, used to prevent re-emitting original decl
-
-	sema := make(chan struct{}, 20) // counting semaphore to limit I/O concurrency
-	var wg sync.WaitGroup
-
-	for u := range users {
-		u := u
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			uIsXTest := strings.HasSuffix(u, "!test") // indicates whether this package is the special defpkg xtest package
-			u = strings.TrimSuffix(u, "!test")
-
-			// Resolve package.
-			sema <- struct{}{} // acquire token
-			pkg, err := q.Build.Import(u, cwd, build.IgnoreVendor)
-			<-sema // release token
-			if err != nil {
-				return
-			}
-
-			// If we're not in the query package,
-			// the object is in another package regardless,
-			// so we want to process all files.
-			// If we are in the query package,
-			// we want to only process the files that are
-			// part of that query package;
-			// that set depends on whether the query package itself is an xtest.
-			inQueryPkg := u == defpkg && isxtest == uIsXTest
-			var files []string
-			if !inQueryPkg || !isxtest {
-				files = append(files, pkg.GoFiles...)
-				files = append(files, pkg.TestGoFiles...)
-				files = append(files, pkg.CgoFiles...) // use raw cgo files, as we're only parsing
-			}
-			if !inQueryPkg || isxtest {
-				files = append(files, pkg.XTestGoFiles...)
-			}
-
-			if len(files) == 0 {
-				return
-			}
-
-			var deffiles map[string]*ast.File
-			if inQueryPkg {
-				deffiles = make(map[string]*ast.File)
-			}
-
-			buf := new(bytes.Buffer) // reusable buffer for reading files
-
-			for _, file := range files {
-				if !buildutil.IsAbsPath(q.Build, file) {
-					file = buildutil.JoinPath(q.Build, pkg.Dir, file)
-				}
-				buf.Reset()
-				sema <- struct{}{} // acquire token
-				src, err := readFile(q.Build, file, buf)
-				<-sema // release token
-				if err != nil {
-					continue
-				}
-
-				// Fast path: If the object's name isn't present anywhere in the source, ignore the file.
-				if !bytes.Contains(src, namebytes) {
-					continue
-				}
-
-				if inQueryPkg {
-					// If we're in the query package, we defer final processing until we have
-					// parsed all of the candidate files in the package.
-					// Best effort; allow errors and use what we can from what remains.
-					f, _ := parser.ParseFile(fset, file, src, parser.AllErrors)
-					if f != nil {
-						deffiles[file] = f
-					}
-					continue
-				}
-
-				// We aren't in the query package. Go file by file.
-
-				// Parse out only the imports, to check whether the defining package
-				// was imported, and if so, under what names.
-				// Best effort; allow errors and use what we can from what remains.
-				f, _ := parser.ParseFile(fset, file, src, parser.ImportsOnly|parser.AllErrors)
-				if f == nil {
-					continue
-				}
-
-				// pkgnames is the set of names by which defpkg is imported in this file.
-				// (Multiple imports in the same file are legal but vanishingly rare.)
-				pkgnames := make([]string, 0, 1)
-				var isdotimport bool
-				for _, imp := range f.Imports {
-					path, err := strconv.Unquote(imp.Path.Value)
-					if err != nil || path != defpkg {
-						continue
-					}
-					switch {
-					case imp.Name == nil:
-						pkgnames = append(pkgnames, defname)
-					case imp.Name.Name == ".":
-						isdotimport = true
-					default:
-						pkgnames = append(pkgnames, imp.Name.Name)
-					}
-				}
-				if len(pkgnames) == 0 && !isdotimport {
-					// Defining package not imported, bail.
-					continue
-				}
-
-				// Re-parse the entire file.
-				// Parse errors are ok; we'll do the best we can with a partial AST, if we have one.
-				f, _ = parser.ParseFile(fset, file, src, parser.AllErrors)
-				if f == nil {
-					continue
-				}
-
-				// Walk the AST looking for references.
-				var refs []*ast.Ident
-				ast.Inspect(f, func(n ast.Node) bool {
-					// Check selector expressions.
-					// If the selector matches the target name,
-					// and the expression is one of the names
-					// that the defining package was imported under,
-					// then we have a match.
-					if sel, ok := n.(*ast.SelectorExpr); ok && sel.Sel.Name == name {
-						if id, ok := sel.X.(*ast.Ident); ok {
-							for _, n := range pkgnames {
-								if n == id.Name {
-									refs = append(refs, sel.Sel)
-									// Don't recurse further, to avoid duplicate entries
-									// from the dot import check below.
-									return false
-								}
-							}
-						}
-					}
-					// Dot imports are special.
-					// Objects imported from the defining package are placed in the package scope.
-					// go/ast does not resolve them to an object.
-					// At all other scopes (file, local), go/ast can do the resolution.
-					// So we're looking for object-free idents with the right name.
-					// The only other way to get something with the right name at the package scope
-					// is to *be* the defining package. We handle that case separately (inQueryPkg).
-					if isdotimport {
-						if id, ok := n.(*ast.Ident); ok && id.Obj == nil && id.Name == name {
-							refs = append(refs, id)
-							return false
-						}
-					}
-					return true
-				})
-
-				// Emit any references we found.
-				if len(refs) > 0 {
-					q.Output(fset, &referrersPackageResult{
-						pkg:   types.NewPackage(pkg.ImportPath, pkg.Name),
-						build: q.Build,
-						fset:  fset,
-						refs:  refs,
-					})
-				}
-			}
-
-			// If we're in the query package, we've now collected all the files in the package.
-			// (Or at least the ones that might contain references to the object.)
-			// Find and emit refs.
-			if inQueryPkg {
-				// Bundle the files together into a package.
-				// This does package-level object resolution.
-				qpkg, _ := ast.NewPackage(fset, deffiles, nil, nil)
-				// Look up the query object; we know that it is defined in the package scope.
-				pkgobj := qpkg.Scope.Objects[name]
-				if pkgobj == nil {
-					panic("missing defpkg object for " + defpkg + "." + name)
-				}
-				// Find all references to the query object.
-				var refs []*ast.Ident
-				ast.Inspect(qpkg, func(n ast.Node) bool {
-					if id, ok := n.(*ast.Ident); ok {
-						// Check both that this is a reference to the query object
-						// and that it is not the query object itself;
-						// the query object itself was already emitted.
-						if id.Obj == pkgobj && objpos != fset.Position(id.Pos()) {
-							refs = append(refs, id)
-							return false
-						}
-					}
-					return true
-				})
-				if len(refs) > 0 {
-					q.Output(fset, &referrersPackageResult{
-						pkg:   types.NewPackage(pkg.ImportPath, pkg.Name),
-						build: q.Build,
-						fset:  fset,
-						refs:  refs,
-					})
-				}
-				deffiles = nil // allow GC
-			}
-		}()
-	}
-
-	wg.Wait()
-
-	return nil
-}
-
 // findObject returns the object defined at the specified position.
 func findObject(fset *token.FileSet, info *types.Info, objposn token.Position) types.Object {
 	good := func(obj types.Object) bool {
@@ -730,7 +453,7 @@ func (r *referrersPackageResult) foreachRef(f func(id *ast.Ident, text string))
 			// start asynchronous read.
 			go func() {
 				sema <- struct{}{} // acquire token
-				content, err := readFile(r.build, posn.Filename, nil)
+				content, err := readFile(r.build, posn.Filename)
 				<-sema // release token
 				if err != nil {
 					fi.data <- err
@@ -768,17 +491,14 @@ func (r *referrersPackageResult) foreachRef(f func(id *ast.Ident, text string))
 
 // readFile is like ioutil.ReadFile, but
 // it goes through the virtualized build.Context.
-// If non-nil, buf must have been reset.
+func readFile(ctxt *build.Context, filename string) ([]byte, error) {
-func readFile(ctxt *build.Context, filename string, buf *bytes.Buffer) ([]byte, error) {
 	rc, err := buildutil.OpenFile(ctxt, filename)
 	if err != nil {
 		return nil, err
 	}
 	defer rc.Close()
-	if buf == nil {
+	var buf bytes.Buffer
-		buf = new(bytes.Buffer)
+	if _, err := io.Copy(&buf, rc); err != nil {
-	}
-	if _, err := io.Copy(buf, rc); err != nil {
 		return nil, err
 	}
 	return buf.Bytes(), nil

cmd/guru/serial/serial.go

@@ -193,10 +193,9 @@ type PointsTo struct {
 // A DescribeValue is the additional result of a 'describe' query
 // if the selection indicates a value or expression.
 type DescribeValue struct {
-	Type     string       `json:"type"`               // type of the expression
+	Type   string `json:"type"`             // type of the expression
-	Value    string       `json:"value,omitempty"`    // value of the expression, if constant
+	Value  string `json:"value,omitempty"`  // value of the expression, if constant
-	ObjPos   string       `json:"objpos,omitempty"`   // location of the definition, if an Ident
+	ObjPos string `json:"objpos,omitempty"` // location of the definition, if an Ident
-	TypesPos []Definition `json:"typespos,omitempty"` // location of the named types, that type consist of
 }
 
 type DescribeMethod struct {

cmd/guru/testdata/src/definition-json/main.go

@@ -9,7 +9,6 @@ package definition
 import (
 	"lib"
 	lib2 "lib"
-	"nosuchpkg"
 )
 
 func main() {
@@ -28,7 +27,6 @@ func main() {
 	var _ lib.Const    // @definition qualified-const "Const"
 	var _ lib2.Type    // @definition qualified-type-renaming "Type"
 	var _ lib.Nonesuch // @definition qualified-nomember "Nonesuch"
-	var _ nosuchpkg.T  // @definition qualified-nopkg "nosuchpkg"
 
 	var u U
 	print(u.field) // @definition select-field "field"

cmd/guru/testdata/src/definition-json/main.golden

@@ -11,17 +11,17 @@ Error: no object for identifier
 }
 -------- @definition lexical-func --------
 {
-	"objpos": "$GOPATH/src/definition-json/main.go:38:6",
+	"objpos": "$GOPATH/src/definition-json/main.go:36:6",
 	"desc": "func f"
 }
 -------- @definition lexical-var --------
 {
-	"objpos": "$GOPATH/src/definition-json/main.go:19:6",
+	"objpos": "$GOPATH/src/definition-json/main.go:18:6",
 	"desc": "var x"
 }
 -------- @definition lexical-shadowing --------
 {
-	"objpos": "$GOPATH/src/definition-json/main.go:22:5",
+	"objpos": "$GOPATH/src/definition-json/main.go:21:5",
 	"desc": "var x"
 }
 -------- @definition qualified-type --------
@@ -52,19 +52,14 @@ Error: no object for identifier
 -------- @definition qualified-nomember --------
 
 Error: couldn't find declaration of Nonesuch in "lib"
--------- @definition qualified-nopkg --------
-{
-	"objpos": "testdata/src/definition-json/main.go:12:2",
-	"desc": "package nosuchpkg"
-}
 -------- @definition select-field --------
 {
-	"objpos": "testdata/src/definition-json/main.go:40:16",
+	"objpos": "testdata/src/definition-json/main.go:38:16",
 	"desc": "field field int"
 }
 -------- @definition select-method --------
 {
-	"objpos": "testdata/src/definition-json/main.go:42:10",
+	"objpos": "testdata/src/definition-json/main.go:40:10",
 	"desc": "func (T).method()"
 }
 -------- @definition embedded-other-file --------
@@ -90,6 +85,6 @@ Error: int is built in
 }
 -------- @definition embedded-same-file --------
 {
-	"objpos": "$GOPATH/src/definition-json/main.go:40:6",
+	"objpos": "$GOPATH/src/definition-json/main.go:38:6",
 	"desc": "type T"
 }

cmd/guru/testdata/src/definition-json/main19.go

@@ -0,0 +1,5 @@
+package definition
+
+import "nosuchpkg"
+
+var _ nosuchpkg.T // @definition qualified-nopkg "nosuchpkg"

cmd/guru/testdata/src/definition-json/main19.golden

@@ -0,0 +1,5 @@
+-------- @definition qualified-nopkg --------
+{
+	"objpos": "testdata/src/definition-json/main19.go:3:8",
+	"desc": "package nosuchpkg"
+}

cmd/guru/testdata/src/describe-json/main.go

@@ -25,5 +25,5 @@ type I interface {
 type C int // @describe desc-type-C "C"
 type D struct{}
 
-func (c C) f()  {} // @describe desc-param-c "\\bc\\b"
+func (c C) f()  {}
-func (d *D) f() {} // @describe desc-param-d "\\bd\\b"
+func (d *D) f() {}

cmd/guru/testdata/src/describe-json/main.golden

@@ -68,13 +68,7 @@
 	"detail": "value",
 	"value": {
 		"type": "I",
-		"objpos": "testdata/src/describe-json/main.go:12:6",
+		"objpos": "testdata/src/describe-json/main.go:12:6"
-		"typespos": [
-			{
-				"objpos": "testdata/src/describe-json/main.go:21:6",
-				"desc": "I"
-			}
-		]
 	}
 }
 -------- @describe desc-stmt --------
@@ -100,35 +94,3 @@
 		]
 	}
 }
--------- @describe desc-param-c --------
-{
-	"desc": "identifier",
-	"pos": "testdata/src/describe-json/main.go:28:7",
-	"detail": "value",
-	"value": {
-		"type": "C",
-		"objpos": "testdata/src/describe-json/main.go:28:7",
-		"typespos": [
-			{
-				"objpos": "testdata/src/describe-json/main.go:25:6",
-				"desc": "C"
-			}
-		]
-	}
-}
--------- @describe desc-param-d --------
-{
-	"desc": "identifier",
-	"pos": "testdata/src/describe-json/main.go:29:7",
-	"detail": "value",
-	"value": {
-		"type": "*D",
-		"objpos": "testdata/src/describe-json/main.go:29:7",
-		"typespos": [
-			{
-				"objpos": "testdata/src/describe-json/main.go:26:6",
-				"desc": "D"
-			}
-		]
-	}
-}

cmd/guru/testdata/src/describe/main.go

@@ -8,14 +8,9 @@ package describe // @describe pkgdecl "describe"
 
 import (
 	"lib"
-	"nosuchpkg"            // @describe badimport1 "nosuchpkg"
+	_ "unsafe" // @describe unsafe "unsafe"
-	nosuchpkg2 "nosuchpkg" // @describe badimport2 "nosuchpkg2"
-	_ "unsafe"             // @describe unsafe "unsafe"
 )
 
-var _ nosuchpkg.T
-var _ nosuchpkg2.T
-
 type cake float64 // @describe type-ref-builtin "float64"
 
 const c = iota // @describe const-ref-iota "iota"
@@ -28,15 +23,14 @@ var global = new(string) // NB: ssa.Global is indirect, i.e. **string
 
 func main() { // @describe func-def-main "main"
 	// func objects
-	_ = main      // @describe func-ref-main "main"
+	_ = main   // @describe func-ref-main "main"
-	_ = (*C).f    // @describe func-ref-*C.f "..C..f"
+	_ = (*C).f // @describe func-ref-*C.f "..C..f"
-	_ = D.f       // @describe func-ref-D.f "D.f"
+	_ = D.f    // @describe func-ref-D.f "D.f"
-	_ = I.f       // @describe func-ref-I.f "I.f"
+	_ = I.f    // @describe func-ref-I.f "I.f"
-	var d D       // @describe type-D "D"
+	var d D    // @describe type-D "D"
-	var i I       // @describe type-I "I"
+	var i I    // @describe type-I "I"
-	_ = d.f       // @describe func-ref-d.f "d.f"
+	_ = d.f    // @describe func-ref-d.f "d.f"
-	_ = i.f       // @describe func-ref-i.f "i.f"
+	_ = i.f    // @describe func-ref-i.f "i.f"
-	var slice []D // @describe slice-of-D "slice"
 
 	var dptr *D // @describe ptr-with-nonptr-methods "dptr"
 	_ = dptr
@@ -91,11 +85,6 @@ func main() { // @describe func-def-main "main"
 
 	var _ lib.Outer // @describe lib-outer "Outer"
 
-	var mmm map[C]D // @describe var-map-of-C-D "mmm"
-
-	d := newD().ThirdField // @describe field-access "ThirdField"
-
-	astCopy := ast
 	unknown() // @describe call-unknown "\\("
 }
 
@@ -107,10 +96,7 @@ type C int
 type D struct {
 	Field        int
 	AnotherField string
-	ThirdField   C
 }
 
 func (c *C) f() {}
 func (d D) f()  {}
-
-func newD() D { return D{} }

cmd/guru/testdata/src/describe/main.golden

@@ -10,16 +10,9 @@ definition of package "describe"
 	type  cake   float64
 	var   global *string
 	func  main   func()
-	func  newD   func() D
 	const pi     untyped float = 3.141
 	const pie    cake = 3.141
 
--------- @describe badimport1 --------
-import of package "nosuchpkg"
-
--------- @describe badimport2 --------
-reference to package "nosuchpkg"
-
 -------- @describe unsafe --------
 import of package "unsafe"
 	builtin Alignof 
@@ -38,8 +31,6 @@ definition of const pi untyped float of value 3.141
 
 -------- @describe const-def-pie --------
 definition of const pie cake of value 3.141
-Named types:
-	type cake defined here
 
 -------- @describe const-ref-pi --------
 reference to const pi untyped float of value 3.141
@@ -65,14 +56,13 @@ reference to interface method func (I).f()
 defined here
 
 -------- @describe type-D --------
-reference to type D (size 32, align 8)
+reference to type D (size 24, align 8)
-defined as struct{Field int; AnotherField string; ThirdField C}
+defined as struct{Field int; AnotherField string}
 Methods:
 	method (D) f()
 Fields:
 	Field        int
 	AnotherField string
-	ThirdField   C
 
 -------- @describe type-I --------
 reference to type I (size 16, align 8)
@@ -88,11 +78,6 @@ defined here
 reference to interface method func (I).f()
 defined here
 
--------- @describe slice-of-D --------
-definition of var slice []D
-Named types:
-	type D defined here
-
 -------- @describe ptr-with-nonptr-methods --------
 definition of var dptr *D
 Methods:
@@ -100,9 +85,6 @@ Methods:
 Fields:
 	Field        int
 	AnotherField string
-	ThirdField   C
-Named types:
-	type D defined here
 
 -------- @describe ref-lexical-d --------
 reference to var d D
@@ -112,9 +94,6 @@ Methods:
 Fields:
 	Field        int
 	AnotherField string
-	ThirdField   C
-Named types:
-	type D defined here
 
 -------- @describe ref-anon --------
 reference to var anon func()
@@ -144,32 +123,24 @@ reference to var i I
 defined here
 Methods:
 	method (I) f()
-Named types:
-	type I defined here
 
 -------- @describe var-ref-i-D --------
 reference to var i I
 defined here
 Methods:
 	method (I) f()
-Named types:
-	type I defined here
 
 -------- @describe var-ref-i --------
 reference to var i I
 defined here
 Methods:
 	method (I) f()
-Named types:
-	type I defined here
 
 -------- @describe const-local-pi --------
 definition of const localpi untyped float of value 3.141
 
 -------- @describe const-local-pie --------
 definition of const localpie cake of value 3.141
-Named types:
-	type cake defined here
 
 -------- @describe const-ref-localpi --------
 reference to const localpi untyped float of value 3.141
@@ -228,20 +199,6 @@ Fields:
 	inner.C           bool
 	inner.recursive.E bool
 
--------- @describe var-map-of-C-D --------
-definition of var mmm map[C]D
-Named types:
-	type C defined here
-	type D defined here
-
--------- @describe field-access --------
-reference to field ThirdField C
-defined here
-Methods:
-	method (*C) f()
-Named types:
-	type C defined here
-
 -------- @describe call-unknown --------
 function call of type invalid type
 

cmd/guru/testdata/src/describe/main19.go

@@ -0,0 +1,13 @@
+package describe
+
+// The behavior of "describe" on a non-existent import changed
+// when go/types started returning fake packages, so this test
+// is executed only under go1.9.
+
+import (
+	"nosuchpkg"            // @describe badimport1 "nosuchpkg"
+	nosuchpkg2 "nosuchpkg" // @describe badimport2 "nosuchpkg2"
+)
+
+var _ nosuchpkg.T
+var _ nosuchpkg2.T

cmd/guru/testdata/src/describe/main19.golden

@@ -0,0 +1,6 @@
+-------- @describe badimport1 --------
+import of package "nosuchpkg"
+
+-------- @describe badimport2 --------
+reference to package "nosuchpkg"
+

cmd/guru/testdata/src/referrers-json/main.golden

@@ -6,35 +6,35 @@
 	"package": "definition-json",
 	"refs": [
 		{
-			"pos": "testdata/src/definition-json/main.go:19:8",
+			"pos": "testdata/src/definition-json/main.go:18:8",
 			"text": "\tvar x lib.T           // @definition lexical-pkgname \"lib\""
 		},
 		{
-			"pos": "testdata/src/definition-json/main.go:25:8",
+			"pos": "testdata/src/definition-json/main.go:24:8",
 			"text": "\tvar _ lib.Type     // @definition qualified-type \"Type\""
 		},
 		{
-			"pos": "testdata/src/definition-json/main.go:26:8",
+			"pos": "testdata/src/definition-json/main.go:25:8",
 			"text": "\tvar _ lib.Func     // @definition qualified-func \"Func\""
 		},
 		{
-			"pos": "testdata/src/definition-json/main.go:27:8",
+			"pos": "testdata/src/definition-json/main.go:26:8",
 			"text": "\tvar _ lib.Var      // @definition qualified-var \"Var\""
 		},
 		{
-			"pos": "testdata/src/definition-json/main.go:28:8",
+			"pos": "testdata/src/definition-json/main.go:27:8",
 			"text": "\tvar _ lib.Const    // @definition qualified-const \"Const\""
 		},
 		{
-			"pos": "testdata/src/definition-json/main.go:29:8",
+			"pos": "testdata/src/definition-json/main.go:28:8",
 			"text": "\tvar _ lib2.Type    // @definition qualified-type-renaming \"Type\""
 		},
 		{
-			"pos": "testdata/src/definition-json/main.go:30:8",
+			"pos": "testdata/src/definition-json/main.go:29:8",
 			"text": "\tvar _ lib.Nonesuch // @definition qualified-nomember \"Nonesuch\""
 		},
 		{
-			"pos": "testdata/src/definition-json/main.go:63:2",
+			"pos": "testdata/src/definition-json/main.go:61:2",
 			"text": "\tlib.Type // @definition embedded-other-pkg \"Type\""
 		}
 	]
@@ -43,7 +43,7 @@
 	"package": "describe",
 	"refs": [
 		{
-			"pos": "testdata/src/describe/main.go:92:8",
+			"pos": "testdata/src/describe/main.go:86:8",
 			"text": "\tvar _ lib.Outer // @describe lib-outer \"Outer\""
 		}
 	]

cmd/guru/testdata/src/referrers/int_test.go

@@ -5,6 +5,4 @@ import "lib"
 func _() {
 	// This reference should be found by the ref-method query.
 	_ = (lib.Type).Method // ref from internal test package
-
-	_ = notexported
 }

cmd/guru/testdata/src/referrers/main.go

@@ -25,8 +25,6 @@ func main() {
 	s2.f = 1
 }
 
-var notexported int // @referrers unexported-from-test "notexported"
-
 // Test //line directives:
 
 type U int // @referrers ref-type-U "U"

cmd/guru/testdata/src/referrers/main.golden

@@ -33,7 +33,7 @@ type _ lib.T
 var _ lib.Var // @what pkg "lib"
 
 -------- @referrers ref-method --------
-references to func (lib.Type).Method(x *int) *int
+references to func (Type).Method(x *int) *int
 	_ = (lib.Type).Method // ref from external test package
 	_ = (lib.Type).Method // ref from internal test package
 	_ = v.Method
@@ -54,10 +54,6 @@ references to field f int
 	_ = s{}.f // @referrers ref-field "f"
 	s2.f = 1
 
--------- @referrers unexported-from-test --------
-references to var notexported int
-	_ = notexported
-
 -------- @referrers ref-type-U --------
 references to type U int
 open testdata/src/referrers/nosuchfile.y: no such file or directory (+ 1 more refs in this file)

cmd/guru/what.go

@@ -10,7 +10,6 @@ import (
 	"go/build"
 	"go/token"
 	"os"
-	"path"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -203,7 +202,7 @@ func guessImportPath(filename string, buildContext *build.Context) (srcdir, impo
 		if d >= 0 && d < minD {
 			minD = d
 			srcdir = gopathDir
-			importPath = path.Join(segmentedAbsFileDir[len(segmentedAbsFileDir)-minD:]...)
+			importPath = strings.Join(segmentedAbsFileDir[len(segmentedAbsFileDir)-minD:], string(os.PathSeparator))
 		}
 	}
 	if srcdir == "" {

cmd/heapview/client/.clang-format

@@ -0,0 +1 @@
+BasedOnStyle: Google

cmd/heapview/client/.gitignore

@@ -0,0 +1 @@
+/node_modules/

cmd/heapview/client/README.md

@@ -0,0 +1,45 @@
+# Go Heap Viewer Client
+
+This directory contains the client Typescript code for the Go
+heap viewer.
+
+## Typescript Tooling
+
+Below are instructions for downloading tooling and files to
+help make the development process more convenient. These tools
+are not required for contributing or running the heap viewer-
+they are just meant as development aids.
+
+## Node and NPM
+
+We use npm to manage the dependencies for these tools. There are
+a couple of ways of installing npm on your system, but we recommend
+using nvm.
+
+Run the following command to install nvm:
+
+    [shell]$ curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.31.3/install.sh | bash
+
+or see the instructions on [the nvm github page](github.com/creationix/nvm)
+for alternative methods. This will put the nvm tool in your home directory
+and edit your path to add nvm, node and other tools you install using them.
+Once nvm is installed, use
+
+    [shell]$ nvm install node
+
+then
+
+    [shell]$ nvm use node
+
+to install node.js.
+
+Once node is installed, you can install typescript using
+
+    [shell]$ npm install -g typescript
+
+Finally, import type definitions into this project by running
+
+    [shell]$ npm install
+
+in this directory. They will be imported into the node_packages directory
+and be automatically available to the Typescript compiler.

cmd/heapview/client/main.ts

@@ -0,0 +1,195 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/**
+ * An enum of types of actions that might be requested
+ * by the app.
+ */
+enum Action {
+  TOGGLE_SIDEBAR,  // Toggle the sidebar.
+  NAVIGATE_ABOUT,  // Go to the about page.
+}
+
+const TITLE = 'Go Heap Viewer';
+
+/**
+ * A type of event that signals to the AppElement controller
+ * that something shoud be done. For the most part, the structure
+ * of the app will be that elements' state will mostly be controlled
+ * by parent elements. Elements will issue actions that the AppElement
+ * will handle, and the app will be re-rendered down the DOM
+ * hierarchy.
+ */
+class ActionEvent extends Event {
+  static readonly EVENT_TYPE = 'action-event'
+  constructor(public readonly action: Action) { super(ActionEvent.EVENT_TYPE); }
+}
+
+/**
+ * A hamburger menu element. Triggers a TOGGLE_SIDE action to toggle the
+ * sidebar.
+ */
+export class HamburgerElement extends HTMLElement {
+  static readonly NAME = 'heap-hamburger';
+
+  createdCallback() {
+    this.appendChild(document.createTextNode('☰'));
+    this.onclick =
+        () => { this.dispatchEvent(new ActionEvent(Action.TOGGLE_SIDEBAR)) };
+  }
+}
+document.registerElement(HamburgerElement.NAME, HamburgerElement);
+
+/**
+ * A heading for the page with a hamburger menu and a title.
+ */
+export class HeadingElement extends HTMLElement {
+  static readonly NAME = 'heap-heading';
+
+  createdCallback() {
+    this.style.display = 'block';
+    this.style.backgroundColor = '#2196F3';
+    this.style.webkitUserSelect = 'none';
+    this.style.cursor = 'default';
+    this.style.color = '#FFFFFF';
+    this.style.padding = '10px';
+
+    const div = document.createElement('div');
+    div.style.margin = '0px';
+    div.style.fontSize = '2em';
+    div.appendChild(document.createElement(HamburgerElement.NAME));
+    div.appendChild(document.createTextNode(' ' + TITLE));
+    this.appendChild(div);
+  }
+}
+document.registerElement(HeadingElement.NAME, HeadingElement);
+
+/**
+ * A sidebar that has navigation for the app.
+ */
+export class SidebarElement extends HTMLElement {
+  static readonly NAME = 'heap-sidebar';
+
+  createdCallback() {
+    this.style.display = 'none';
+    this.style.backgroundColor = '#9E9E9E';
+    this.style.width = '15em';
+
+    const aboutButton = document.createElement('button');
+    aboutButton.innerText = 'about';
+    aboutButton.onclick =
+        () => { this.dispatchEvent(new ActionEvent(Action.NAVIGATE_ABOUT)) };
+    this.appendChild(aboutButton);
+  }
+
+  toggle() {
+    this.style.display = this.style.display === 'none' ? 'block' : 'none';
+  }
+}
+document.registerElement(SidebarElement.NAME, SidebarElement);
+
+/**
+ * A Container for the main content in the app.
+ * TODO(matloob): Implement main content.
+ */
+export class MainContentElement extends HTMLElement {
+  static readonly NAME = 'heap-container';
+
+  attachedCallback() {
+    this.style.backgroundColor = '#E0E0E0';
+    this.style.height = '100%';
+    this.style.flex = '1';
+  }
+}
+document.registerElement(MainContentElement.NAME, MainContentElement);
+
+/**
+ * A container and controller for the whole app.
+ * Contains the heading, side drawer and main panel.
+ */
+class AppElement extends HTMLElement {
+  static readonly NAME = 'heap-app';
+  private sidebar: SidebarElement;
+  private mainContent: MainContentElement;
+
+  attachedCallback() {
+    document.title = TITLE;
+
+    this.addEventListener(
+        ActionEvent.EVENT_TYPE, e => this.handleAction(e as ActionEvent),
+        /* capture */ true);
+
+    this.render();
+  }
+
+  render() {
+    this.style.display = 'block';
+    this.style.height = '100vh';
+    this.style.width = '100vw';
+    this.appendChild(document.createElement(HeadingElement.NAME));
+
+    const bodyDiv = document.createElement('div');
+    bodyDiv.style.height = '100%';
+    bodyDiv.style.display = 'flex';
+    this.sidebar =
+        document.createElement(SidebarElement.NAME) as SidebarElement;
+    bodyDiv.appendChild(this.sidebar);
+    this.mainContent =
+        document.createElement(MainContentElement.NAME) as MainContentElement;
+    bodyDiv.appendChild(this.mainContent);
+    this.appendChild(bodyDiv);
+
+    this.renderRoute();
+  }
+
+  renderRoute() {
+    this.mainContent.innerHTML = ''
+    switch (window.location.pathname) {
+      case '/about':
+        this.mainContent.appendChild(
+            document.createElement(AboutPageElement.NAME));
+        break;
+    }
+  }
+
+  handleAction(event: ActionEvent) {
+    switch (event.action) {
+      case Action.TOGGLE_SIDEBAR:
+        this.sidebar.toggle();
+        break;
+      case Action.NAVIGATE_ABOUT:
+        window.history.pushState({}, '', '/about');
+        this.renderRoute();
+        break;
+    }
+  }
+}
+document.registerElement(AppElement.NAME, AppElement);
+
+/**
+ * An about page.
+ */
+class AboutPageElement extends HTMLElement {
+  static readonly NAME = 'heap-about';
+
+  createdCallback() { this.textContent = TITLE; }
+}
+document.registerElement(AboutPageElement.NAME, AboutPageElement);
+
+/**
+ * Resets body's margin and padding, and sets font.
+ */
+function clearStyle(document: Document) {
+  const styleElement = document.createElement('style') as HTMLStyleElement;
+  document.head.appendChild(styleElement);
+  const styleSheet = styleElement.sheet as CSSStyleSheet;
+  styleSheet.insertRule(
+      '* {font-family: Roboto,Helvetica; box-sizing: border-box}', 0);
+  styleSheet.insertRule('body {margin: 0px; padding:0px}', 0);
+}
+
+export function main() {
+  clearStyle(document);
+  document.body.appendChild(document.createElement(AppElement.NAME));
+}

cmd/heapview/client/main_test.ts

@@ -0,0 +1,29 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import {HamburgerElement, HeadingElement, SidebarElement, main} from './main';
+
+describe('main', () => {
+  it('sets the document\'s title', () => {
+    main();
+    expect(document.title).toBe('Go Heap Viewer');
+  });
+
+  it('has a heading', () => {
+    main();
+    expect(document.querySelector(HeadingElement.NAME)).toBeDefined();
+  });
+
+  it('has a sidebar', () => {
+    main();
+    const hamburger = document.querySelector(HamburgerElement.NAME);
+    const sidebar =
+        document.querySelector(SidebarElement.NAME) as SidebarElement;
+    expect(sidebar.style.display).toBe('none');
+
+    // Click on the hamburger. Sidebar should then be visible.
+    hamburger.dispatchEvent(new Event('click'));
+    expect(sidebar.style.display).toBe('block');
+  })
+});

cmd/heapview/client/package.json

@@ -0,0 +1,35 @@
+{
+  "//": [
+    "Copyright 2016 The Go Authors. All rights reserved.",
+    "Use of this source code is governed by a BSD-style",
+    "license that can be found in the LICENSE file.",
+
+    "This file exists to help import typescript typings",
+    "for web features used in this project. Neither the",
+    "typings nor node or npm are required to do development",
+    "on the code in this project.",
+
+    "If you do have npm installed, use the `npm i` command",
+    "in this directory to install the typings."
+  ],
+  "private": true,
+  "name": "@golangtools/heapview",
+  "version": "0.0.0",
+  "devDependencies": {
+    "@types/webcomponents.js": "latest",
+    "@types/whatwg-fetch": "latest",
+    "@types/jasmine": "latest",
+
+    "jasmine-core": "latest",
+    "karma": "latest",
+    "karma-jasmine": "latest",
+    "karma-chrome-launcher": "latest",
+
+    "clang-format": "latest"
+  },
+  "scripts": {
+    "test": "karma start testing/karma.conf.js",
+    "format": "find . | grep '\\(test_main\\.js\\|\\.ts\\)$' | xargs clang-format -i",
+    "lint": "tslint --project ."
+  }
+}

cmd/heapview/client/testing/karma.conf.js

@@ -0,0 +1,22 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+module.exports = config => {
+    config.set({
+        frameworks: ['jasmine'],
+        basePath: '../../../..',
+        files: [
+            'third_party/webcomponents/customelements.js',
+            'third_party/typescript/typescript.js',
+            'third_party/moduleloader/moduleloader.js',
+            'cmd/heapview/client/testing/test_main.js',
+            {pattern: 'cmd/heapview/client/**/*.ts', included: false},
+        ],
+        browsers: ['Chrome'],
+        plugins: [
+            'karma-jasmine',
+            'karma-chrome-launcher'
+        ],
+    })
+}

cmd/heapview/client/testing/test_main.js

@@ -0,0 +1,32 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Configure module loader.
+System.transpiler = 'typescript'
+System.typescriptOptions = {
+  target: ts.ScriptTarget.ES2015
+};
+System.locate = (load) => load.name + '.ts';
+
+// Determine set of test files.
+var tests = [];
+for (var file in window.__karma__.files) {
+  if (window.__karma__.files.hasOwnProperty(file)) {
+    if (/_test\.ts$/.test(file)) {
+      tests.push(file.slice(0, -3));
+    }
+  }
+}
+
+// Steal loaded callback so we can block until we're
+// done loading all test modules.
+var loadedCallback = window.__karma__.loaded.bind(window.__karma__);
+window.__karma__.loaded = () => {};
+
+// Load all test modules, and then call loadedCallback.
+var promises = [];
+for (var i = 0; i < tests.length; i++) {
+  promises.push(System.import(tests[i]));
+}
+Promise.all(promises).then(loadedCallback);

cmd/heapview/client/tsconfig.json

@@ -0,0 +1,16 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file contains configuration for the Typescript
+// compiler if you're running it locally for typechecking
+// and other tooling. The Typescript compiler is
+// not necessary to do development on this project.
+
+{
+  "compilerOptions": {
+    "noEmit": true,
+    "strictNullChecks": true,
+    "target": "es2015"
+  }
+}

cmd/heapview/client/tslint.json

@@ -0,0 +1,40 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This tslint file is based on a configuration used at
+// Google.
+
+{
+  "rules": {
+    "class-name": true,
+    "forin": true,
+    "interface-name": [true, "never-prefix"],
+    "jsdoc-format": true,
+    "label-position": true,
+    "label-undefined": true,
+    "new-parens": true,
+    "no-angle-bracket-type-assertion": true,
+    "no-construct": true,
+    "no-debugger": true,
+    "no-namespace": [true, "allow-declarations"],
+    "no-reference": true,
+    "no-require-imports": true,
+    "no-unused-expression": true,
+    "no-unused-variable": true,
+    "no-use-before-declare": true,
+    "no-var-keyword": true,
+    "semicolon": [true, "always"],
+    "switch-default": true,
+    "triple-equals": [true, "allow-null-check"],
+    "use-isnan": true,
+    "variable-name": [
+      true,
+      "check-format",
+      "ban-keywords",
+      "allow-leading-underscore",
+      "allow-trailing-underscore",
+      "allow-pascal-case"
+    ]
+  }
+}

cmd/heapview/internal/core/mmapfile.go

@@ -0,0 +1,145 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin linux
+
+package core
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"syscall"
+)
+
+var errMmapClosed = errors.New("mmap: closed")
+
+// mmapFile wraps a memory-mapped file.
+type mmapFile struct {
+	data     []byte
+	pos      uint64
+	writable bool
+}
+
+// mmapOpen opens the named file for reading.
+// If writable is true, the file is also open for writing.
+func mmapOpen(filename string, writable bool) (*mmapFile, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	st, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	size := st.Size()
+	if size == 0 {
+		return &mmapFile{data: []byte{}}, nil
+	}
+	if size < 0 {
+		return nil, fmt.Errorf("mmap: file %q has negative size: %d", filename, size)
+	}
+	if size != int64(int(size)) {
+		return nil, fmt.Errorf("mmap: file %q is too large", filename)
+	}
+
+	prot := syscall.PROT_READ
+	if writable {
+		prot |= syscall.PROT_WRITE
+	}
+	data, err := syscall.Mmap(int(f.Fd()), 0, int(size), prot, syscall.MAP_SHARED)
+	if err != nil {
+		return nil, err
+	}
+	return &mmapFile{data: data, writable: writable}, nil
+}
+
+// Size returns the size of the mapped file.
+func (f *mmapFile) Size() uint64 {
+	return uint64(len(f.data))
+}
+
+// Pos returns the current file pointer.
+func (f *mmapFile) Pos() uint64 {
+	return f.pos
+}
+
+// SeekTo sets the current file pointer relative to the start of the file.
+func (f *mmapFile) SeekTo(offset uint64) {
+	f.pos = offset
+}
+
+// Read implements io.Reader.
+func (f *mmapFile) Read(p []byte) (int, error) {
+	if f.data == nil {
+		return 0, errMmapClosed
+	}
+	if f.pos >= f.Size() {
+		return 0, io.EOF
+	}
+	n := copy(p, f.data[f.pos:])
+	f.pos += uint64(n)
+	if n < len(p) {
+		return n, io.EOF
+	}
+	return n, nil
+}
+
+// ReadByte implements io.ByteReader.
+func (f *mmapFile) ReadByte() (byte, error) {
+	if f.data == nil {
+		return 0, errMmapClosed
+	}
+	if f.pos >= f.Size() {
+		return 0, io.EOF
+	}
+	b := f.data[f.pos]
+	f.pos++
+	return b, nil
+}
+
+// ReadSlice returns a slice of size n that points directly at the
+// underlying mapped file. There is no copying. Fails if it cannot
+// read at least n bytes.
+func (f *mmapFile) ReadSlice(n uint64) ([]byte, error) {
+	if f.data == nil {
+		return nil, errMmapClosed
+	}
+	if f.pos+n >= f.Size() {
+		return nil, io.EOF
+	}
+	first := f.pos
+	f.pos += n
+	return f.data[first:f.pos:f.pos], nil
+}
+
+// ReadSliceAt is like ReadSlice, but reads from a specific offset.
+// The file pointer is not used or advanced.
+func (f *mmapFile) ReadSliceAt(offset, n uint64) ([]byte, error) {
+	if f.data == nil {
+		return nil, errMmapClosed
+	}
+	if f.Size() < offset {
+		return nil, fmt.Errorf("mmap: out-of-bounds ReadSliceAt offset %d, size is %d", offset, f.Size())
+	}
+	if offset+n >= f.Size() {
+		return nil, io.EOF
+	}
+	end := offset + n
+	return f.data[offset:end:end], nil
+}
+
+// Close closes the file.
+func (f *mmapFile) Close() error {
+	if f.data == nil {
+		return nil
+	}
+	err := syscall.Munmap(f.data)
+	f.data = nil
+	f.pos = 0
+	return err
+}

cmd/heapview/internal/core/mmapfile_other.go

@@ -0,0 +1,14 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !darwin,!linux
+
+package core
+
+// TODO(matloob): perhaps use the more portable golang.org/x/exp/mmap
+// instead of the mmap code in mmapfile.go.
+
+type mmapFile struct{}
+
+func (m *mmapFile) Close() error { return nil }

cmd/heapview/internal/core/raw.go

@@ -0,0 +1,308 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package core provides functions for reading core dumps
+// and examining their contained heaps.
+package core
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"runtime"
+	"sort"
+)
+
+// RawDump provides raw access to the heap records in a core file.
+// The raw records in this file are described by other structs named Raw{*}.
+// All []byte slices are direct references to the underlying mmap'd file.
+// These references will become invalid as soon as the RawDump is closed.
+type RawDump struct {
+	Params   *RawParams
+	MemStats *runtime.MemStats
+
+	HeapObjects    []RawSegment // heap objects sorted by Addr, low-to-high
+	GlobalSegments []RawSegment // data, bss, and noptrbss segments
+
+	OSThreads   []*RawOSThread
+	Goroutines  []*RawGoroutine
+	StackFrames []*RawStackFrame
+	OtherRoots  []*RawOtherRoot
+	Finalizers  []*RawFinalizer
+	Defers      []*RawDefer
+	Panics      []*RawPanic
+
+	TypeFromItab map[uint64]uint64   // map from itab address to the type address that itab represents
+	TypeFromAddr map[uint64]*RawType // map from RawType.Addr to RawType
+
+	MemProfMap   map[uint64]*RawMemProfEntry
+	AllocSamples []*RawAllocSample
+
+	fmap *mmapFile
+}
+
+// RawParams holds metadata about the program that generated the dump.
+type RawParams struct {
+	// Info about the memory space
+
+	ByteOrder binary.ByteOrder // byte order of all memory in this dump
+	PtrSize   uint64           // in bytes
+	HeapStart uint64           // heap start address
+	HeapEnd   uint64           // heap end address (this is the last byte in the heap + 1)
+
+	// Info about the program that generated this heapdump
+
+	GoArch       string // GOARCH of the runtime library that generated this dump
+	GoExperiment string // GOEXPERIMENT of the toolchain that build the runtime library
+	NCPU         uint64 // number of physical cpus available to the program
+}
+
+// RawSegment represents a segment of memory.
+type RawSegment struct {
+	Addr      uint64       // base address of the segment
+	Data      []byte       // data for this segment
+	PtrFields RawPtrFields // offsets of ptr fields within this segment
+}
+
+// RawPtrFields represents a pointer field.
+type RawPtrFields struct {
+	encoded          []byte // list of uvarint-encoded offsets, or nil if none
+	startOff, endOff uint64 // decoded offsets are translated and clipped to [startOff,endOff)
+}
+
+// RawOSThread represents an OS thread.
+type RawOSThread struct {
+	MAddr  uint64 // address of the OS thread descriptor (M)
+	GoID   uint64 // go's internal ID for the thread
+	ProcID uint64 // kernel's ID for the thread
+}
+
+// RawGoroutine represents a goroutine structure.
+type RawGoroutine struct {
+	GAddr        uint64 // address of the goroutine descriptor
+	SP           uint64 // current stack pointer (lowest address in the currently running frame)
+	GoID         uint64 // goroutine ID
+	GoPC         uint64 // PC of the go statement that created this goroutine
+	Status       uint64
+	IsSystem     bool   // true if started by the system
+	IsBackground bool   // always false in go1.7
+	WaitSince    uint64 // time the goroutine started waiting, in nanoseconds since the Unix epoch
+	WaitReason   string
+	CtxtAddr     uint64 // address of the scheduling ctxt
+	MAddr        uint64 // address of the OS thread descriptor (M)
+	TopDeferAddr uint64 // address of the top defer record
+	TopPanicAddr uint64 // address of the top panic record
+}
+
+// RawStackFrame represents a stack frame.
+type RawStackFrame struct {
+	Name     string
+	Depth    uint64     // 0 = bottom of stack (currently running frame)
+	CalleeSP uint64     // stack pointer of the child frame (or 0 for the bottom-most frame)
+	EntryPC  uint64     // entry PC for the function
+	PC       uint64     // current PC being executed
+	NextPC   uint64     // for callers, where the function resumes (if anywhere) after the callee is done
+	Segment  RawSegment // local vars (Segment.Addr is the stack pointer, i.e., lowest address in the frame)
+}
+
+// RawOtherRoot represents the other roots not in RawDump's other fields.
+type RawOtherRoot struct {
+	Description string
+	Addr        uint64 // address pointed to by this root
+}
+
+// RawFinalizer represents a finalizer.
+type RawFinalizer struct {
+	IsQueued      bool   // if true, the object is unreachable and the finalizer is ready to run
+	ObjAddr       uint64 // address of the object to finalize
+	ObjTypeAddr   uint64 // address of the descriptor for typeof(obj)
+	FnAddr        uint64 // function to be run (a FuncVal*)
+	FnArgTypeAddr uint64 // address of the descriptor for the type of the function argument
+	FnPC          uint64 // PC of finalizer entry point
+}
+
+// RawDefer represents a defer.
+type RawDefer struct {
+	Addr     uint64 // address of the defer record
+	GAddr    uint64 // address of the containing goroutine's descriptor
+	ArgP     uint64 // stack pointer giving the args for defer (TODO: is this right?)
+	PC       uint64 // PC of the defer instruction
+	FnAddr   uint64 // function to be run (a FuncVal*)
+	FnPC     uint64 // PC of the defered function's entry point
+	LinkAddr uint64 // address of the next defer record in this chain
+}
+
+// RawPanic represents a panic.
+type RawPanic struct {
+	Addr        uint64 // address of the panic record
+	GAddr       uint64 // address of the containing goroutine's descriptor
+	ArgTypeAddr uint64 // type of the panic arg
+	ArgAddr     uint64 // address of the panic arg
+	DeferAddr   uint64 // address of the defer record that is currently running
+	LinkAddr    uint64 // address of the next panic record in this chain
+}
+
+// RawType repesents the Go runtime's representation of a type.
+type RawType struct {
+	Addr uint64 // address of the type descriptor
+	Size uint64 // in bytes
+	Name string // not necessarily unique
+	// If true, this type is equivalent to a single pointer, so ifaces can store
+	// this type directly in the data field (without indirection).
+	DirectIFace bool
+}
+
+// RawMemProfEntry represents a memory profiler entry.
+type RawMemProfEntry struct {
+	Size      uint64            // size of the allocated object
+	NumAllocs uint64            // number of allocations
+	NumFrees  uint64            // number of frees
+	Stacks    []RawMemProfFrame // call stacks
+}
+
+// RawMemProfFrame represents a memory profiler frame.
+type RawMemProfFrame struct {
+	Func []byte // string left as []byte reference to save memory
+	File []byte // string left as []byte reference to save memory
+	Line uint64
+}
+
+// RawAllocSample represents a memory profiler allocation sample.
+type RawAllocSample struct {
+	Addr uint64           // address of object
+	Prof *RawMemProfEntry // record of allocation site
+}
+
+// Close closes the file.
+func (r *RawDump) Close() error {
+	return r.fmap.Close()
+}
+
+// FindSegment returns the segment that contains the given address, or
+// nil of no segment contains the address.
+func (r *RawDump) FindSegment(addr uint64) *RawSegment {
+	// Binary search for an upper-bound heap object, then check
+	// if the previous object contains addr.
+	k := sort.Search(len(r.HeapObjects), func(k int) bool {
+		return addr < r.HeapObjects[k].Addr
+	})
+	k--
+	if k >= 0 && r.HeapObjects[k].Contains(addr) {
+		return &r.HeapObjects[k]
+	}
+
+	// Check all global segments.
+	for k := range r.GlobalSegments {
+		if r.GlobalSegments[k].Contains(addr) {
+			return &r.GlobalSegments[k]
+		}
+	}
+
+	// NB: Stack-local vars are technically allocated in the heap, since stack frames are
+	// allocated in the heap space, however, stack frames don't show up in r.HeapObjects.
+	for _, f := range r.StackFrames {
+		if f.Segment.Contains(addr) {
+			return &f.Segment
+		}
+	}
+
+	return nil
+}
+
+// Contains returns true if the segment contains the given address.
+func (r RawSegment) Contains(addr uint64) bool {
+	return r.Addr <= addr && addr < r.Addr+r.Size()
+}
+
+// ContainsRange returns true if the segment contains the range [addr, addr+size).
+func (r RawSegment) ContainsRange(addr, size uint64) bool {
+	if !r.Contains(addr) {
+		return false
+	}
+	if size > 0 && !r.Contains(addr+size-1) {
+		return false
+	}
+	return true
+}
+
+// Size returns the size of the segment in bytes.
+func (r RawSegment) Size() uint64 {
+	return uint64(len(r.Data))
+}
+
+// Slice takes a slice of the given segment. Panics if [offset,offset+size)
+// is out-of-bounds. The resulting RawSegment.PtrOffsets will clipped and
+// translated into the new segment.
+func (r RawSegment) Slice(offset, size uint64) *RawSegment {
+	if offset+size > uint64(len(r.Data)) {
+		panic(fmt.Errorf("slice(%d,%d) out-of-bounds of segment @%x sz=%d", offset, size, r.Addr, len(r.Data)))
+	}
+	return &RawSegment{
+		Addr: r.Addr + offset,
+		Data: r.Data[offset : offset+size : offset+size],
+		PtrFields: RawPtrFields{
+			encoded:  r.PtrFields.encoded,
+			startOff: r.PtrFields.startOff + offset,
+			endOff:   r.PtrFields.startOff + offset + size,
+		},
+	}
+}
+
+// Offsets decodes the list of ptr field offsets.
+func (r RawPtrFields) Offsets() []uint64 {
+	if r.encoded == nil {
+		return nil
+	}
+
+	// NB: This should never fail since we already decoded the varints once
+	// when parsing the file originally. Hence we panic on failure.
+	reader := bytes.NewReader(r.encoded)
+	readUint64 := func() uint64 {
+		x, err := binary.ReadUvarint(reader)
+		if err != nil {
+			panic(fmt.Errorf("unexpected failure decoding uvarint: %v", err))
+		}
+		return x
+	}
+
+	var out []uint64
+	for {
+		k := readUint64()
+		switch k {
+		case 0: // end
+			return out
+		case 1: // ptr
+			x := readUint64()
+			if r.startOff <= x && x < r.endOff {
+				out = append(out, x-r.startOff)
+			}
+		default:
+			panic(fmt.Errorf("unexpected FieldKind %d", k))
+		}
+	}
+}
+
+// ReadPtr decodes a ptr from the given byte slice.
+func (r *RawParams) ReadPtr(b []byte) uint64 {
+	switch r.PtrSize {
+	case 4:
+		return uint64(r.ByteOrder.Uint32(b))
+	case 8:
+		return r.ByteOrder.Uint64(b)
+	default:
+		panic(fmt.Errorf("unsupported PtrSize=%d", r.PtrSize))
+	}
+}
+
+// WritePtr encodes a ptr into the given byte slice.
+func (r *RawParams) WritePtr(b []byte, addr uint64) {
+	switch r.PtrSize {
+	case 4:
+		r.ByteOrder.PutUint32(b, uint32(addr))
+	case 8:
+		r.ByteOrder.PutUint64(b, addr)
+	default:
+		panic(fmt.Errorf("unsupported PtrSize=%d", r.PtrSize))
+	}
+}

cmd/heapview/main.go

@@ -0,0 +1,83 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// heapview is a tool for viewing Go heap dumps.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"go/build"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+var host = flag.String("host", "", "host addr to listen on")
+var port = flag.Int("port", 8080, "service port")
+
+var index = `<!DOCTYPE html>
+<script src="js/customelements.js"></script>
+<script src="js/typescript.js"></script>
+<script src="js/moduleloader.js"></script>
+<script>
+  System.transpiler = 'typescript';
+  System.typescriptOptions = {target: ts.ScriptTarget.ES2015};
+  System.locate = (load) => load.name + '.ts';
+</script>
+<script type="module">
+  import {main} from './client/main';
+  main();
+</script>
+`
+
+func toolsDir() string {
+	p, err := build.Import("golang.org/x/tools", "", build.FindOnly)
+	if err != nil {
+		log.Println("error: can't find client files:", err)
+		os.Exit(1)
+	}
+	return p.Dir
+}
+
+var parseFlags = func() {
+	flag.Parse()
+}
+
+var addHandlers = func() {
+	toolsDir := toolsDir()
+
+	// Directly serve typescript code in client directory for development.
+	http.Handle("/client/", http.StripPrefix("/client",
+		http.FileServer(http.Dir(filepath.Join(toolsDir, "cmd/heapview/client")))))
+
+	// Serve typescript.js and moduleloader.js for development.
+	http.HandleFunc("/js/typescript.js", func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, filepath.Join(toolsDir, "third_party/typescript/typescript.js"))
+	})
+	http.HandleFunc("/js/moduleloader.js", func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, filepath.Join(toolsDir, "third_party/moduleloader/moduleloader.js"))
+	})
+	http.HandleFunc("/js/customelements.js", func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, filepath.Join(toolsDir, "third_party/webcomponents/customelements.js"))
+	})
+
+	// Serve index.html using html string above.
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		io.WriteString(w, index)
+	})
+}
+
+var listenAndServe = func() error {
+	return http.ListenAndServe(fmt.Sprintf("%s:%d", *host, *port), nil)
+}
+
+func main() {
+	parseFlags()
+	addHandlers()
+	log.Fatal(listenAndServe())
+}

cmd/present/appengine.go

@@ -0,0 +1,22 @@
+// Copyright 2012 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine
+
+package main
+
+import (
+	"mime"
+
+	"golang.org/x/tools/present"
+)
+
+func init() {
+	initTemplates("./present/")
+	present.PlayEnabled = true
+	initPlayground("./present/", nil)
+
+	// App Engine has no /etc/mime.types
+	mime.AddExtensionType(".svg", "image/svg+xml")
+}

cmd/present/dir.go

@@ -13,7 +13,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strings"
 
 	"golang.org/x/tools/present"
 )
@@ -22,18 +21,19 @@ func init() {
 	http.HandleFunc("/", dirHandler)
 }
 
-// dirHandler serves a directory listing for the requested path, rooted at *contentPath.
+// dirHandler serves a directory listing for the requested path, rooted at basePath.
 func dirHandler(w http.ResponseWriter, r *http.Request) {
 	if r.URL.Path == "/favicon.ico" {
-		http.NotFound(w, r)
+		http.Error(w, "not found", 404)
 		return
 	}
-	name := filepath.Join(*contentPath, r.URL.Path)
+	const base = "."
+	name := filepath.Join(base, r.URL.Path)
 	if isDoc(name) {
 		err := renderDoc(w, name)
 		if err != nil {
 			log.Println(err)
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+			http.Error(w, err.Error(), 500)
 		}
 		return
 	}
@@ -43,12 +43,12 @@ func dirHandler(w http.ResponseWriter, r *http.Request) {
 			addr = r.RemoteAddr
 		}
 		log.Printf("request from %s: %s", addr, err)
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+		http.Error(w, err.Error(), 500)
 		return
 	} else if isDir {
 		return
 	}
-	http.FileServer(http.Dir(*contentPath)).ServeHTTP(w, r)
+	http.FileServer(http.Dir(base)).ServeHTTP(w, r)
 }
 
 func isDoc(path string) bool {
@@ -113,7 +113,7 @@ func parse(name string, mode present.ParseMode) (*present.Doc, error) {
 		return nil, err
 	}
 	defer f.Close()
-	return present.Parse(f, name, mode)
+	return present.Parse(f, name, 0)
 }
 
 // dirList scans the given path and writes a directory listing to w.
@@ -138,9 +138,7 @@ func dirList(w io.Writer, name string) (isDir bool, err error) {
 	if err != nil {
 		return false, err
 	}
-	strippedPath := strings.TrimPrefix(name, filepath.Clean(*contentPath))
+	d := &dirListData{Path: name}
-	strippedPath = strings.TrimPrefix(strippedPath, "/")
-	d := &dirListData{Path: strippedPath}
 	for _, fi := range fis {
 		// skip the golang.org directory
 		if name == "." && fi.Name() == "golang.org" {
@@ -148,16 +146,15 @@ func dirList(w io.Writer, name string) (isDir bool, err error) {
 		}
 		e := dirEntry{
 			Name: fi.Name(),
-			Path: filepath.ToSlash(filepath.Join(strippedPath, fi.Name())),
+			Path: filepath.ToSlash(filepath.Join(name, fi.Name())),
 		}
 		if fi.IsDir() && showDir(e.Name) {
 			d.Dirs = append(d.Dirs, e)
 			continue
 		}
 		if isDoc(e.Name) {
-			fn := filepath.ToSlash(filepath.Join(name, fi.Name()))
+			if p, err := parse(e.Path, present.TitlesOnly); err != nil {
-			if p, err := parse(fn, present.TitlesOnly); err != nil {
+				log.Println(err)
-				log.Printf("parse(%q, present.TitlesOnly): %v", fn, err)
 			} else {
 				e.Title = p.Title
 			}