go.tools/cmd/gotype: make gotype use go/build

This CL makes gotype usable again. Removed -r
(recursive) mode; use go/build to determine
the correct set of Go files when processing
a directory. The -v (verbose) mode now prints
some basic stats (duration, number of files,
lines, and lines/s).

Thoroughly restructured the code.

Applying gotype -v -a . to the go/types directory:
128.94141ms (40 files, 12008 lines, 93127 lines/s)

On a 2.8 GHz Quad-Core Intel Xeon, 800 MHz DDR2 FB-DIMM,
with go/types built with the (interal) debug flag set to
false. There's still quite a bit of room for performance
improvement in all parts of the code since no tuning has
been done.

R=golang-dev, adonovan
CC=golang-dev
https://golang.org/cl/19930043

cmd/gotype/doc.go

@@ -4,54 +4,51 @@
 
 /*
 The gotype command does syntactic and semantic analysis of Go files
-and packages similar to the analysis performed by the front-end of
+and packages like the front-end of a Go compiler. Errors are reported
-a Go compiler. Errors are reported if the analysis fails; otherwise
+if the analysis fails; otherwise gotype is quiet (unless -v is set).
-gotype is quiet (unless -v is set).
 
-Without a list of paths, gotype processes the standard input, which must
+Without a list of paths, gotype reads from standard input, which
-be the source of a single package file.
+must provide a single Go source file defining a complete package.
 
-Given a list of file names, each file must be a source file belonging to
+If a single path is specified that is a directory, gotype checks
-the same package unless the package name is explicitly specified with the
+the Go files in that directory; they must all belong to the same
--p flag.
+package.
 
-Given a directory name, gotype collects all .go files in the directory
+Otherwise, each path must be the filename of Go file belonging to
-and processes them as if they were provided as an explicit list of file
+the same package.
-names. Each directory is processed independently. Files starting with .
-or not ending in .go are ignored.
 
 Usage:
-	gotype [flags] [path ...]
+	gotype [flags] [path...]
 
 The flags are:
+	-a
+		use all (incl. _test.go) files when processing a directory
 	-e
-		Print all (including spurious) errors.
+		report all errors (not just the first 10)
-	-p pkgName
-		Process only those files in package pkgName.
-	-r
-		Recursively process subdirectories.
 	-v
-		Verbose mode.
+		verbose mode
 
 Debugging flags:
-	-comments
-		Parse comments (ignored if -ast not set).
 	-ast
-		Print AST (disables concurrent parsing).
+		print AST
 	-trace
-		Print parse trace (disables concurrent parsing).
+		print parse trace
+	-comments
+		parse comments (ignored unless -ast or -trace is provided)
 
+Examples:
 
-Examples
+To check the files a.go, b.go, and c.go:
 
-To check the files file.go, old.saved, and .ignored:
+	gotype a.go b.go c.go
 
-	gotype file.go old.saved .ignored
+To check an entire package in the directory dir and print the processed files:
 
-To check all .go files belonging to package main in the current directory
+	gotype -v dir
-and recursively in all subdirectories:
 
-	gotype -p main -r .
+To check an entire package including tests in the local directory:
+
+	gotype -a .
 
 To verify the output of a pipe:
 

cmd/gotype/gotype.go

@@ -8,34 +8,63 @@ import (
 	"flag"
 	"fmt"
 	"go/ast"
+	"go/build"
 	"go/parser"
 	"go/scanner"
 	"go/token"
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
+	"time"
 
 	"code.google.com/p/go.tools/go/types"
 )
 
 var (
 	// main operation modes
-	pkgName   = flag.String("p", "", "process only those files in package pkgName")
+	allFiles  = flag.Bool("a", false, "use all (incl. _test.go) files when processing a directory")
-	recursive = flag.Bool("r", false, "recursively process subdirectories")
+	allErrors = flag.Bool("e", false, "report all errors (not just the first 10)")
 	verbose   = flag.Bool("v", false, "verbose mode")
-	allErrors = flag.Bool("e", false, "report all errors (not just the first 10 on different lines)")
 
 	// debugging support
-	parseComments = flag.Bool("comments", false, "parse comments (ignored if -ast not set)")
-	printTrace    = flag.Bool("trace", false, "print parse trace")
 	printAST      = flag.Bool("ast", false, "print AST")
+	printTrace    = flag.Bool("trace", false, "print parse trace")
+	parseComments = flag.Bool("comments", false, "parse comments (ignored unless -ast or -trace is provided)")
 )
 
-var errorCount int
+var (
+	fset       = token.NewFileSet()
+	errorCount = 0
+	parserMode parser.Mode
+	sizes      types.Sizes
+)
+
+func initParserMode() {
+	if *allErrors {
+		parserMode |= parser.AllErrors
+	}
+	if *printTrace {
+		parserMode |= parser.Trace
+	}
+	if *parseComments && (*printAST || *printTrace) {
+		parserMode |= parser.ParseComments
+	}
+}
+
+func initSizes() {
+	wordSize := 8
+	maxAlign := 8
+	switch build.Default.GOARCH {
+	case "386", "arm":
+		wordSize = 4
+		maxAlign = 4
+		// add more cases as needed
+	}
+	sizes = &types.StdSizes{WordSize: int64(wordSize), MaxAlign: int64(maxAlign)}
+}
 
 func usage() {
-	fmt.Fprintf(os.Stderr, "usage: gotype [flags] [path ...]\n")
+	fmt.Fprintln(os.Stderr, "usage: gotype [flags] [path ...]")
 	flag.PrintDefaults()
 	os.Exit(2)
 }
@@ -49,133 +78,94 @@ func report(err error) {
 	errorCount++
 }
 
-// parse returns the AST for the Go source src.
+func parse(filename string, src interface{}) (*ast.File, error) {
-// The filename is for error reporting only.
-// The result is nil if there were errors or if
-// the file does not belong to the -p package.
-func parse(fset *token.FileSet, filename string, src []byte) *ast.File {
 	if *verbose {
 		fmt.Println(filename)
 	}
-
+	file, err := parser.ParseFile(fset, filename, src, parserMode)
-	// ignore files with different package name
-	if *pkgName != "" {
-		file, err := parser.ParseFile(fset, filename, src, parser.PackageClauseOnly)
-		if err != nil {
-			report(err)
-			return nil
-		}
-		if file.Name.Name != *pkgName {
-			if *verbose {
-				fmt.Printf("\tignored (package %s)\n", file.Name.Name)
-			}
-			return nil
-		}
-	}
-
-	// parse entire file
-	var mode parser.Mode
-	if *allErrors {
-		mode |= parser.AllErrors
-	}
-	if *parseComments && *printAST {
-		mode |= parser.ParseComments
-	}
-	if *printTrace {
-		mode |= parser.Trace
-	}
-	file, err := parser.ParseFile(fset, filename, src, mode)
-	if err != nil {
-		report(err)
-		return nil
-	}
 	if *printAST {
 		ast.Print(fset, file)
 	}
-
+	return file, err
-	return file
 }
 
-func parseStdin(fset *token.FileSet) (files []*ast.File) {
+func parseStdin() (*ast.File, error) {
 	src, err := ioutil.ReadAll(os.Stdin)
 	if err != nil {
-		report(err)
+		return nil, err
-		return
 	}
-	const filename = "<standard input>"
+	return parse("<standard input>", src)
-	if file := parse(fset, filename, src); file != nil {
-		files = []*ast.File{file}
-	}
-	return
 }
 
-func parseFiles(fset *token.FileSet, filenames []string) (files []*ast.File) {
+func parseFiles(filenames []string) ([]*ast.File, error) {
+	var files []*ast.File
 	for _, filename := range filenames {
-		src, err := ioutil.ReadFile(filename)
+		file, err := parse(filename, nil)
 		if err != nil {
-			report(err)
+			return nil, err
-			continue
-		}
-		if file := parse(fset, filename, src); file != nil {
-			files = append(files, file)
 		}
+		files = append(files, file)
 	}
-	return
+	return files, nil
 }
 
-func isGoFilename(filename string) bool {
+func parseDir(dirname string) ([]*ast.File, error) {
-	// ignore non-Go files
+	ctxt := build.Default
-	return !strings.HasPrefix(filename, ".") && strings.HasSuffix(filename, ".go")
+	pkginfo, err := ctxt.ImportDir(dirname, 0)
-}
+	if _, nogo := err.(*build.NoGoError); err != nil && !nogo {
+		return nil, err
+	}
+	filenames := append(pkginfo.GoFiles, pkginfo.CgoFiles...)
+	if *allFiles {
+		filenames = append(filenames, pkginfo.TestGoFiles...)
+	}
 
-func processDirectory(dirname string) {
+	// complete file names
-	f, err := os.Open(dirname)
-	if err != nil {
-		report(err)
-		return
-	}
-	filenames, err := f.Readdirnames(-1)
-	f.Close()
-	if err != nil {
-		report(err)
-		// continue since filenames may not be empty
-	}
 	for i, filename := range filenames {
 		filenames[i] = filepath.Join(dirname, filename)
 	}
-	processFiles(dirname, filenames, false)
+
+	return parseFiles(filenames)
 }
 
-func processFiles(path string, filenames []string, allFiles bool) {
+func getPkgFiles(args []string) ([]*ast.File, error) {
-	i := 0
+	if len(args) == 0 {
-	for _, filename := range filenames {
+		// stdin
-		switch info, err := os.Stat(filename); {
+		var file *ast.File
-		case err != nil:
+		file, err := parseStdin()
-			report(err)
+		if err != nil {
-		case info.IsDir():
+			return nil, err
-			if allFiles || *recursive {
+		}
-				processDirectory(filename)
+		return []*ast.File{file}, nil
-			}
+	}
-		default:
+
-			if allFiles || isGoFilename(info.Name()) {
+	if len(args) == 1 {
-				filenames[i] = filename
+		// possibly a directory
-				i++
+		path := args[0]
-			}
+		info, err := os.Stat(path)
+		if err != nil {
+			return nil, err
+		}
+		if info.IsDir() {
+			return parseDir(path)
 		}
 	}
-	fset := token.NewFileSet()
+
-	processPackage(path, fset, parseFiles(fset, filenames[0:i]))
+	// list of files
+	return parseFiles(args)
 }
 
-func processPackage(path string, fset *token.FileSet, files []*ast.File) {
+func checkPkgFiles(files []*ast.File) {
 	type bailout struct{}
 	conf := types.Config{
+		FakeImportC: true,
 		Error: func(err error) {
 			if !*allErrors && errorCount >= 10 {
 				panic(bailout{})
 			}
 			report(err)
 		},
+		Sizes: sizes,
 	}
 
 	defer func() {
@@ -186,21 +176,44 @@ func processPackage(path string, fset *token.FileSet, files []*ast.File) {
 		}
 	}()
 
+	const path = "pkg" // any non-empty string will do for now
 	conf.Check(path, fset, files, nil)
 }
 
+func printStats(d time.Duration) {
+	fileCount := 0
+	lineCount := 0
+	fset.Iterate(func(f *token.File) bool {
+		fileCount++
+		lineCount += f.LineCount()
+		return true
+	})
+
+	fmt.Printf(
+		"%s (%d files, %d lines, %d lines/s)\n",
+		d, fileCount, lineCount, int64(float64(lineCount)/d.Seconds()),
+	)
+}
+
 func main() {
 	flag.Usage = usage
 	flag.Parse()
+	initParserMode()
+	initSizes()
+	start := time.Now()
 
-	if flag.NArg() == 0 {
+	files, err := getPkgFiles(flag.Args())
-		fset := token.NewFileSet()
+	if err != nil {
-		processPackage("<stdin>", fset, parseStdin(fset))
+		report(err)
-	} else {
+		os.Exit(2)
-		processFiles("<files>", flag.Args(), true)
 	}
 
+	checkPkgFiles(files)
 	if errorCount > 0 {
 		os.Exit(2)
 	}
+
+	if *verbose {
+		printStats(time.Since(start))
+	}
 }