From 2a3a12930b2759c2d7f3b1e355fcf8fbc3a7c336 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 30 Jul 2013 14:28:14 -0400 Subject: [PATCH] go.tools/ssa: add test of SSA construction on $GOROOT/src/pkg/... stdlib_test runs the builder (in sanity-checking mode) over the Go standard library. It also prints some stats about the time and memory usage. Also: - importer.LoadPackage too (not just doImport) must consult the cache to avoid creating duplicate Package instances for the same import path when called serially from a test. - importer: skip empty directories without an error. - importer: print all errors, not just the first. - visit.go: added AllFunctions utility for enumerating all Functions in a Program. - ssa.MethodSet is not safe to expose from the package since it must be accessed under an (inaccessible) lock. (!!!) This CL makes it unexported and restricts its use to the single function Program.LookupMethod(). - Program.MethodSet() has gone. Clients should instead iterate over the types.MethodSet and call LookupMethod. - Package.DumpTo(): improved efficiency of methodset printing (by not creating wrappers) and accuracy (by showing * on receiver type only when necessary). - Program.CreatePackage: documented precondition and added assertion. R=gri CC=golang-dev https://golang.org/cl/12058048 --- importer/importer.go | 44 ++++++++++------ importer/util.go | 3 ++ ssa/builder.go | 10 +++- ssa/builder_test.go | 9 +++- ssa/interp/interp.go | 6 ++- ssa/interp/reflect.go | 4 +- ssa/print.go | 33 +++++------- ssa/promote.go | 102 ++++++++++-------------------------- ssa/source.go | 17 +++--- ssa/ssa.go | 21 ++------ ssa/stdlib_test.go | 118 ++++++++++++++++++++++++++++++++++++++++++ ssa/visit.go | 73 ++++++++++++++++++++++++++ 12 files changed, 294 insertions(+), 146 deletions(-) create mode 100644 ssa/stdlib_test.go create mode 100644 ssa/visit.go diff --git a/importer/importer.go b/importer/importer.go index eb68c8fc..829ced54 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -8,8 +8,10 @@ package importer import ( + "fmt" "go/ast" "go/token" + "os" "code.google.com/p/go.tools/go/exact" "code.google.com/p/go.tools/go/types" @@ -28,8 +30,8 @@ type Importer struct { type Config struct { // TypeChecker contains options relating to the type checker. // The Importer will override any user-supplied values for its - // Expr, Ident, ImplicitObj and Import fields; other fields - // will be passed through to the type checker. + // Error and Import fields; other fields will be passed + // through to the type checker. TypeChecker types.Config // If Loader is non-nil, it is used to satisfy imports. @@ -68,6 +70,7 @@ func New(config *Config) *Importer { Packages: make(map[string]*PackageInfo), errors: make(map[string]error), } + imp.config.TypeChecker.Error = func(e error) { fmt.Fprintln(os.Stderr, e) } imp.config.TypeChecker.Import = imp.doImport return imp } @@ -81,25 +84,27 @@ func (imp *Importer) doImport(imports map[string]*types.Package, path string) (p return types.Unsafe, nil } - if info, ok := imp.Packages[path]; ok { - imports[path] = info.Pkg - pkg = info.Pkg - return // positive cache hit - } - - if err = imp.errors[path]; err != nil { - return // negative cache hit - } - // Load the source/binary for 'path', type-check it, construct // a PackageInfo and update our map (imp.Packages) and the // type-checker's map (imports). var info *PackageInfo if imp.config.Loader != nil { info, err = imp.LoadPackage(path) - } else if pkg, err = types.GcImport(imports, path); err == nil { - info = &PackageInfo{Pkg: pkg} - imp.Packages[path] = info + } else { + if info, ok := imp.Packages[path]; ok { + imports[path] = info.Pkg + pkg = info.Pkg + return // positive cache hit + } + + if err = imp.errors[path]; err != nil { + return // negative cache hit + } + + if pkg, err = types.GcImport(imports, path); err == nil { + info = &PackageInfo{Pkg: pkg} + imp.Packages[path] = info + } } if err == nil { @@ -124,8 +129,15 @@ func (imp *Importer) doImport(imports map[string]*types.Package, path string) (p // Not thread-safe! // TODO(adonovan): rethink this API. // -// func (imp *Importer) LoadPackage(importPath string) (*PackageInfo, error) { + if info, ok := imp.Packages[importPath]; ok { + return info, nil // positive cache hit + } + + if err := imp.errors[importPath]; err != nil { + return nil, err // negative cache hit + } + if imp.config.Loader == nil { panic("Importer.LoadPackage without a SourceLoader") } diff --git a/importer/util.go b/importer/util.go index 8a04e35e..a2865c82 100644 --- a/importer/util.go +++ b/importer/util.go @@ -73,6 +73,9 @@ func MakeGoBuildLoader(ctxt *build.Context) SourceLoader { // TODO(adonovan): fix: Do we need cwd? Shouldn't // ImportDir(path) / $GOROOT suffice? bp, err := ctxt.Import(path, srcDir, 0) + if _, ok := err.(*build.NoGoError); ok { + return nil, nil // empty directory + } if err != nil { return // import failed } diff --git a/ssa/builder.go b/ssa/builder.go index 2fe0eb24..3bc8203f 100644 --- a/ssa/builder.go +++ b/ssa/builder.go @@ -2257,6 +2257,10 @@ func (prog *Program) BuildAll() { // Build builds SSA code for all functions and vars in package p. // +// Precondition: CreatePackage must have been called for all of p's +// direct imports (and hence its direct imports must have been +// error-free). +// // Build is idempotent and thread-safe. // func (p *Package) Build() { @@ -2283,8 +2287,12 @@ func (p *Package) Build() { // Call the init() function of each package we import. for _, obj := range p.info.Imports() { + prereq := p.Prog.packages[obj] + if prereq == nil { + panic(fmt.Sprintf("Package(%q).Build(): unsatisified import: Program.CreatePackage(%q) was not called", p.Object.Path(), obj.Path())) + } var v Call - v.Call.Value = p.Prog.packages[obj].init + v.Call.Value = prereq.init v.Call.pos = init.pos v.setType(types.NewTuple()) init.emit(&v) diff --git a/ssa/builder_test.go b/ssa/builder_test.go index 053d8e3a..61d225e6 100644 --- a/ssa/builder_test.go +++ b/ssa/builder_test.go @@ -93,7 +93,14 @@ func main() { if !isExt { t.Fatalf("unexpected name type in main package: %s", mem) } - for _, m := range prog.MethodSet(types.NewPointer(mem.Type())) { + if _, ok := mem.Type().Underlying().(*types.Interface); ok { + // TODO(adonovan): workaround bug in types.MethodSet + // whereby mset(*I) is nonempty if I is an interface. + continue + } + mset := types.NewPointer(mem.Type()).MethodSet() + for i, n := 0, mset.Len(); i < n; i++ { + m := prog.LookupMethod(mset.At(i)) // For external types, only synthetic wrappers have code. expExt := !strings.Contains(m.Synthetic, "wrapper") if expExt && !isEmpty(m) { diff --git a/ssa/interp/interp.go b/ssa/interp/interp.go index e7e891cc..96831913 100644 --- a/ssa/interp/interp.go +++ b/ssa/interp/interp.go @@ -79,14 +79,16 @@ const ( EnableTracing // Print a trace of all instructions as they are interpreted. ) +type methodSet map[string]*ssa.Function + // State shared between all interpreted goroutines. type interpreter struct { prog *ssa.Program // the SSA program globals map[ssa.Value]*value // addresses of global variables (immutable) mode Mode // interpreter options reflectPackage *ssa.Package // the fake reflect package - errorMethods ssa.MethodSet // the method set of reflect.error, which implements the error interface. - rtypeMethods ssa.MethodSet // the method set of rtype, which implements the reflect.Type interface. + errorMethods methodSet // the method set of reflect.error, which implements the error interface. + rtypeMethods methodSet // the method set of rtype, which implements the reflect.Type interface. } type frame struct { diff --git a/ssa/interp/reflect.go b/ssa/interp/reflect.go index 43632751..51644953 100644 --- a/ssa/interp/reflect.go +++ b/ssa/interp/reflect.go @@ -408,7 +408,7 @@ func initReflect(i *interpreter) { Members: make(map[string]ssa.Member), } - i.rtypeMethods = ssa.MethodSet{ + i.rtypeMethods = methodSet{ "Bits": newMethod(i.reflectPackage, rtypeType, "Bits"), "Elem": newMethod(i.reflectPackage, rtypeType, "Elem"), "Kind": newMethod(i.reflectPackage, rtypeType, "Kind"), @@ -416,7 +416,7 @@ func initReflect(i *interpreter) { "Out": newMethod(i.reflectPackage, rtypeType, "Out"), "String": newMethod(i.reflectPackage, rtypeType, "String"), } - i.errorMethods = ssa.MethodSet{ + i.errorMethods = methodSet{ "Error": newMethod(i.reflectPackage, errorType, "Error"), } } diff --git a/ssa/print.go b/ssa/print.go index 0ced85dc..b1310142 100644 --- a/ssa/print.go +++ b/ssa/print.go @@ -379,31 +379,22 @@ func (p *Package) DumpTo(w io.Writer) { case *Type: fmt.Fprintf(w, " type %-*s %s\n", maxname, name, mem.Type().Underlying()) - // We display only mset(*T) since its keys - // are a superset of mset(T)'s keys, though the - // methods themselves may differ, - // e.g. promotion wrappers. - // NB: if mem.Type() is a pointer, mset is empty. - // - // TODO(adonovan): opt: avoid constructing the - // entire ssa.MethodSet by using the - // types.MethodSet if possible. - mset := p.Prog.MethodSet(types.NewPointer(mem.Type())) - var keys []string - for id := range mset { - keys = append(keys, id) - } - sort.Strings(keys) - for _, id := range keys { - method := mset[id] - // TODO(adonovan): show pointerness of receiver of declared method, not the index - - fmt.Fprintf(w, " method %s %s\n", id, method.Signature) + // Iterate over the keys of mset(*T) since they + // are a superset of mset(T)'s keys. + // The keys of a types.MethodSet are sorted (by Id). + mset := methodSetOf(mem.Type()) + pmset := methodSetOf(types.NewPointer(mem.Type())) + for i, n := 0, pmset.Len(); i < n; i++ { + meth := pmset.At(i) + // If the method also exists in mset(T), show that instead. + if m := mset.Lookup(meth.Obj().Pkg(), meth.Obj().Name()); m != nil { + meth = m + } + fmt.Fprintf(w, " %s\n", meth) } case *Global: fmt.Fprintf(w, " var %-*s %s\n", maxname, name, mem.Type()) - } } } diff --git a/ssa/promote.go b/ssa/promote.go index a67492b4..310384c0 100644 --- a/ssa/promote.go +++ b/ssa/promote.go @@ -22,81 +22,7 @@ func recvType(obj *types.Func) types.Type { return obj.Type().(*types.Signature).Recv().Type() } -// MethodSet returns the method set for type typ, building wrapper -// methods as needed for embedded field promotion, and indirection for -// *T receiver types, etc. -// A nil result indicates an empty set. -// -// This function should only be called when you need to construct the -// entire method set, synthesizing all wrappers, for example during -// the processing of a MakeInterface instruction or when visiting all -// reachable functions. -// -// If you only need to look up a single method (obj), avoid this -// function and use LookupMethod instead: -// -// meth := types.MethodSet(typ).Lookup(pkg, name) -// m := prog.MethodSet(typ)[meth.Id()] // don't do this -// m := prog.LookupMethod(meth) // use this instead -// -// If you only need to enumerate the keys, use types.MethodSet -// instead. -// -// EXCLUSIVE_LOCKS_ACQUIRED(prog.methodsMu) -// -// Thread-safe. -// -func (prog *Program) MethodSet(typ types.Type) MethodSet { - return prog.populateMethodSet(typ, nil) -} - -// populateMethodSet returns the method set for typ, ensuring that it -// contains at least the function for meth, if that is a key. -// If meth is nil, the entire method set is populated. -// -// EXCLUSIVE_LOCKS_ACQUIRED(prog.methodsMu) -// -func (prog *Program) populateMethodSet(typ types.Type, meth *types.Selection) MethodSet { - tmset := methodSet(typ) - n := tmset.Len() - if n == 0 { - return nil - } - - if prog.mode&LogSource != 0 { - defer logStack("populateMethodSet %s meth=%v", typ, meth)() - } - - prog.methodsMu.Lock() - defer prog.methodsMu.Unlock() - - mset, _ := prog.methodSets.At(typ).(MethodSet) - if mset == nil { - mset = make(MethodSet) - prog.methodSets.Set(typ, mset) - } - - if len(mset) < n { - if meth != nil { // single method - id := meth.Obj().Id() - if mset[id] == nil { - mset[id] = findMethod(prog, meth) - } - } else { - // complete set - for i := 0; i < n; i++ { - meth := tmset.At(i) - if id := meth.Obj().Id(); mset[id] == nil { - mset[id] = findMethod(prog, meth) - } - } - } - } - - return mset -} - -func methodSet(typ types.Type) *types.MethodSet { +func methodSetOf(typ types.Type) *types.MethodSet { // TODO(adonovan): temporary workaround. Inline it away when fixed. if _, ok := deref(typ).Underlying().(*types.Interface); ok && isPointer(typ) { // TODO(gri): fix: go/types bug: pointer-to-interface @@ -115,7 +41,31 @@ func methodSet(typ types.Type) *types.MethodSet { // EXCLUSIVE_LOCKS_ACQUIRED(prog.methodsMu) // func (prog *Program) LookupMethod(meth *types.Selection) *Function { - return prog.populateMethodSet(meth.Recv(), meth)[meth.Obj().Id()] + if meth == nil { + panic("LookupMethod(nil)") + } + typ := meth.Recv() + if prog.mode&LogSource != 0 { + defer logStack("LookupMethod %s %v", typ, meth)() + } + + prog.methodsMu.Lock() + defer prog.methodsMu.Unlock() + + type methodSet map[string]*Function + mset, _ := prog.methodSets.At(typ).(methodSet) + if mset == nil { + mset = make(methodSet) + prog.methodSets.Set(typ, mset) + } + + id := meth.Obj().Id() + fn := mset[id] + if fn == nil { + fn = findMethod(prog, meth) + mset[id] = fn + } + return fn } // declaredFunc returns the concrete function/method denoted by obj. diff --git a/ssa/source.go b/ssa/source.go index d40e15dc..46e0396b 100644 --- a/ssa/source.go +++ b/ssa/source.go @@ -1,6 +1,7 @@ package ssa -// This file defines utilities for working with source positions. +// This file defines utilities for working with source positions +// or source-level named entities ("objects"). import ( "go/ast" @@ -100,14 +101,12 @@ func findNamedFunc(pkg *Package, pos token.Pos) *Function { return mem } case *Type: - for _, meth := range pkg.Prog.MethodSet(mem.Type()) { - if meth.Synthetic == "" && meth.Pos() == pos { - return meth - } - } - for _, meth := range pkg.Prog.MethodSet(types.NewPointer(mem.Type())) { - if meth.Synthetic == "" && meth.Pos() == pos { - return meth + mset := methodSetOf(types.NewPointer(mem.Type())) + for i, n := 0, mset.Len(); i < n; i++ { + // Don't call LookupMethod: avoid creating wrappers. + obj := mset.At(i).Obj().(*types.Func) + if obj.Pos() == pos { + return pkg.values[obj].(*Function) } } } diff --git a/ssa/ssa.go b/ssa/ssa.go index 78c5d0ad..453b5a01 100644 --- a/ssa/ssa.go +++ b/ssa/ssa.go @@ -25,7 +25,7 @@ type Program struct { mode BuilderMode // set of mode bits for SSA construction methodsMu sync.Mutex // guards the following maps: - methodSets typemap.M // maps type to its concrete MethodSet + methodSets typemap.M // maps type to its concrete methodSet boundMethodWrappers map[*types.Func]*Function // wrappers for curried x.Method closures ifaceMethodWrappers map[*types.Func]*Function // wrappers for curried I.Method functions } @@ -61,22 +61,6 @@ type Member interface { Token() token.Token // token.{VAR,FUNC,CONST,TYPE} } -// A MethodSet contains all the methods for a particular type T. -// The method sets for T and *T are distinct entities. -// -// All methods in the method set for T have a receiver type of exactly -// T. The method set of *T may contain synthetic indirection methods -// that wrap methods whose receiver type is T. -// -// The keys of a method set are strings returned by the types.Id() -// function. -// -// TODO(adonovan): encapsulate the representation behind both Id-based -// and types.Method-based accessors and enable lazy population. -// Perhaps hide it entirely within the Program API. -// -type MethodSet map[string]*Function - // A Type is a Member of a Package representing a package-level named type. // // Type() returns a *types.Named. @@ -599,7 +583,8 @@ type ChangeInterface struct { // MakeInterface constructs an instance of an interface type from a // value of a concrete type. // -// Use Program.MethodSet(X.Type()) to find the method-set of X. +// Use X.Type().MethodSet() to find the method-set of X, and +// Program.LookupMethod(m) to find the implementation of a method. // // To construct the zero value of an interface type T, use: // NewConst(exact.MakeNil(), T, pos) diff --git a/ssa/stdlib_test.go b/ssa/stdlib_test.go new file mode 100644 index 00000000..97a7c133 --- /dev/null +++ b/ssa/stdlib_test.go @@ -0,0 +1,118 @@ +package ssa_test + +// This file runs the SSA builder in sanity-checking mode on all +// packages beneath $GOROOT and prints some summary information. +// +// Run test with GOMAXPROCS=8 and CGO_ENABLED=0. The latter cannot be +// set from the test because it's too late to stop go/build.init() +// from picking up the value from the parent's environment. + +import ( + "go/build" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "code.google.com/p/go.tools/importer" + "code.google.com/p/go.tools/ssa" +) + +func allPackages() []string { + var pkgs []string + root := filepath.Join(runtime.GOROOT(), "src/pkg") + "/" + filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + // Prune the search if we encounter any of these names: + switch filepath.Base(path) { + case "testdata", ".hg": + return filepath.SkipDir + } + if info.IsDir() { + pkg := strings.TrimPrefix(path, root) + switch pkg { + case "builtin", "pkg", "code.google.com": + return filepath.SkipDir // skip these subtrees + case "": + return nil // ignore root of tree + } + pkgs = append(pkgs, pkg) + } + + return nil + }) + return pkgs +} + +func TestStdlib(t *testing.T) { + ctxt := build.Default + ctxt.CgoEnabled = false + impctx := importer.Config{Loader: importer.MakeGoBuildLoader(&ctxt)} + + // Load, parse and type-check the program. + t0 := time.Now() + + var hasErrors bool + imp := importer.New(&impctx) + for _, importPath := range allPackages() { + if _, err := imp.LoadPackage(importPath); err != nil { + t.Errorf("LoadPackage(%s): %s", importPath, err) + hasErrors = true + } + } + + t1 := time.Now() + + runtime.GC() + var memstats runtime.MemStats + runtime.ReadMemStats(&memstats) + alloc := memstats.Alloc + + // Create SSA packages. + prog := ssa.NewProgram(imp.Fset, ssa.DebugInfo|ssa.SanityCheckFunctions) + for _, info := range imp.Packages { + if info.Err == nil { + prog.CreatePackage(info) + } + } + + t2 := time.Now() + + // Build SSA IR... if it's safe. + if !hasErrors { + prog.BuildAll() + } + + t3 := time.Now() + + runtime.GC() + runtime.ReadMemStats(&memstats) + + numPkgs := len(prog.PackagesByPath) + if want := 140; numPkgs < want { + t.Errorf("Loaded only %d packages, want at least %d", numPkgs, want) + } + + // Dump some statistics. + allFuncs := ssa.AllFunctions(prog) + var numInstrs int + for fn := range allFuncs { + for _, b := range fn.Blocks { + numInstrs += len(b.Instrs) + } + } + + t.Log("GOMAXPROCS: ", runtime.GOMAXPROCS(0)) + t.Log("Load/parse/typecheck: ", t1.Sub(t0)) + t.Log("SSA create: ", t2.Sub(t1)) + if !hasErrors { + t.Log("SSA build: ", t3.Sub(t2)) + } + + // SSA stats: + t.Log("#Packages: ", numPkgs) + t.Log("#Functions: ", len(allFuncs)) + t.Log("#Instructions: ", numInstrs) + t.Log("#MB: ", (memstats.Alloc-alloc)/1000000) +} diff --git a/ssa/visit.go b/ssa/visit.go new file mode 100644 index 00000000..11dd603d --- /dev/null +++ b/ssa/visit.go @@ -0,0 +1,73 @@ +package ssa + +// This file defines utilities for visiting the SSA representation of +// a Program. +// +// TODO(adonovan): improve the API: +// - permit client to supply a callback for each function, +// instruction, type with methods, etc? +// - return graph information about the traversal? +// - test coverage. + +import "code.google.com/p/go.tools/go/types" + +// AllFunctions returns the set of all functions (including anonymous +// functions and synthetic wrappers) in program prog. +// +// Precondition: all packages are built. +// +func AllFunctions(prog *Program) map[*Function]bool { + visit := visitor{ + prog: prog, + seen: make(map[*Function]bool), + } + visit.program() + return visit.seen +} + +type visitor struct { + prog *Program + seen map[*Function]bool +} + +func (visit *visitor) program() { + for _, pkg := range visit.prog.PackagesByPath { + for _, mem := range pkg.Members { + switch mem := mem.(type) { + case *Function: + visit.function(mem) + case *Type: + visit.methodSet(mem.Type()) + visit.methodSet(types.NewPointer(mem.Type())) + } + } + } +} + +func (visit *visitor) methodSet(typ types.Type) { + mset := methodSetOf(typ) + for i, n := 0, mset.Len(); i < n; i++ { + // Side-effect: creates all wrapper methods. + visit.function(visit.prog.LookupMethod(mset.At(i))) + } +} + +func (visit *visitor) function(fn *Function) { + if !visit.seen[fn] { + visit.seen[fn] = true + for _, b := range fn.Blocks { + for _, instr := range b.Instrs { + switch instr := instr.(type) { + case *MakeInterface: + visit.methodSet(instr.X.Type()) + } + var buf [10]*Value // avoid alloc in common case + for _, op := range instr.Operands(buf[:0]) { + if fn, ok := (*op).(*Function); ok { + visit.function(fn) + } + } + } + } + } +}