From 02990bd4944bae4461a455fb1c0a478ddcc1c532 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Tue, 17 Jun 2014 10:56:47 -0700 Subject: [PATCH] go.tools/go/gccgoimporter: keep track of package and import priority Clients such as compilers need this information in order to correctly link against imported packages. This also adds support for the condensed import data format where the priority information is stored as a suffix of the condensed import data, as well as support for archive files. LGTM=gri R=gri CC=golang-codereviews, iant https://golang.org/cl/78740043 --- cmd/godex/gccgo.go | 115 +++----------------- cmd/godex/print.go | 1 + go/gccgoimporter/gccgoinstallation.go | 4 +- go/gccgoimporter/gccgoinstallation_test.go | 4 +- go/gccgoimporter/importer.go | 114 ++++++++++++++++++-- go/gccgoimporter/importer_test.go | 105 ++++++++++++++---- go/gccgoimporter/parser.go | 118 ++++++++++++++------- go/gccgoimporter/testdata/imports.go | 5 + go/gccgoimporter/testdata/imports.gox | 7 ++ go/importer/import.go | 44 ++++---- go/importer/import_test.go | 6 +- 11 files changed, 327 insertions(+), 196 deletions(-) create mode 100644 go/gccgoimporter/testdata/imports.go create mode 100644 go/gccgoimporter/testdata/imports.gox diff --git a/cmd/godex/gccgo.go b/cmd/godex/gccgo.go index 1f3916c9..c2c10b1e 100644 --- a/cmd/godex/gccgo.go +++ b/cmd/godex/gccgo.go @@ -7,119 +7,34 @@ package main import ( - "debug/elf" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "code.google.com/p/go.tools/go/gccgoimporter" - "code.google.com/p/go.tools/go/importer" "code.google.com/p/go.tools/go/types" ) +var ( + initmap = make(map[*types.Package]gccgoimporter.InitData) +) + func init() { incpaths := []string{"/"} // importer for default gccgo var inst gccgoimporter.GccgoInstallation inst.InitFromDriver("gccgo") - register("gccgo", inst.GetImporter(incpaths)) - - // importer for gccgo using condensed export format (experimental) - register("gccgo-new", getNewImporter(append(append(incpaths, inst.SearchPaths()...), "."))) + register("gccgo", inst.GetImporter(incpaths, initmap)) } -// This function is an adjusted variant of gccgoimporter.GccgoInstallation.GetImporter. -func getNewImporter(searchpaths []string) types.Importer { - return func(imports map[string]*types.Package, pkgpath string) (pkg *types.Package, err error) { - if pkgpath == "unsafe" { - return types.Unsafe, nil - } +// Print the extra gccgo compiler data for this package, if it exists. +func (p *printer) printGccgoExtra(pkg *types.Package) { + if initdata, ok := initmap[pkg]; ok { + p.printf("/*\npriority %d\n", initdata.Priority) - fpath, err := findExportFile(searchpaths, pkgpath) - if err != nil { - return - } - - reader, closer, err := openExportFile(fpath) - if err != nil { - return nil, err - } - defer closer.Close() - - // TODO(gri) At the moment we just read the entire file. - // We should change importer.ImportData to take an io.Reader instead. - data, err := ioutil.ReadAll(reader) - if err != nil && err != io.EOF { - return nil, err - } - - return importer.ImportData(packages, data) - } -} - -// This function is an exact copy of gccgoimporter.findExportFile. -func findExportFile(searchpaths []string, pkgpath string) (string, error) { - for _, spath := range searchpaths { - pkgfullpath := filepath.Join(spath, pkgpath) - pkgdir, name := filepath.Split(pkgfullpath) - - for _, filepath := range [...]string{ - pkgfullpath, - pkgfullpath + ".gox", - pkgdir + "lib" + name + ".so", - pkgdir + "lib" + name + ".a", - pkgfullpath + ".o", - } { - fi, err := os.Stat(filepath) - if err == nil && !fi.IsDir() { - return filepath, nil + p.printDecl("init", len(initdata.Inits), func() { + for _, init := range initdata.Inits { + p.printf("%s %s %d\n", init.Name, init.InitFunc, init.Priority) } - } - } + }) - return "", fmt.Errorf("%s: could not find export data (tried %s)", pkgpath, strings.Join(searchpaths, ":")) -} - -// This function is an exact copy of gccgoimporter.openExportFile. -func openExportFile(fpath string) (reader io.ReadSeeker, closer io.Closer, err error) { - f, err := os.Open(fpath) - if err != nil { - return - } - defer func() { - if err != nil { - f.Close() - } - }() - closer = f - - var magic [4]byte - _, err = f.ReadAt(magic[:], 0) - if err != nil { - return - } - - if string(magic[:]) == "v1;\n" { - // Raw export data. - reader = f - return - } - - ef, err := elf.NewFile(f) - if err != nil { - return - } - - sec := ef.Section(".go_export") - if sec == nil { - err = fmt.Errorf("%s: .go_export section not found", fpath) - return - } - - reader = sec.Open() - return + p.print("*/\n") + } } diff --git a/cmd/godex/print.go b/cmd/godex/print.go index e8b87c5f..d40160d6 100644 --- a/cmd/godex/print.go +++ b/cmd/godex/print.go @@ -21,6 +21,7 @@ func print(w io.Writer, pkg *types.Package, filter func(types.Object) bool) { var p printer p.pkg = pkg p.printPackage(pkg, filter) + p.printGccgoExtra(pkg) io.Copy(w, &p.buf) } diff --git a/go/gccgoimporter/gccgoinstallation.go b/go/gccgoimporter/gccgoinstallation.go index 6e3f7ffb..266fbac8 100644 --- a/go/gccgoimporter/gccgoinstallation.go +++ b/go/gccgoimporter/gccgoinstallation.go @@ -90,6 +90,6 @@ func (inst *GccgoInstallation) SearchPaths() (paths []string) { // Return an importer that searches incpaths followed by the gcc installation's // built-in search paths and the current directory. -func (inst *GccgoInstallation) GetImporter(incpaths []string) types.Importer { - return GetImporter(append(append(incpaths, inst.SearchPaths()...), ".")) +func (inst *GccgoInstallation) GetImporter(incpaths []string, initmap map[*types.Package]InitData) types.Importer { + return GetImporter(append(append(incpaths, inst.SearchPaths()...), "."), initmap) } diff --git a/go/gccgoimporter/gccgoinstallation_test.go b/go/gccgoimporter/gccgoinstallation_test.go index a9e39dc3..64c9890d 100644 --- a/go/gccgoimporter/gccgoinstallation_test.go +++ b/go/gccgoimporter/gccgoinstallation_test.go @@ -161,7 +161,7 @@ func TestInstallationImporter(t *testing.T) { if err != nil { t.Fatal(err) } - imp := inst.GetImporter(nil) + imp := inst.GetImporter(nil, nil) // Ensure we don't regress the number of packages we can parse. First import // all packages into the same map and then each individually. @@ -189,6 +189,6 @@ func TestInstallationImporter(t *testing.T) { {pkgpath: "sort", name: "Ints", want: "func Ints(a []int)"}, {pkgpath: "unsafe", name: "Pointer", want: "type Pointer unsafe.Pointer"}, } { - runImporterTest(t, imp, &test) + runImporterTest(t, imp, nil, &test) } } diff --git a/go/gccgoimporter/importer.go b/go/gccgoimporter/importer.go index 17504e2e..dc02383c 100644 --- a/go/gccgoimporter/importer.go +++ b/go/gccgoimporter/importer.go @@ -6,16 +6,40 @@ package gccgoimporter import ( + "bytes" "debug/elf" "fmt" "io" + "io/ioutil" "os" + "os/exec" "path/filepath" "strings" + "code.google.com/p/go.tools/go/importer" "code.google.com/p/go.tools/go/types" ) +// A PackageInit describes an imported package that needs initialization. +type PackageInit struct { + Name string // short package name + InitFunc string // name of init function + Priority int // priority of init function, see InitData.Priority +} + +// The gccgo-specific init data for a package. +type InitData struct { + // Initialization priority of this package relative to other packages. + // This is based on the maximum depth of the package's dependency graph; + // it is guaranteed to be greater than that of its dependencies. + Priority int + + // The list of packages which this package depends on to be initialized, + // including itself if needed. This is the subset of the transitive closure of + // the package's dependencies that need initialization. + Inits []PackageInit +} + // Locate the file from which to read export data. // This is intended to replicate the logic in gofrontend. func findExportFile(searchpaths []string, pkgpath string) (string, error) { @@ -40,20 +64,27 @@ func findExportFile(searchpaths []string, pkgpath string) (string, error) { return "", fmt.Errorf("%s: could not find export data (tried %s)", pkgpath, strings.Join(searchpaths, ":")) } +const ( + gccgov1Magic = "v1;\n" + goimporterMagic = "\n$$ " + archiveMagic = "! 0 { + initdata := initmap[pkg] + found := false + // Check that the package's own init function has the package's priority + for _, pkginit := range initdata.Inits { + if pkginit.InitFunc == test.wantinits[0] { + if initdata.Priority != pkginit.Priority { + t.Errorf("%s: got self priority %d; want %d", test.pkgpath, pkginit.Priority, initdata.Priority) + } + found = true + break + } + } - if test.wantval != "" { - gotval := obj.(*types.Const).Val().String() - if gotval != test.wantval { - t.Errorf("%s: got val %q; want val %q", test.name, gotval, test.wantval) + if !found { + t.Errorf("%s: could not find expected function %q", test.pkgpath, test.wantinits[0]) + } + + // Each init function in the list other than the first one is a + // dependency of the function immediately before it. Check that + // the init functions appear in descending priority order. + priority := initdata.Priority + for _, wantdepinit := range test.wantinits[1:] { + found = false + for _, pkginit := range initdata.Inits { + if pkginit.InitFunc == wantdepinit { + if priority <= pkginit.Priority { + t.Errorf("%s: got dep priority %d; want less than %d", test.pkgpath, pkginit.Priority, priority) + } + found = true + priority = pkginit.Priority + break + } + } + + if !found { + t.Errorf("%s: could not find expected function %q", test.pkgpath, wantdepinit) + } } } } @@ -51,13 +95,15 @@ var importerTests = [...]importerTest{ {pkgpath: "complexnums", name: "NP", want: "const NP untyped complex", wantval: "(-1/1 + 1/1i)"}, {pkgpath: "complexnums", name: "PN", want: "const PN untyped complex", wantval: "(1/1 + -1/1i)"}, {pkgpath: "complexnums", name: "PP", want: "const PP untyped complex", wantval: "(1/1 + 1/1i)"}, + {pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import", "math..import"}}, } func TestGoxImporter(t *testing.T) { - imp := GetImporter([]string{"testdata"}) + initmap := make(map[*types.Package]InitData) + imp := GetImporter([]string{"testdata"}, initmap) for _, test := range importerTests { - runImporterTest(t, imp, &test) + runImporterTest(t, imp, initmap, &test) } } @@ -73,22 +119,43 @@ func TestObjImporter(t *testing.T) { if err != nil { t.Fatal(err) } - imp := GetImporter([]string{tmpdir}) + initmap := make(map[*types.Package]InitData) + imp := GetImporter([]string{tmpdir}, initmap) + + artmpdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + arinitmap := make(map[*types.Package]InitData) + arimp := GetImporter([]string{artmpdir}, arinitmap) for _, test := range importerTests { gofile := filepath.Join("testdata", test.pkgpath+".go") ofile := filepath.Join(tmpdir, test.pkgpath+".o") + afile := filepath.Join(artmpdir, "lib"+test.pkgpath+".a") - cmd := exec.Command("gccgo", "-c", "-o", ofile, gofile) + cmd := exec.Command("gccgo", "-fgo-pkgpath="+test.pkgpath, "-c", "-o", ofile, gofile) out, err := cmd.CombinedOutput() if err != nil { t.Logf("%s", out) t.Fatalf("gccgo %s failed: %s", gofile, err) } - runImporterTest(t, imp, &test) + runImporterTest(t, imp, initmap, &test) - if err := os.Remove(ofile); err != nil { + cmd = exec.Command("ar", "cr", afile, ofile) + out, err = cmd.CombinedOutput() + if err != nil { + t.Logf("%s", out) + t.Fatalf("ar cr %s %s failed: %s", afile, ofile, err) + } + + runImporterTest(t, arimp, arinitmap, &test) + + if err = os.Remove(ofile); err != nil { + t.Fatal(err) + } + if err = os.Remove(afile); err != nil { t.Fatal(err) } } diff --git a/go/gccgoimporter/parser.go b/go/gccgoimporter/parser.go index b09b1376..6291e87a 100644 --- a/go/gccgoimporter/parser.go +++ b/go/gccgoimporter/parser.go @@ -19,14 +19,15 @@ import ( ) type parser struct { - scanner scanner.Scanner - tok rune // current token - lit string // literal string; only valid for Ident, Int, String tokens - pkgpath string // package path of imported package - pkgname string // name of imported package - pkg *types.Package // reference to imported package - imports map[string]*types.Package // package path -> package object - typeMap map[int]types.Type // type number -> type + scanner scanner.Scanner + tok rune // current token + lit string // literal string; only valid for Ident, Int, String tokens + pkgpath string // package path of imported package + pkgname string // name of imported package + pkg *types.Package // reference to imported package + imports map[string]*types.Package // package path -> package object + typeMap map[int]types.Type // type number -> type + initdata InitData // package init priority data } func (p *parser) init(filename string, src io.Reader, imports map[string]*types.Package) { @@ -412,6 +413,15 @@ func (p *parser) parseNamedType(n int) types.Type { return nt } +func (p *parser) parseInt() int64 { + lit := p.expect(scanner.Int) + n, err := strconv.ParseInt(lit, 10, 0) + if err != nil { + p.error(err) + } + return n +} + // ArrayOrSliceType = "[" [ int ] "]" Type . func (p *parser) parseArrayOrSliceType(pkg *types.Package) types.Type { p.expect('[') @@ -420,11 +430,7 @@ func (p *parser) parseArrayOrSliceType(pkg *types.Package) types.Type { return types.NewSlice(p.parseType(pkg)) } - lit := p.expect(scanner.Int) - n, err := strconv.ParseInt(lit, 10, 0) - if err != nil { - p.error(err) - } + n := p.parseInt() p.expect(']') return types.NewArray(p.parseType(pkg), n) } @@ -665,11 +671,7 @@ func (p *parser) parseType(pkg *types.Package) (t types.Type) { switch p.tok { case scanner.Int: - n, err := strconv.ParseInt(p.lit, 10, 0) - if err != nil { - p.error(err) - } - p.next() + n := p.parseInt() if p.tok == '>' { t = p.typeMap[int(n)] @@ -679,11 +681,7 @@ func (p *parser) parseType(pkg *types.Package) (t types.Type) { case '-': p.next() - lit := p.expect(scanner.Int) - n, err := strconv.ParseInt(lit, 10, 0) - if err != nil { - p.error(err) - } + n := p.parseInt() t = lookupBuiltinType(int(n)) default: @@ -695,6 +693,14 @@ func (p *parser) parseType(pkg *types.Package) (t types.Type) { return } +// PackageInit = unquotedString unquotedString int . +func (p *parser) parsePackageInit() PackageInit { + name := p.parseUnquotedString() + initfunc := p.parseUnquotedString() + priority := int(p.parseInt()) + return PackageInit{Name: name, InitFunc: initfunc, Priority: priority} +} + // Throw away tokens until we see a ';'. If we see a '<', attempt to parse as a type. func (p *parser) discardDirectiveWhileParsingTypes(pkg *types.Package) { for { @@ -718,7 +724,49 @@ func (p *parser) maybeCreatePackage() { } } -// Directive = ("v1" | "priority" | "init" | "checksum") { } ";" | +// InitDataDirective = "v1" ";" | +// "priority" int ";" | +// "init" { PackageInit } ";" | +// "checksum" unquotedString ";" . +func (p *parser) parseInitDataDirective() { + if p.tok != scanner.Ident { + // unexpected token kind; panic + p.expect(scanner.Ident) + } + + switch p.lit { + case "v1": + p.next() + p.expect(';') + + case "priority": + p.next() + p.initdata.Priority = int(p.parseInt()) + p.expect(';') + + case "init": + p.next() + for p.tok != ';' && p.tok != scanner.EOF { + p.initdata.Inits = append(p.initdata.Inits, p.parsePackageInit()) + } + p.expect(';') + + case "checksum": + // Don't let the scanner try to parse the checksum as a number. + defer func(mode uint) { + p.scanner.Mode = mode + }(p.scanner.Mode) + p.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats + p.next() + p.parseUnquotedString() + p.expect(';') + + default: + p.errorf("unexpected identifier: %q", p.lit) + } +} + +// Directive = InitDataDirective | // "package" unquotedString ";" | // "pkgpath" unquotedString ";" | // "import" unquotedString unquotedString string ";" | @@ -728,14 +776,13 @@ func (p *parser) maybeCreatePackage() { // "const" Const ";" . func (p *parser) parseDirective() { if p.tok != scanner.Ident { + // unexpected token kind; panic p.expect(scanner.Ident) } switch p.lit { - case "v1", "priority", "init": - // We can't parse these yet. - p.discardDirectiveWhileParsingTypes(p.pkg) - p.next() + case "v1", "priority", "init", "checksum": + p.parseInitDataDirective() case "package": p.next() @@ -782,14 +829,6 @@ func (p *parser) parseDirective() { p.pkg.Scope().Insert(c) p.expect(';') - case "checksum": - // Don't let the scanner try to parse the checksum as a number. - p.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats - p.next() - p.parseUnquotedString() - p.expect(';') - p.scanner.Mode |= scanner.ScanInts | scanner.ScanFloats - default: p.errorf("unexpected identifier: %q", p.lit) } @@ -803,3 +842,10 @@ func (p *parser) parsePackage() *types.Package { p.pkg.MarkComplete() return p.pkg } + +// InitData = { InitDataDirective } . +func (p *parser) parseInitData() { + for p.tok != scanner.EOF { + p.parseInitDataDirective() + } +} diff --git a/go/gccgoimporter/testdata/imports.go b/go/gccgoimporter/testdata/imports.go new file mode 100644 index 00000000..7907316a --- /dev/null +++ b/go/gccgoimporter/testdata/imports.go @@ -0,0 +1,5 @@ +package imports + +import "fmt" + +var Hello = fmt.Sprintf("Hello, world") diff --git a/go/gccgoimporter/testdata/imports.gox b/go/gccgoimporter/testdata/imports.gox new file mode 100644 index 00000000..958a4f5b --- /dev/null +++ b/go/gccgoimporter/testdata/imports.gox @@ -0,0 +1,7 @@ +v1; +package imports; +pkgpath imports; +priority 7; +import fmt fmt "fmt"; +init imports imports..import 7 math math..import 1 runtime runtime..import 1 strconv strconv..import 2 io io..import 3 reflect reflect..import 3 syscall syscall..import 3 time time..import 4 os os..import 5 fmt fmt..import 6; +var Hello ; diff --git a/go/importer/import.go b/go/importer/import.go index 54207ab9..7fce5a53 100644 --- a/go/importer/import.go +++ b/go/importer/import.go @@ -18,11 +18,14 @@ import ( "code.google.com/p/go.tools/go/types" ) -// ImportData imports a package from the serialized package data. +// ImportData imports a package from the serialized package data +// and returns the number of bytes consumed and a reference to the package. // If data is obviously malformed, an error is returned but in // general it is not recommended to call ImportData on untrusted // data. -func ImportData(imports map[string]*types.Package, data []byte) (*types.Package, error) { +func ImportData(imports map[string]*types.Package, data []byte) (int, *types.Package, error) { + datalen := len(data) + // check magic string var s string if len(data) >= len(magic) { @@ -30,7 +33,7 @@ func ImportData(imports map[string]*types.Package, data []byte) (*types.Package, data = data[len(magic):] } if s != magic { - return nil, fmt.Errorf("incorrect magic string: got %q; want %q", s, magic) + return 0, nil, fmt.Errorf("incorrect magic string: got %q; want %q", s, magic) } // check low-level encoding format @@ -40,13 +43,13 @@ func ImportData(imports map[string]*types.Package, data []byte) (*types.Package, data = data[1:] } if m != format() { - return nil, fmt.Errorf("incorrect low-level encoding format: got %c; want %c", m, format()) + return 0, nil, fmt.Errorf("incorrect low-level encoding format: got %c; want %c", m, format()) } p := importer{ - data: data, - imports: imports, - consumed: len(magic) + 1, // for debugging only + data: data, + datalen: datalen, + imports: imports, } // populate typList with predeclared types @@ -55,7 +58,7 @@ func ImportData(imports map[string]*types.Package, data []byte) (*types.Package, } if v := p.string(); v != version { - return nil, fmt.Errorf("unknown version: got %s; want %s", v, version) + return 0, nil, fmt.Errorf("unknown version: got %s; want %s", v, version) } pkg := p.pkg() @@ -69,24 +72,18 @@ func ImportData(imports map[string]*types.Package, data []byte) (*types.Package, p.obj(pkg) } - if len(p.data) > 0 { - return nil, fmt.Errorf("not all input data consumed") - } - // package was imported completely and without errors pkg.MarkComplete() - return pkg, nil + return p.consumed(), pkg, nil } type importer struct { data []byte + datalen int imports map[string]*types.Package pkgList []*types.Package typList []types.Type - - // debugging support - consumed int } func (p *importer) pkg() *types.Package { @@ -417,9 +414,6 @@ func (p *importer) bytes() []byte { if n := int(p.rawInt64()); n > 0 { b = p.data[:n] p.data = p.data[n:] - if debug { - p.consumed += n - } } return b } @@ -427,12 +421,11 @@ func (p *importer) bytes() []byte { func (p *importer) marker(want byte) { if debug { if got := p.data[0]; got != want { - panic(fmt.Sprintf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.consumed)) + panic(fmt.Sprintf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.consumed())) } p.data = p.data[1:] - p.consumed++ - pos := p.consumed + pos := p.consumed() if n := int(p.rawInt64()); n != pos { panic(fmt.Sprintf("incorrect position: got %d; want %d", n, pos)) } @@ -443,8 +436,9 @@ func (p *importer) marker(want byte) { func (p *importer) rawInt64() int64 { i, n := binary.Varint(p.data) p.data = p.data[n:] - if debug { - p.consumed += n - } return i } + +func (p *importer) consumed() int { + return p.datalen - len(p.data) +} diff --git a/go/importer/import_test.go b/go/importer/import_test.go index cd2af5d2..7950d7e4 100644 --- a/go/importer/import_test.go +++ b/go/importer/import_test.go @@ -144,11 +144,15 @@ func testExportImport(t *testing.T, pkg0 *types.Package, path string) (size, gcs size = len(data) imports := make(map[string]*types.Package) - pkg1, err := ImportData(imports, data) + n, pkg1, err := ImportData(imports, data) if err != nil { t.Errorf("package %s: import failed: %s", pkg0.Name(), err) return } + if n != size { + t.Errorf("package %s: not all input data consumed", pkg0.Name()) + return + } s0 := pkgString(pkg0) s1 := pkgString(pkg1)