go.tools/go/gccgoimporter: keep track of package and import priority

Clients such as compilers need this information in order
to correctly link against imported packages.

This also adds support for the condensed import data format
where the priority information is stored as a suffix of the
condensed import data, as well as support for archive files.

LGTM=gri
R=gri
CC=golang-codereviews, iant
https://golang.org/cl/78740043

cmd/godex/gccgo.go

@@ -7,119 +7,34 @@
 package main
 
 import (
-	"debug/elf"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-
 	"code.google.com/p/go.tools/go/gccgoimporter"
-	"code.google.com/p/go.tools/go/importer"
 	"code.google.com/p/go.tools/go/types"
 )
 
+var (
+	initmap = make(map[*types.Package]gccgoimporter.InitData)
+)
+
 func init() {
 	incpaths := []string{"/"}
 
 	// importer for default gccgo
 	var inst gccgoimporter.GccgoInstallation
 	inst.InitFromDriver("gccgo")
-	register("gccgo", inst.GetImporter(incpaths))
+	register("gccgo", inst.GetImporter(incpaths, initmap))
-
-	// importer for gccgo using condensed export format (experimental)
-	register("gccgo-new", getNewImporter(append(append(incpaths, inst.SearchPaths()...), ".")))
 }
 
-// This function is an adjusted variant of gccgoimporter.GccgoInstallation.GetImporter.
+// Print the extra gccgo compiler data for this package, if it exists.
-func getNewImporter(searchpaths []string) types.Importer {
+func (p *printer) printGccgoExtra(pkg *types.Package) {
-	return func(imports map[string]*types.Package, pkgpath string) (pkg *types.Package, err error) {
+	if initdata, ok := initmap[pkg]; ok {
-		if pkgpath == "unsafe" {
+		p.printf("/*\npriority %d\n", initdata.Priority)
-			return types.Unsafe, nil
-		}
 
-		fpath, err := findExportFile(searchpaths, pkgpath)
+		p.printDecl("init", len(initdata.Inits), func() {
-		if err != nil {
+			for _, init := range initdata.Inits {
-			return
+				p.printf("%s %s %d\n", init.Name, init.InitFunc, init.Priority)
 			}
+		})
 
-		reader, closer, err := openExportFile(fpath)
+		p.print("*/\n")
-		if err != nil {
-			return nil, err
-		}
-		defer closer.Close()
-
-		// TODO(gri) At the moment we just read the entire file.
-		// We should change importer.ImportData to take an io.Reader instead.
-		data, err := ioutil.ReadAll(reader)
-		if err != nil && err != io.EOF {
-			return nil, err
-		}
-
-		return importer.ImportData(packages, data)
 	}
 }
-
-// This function is an exact copy of gccgoimporter.findExportFile.
-func findExportFile(searchpaths []string, pkgpath string) (string, error) {
-	for _, spath := range searchpaths {
-		pkgfullpath := filepath.Join(spath, pkgpath)
-		pkgdir, name := filepath.Split(pkgfullpath)
-
-		for _, filepath := range [...]string{
-			pkgfullpath,
-			pkgfullpath + ".gox",
-			pkgdir + "lib" + name + ".so",
-			pkgdir + "lib" + name + ".a",
-			pkgfullpath + ".o",
-		} {
-			fi, err := os.Stat(filepath)
-			if err == nil && !fi.IsDir() {
-				return filepath, nil
-			}
-		}
-	}
-
-	return "", fmt.Errorf("%s: could not find export data (tried %s)", pkgpath, strings.Join(searchpaths, ":"))
-}
-
-// This function is an exact copy of gccgoimporter.openExportFile.
-func openExportFile(fpath string) (reader io.ReadSeeker, closer io.Closer, err error) {
-	f, err := os.Open(fpath)
-	if err != nil {
-		return
-	}
-	defer func() {
-		if err != nil {
-			f.Close()
-		}
-	}()
-	closer = f
-
-	var magic [4]byte
-	_, err = f.ReadAt(magic[:], 0)
-	if err != nil {
-		return
-	}
-
-	if string(magic[:]) == "v1;\n" {
-		// Raw export data.
-		reader = f
-		return
-	}
-
-	ef, err := elf.NewFile(f)
-	if err != nil {
-		return
-	}
-
-	sec := ef.Section(".go_export")
-	if sec == nil {
-		err = fmt.Errorf("%s: .go_export section not found", fpath)
-		return
-	}
-
-	reader = sec.Open()
-	return
-}

cmd/godex/print.go

@@ -21,6 +21,7 @@ func print(w io.Writer, pkg *types.Package, filter func(types.Object) bool) {
 	var p printer
 	p.pkg = pkg
 	p.printPackage(pkg, filter)
+	p.printGccgoExtra(pkg)
 	io.Copy(w, &p.buf)
 }
 

go/gccgoimporter/gccgoinstallation.go

@@ -90,6 +90,6 @@ func (inst *GccgoInstallation) SearchPaths() (paths []string) {
 
 // Return an importer that searches incpaths followed by the gcc installation's
 // built-in search paths and the current directory.
-func (inst *GccgoInstallation) GetImporter(incpaths []string) types.Importer {
+func (inst *GccgoInstallation) GetImporter(incpaths []string, initmap map[*types.Package]InitData) types.Importer {
-	return GetImporter(append(append(incpaths, inst.SearchPaths()...), "."))
+	return GetImporter(append(append(incpaths, inst.SearchPaths()...), "."), initmap)
 }

go/gccgoimporter/gccgoinstallation_test.go

@@ -161,7 +161,7 @@ func TestInstallationImporter(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	imp := inst.GetImporter(nil)
+	imp := inst.GetImporter(nil, nil)
 
 	// Ensure we don't regress the number of packages we can parse. First import
 	// all packages into the same map and then each individually.
@@ -189,6 +189,6 @@ func TestInstallationImporter(t *testing.T) {
 		{pkgpath: "sort", name: "Ints", want: "func Ints(a []int)"},
 		{pkgpath: "unsafe", name: "Pointer", want: "type Pointer unsafe.Pointer"},
 	} {
-		runImporterTest(t, imp, &test)
+		runImporterTest(t, imp, nil, &test)
 	}
 }

go/gccgoimporter/importer.go

@@ -6,16 +6,40 @@
 package gccgoimporter
 
 import (
+	"bytes"
 	"debug/elf"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
+	"code.google.com/p/go.tools/go/importer"
 	"code.google.com/p/go.tools/go/types"
 )
 
+// A PackageInit describes an imported package that needs initialization.
+type PackageInit struct {
+	Name     string // short package name
+	InitFunc string // name of init function
+	Priority int    // priority of init function, see InitData.Priority
+}
+
+// The gccgo-specific init data for a package.
+type InitData struct {
+	// Initialization priority of this package relative to other packages.
+	// This is based on the maximum depth of the package's dependency graph;
+	// it is guaranteed to be greater than that of its dependencies.
+	Priority int
+
+	// The list of packages which this package depends on to be initialized,
+	// including itself if needed. This is the subset of the transitive closure of
+	// the package's dependencies that need initialization.
+	Inits []PackageInit
+}
+
 // Locate the file from which to read export data.
 // This is intended to replicate the logic in gofrontend.
 func findExportFile(searchpaths []string, pkgpath string) (string, error) {
@@ -40,20 +64,27 @@ func findExportFile(searchpaths []string, pkgpath string) (string, error) {
 	return "", fmt.Errorf("%s: could not find export data (tried %s)", pkgpath, strings.Join(searchpaths, ":"))
 }
 
+const (
+	gccgov1Magic    = "v1;\n"
+	goimporterMagic = "\n$$ "
+	archiveMagic    = "!<ar"
+)
+
 // Opens the export data file at the given path. If this is an ELF file,
-// searches for and opens the .go_export section.
+// searches for and opens the .go_export section. If this is an archive,
-// This is intended to replicate the logic in gofrontend, although it doesn't handle archive files yet.
+// reads the export data from the first member, which is assumed to be an ELF file.
+// This is intended to replicate the logic in gofrontend.
 func openExportFile(fpath string) (reader io.ReadSeeker, closer io.Closer, err error) {
 	f, err := os.Open(fpath)
 	if err != nil {
 		return
 	}
+	closer = f
 	defer func() {
-		if err != nil {
+		if err != nil && closer != nil {
 			f.Close()
 		}
 	}()
-	closer = f
 
 	var magic [4]byte
 	_, err = f.ReadAt(magic[:], 0)
@@ -61,13 +92,32 @@ func openExportFile(fpath string) (reader io.ReadSeeker, closer io.Closer, err e
 		return
 	}
 
-	if string(magic[:]) == "v1;\n" {
+	var elfreader io.ReaderAt
+	switch string(magic[:]) {
+	case gccgov1Magic, goimporterMagic:
 		// Raw export data.
 		reader = f
 		return
+
+	case archiveMagic:
+		// TODO(pcc): Read the archive directly instead of using "ar".
+		f.Close()
+		closer = nil
+
+		cmd := exec.Command("ar", "p", fpath)
+		var out []byte
+		out, err = cmd.Output()
+		if err != nil {
+			return
 		}
 
-	ef, err := elf.NewFile(f)
+		elfreader = bytes.NewReader(out)
+
+	default:
+		elfreader = f
+	}
+
+	ef, err := elf.NewFile(elfreader)
 	if err != nil {
 		return
 	}
@@ -82,7 +132,7 @@ func openExportFile(fpath string) (reader io.ReadSeeker, closer io.Closer, err e
 	return
 }
 
-func GetImporter(searchpaths []string) types.Importer {
+func GetImporter(searchpaths []string, initmap map[*types.Package]InitData) types.Importer {
 	return func(imports map[string]*types.Package, pkgpath string) (pkg *types.Package, err error) {
 		if pkgpath == "unsafe" {
 			return types.Unsafe, nil
@@ -97,11 +147,53 @@ func GetImporter(searchpaths []string) types.Importer {
 		if err != nil {
 			return
 		}
+		if closer != nil {
 			defer closer.Close()
+		}
 
+		var magic [4]byte
+		_, err = reader.Read(magic[:])
+		if err != nil {
+			return
+		}
+		_, err = reader.Seek(0, 0)
+		if err != nil {
+			return
+		}
+
+		switch string(magic[:]) {
+		case gccgov1Magic:
 			var p parser
 			p.init(fpath, reader, imports)
 			pkg = p.parsePackage()
+			if initmap != nil {
+				initmap[pkg] = p.initdata
+			}
+
+		case goimporterMagic:
+			var data []byte
+			data, err = ioutil.ReadAll(reader)
+			if err != nil {
+				return
+			}
+			var n int
+			n, pkg, err = importer.ImportData(imports, data)
+			if err != nil {
+				return
+			}
+
+			if initmap != nil {
+				suffixreader := bytes.NewReader(data[n:])
+				var p parser
+				p.init(fpath, suffixreader, nil)
+				p.parseInitData()
+				initmap[pkg] = p.initdata
+			}
+
+		default:
+			err = fmt.Errorf("unrecognized magic string: %q", string(magic[:]))
+		}
+
 		return
 	}
 }

go/gccgoimporter/importer_test.go

@@ -17,15 +17,17 @@ import (
 
 type importerTest struct {
 	pkgpath, name, want, wantval string
+	wantinits                    []string
 }
 
-func runImporterTest(t *testing.T, imp types.Importer, test *importerTest) {
+func runImporterTest(t *testing.T, imp types.Importer, initmap map[*types.Package]InitData, test *importerTest) {
 	pkg, err := imp(make(map[string]*types.Package), test.pkgpath)
 	if err != nil {
 		t.Error(err)
 		return
 	}
 
+	if test.name != "" {
 		obj := pkg.Scope().Lookup(test.name)
 		if obj == nil {
 			t.Errorf("%s: object not found", test.name)
@@ -45,19 +47,63 @@ func runImporterTest(t *testing.T, imp types.Importer, test *importerTest) {
 		}
 	}
 
+	if len(test.wantinits) > 0 {
+		initdata := initmap[pkg]
+		found := false
+		// Check that the package's own init function has the package's priority
+		for _, pkginit := range initdata.Inits {
+			if pkginit.InitFunc == test.wantinits[0] {
+				if initdata.Priority != pkginit.Priority {
+					t.Errorf("%s: got self priority %d; want %d", test.pkgpath, pkginit.Priority, initdata.Priority)
+				}
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			t.Errorf("%s: could not find expected function %q", test.pkgpath, test.wantinits[0])
+		}
+
+		// Each init function in the list other than the first one is a
+		// dependency of the function immediately before it. Check that
+		// the init functions appear in descending priority order.
+		priority := initdata.Priority
+		for _, wantdepinit := range test.wantinits[1:] {
+			found = false
+			for _, pkginit := range initdata.Inits {
+				if pkginit.InitFunc == wantdepinit {
+					if priority <= pkginit.Priority {
+						t.Errorf("%s: got dep priority %d; want less than %d", test.pkgpath, pkginit.Priority, priority)
+					}
+					found = true
+					priority = pkginit.Priority
+					break
+				}
+			}
+
+			if !found {
+				t.Errorf("%s: could not find expected function %q", test.pkgpath, wantdepinit)
+			}
+		}
+	}
+}
+
 var importerTests = [...]importerTest{
 	{pkgpath: "pointer", name: "Int8Ptr", want: "type Int8Ptr *int8"},
 	{pkgpath: "complexnums", name: "NN", want: "const NN untyped complex", wantval: "(-1/1 + -1/1i)"},
 	{pkgpath: "complexnums", name: "NP", want: "const NP untyped complex", wantval: "(-1/1 + 1/1i)"},
 	{pkgpath: "complexnums", name: "PN", want: "const PN untyped complex", wantval: "(1/1 + -1/1i)"},
 	{pkgpath: "complexnums", name: "PP", want: "const PP untyped complex", wantval: "(1/1 + 1/1i)"},
+	{pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import", "math..import"}},
 }
 
 func TestGoxImporter(t *testing.T) {
-	imp := GetImporter([]string{"testdata"})
+	initmap := make(map[*types.Package]InitData)
+	imp := GetImporter([]string{"testdata"}, initmap)
 
 	for _, test := range importerTests {
-		runImporterTest(t, imp, &test)
+		runImporterTest(t, imp, initmap, &test)
 	}
 }
 
@@ -73,22 +119,43 @@ func TestObjImporter(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	imp := GetImporter([]string{tmpdir})
+	initmap := make(map[*types.Package]InitData)
+	imp := GetImporter([]string{tmpdir}, initmap)
+
+	artmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	arinitmap := make(map[*types.Package]InitData)
+	arimp := GetImporter([]string{artmpdir}, arinitmap)
 
 	for _, test := range importerTests {
 		gofile := filepath.Join("testdata", test.pkgpath+".go")
 		ofile := filepath.Join(tmpdir, test.pkgpath+".o")
+		afile := filepath.Join(artmpdir, "lib"+test.pkgpath+".a")
 
-		cmd := exec.Command("gccgo", "-c", "-o", ofile, gofile)
+		cmd := exec.Command("gccgo", "-fgo-pkgpath="+test.pkgpath, "-c", "-o", ofile, gofile)
 		out, err := cmd.CombinedOutput()
 		if err != nil {
 			t.Logf("%s", out)
 			t.Fatalf("gccgo %s failed: %s", gofile, err)
 		}
 
-		runImporterTest(t, imp, &test)
+		runImporterTest(t, imp, initmap, &test)
 
-		if err := os.Remove(ofile); err != nil {
+		cmd = exec.Command("ar", "cr", afile, ofile)
+		out, err = cmd.CombinedOutput()
+		if err != nil {
+			t.Logf("%s", out)
+			t.Fatalf("ar cr %s %s failed: %s", afile, ofile, err)
+		}
+
+		runImporterTest(t, arimp, arinitmap, &test)
+
+		if err = os.Remove(ofile); err != nil {
+			t.Fatal(err)
+		}
+		if err = os.Remove(afile); err != nil {
 			t.Fatal(err)
 		}
 	}

go/gccgoimporter/parser.go

@@ -27,6 +27,7 @@ type parser struct {
 	pkg      *types.Package            // reference to imported package
 	imports  map[string]*types.Package // package path -> package object
 	typeMap  map[int]types.Type        // type number -> type
+	initdata InitData                  // package init priority data
 }
 
 func (p *parser) init(filename string, src io.Reader, imports map[string]*types.Package) {
@@ -412,6 +413,15 @@ func (p *parser) parseNamedType(n int) types.Type {
 	return nt
 }
 
+func (p *parser) parseInt() int64 {
+	lit := p.expect(scanner.Int)
+	n, err := strconv.ParseInt(lit, 10, 0)
+	if err != nil {
+		p.error(err)
+	}
+	return n
+}
+
 // ArrayOrSliceType = "[" [ int ] "]" Type .
 func (p *parser) parseArrayOrSliceType(pkg *types.Package) types.Type {
 	p.expect('[')
@@ -420,11 +430,7 @@ func (p *parser) parseArrayOrSliceType(pkg *types.Package) types.Type {
 		return types.NewSlice(p.parseType(pkg))
 	}
 
-	lit := p.expect(scanner.Int)
+	n := p.parseInt()
-	n, err := strconv.ParseInt(lit, 10, 0)
-	if err != nil {
-		p.error(err)
-	}
 	p.expect(']')
 	return types.NewArray(p.parseType(pkg), n)
 }
@@ -665,11 +671,7 @@ func (p *parser) parseType(pkg *types.Package) (t types.Type) {
 
 	switch p.tok {
 	case scanner.Int:
-		n, err := strconv.ParseInt(p.lit, 10, 0)
+		n := p.parseInt()
-		if err != nil {
-			p.error(err)
-		}
-		p.next()
 
 		if p.tok == '>' {
 			t = p.typeMap[int(n)]
@@ -679,11 +681,7 @@ func (p *parser) parseType(pkg *types.Package) (t types.Type) {
 
 	case '-':
 		p.next()
-		lit := p.expect(scanner.Int)
+		n := p.parseInt()
-		n, err := strconv.ParseInt(lit, 10, 0)
-		if err != nil {
-			p.error(err)
-		}
 		t = lookupBuiltinType(int(n))
 
 	default:
@@ -695,6 +693,14 @@ func (p *parser) parseType(pkg *types.Package) (t types.Type) {
 	return
 }
 
+// PackageInit = unquotedString unquotedString int .
+func (p *parser) parsePackageInit() PackageInit {
+	name := p.parseUnquotedString()
+	initfunc := p.parseUnquotedString()
+	priority := int(p.parseInt())
+	return PackageInit{Name: name, InitFunc: initfunc, Priority: priority}
+}
+
 // Throw away tokens until we see a ';'. If we see a '<', attempt to parse as a type.
 func (p *parser) discardDirectiveWhileParsingTypes(pkg *types.Package) {
 	for {
@@ -718,7 +724,49 @@ func (p *parser) maybeCreatePackage() {
 	}
 }
 
-// Directive = ("v1" | "priority" | "init" | "checksum") { <any token> } ";" |
+// InitDataDirective = "v1" ";" |
+//                     "priority" int ";" |
+//                     "init" { PackageInit } ";" |
+//                     "checksum" unquotedString ";" .
+func (p *parser) parseInitDataDirective() {
+	if p.tok != scanner.Ident {
+		// unexpected token kind; panic
+		p.expect(scanner.Ident)
+	}
+
+	switch p.lit {
+	case "v1":
+		p.next()
+		p.expect(';')
+
+	case "priority":
+		p.next()
+		p.initdata.Priority = int(p.parseInt())
+		p.expect(';')
+
+	case "init":
+		p.next()
+		for p.tok != ';' && p.tok != scanner.EOF {
+			p.initdata.Inits = append(p.initdata.Inits, p.parsePackageInit())
+		}
+		p.expect(';')
+
+	case "checksum":
+		// Don't let the scanner try to parse the checksum as a number.
+		defer func(mode uint) {
+			p.scanner.Mode = mode
+		}(p.scanner.Mode)
+		p.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats
+		p.next()
+		p.parseUnquotedString()
+		p.expect(';')
+
+	default:
+		p.errorf("unexpected identifier: %q", p.lit)
+	}
+}
+
+// Directive = InitDataDirective |
 //             "package" unquotedString ";" |
 //             "pkgpath" unquotedString ";" |
 //             "import" unquotedString unquotedString string ";" |
@@ -728,14 +776,13 @@ func (p *parser) maybeCreatePackage() {
 //             "const" Const ";" .
 func (p *parser) parseDirective() {
 	if p.tok != scanner.Ident {
+		// unexpected token kind; panic
 		p.expect(scanner.Ident)
 	}
 
 	switch p.lit {
-	case "v1", "priority", "init":
+	case "v1", "priority", "init", "checksum":
-		// We can't parse these yet.
+		p.parseInitDataDirective()
-		p.discardDirectiveWhileParsingTypes(p.pkg)
-		p.next()
 
 	case "package":
 		p.next()
@@ -782,14 +829,6 @@ func (p *parser) parseDirective() {
 		p.pkg.Scope().Insert(c)
 		p.expect(';')
 
-	case "checksum":
-		// Don't let the scanner try to parse the checksum as a number.
-		p.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats
-		p.next()
-		p.parseUnquotedString()
-		p.expect(';')
-		p.scanner.Mode |= scanner.ScanInts | scanner.ScanFloats
-
 	default:
 		p.errorf("unexpected identifier: %q", p.lit)
 	}
@@ -803,3 +842,10 @@ func (p *parser) parsePackage() *types.Package {
 	p.pkg.MarkComplete()
 	return p.pkg
 }
+
+// InitData = { InitDataDirective } .
+func (p *parser) parseInitData() {
+	for p.tok != scanner.EOF {
+		p.parseInitDataDirective()
+	}
+}

go/gccgoimporter/testdata/imports.go

@@ -0,0 +1,5 @@
+package imports
+
+import "fmt"
+
+var Hello = fmt.Sprintf("Hello, world")

go/gccgoimporter/testdata/imports.gox

@@ -0,0 +1,7 @@
+v1;
+package imports;
+pkgpath imports;
+priority 7;
+import fmt fmt "fmt";
+init imports imports..import 7 math math..import 1 runtime runtime..import 1 strconv strconv..import 2 io io..import 3 reflect reflect..import 3 syscall syscall..import 3 time time..import 4 os os..import 5 fmt fmt..import 6;
+var Hello <type -16>;

go/importer/import.go

@@ -18,11 +18,14 @@ import (
 	"code.google.com/p/go.tools/go/types"
 )
 
-// ImportData imports a package from the serialized package data.
+// ImportData imports a package from the serialized package data
+// and returns the number of bytes consumed and a reference to the package.
 // If data is obviously malformed, an error is returned but in
 // general it is not recommended to call ImportData on untrusted
 // data.
-func ImportData(imports map[string]*types.Package, data []byte) (*types.Package, error) {
+func ImportData(imports map[string]*types.Package, data []byte) (int, *types.Package, error) {
+	datalen := len(data)
+
 	// check magic string
 	var s string
 	if len(data) >= len(magic) {
@@ -30,7 +33,7 @@ func ImportData(imports map[string]*types.Package, data []byte) (*types.Package,
 		data = data[len(magic):]
 	}
 	if s != magic {
-		return nil, fmt.Errorf("incorrect magic string: got %q; want %q", s, magic)
+		return 0, nil, fmt.Errorf("incorrect magic string: got %q; want %q", s, magic)
 	}
 
 	// check low-level encoding format
@@ -40,13 +43,13 @@ func ImportData(imports map[string]*types.Package, data []byte) (*types.Package,
 		data = data[1:]
 	}
 	if m != format() {
-		return nil, fmt.Errorf("incorrect low-level encoding format: got %c; want %c", m, format())
+		return 0, nil, fmt.Errorf("incorrect low-level encoding format: got %c; want %c", m, format())
 	}
 
 	p := importer{
 		data:    data,
+		datalen: datalen,
 		imports: imports,
-		consumed: len(magic) + 1, // for debugging only
 	}
 
 	// populate typList with predeclared types
@@ -55,7 +58,7 @@ func ImportData(imports map[string]*types.Package, data []byte) (*types.Package,
 	}
 
 	if v := p.string(); v != version {
-		return nil, fmt.Errorf("unknown version: got %s; want %s", v, version)
+		return 0, nil, fmt.Errorf("unknown version: got %s; want %s", v, version)
 	}
 
 	pkg := p.pkg()
@@ -69,24 +72,18 @@ func ImportData(imports map[string]*types.Package, data []byte) (*types.Package,
 		p.obj(pkg)
 	}
 
-	if len(p.data) > 0 {
-		return nil, fmt.Errorf("not all input data consumed")
-	}
-
 	// package was imported completely and without errors
 	pkg.MarkComplete()
 
-	return pkg, nil
+	return p.consumed(), pkg, nil
 }
 
 type importer struct {
 	data    []byte
+	datalen int
 	imports map[string]*types.Package
 	pkgList []*types.Package
 	typList []types.Type
-
-	// debugging support
-	consumed int
 }
 
 func (p *importer) pkg() *types.Package {
@@ -417,9 +414,6 @@ func (p *importer) bytes() []byte {
 	if n := int(p.rawInt64()); n > 0 {
 		b = p.data[:n]
 		p.data = p.data[n:]
-		if debug {
-			p.consumed += n
-		}
 	}
 	return b
 }
@@ -427,12 +421,11 @@ func (p *importer) bytes() []byte {
 func (p *importer) marker(want byte) {
 	if debug {
 		if got := p.data[0]; got != want {
-			panic(fmt.Sprintf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.consumed))
+			panic(fmt.Sprintf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.consumed()))
 		}
 		p.data = p.data[1:]
-		p.consumed++
 
-		pos := p.consumed
+		pos := p.consumed()
 		if n := int(p.rawInt64()); n != pos {
 			panic(fmt.Sprintf("incorrect position: got %d; want %d", n, pos))
 		}
@@ -443,8 +436,9 @@ func (p *importer) marker(want byte) {
 func (p *importer) rawInt64() int64 {
 	i, n := binary.Varint(p.data)
 	p.data = p.data[n:]
-	if debug {
-		p.consumed += n
-	}
 	return i
 }
+
+func (p *importer) consumed() int {
+	return p.datalen - len(p.data)
+}

go/importer/import_test.go

@@ -144,11 +144,15 @@ func testExportImport(t *testing.T, pkg0 *types.Package, path string) (size, gcs
 	size = len(data)
 
 	imports := make(map[string]*types.Package)
-	pkg1, err := ImportData(imports, data)
+	n, pkg1, err := ImportData(imports, data)
 	if err != nil {
 		t.Errorf("package %s: import failed: %s", pkg0.Name(), err)
 		return
 	}
+	if n != size {
+		t.Errorf("package %s: not all input data consumed", pkg0.Name())
+		return
+	}
 
 	s0 := pkgString(pkg0)
 	s1 := pkgString(pkg1)