go/internal/gcimporter, gccgoimporter: updated to match latest version in std lib

This CL brings over the changes from:

https://go-review.googlesource.com/118496 (better error message when importer is out of date)
https://go-review.googlesource.com/114317 (permit embedding of non-defined interfaces via alias type names)
https://go-review.googlesource.com/85318  (use named receiver types for methods of named interfaces)
https://go-review.googlesource.com/42870  (report import path if package is not found)
https://go-review.googlesource.com/41710  (version tests for 1.8, v4 and v5)

Also updated go/gcexportdata to select between binary and new indexed export format.

For golang/go#25856.
For golang/go#25301.
For golang/go#20230.
For golang/go#13829.

Change-Id: Ibf77c50f86e767cef411bd1d3809e12397678958
Reviewed-on: https://go-review.googlesource.com/118555
Run-TryBot: Robert Griesemer <gri@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Alan Donovan <adonovan@google.com>

go/gcexportdata/gcexportdata.go

@@ -85,6 +85,14 @@ func Read(in io.Reader, fset *token.FileSet, imports map[string]*types.Package,
 		return gcimporter.ImportData(imports, path, path, bytes.NewReader(data))
 	}
 
+	// The indexed export format starts with an 'i'; the older
+	// binary export format starts with a 'c', 'd', or 'v'
+	// (from "version"). Select appropriate importer.
+	if len(data) > 0 && data[0] == 'i' {
+		_, pkg, err := gcimporter.IImportData(fset, imports, data[1:], path)
+		return pkg, err
+	}
+
 	_, pkg, err := gcimporter.BImportData(fset, imports, data, path)
 	return pkg, err
 }

go/internal/gccgoimporter/parser.go

@@ -587,13 +587,13 @@ func (p *parser) parseInterfaceType(pkg *types.Package) types.Type {
 	p.expectKeyword("interface")
 
 	var methods []*types.Func
-	var typs []*types.Named
+	var embeddeds []types.Type
 
 	p.expect('{')
 	for p.tok != '}' && p.tok != scanner.EOF {
 		if p.tok == '?' {
 			p.next()
-			typs = append(typs, p.parseType(pkg).(*types.Named))
+			embeddeds = append(embeddeds, p.parseType(pkg))
 		} else {
 			method := p.parseFunc(pkg)
 			methods = append(methods, method)
@@ -602,7 +602,7 @@ func (p *parser) parseInterfaceType(pkg *types.Package) types.Type {
 	}
 	p.expect('}')
 
-	return types.NewInterface(methods, typs)
+	return types.NewInterface2(methods, embeddeds)
 }
 
 // PointerType = "*" ("any" | Type) .

go/internal/gcimporter/bexport.go

@@ -38,6 +38,11 @@ const debugFormat = false // default: false
 const trace = false // default: false
 
 // Current export format version. Increase with each format change.
+// Note: The latest binary (non-indexed) export format is at version 6.
+//       This exporter is still at level 4, but it doesn't matter since
+//       the binary importer can handle older versions just fine.
+// 6: package height (CL 105038) -- NOT IMPLEMENTED HERE
+// 5: improved position encoding efficiency (issue 20080, CL 41619) -- NOT IMPLEMEMTED HERE
 // 4: type name objects support type aliases, uses aliasTag
 // 3: Go1.8 encoding (same as version 2, aliasTag defined but never used)
 // 2: removed unused bool in ODCL export (compiler only)

go/internal/gcimporter/bimport.go

@@ -52,24 +52,24 @@ type importer struct {
 // compromised, an error is returned.
 func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) {
 	// catch panics and return them as errors
+	const currentVersion = 6
+	version := -1 // unknown version
 	defer func() {
 		if e := recover(); e != nil {
-			// The package (filename) causing the problem is added to this
-			// error by a wrapper in the caller (Import in gcimporter.go).
 			// Return a (possibly nil or incomplete) package unchanged (see #16088).
-			err = fmt.Errorf("cannot import, possibly version skew (%v) - reinstall package", e)
+			if version > currentVersion {
+				err = fmt.Errorf("cannot import %q (%v), export data is newer version - update tool", path, e)
+			} else {
+				err = fmt.Errorf("cannot import %q (%v), possibly version skew - reinstall package", path, e)
+			}
 		}
 	}()
 
-	if len(data) > 0 && data[0] == 'i' {
-		return iImportData(fset, imports, data[1:], path)
-	}
-
 	p := importer{
 		imports:    imports,
 		data:       data,
 		importpath: path,
-		version:    -1,           // unknown version
+		version:    version,
 		strList:    []string{""}, // empty string is mapped to 0
 		pathList:   []string{""}, // empty string is mapped to 0
 		fake: fakeFileSet{
@@ -94,7 +94,7 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 		p.posInfoFormat = p.int() != 0
 		versionstr = p.string()
 		if versionstr == "v1" {
-			p.version = 0
+			version = 0
 		}
 	} else {
 		// Go1.8 extensible encoding
@@ -102,24 +102,25 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 		versionstr = p.rawStringln(b)
 		if s := strings.SplitN(versionstr, " ", 3); len(s) >= 2 && s[0] == "version" {
 			if v, err := strconv.Atoi(s[1]); err == nil && v > 0 {
-				p.version = v
+				version = v
 			}
 		}
 	}
+	p.version = version
 
 	// read version specific flags - extend as necessary
 	switch p.version {
-	// case 7:
+	// case currentVersion:
 	// 	...
 	//	fallthrough
-	case 6, 5, 4, 3, 2, 1:
+	case currentVersion, 5, 4, 3, 2, 1:
 		p.debugFormat = p.rawStringln(p.rawByte()) == "debug"
 		p.trackAllTypes = p.int() != 0
 		p.posInfoFormat = p.int() != 0
 	case 0:
 		// Go1.7 encoding format - nothing to do here
 	default:
-		errorf("unknown export format version %d (%q)", p.version, versionstr)
+		errorf("unknown bexport format version %d (%q)", p.version, versionstr)
 	}
 
 	// --- generic export data ---
@@ -531,13 +532,13 @@ func (p *importer) typ(parent *types.Package, tname *types.Named) types.Type {
 			p.record(nil)
 		}
 
-		var embeddeds []*types.Named
+		var embeddeds []types.Type
 		for n := p.int(); n > 0; n-- {
 			p.pos()
-			embeddeds = append(embeddeds, p.typ(parent, nil).(*types.Named))
+			embeddeds = append(embeddeds, p.typ(parent, nil))
 		}
 
-		t := types.NewInterface(p.methodList(parent, tname), embeddeds)
+		t := types.NewInterface2(p.methodList(parent, tname), embeddeds)
 		p.interfaceList = append(p.interfaceList, t)
 		if p.trackAllTypes {
 			p.typList[n] = t

go/internal/gcimporter/gcimporter.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// This file is a copy of $GOROOT/src/go/internal/gcimporter/gcimporter.go,
+// This file is a modified copy of $GOROOT/src/go/internal/gcimporter/gcimporter.go,
 // but it also contains the original source-based importer code for Go1.6.
 // Once we stop supporting 1.6, we can remove that code.
 
@@ -55,6 +55,7 @@ func FindPkg(path, srcDir string) (filename, id string) {
 		}
 		bp, _ := build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
 		if bp.PkgObj == "" {
+			id = path // make sure we have an id to print in error message
 			return
 		}
 		noext = strings.TrimSuffix(bp.PkgObj, ".a")
@@ -133,7 +134,7 @@ func Import(packages map[string]*types.Package, path, srcDir string) (pkg *types
 		if path == "unsafe" {
 			return types.Unsafe, nil
 		}
-		err = fmt.Errorf("can't find import: %s", id)
+		err = fmt.Errorf("can't find import: %q", id)
 		return
 	}
 
@@ -164,14 +165,27 @@ func Import(packages map[string]*types.Package, path, srcDir string) (pkg *types
 	switch hdr {
 	case "$$\n":
 		return ImportData(packages, filename, id, buf)
+
 	case "$$B\n":
 		var data []byte
 		data, err = ioutil.ReadAll(buf)
-		if err == nil {
-			fset := token.NewFileSet()
-			_, pkg, err = BImportData(fset, packages, data, id)
-			return
+		if err != nil {
+			break
 		}
+
+		// TODO(gri): allow clients of go/importer to provide a FileSet.
+		// Or, define a new standard go/types/gcexportdata package.
+		fset := token.NewFileSet()
+
+		// The indexed export format starts with an 'i'; the older
+		// binary export format starts with a 'c', 'd', or 'v'
+		// (from "version"). Select appropriate importer.
+		if len(data) > 0 && data[0] == 'i' {
+			_, pkg, err = IImportData(fset, packages, data[1:], id)
+		} else {
+			_, pkg, err = BImportData(fset, packages, data, id)
+		}
+
 	default:
 		err = fmt.Errorf("unknown export data header: %q", hdr)
 	}

go/internal/gcimporter/gcimporter_test.go

@@ -185,9 +185,21 @@ func TestVersionHandling(t *testing.T) {
 		}
 		pkgpath := "./" + name[:len(name)-2]
 
+		if testing.Verbose() {
+			t.Logf("importing %s", name)
+		}
+
 		// test that export data can be imported
 		_, err := Import(make(map[string]*types.Package), pkgpath, dir)
 		if err != nil {
+			// ok to fail if it fails with a newer version error for select files
+			if strings.Contains(err.Error(), "newer version") {
+				switch name {
+				case "test_go1.11_999b.a", "test_go1.11_999i.a":
+					continue
+				}
+				// fall through
+			}
 			t.Errorf("import %q failed: %v", pkgpath, err)
 			continue
 		}
@@ -251,7 +263,8 @@ var importedObjectTests = []struct {
 	// TODO(gri) enable again once we're off 1.7 and 1.8.
 	// {"io.ReadWriter", "type ReadWriter interface{Reader; Writer}"},
 	{"math.Sin", "func Sin(x float64) float64"},
-	// TODO(gri) add more tests
+	// TODO(gri) Add additional tests which are now present in the
+	//           corresponding std library version of this file.
 }
 
 func TestImportedTypes(t *testing.T) {
@@ -286,9 +299,48 @@ func TestImportedTypes(t *testing.T) {
 		if got != test.want {
 			t.Errorf("%s: got %q; want %q", test.name, got, test.want)
 		}
+
+		if named, _ := obj.Type().(*types.Named); named != nil {
+			verifyInterfaceMethodRecvs(t, named, 0)
+		}
 	}
 }
 
+// verifyInterfaceMethodRecvs verifies that method receiver types
+// are named if the methods belong to a named interface type.
+func verifyInterfaceMethodRecvs(t *testing.T, named *types.Named, level int) {
+	// avoid endless recursion in case of an embedding bug that lead to a cycle
+	if level > 10 {
+		t.Errorf("%s: embeds itself", named)
+		return
+	}
+
+	iface, _ := named.Underlying().(*types.Interface)
+	if iface == nil {
+		return // not an interface
+	}
+
+	// check explicitly declared methods
+	for i := 0; i < iface.NumExplicitMethods(); i++ {
+		m := iface.ExplicitMethod(i)
+		recv := m.Type().(*types.Signature).Recv()
+		if recv == nil {
+			t.Errorf("%s: missing receiver type", m)
+			continue
+		}
+		if recv.Type() != named {
+			t.Errorf("%s: got recv type %s; want %s", m, recv.Type(), named)
+		}
+	}
+
+	// check embedded interfaces (if they are named, too)
+	for i := 0; i < iface.NumEmbeddeds(); i++ {
+		// embedding of interfaces cannot have cycles; recursion will terminate
+		if etype, _ := iface.EmbeddedType(i).(*types.Named); etype != nil {
+			verifyInterfaceMethodRecvs(t, etype, level+1)
+		}
+	}
+}
 func TestIssue5815(t *testing.T) {
 	skipSpecialPlatforms(t)
 
@@ -504,6 +556,27 @@ func TestIssue20046(t *testing.T) {
 	}
 }
 
+func TestIssue25301(t *testing.T) {
+	skipSpecialPlatforms(t)
+
+	// This package only handles gc export data.
+	if runtime.Compiler != "gc" {
+		t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
+	}
+
+	// On windows, we have to set the -D option for the compiler to avoid having a drive
+	// letter and an illegal ':' in the import path - just skip it (see also issue #3483).
+	if runtime.GOOS == "windows" {
+		t.Skip("avoid dealing with relative paths/drive letters on windows")
+	}
+
+	if f := compile(t, "testdata", "issue25301.go"); f != "" {
+		defer os.Remove(f)
+	}
+
+	importPkg(t, "./testdata/issue25301")
+}
+
 func importPkg(t *testing.T, path string) *types.Package {
 	pkg, err := Import(make(map[string]*types.Package), path, ".")
 	if err != nil {

go/internal/gcimporter/iimport.go

@@ -12,6 +12,7 @@ package gcimporter
 import (
 	"bytes"
 	"encoding/binary"
+	"fmt"
 	"go/constant"
 	"go/token"
 	"go/types"
@@ -57,18 +58,30 @@ const (
 	interfaceType
 )
 
-// iImportData imports a package from the serialized package data
+// IImportData imports a package from the serialized package data
 // and returns the number of bytes consumed and a reference to the package.
 // If the export data version is not recognized or the format is otherwise
 // compromised, an error is returned.
-func iImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) {
+func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) {
+	const currentVersion = 0
+	version := -1
+	defer func() {
+		if e := recover(); e != nil {
+			if version > currentVersion {
+				err = fmt.Errorf("cannot import %q (%v), export data is newer version - update tool", path, e)
+			} else {
+				err = fmt.Errorf("cannot import %q (%v), possibly version skew - reinstall package", path, e)
+			}
+		}
+	}()
+
 	r := &intReader{bytes.NewReader(data), path}
 
-	version := r.uint64()
+	version = int(r.uint64())
 	switch version {
-	case 0:
+	case currentVersion:
 	default:
-		errorf("cannot import %q: unknown iexport format version %d", path, version)
+		errorf("unknown iexport format version %d", version)
 	}
 
 	sLen := int64(r.uint64())
@@ -502,10 +515,10 @@ func (r *importReader) doType(base *types.Named) types.Type {
 	case interfaceType:
 		r.currPkg = r.pkg()
 
-		embeddeds := make([]*types.Named, r.uint64())
+		embeddeds := make([]types.Type, r.uint64())
 		for i := range embeddeds {
 			_ = r.pos()
-			embeddeds[i] = r.typ().(*types.Named)
+			embeddeds[i] = r.typ()
 		}
 
 		methods := make([]*types.Func, r.uint64())
@@ -524,7 +537,7 @@ func (r *importReader) doType(base *types.Named) types.Type {
 			methods[i] = types.NewFunc(mpos, r.currPkg, mname, msig)
 		}
 
-		typ := types.NewInterface(methods, embeddeds)
+		typ := types.NewInterface2(methods, embeddeds)
 		r.p.interfaceList = append(r.p.interfaceList, typ)
 		return typ
 	}

go/internal/gcimporter/testdata/issue25301.go

@@ -0,0 +1,17 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package issue25301
+
+type (
+	A = interface {
+		M()
+	}
+	T interface {
+		A
+	}
+	S struct{}
+)
+
+func (S) M() { println("m") }

go/internal/gcimporter/testdata/versions/test.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// This file is a copy of $GOROOT/src/go/internal/gcimporter/testdata/versions.test.go.
+
 // To create a test case for a new export format version,
 // build this package with the latest compiler and store
 // the resulting .a file appropriately named in the versions
@@ -11,7 +13,10 @@
 //
 // go build -o test_go1.$X_$Y.a test.go
 //
-// with $X = Go version and $Y = export format version.
+// with $X = Go version and $Y = export format version
+// (add 'b' or 'i' to distinguish between binary and
+// indexed format starting with 1.11 as long as both
+// formats are supported).
 //
 // Make sure this source is extended such that it exercises
 // whatever export format change has taken place.