go/gcimporter15: update to match latest changes to std lib gcimporter

Mostly a backport of https://go-review.googlesource.com/27816.
Includes an update to TestIssue13898 to match the std lib gcimporter
test.

Change-Id: I790123a49deb2d52f19e51c84bbc7ee74c99156e
Reviewed-on: https://go-review.googlesource.com/27913
Reviewed-by: Matthew Dempsky <mdempsky@google.com>

go/gcimporter15/bimport.go

@@ -15,6 +15,7 @@ import (
 	"go/token"
 	"go/types"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"unicode"
@@ -48,19 +49,30 @@ type importer struct {
 
 // BImportData imports a package from the serialized package data
 // and returns the number of bytes consumed and a reference to the package.
-// If data is obviously malformed, an error is returned but in
-// general it is not recommended to call BImportData on untrusted data.
-func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) {
+// If the export data version is not recognized or the format is otherwise
+// compromised, an error is returned.
+func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, _ *types.Package, err error) {
+	// catch panics and return them as errors
+	defer func() {
+		if e := recover(); e != nil {
+			// The package (filename) causing the problem is added to this
+			// error by a wrapper in the caller (Import in gcimporter.go).
+			err = fmt.Errorf("cannot import, possibly version skew (%v) - reinstall package", e)
+		}
+	}()
+
 	p := importer{
 		imports: imports,
 		data:    data,
 		path:    path,
+		version: -1,           // unknown version
 		strList: []string{""}, // empty string is mapped to 0
 		fset:    fset,
 		files:   make(map[string]*token.File),
 	}
 
 	// read version info
+	var versionstr string
 	if b := p.rawByte(); b == 'c' || b == 'd' {
 		// Go1.7 encoding; first byte encodes low-level
 		// encoding format (compact vs debug).
@@ -73,21 +85,34 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 		}
 		p.trackAllTypes = p.rawByte() == 'a'
 		p.posInfoFormat = p.int() != 0
-		const go17version = "v1"
-		if s := p.string(); s != go17version {
-			return p.read, nil, fmt.Errorf("importer: unknown export data format: %s (imported package compiled with old compiler?)", s)
-		}
+		versionstr = p.string()
+		if versionstr == "v1" {
 			p.version = 0
+		}
 	} else {
 		// Go1.8 extensible encoding
-		const exportVersion = "version 1"
-		if s := p.rawStringln(b); s != exportVersion {
-			return p.read, nil, fmt.Errorf("importer: unknown export data format: %s (imported package compiled with old compiler?)", s)
+		// read version string and extract version number (ignore anything after the version number)
+		versionstr = p.rawStringln(b)
+		if s := strings.SplitN(versionstr, " ", 3); len(s) >= 2 && s[0] == "version" {
+			if v, err := strconv.Atoi(s[1]); err == nil && v > 0 {
+				p.version = v
 			}
-		p.version = 1
+		}
+	}
+
+	// read version specific flags - extend as necessary
+	switch p.version {
+	// case 2:
+	// 	...
+	//	fallthrough
+	case 1:
 		p.debugFormat = p.rawStringln(p.rawByte()) == "debug"
 		p.trackAllTypes = p.int() != 0
 		p.posInfoFormat = p.int() != 0
+	case 0:
+		// Go1.7 encoding format - nothing to do here
+	default:
+		errorf("unknown export format version %d (%q)", p.version, versionstr)
 	}
 
 	// --- generic export data ---
@@ -111,7 +136,7 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 
 	// self-verification
 	if count := p.int(); count != objcount {
-		panic(fmt.Sprintf("got %d objects; want %d", objcount, count))
+		errorf("got %d objects; want %d", objcount, count)
 	}
 
 	// ignore compiler-specific import data
@@ -139,6 +164,10 @@ func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 	return p.read, pkg, nil
 }
 
+func errorf(format string, args ...interface{}) {
+	panic(fmt.Sprintf(format, args...))
+}
+
 func (p *importer) pkg() *types.Package {
 	// if the package was seen before, i is its index (>= 0)
 	i := p.tagOrIndex()
@@ -148,7 +177,7 @@ func (p *importer) pkg() *types.Package {
 
 	// otherwise, i is the package tag (< 0)
 	if i != packageTag {
-		panic(fmt.Sprintf("unexpected package tag %d", i))
+		errorf("unexpected package tag %d", i)
 	}
 
 	// read package data
@@ -163,7 +192,7 @@ func (p *importer) pkg() *types.Package {
 	// an empty path denotes the package we are currently importing;
 	// it must be the first package we see
 	if (path == "") != (len(p.pkgList) == 0) {
-		panic(fmt.Sprintf("package path %q for pkg index %d", path, len(p.pkgList)))
+		errorf("package path %q for pkg index %d", path, len(p.pkgList))
 	}
 
 	// if the package was imported before, use that one; otherwise create a new one
@@ -175,7 +204,7 @@ func (p *importer) pkg() *types.Package {
 		pkg = types.NewPackage(path, name)
 		p.imports[path] = pkg
 	} else if pkg.Name() != name {
-		panic(fmt.Sprintf("conflicting names %s and %s for package %q", pkg.Name(), name, path))
+		errorf("conflicting names %s and %s for package %q", pkg.Name(), name, path)
 	}
 	p.pkgList = append(p.pkgList, pkg)
 
@@ -192,7 +221,7 @@ func (p *importer) declare(obj types.Object) {
 		// imported.
 		// (See also the comment in cmd/compile/internal/gc/bimport.go importer.obj,
 		// switch case importing functions).
-		panic(fmt.Sprintf("inconsistent import:\n\t%v\npreviously imported as:\n\t%v\n", alt, obj))
+		errorf("inconsistent import:\n\t%v\npreviously imported as:\n\t%v\n", alt, obj)
 	}
 }
 
@@ -223,7 +252,7 @@ func (p *importer) obj(tag int) {
 		p.declare(types.NewFunc(pos, pkg, name, sig))
 
 	default:
-		panic(fmt.Sprintf("unexpected object tag %d", tag))
+		errorf("unexpected object tag %d", tag)
 	}
 }
 
@@ -326,7 +355,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
 		}
 
 		if _, ok := obj.(*types.TypeName); !ok {
-			panic(fmt.Sprintf("pkg = %s, name = %s => %s", parent, name, obj))
+			errorf("pkg = %s, name = %s => %s", parent, name, obj)
 		}
 
 		// associate new named type with obj if it doesn't exist yet
@@ -469,14 +498,15 @@ func (p *importer) typ(parent *types.Package) types.Type {
 		case 3 /* Cboth */ :
 			dir = types.SendRecv
 		default:
-			panic(fmt.Sprintf("unexpected channel dir %d", d))
+			errorf("unexpected channel dir %d", d)
 		}
 		val := p.typ(parent)
 		*t = *types.NewChan(dir, val)
 		return t
 
 	default:
-		panic(fmt.Sprintf("unexpected type tag %d", i))
+		errorf("unexpected type tag %d", i)
+		panic("unreachable")
 	}
 }
 
@@ -541,9 +571,8 @@ func (p *importer) fieldName(parent *types.Package) (*types.Package, string) {
 		// use the imported package instead
 		pkg = p.pkgList[0]
 	}
-	if p.version < 1 && name == "_" {
-		// versions < 1 don't export a package for _ fields
-		// TODO: remove once versions are not supported anymore
+	if p.version == 0 && name == "_" {
+		// version 0 didn't export a package for _ fields
 		return pkg, name
 	}
 	if name != "" && !exported(name) {
@@ -627,7 +656,8 @@ func (p *importer) value() constant.Value {
 	case unknownTag:
 		return constant.MakeUnknown()
 	default:
-		panic(fmt.Sprintf("unexpected value tag %d", tag))
+		errorf("unexpected value tag %d", tag)
+		panic("unreachable")
 	}
 }
 
@@ -729,12 +759,12 @@ func (p *importer) string() string {
 
 func (p *importer) marker(want byte) {
 	if got := p.rawByte(); got != want {
-		panic(fmt.Sprintf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.read))
+		errorf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.read)
 	}
 
 	pos := p.read
 	if n := int(p.rawInt64()); n != pos {
-		panic(fmt.Sprintf("incorrect position: got %d; want %d", n, pos))
+		errorf("incorrect position: got %d; want %d", n, pos)
 	}
 }
 
@@ -742,7 +772,7 @@ func (p *importer) marker(want byte) {
 func (p *importer) rawInt64() int64 {
 	i, err := binary.ReadVarint(p)
 	if err != nil {
-		panic(fmt.Sprintf("read error: %v", err))
+		errorf("read error: %v", err)
 	}
 	return i
 }

go/gcimporter15/gcimporter_test.go

@@ -10,6 +10,7 @@
 package gcimporter
 
 import (
+	"bytes"
 	"fmt"
 	"go/types"
 	"io/ioutil"
@@ -157,6 +158,70 @@ func TestImportTestdata(t *testing.T) {
 	}
 }
 
+func TestVersionHandling(t *testing.T) {
+	skipSpecialPlatforms(t) // we really only need to exclude nacl platforms, but this is fine
+
+	// This package only handles gc export data.
+	if runtime.Compiler != "gc" {
+		t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
+		return
+	}
+
+	const dir = "./testdata/versions"
+	list, err := ioutil.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, f := range list {
+		name := f.Name()
+		if !strings.HasSuffix(name, ".a") {
+			continue // not a package file
+		}
+		if strings.Contains(name, "corrupted") {
+			continue // don't process a leftover corrupted file
+		}
+		pkgpath := "./" + name[:len(name)-2]
+
+		// test that export data can be imported
+		_, err := Import(make(map[string]*types.Package), pkgpath, dir)
+		if err != nil {
+			t.Errorf("import %q failed: %v", pkgpath, err)
+			continue
+		}
+
+		// create file with corrupted export data
+		// 1) read file
+		data, err := ioutil.ReadFile(filepath.Join(dir, name))
+		if err != nil {
+			t.Fatal(err)
+		}
+		// 2) find export data
+		i := bytes.Index(data, []byte("\n$$B\n")) + 5
+		j := bytes.Index(data[i:], []byte("\n$$\n")) + i
+		if i < 0 || j < 0 || i > j {
+			t.Fatalf("export data section not found (i = %d, j = %d)", i, j)
+		}
+		// 3) corrupt the data (increment every 7th byte)
+		for k := j - 13; k >= i; k -= 7 {
+			data[k]++
+		}
+		// 4) write the file
+		pkgpath += "_corrupted"
+		filename := filepath.Join(dir, pkgpath) + ".a"
+		ioutil.WriteFile(filename, data, 0666)
+		defer os.Remove(filename)
+
+		// test that importing the corrupted file results in an error
+		_, err = Import(make(map[string]*types.Package), pkgpath, dir)
+		if err == nil {
+			t.Errorf("import corrupted %q succeeded", pkgpath)
+		} else if msg := err.Error(); !strings.Contains(msg, "version skew") {
+			t.Errorf("import %q error incorrect (%s)", pkgpath, msg)
+		}
+	}
+}
+
 func TestImportStdLib(t *testing.T) {
 	skipSpecialPlatforms(t)
 
@@ -353,9 +418,9 @@ func TestIssue13898(t *testing.T) {
 	}
 
 	// lookup go/types.Object.Pkg method
-	m, _, _ := types.LookupFieldOrMethod(typ, false, nil, "Pkg")
+	m, index, indirect := types.LookupFieldOrMethod(typ, false, nil, "Pkg")
 	if m == nil {
-		t.Fatal("go/types.Object.Pkg not found")
+		t.Fatalf("go/types.Object.Pkg not found (index = %v, indirect = %v)", index, indirect)
 	}
 
 	// the method must belong to go/types

go/gcimporter15/testdata/versions/test.go

@@ -0,0 +1,25 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// To create a test case for a new export format version,
+// build this package with the latest compiler and store
+// the resulting .a file appropriately named in the versions
+// directory. The VersionHandling test will pick it up.
+//
+// In the testdata/versions:
+//
+// go build -o test_go1.$X_$Y.a test.go
+//
+// with $X = Go version and $Y = export format version.
+//
+// Make sure this source is extended such that it exercises
+// whatever export format change has taken place.
+
+package test
+
+// Any release before and including Go 1.7 didn't encode
+// the package for a blank struct field.
+type BlankField struct {
+	_ int
+}