go/internal/gccgoimporter: update package to match std lib version

This CL makes sure it matches the original code in the std lib
but for the necessary changes to make the code work in x/tools
and with older versions of the std lib.

Notably, it brings over changes from https://golang.org/cl/119895
which were not ported to x/tools.

To simplify future comparisons with the original, streamlined
some comments.

Fixes golang/go#27891.

Change-Id: Iff48c11cb7f0f8a55b4ea33321c686f9d5c707c7
Reviewed-on: https://go-review.googlesource.com/c/142893
Reviewed-by: Alan Donovan <adonovan@google.com>

go/internal/gccgoimporter/ar.go

@@ -0,0 +1,151 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Except for this comment, this file is a verbatim copy of the file
+// with the same name in $GOROOT/src/go/internal/gccgoimporter.
+
+package gccgoimporter
+
+import (
+	"bytes"
+	"debug/elf"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+)
+
+// Magic strings for different archive file formats.
+const (
+	armag  = "!<arch>\n"
+	armagt = "!<thin>\n"
+	armagb = "<bigaf>\n"
+)
+
+// Offsets and sizes for fields in a standard archive header.
+const (
+	arNameOff  = 0
+	arNameSize = 16
+	arDateOff  = arNameOff + arNameSize
+	arDateSize = 12
+	arUIDOff   = arDateOff + arDateSize
+	arUIDSize  = 6
+	arGIDOff   = arUIDOff + arUIDSize
+	arGIDSize  = 6
+	arModeOff  = arGIDOff + arGIDSize
+	arModeSize = 8
+	arSizeOff  = arModeOff + arModeSize
+	arSizeSize = 10
+	arFmagOff  = arSizeOff + arSizeSize
+	arFmagSize = 2
+
+	arHdrSize = arFmagOff + arFmagSize
+)
+
+// The contents of the fmag field of a standard archive header.
+const arfmag = "`\n"
+
+// arExportData takes an archive file and returns a ReadSeeker for the
+// export data in that file. This assumes that there is only one
+// object in the archive containing export data, which is not quite
+// what gccgo does; gccgo concatenates together all the export data
+// for all the objects in the file.  In practice that case does not arise.
+func arExportData(archive io.ReadSeeker) (io.ReadSeeker, error) {
+	if _, err := archive.Seek(0, io.SeekStart); err != nil {
+		return nil, err
+	}
+
+	var buf [len(armag)]byte
+	if _, err := archive.Read(buf[:]); err != nil {
+		return nil, err
+	}
+
+	switch string(buf[:]) {
+	case armag:
+		return standardArExportData(archive)
+	case armagt:
+		return nil, errors.New("unsupported thin archive")
+	case armagb:
+		return nil, errors.New("unsupported AIX big archive")
+	default:
+		return nil, fmt.Errorf("unrecognized archive file format %q", buf[:])
+	}
+}
+
+// standardArExportData returns export data form a standard archive.
+func standardArExportData(archive io.ReadSeeker) (io.ReadSeeker, error) {
+	off := int64(len(armag))
+	for {
+		var hdrBuf [arHdrSize]byte
+		if _, err := archive.Read(hdrBuf[:]); err != nil {
+			return nil, err
+		}
+		off += arHdrSize
+
+		if bytes.Compare(hdrBuf[arFmagOff:arFmagOff+arFmagSize], []byte(arfmag)) != 0 {
+			return nil, fmt.Errorf("archive header format header (%q)", hdrBuf[:])
+		}
+
+		size, err := strconv.ParseInt(strings.TrimSpace(string(hdrBuf[arSizeOff:arSizeOff+arSizeSize])), 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing size in archive header (%q): %v", hdrBuf[:], err)
+		}
+
+		fn := hdrBuf[arNameOff : arNameOff+arNameSize]
+		if fn[0] == '/' && (fn[1] == ' ' || fn[1] == '/' || bytes.Compare(fn[:8], []byte("/SYM64/ ")) == 0) {
+			// Archive symbol table or extended name table,
+			// which we don't care about.
+		} else {
+			archiveAt := readerAtFromSeeker(archive)
+			ret, err := elfFromAr(io.NewSectionReader(archiveAt, off, size))
+			if ret != nil || err != nil {
+				return ret, err
+			}
+		}
+
+		if size&1 != 0 {
+			size++
+		}
+		off += size
+		if _, err := archive.Seek(off, io.SeekStart); err != nil {
+			return nil, err
+		}
+	}
+}
+
+// elfFromAr tries to get export data from an archive member as an ELF file.
+// If there is no export data, this returns nil, nil.
+func elfFromAr(member *io.SectionReader) (io.ReadSeeker, error) {
+	ef, err := elf.NewFile(member)
+	if err != nil {
+		return nil, err
+	}
+	sec := ef.Section(".go_export")
+	if sec == nil {
+		return nil, nil
+	}
+	return sec.Open(), nil
+}
+
+// readerAtFromSeeker turns an io.ReadSeeker into an io.ReaderAt.
+// This is only safe because there won't be any concurrent seeks
+// while this code is executing.
+func readerAtFromSeeker(rs io.ReadSeeker) io.ReaderAt {
+	if ret, ok := rs.(io.ReaderAt); ok {
+		return ret
+	}
+	return seekerReadAt{rs}
+}
+
+type seekerReadAt struct {
+	seeker io.ReadSeeker
+}
+
+func (sra seekerReadAt) ReadAt(p []byte, off int64) (int, error) {
+	if _, err := sra.seeker.Seek(off, io.SeekStart); err != nil {
+		return 0, err
+	}
+	return sra.seeker.Read(p)
+}

go/internal/gccgoimporter/backdoor.go

@@ -2,10 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package gccgoimporter
-
 // This file opens a back door to the parser for golang.org/x/tools/go/gccgoexportdata.
 
+package gccgoimporter
+
 import (
 	"go/types"
 	"io"

go/internal/gccgoimporter/gccgoinstallation.go

@@ -2,9 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package gccgoimporter
+// Except for this comment, this file is a verbatim copy of the file
+// with the same name in $GOROOT/src/go/internal/gccgoimporter.
 
-// This is a verbatim copy of $GOROOT/src/go/internal/gccgoimporter/gccgoinstallation.go.
+package gccgoimporter
 
 import (
 	"bufio"
@@ -28,7 +29,7 @@ type GccgoInstallation struct {
 }
 
 // Ask the driver at the given path for information for this GccgoInstallation.
-// The given arguments are passed directly to the call to the driver.
+// The given arguments are passed directly to the call of the driver.
 func (inst *GccgoInstallation) InitFromDriver(gccgoPath string, args ...string) (err error) {
 	argv := append([]string{"-###", "-S", "-x", "go", "-"}, args...)
 	cmd := exec.Command(gccgoPath, argv...)

go/internal/gccgoimporter/gccgoinstallation_test.go

@@ -2,9 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package gccgoimporter
+// Except for this comment, this file is a verbatim copy of the file
+// with the same name in $GOROOT/src/go/internal/gccgoimporter.
 
-// This is a verbatim copy of $GOROOT/src/go/internal/gccgoimporter/gccgoinstallation_test.go.
+package gccgoimporter
 
 import (
 	"go/types"
@@ -150,7 +151,6 @@ func TestInstallationImporter(t *testing.T) {
 	// were compiled with gccgo.
 	if runtime.Compiler != "gccgo" {
 		t.Skip("This test needs gccgo")
-		return
 	}
 
 	var inst GccgoInstallation
@@ -164,14 +164,14 @@ func TestInstallationImporter(t *testing.T) {
 	// all packages into the same map and then each individually.
 	pkgMap := make(map[string]*types.Package)
 	for _, pkg := range importablePackages {
-		_, err = imp(pkgMap, pkg)
+		_, err = imp(pkgMap, pkg, ".", nil)
 		if err != nil {
 			t.Error(err)
 		}
 	}
 
 	for _, pkg := range importablePackages {
-		_, err = imp(make(map[string]*types.Package), pkg)
+		_, err = imp(make(map[string]*types.Package), pkg, ".", nil)
 		if err != nil {
 			t.Error(err)
 		}
@@ -179,12 +179,12 @@ func TestInstallationImporter(t *testing.T) {
 
 	// Test for certain specific entities in the imported data.
 	for _, test := range [...]importerTest{
-		{pkgpath: "io", name: "Reader", want: "type Reader interface{Read(p []byte) (n int, err error)}"},
+		{pkgpath: "io", name: "Reader", want: "type Reader interface{Read(p []uint8) (n int, err error)}"},
 		{pkgpath: "io", name: "ReadWriter", want: "type ReadWriter interface{Reader; Writer}"},
 		{pkgpath: "math", name: "Pi", want: "const Pi untyped float"},
 		{pkgpath: "math", name: "Sin", want: "func Sin(x float64) float64"},
 		{pkgpath: "sort", name: "Ints", want: "func Ints(a []int)"},
-		{pkgpath: "unsafe", name: "Pointer", want: "type Pointer"},
+		{pkgpath: "unsafe", name: "Pointer", want: "type Pointer unsafe.Pointer"},
 	} {
 		runImporterTest(t, imp, nil, &test)
 	}

go/internal/gccgoimporter/importer.go

@@ -2,19 +2,18 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// Except for this comment and the import path, this file is a verbatim copy of the file
+// with the same name in $GOROOT/src/go/internal/gccgoimporter.
+
 // Package gccgoimporter implements Import for gccgo-generated object files.
 package gccgoimporter // import "golang.org/x/tools/go/internal/gccgoimporter"
 
-// This is a verbatim copy of $GOROOT/src/go/internal/gccgoimporter/importer.go.
-
 import (
-	"bytes"
 	"debug/elf"
 	"fmt"
 	"go/types"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 )
@@ -100,18 +99,8 @@ func openExportFile(fpath string) (reader io.ReadSeeker, closer io.Closer, err e
 		return
 
 	case archiveMagic:
-		// TODO(pcc): Read the archive directly instead of using "ar".
+		reader, err = arExportData(f)
-		f.Close()
-		closer = nil
-
-		cmd := exec.Command("ar", "p", fpath)
-		var out []byte
-		out, err = cmd.Output()
-		if err != nil {
 		return
-		}
-
-		elfreader = bytes.NewReader(out)
 
 	default:
 		elfreader = f
@@ -139,38 +128,76 @@ func openExportFile(fpath string) (reader io.ReadSeeker, closer io.Closer, err e
 // the map entry. Otherwise, the importer must load the package data for the
 // given path into a new *Package, record it in imports map, and return the
 // package.
-type Importer func(imports map[string]*types.Package, path string) (*types.Package, error)
+type Importer func(imports map[string]*types.Package, path, srcDir string, lookup func(string) (io.ReadCloser, error)) (*types.Package, error)
 
 func GetImporter(searchpaths []string, initmap map[*types.Package]InitData) Importer {
-	return func(imports map[string]*types.Package, pkgpath string) (pkg *types.Package, err error) {
+	return func(imports map[string]*types.Package, pkgpath, srcDir string, lookup func(string) (io.ReadCloser, error)) (pkg *types.Package, err error) {
+		// TODO(gri): Use srcDir.
+		// Or not. It's possible that srcDir will fade in importance as
+		// the go command and other tools provide a translation table
+		// for relative imports (like ./foo or vendored imports).
 		if pkgpath == "unsafe" {
 			return types.Unsafe, nil
 		}
 
-		fpath, err := findExportFile(searchpaths, pkgpath)
+		var reader io.ReadSeeker
+		var fpath string
+		var rc io.ReadCloser
+		if lookup != nil {
+			if p := imports[pkgpath]; p != nil && p.Complete() {
+				return p, nil
+			}
+			rc, err = lookup(pkgpath)
 			if err != nil {
-			return
+				return nil, err
+			}
+		}
+		if rc != nil {
+			defer rc.Close()
+			rs, ok := rc.(io.ReadSeeker)
+			if !ok {
+				return nil, fmt.Errorf("gccgo importer requires lookup to return an io.ReadSeeker, have %T", rc)
+			}
+			reader = rs
+			fpath = "<lookup " + pkgpath + ">"
+			// Take name from Name method (like on os.File) if present.
+			if n, ok := rc.(interface{ Name() string }); ok {
+				fpath = n.Name()
+			}
+		} else {
+			fpath, err = findExportFile(searchpaths, pkgpath)
+			if err != nil {
+				return nil, err
 			}
 
-		reader, closer, err := openExportFile(fpath)
+			r, closer, err := openExportFile(fpath)
 			if err != nil {
-			return
+				return nil, err
 			}
 			if closer != nil {
 				defer closer.Close()
 			}
-
+			reader = r
-		var magic [4]byte
-		_, err = reader.Read(magic[:])
-		if err != nil {
-			return
 		}
-		_, err = reader.Seek(0, 0) // 0 bytes after start (use 0, io.SeekStart when we drop Go 1.6)
+
+		var magics string
+		magics, err = readMagic(reader)
 		if err != nil {
 			return
 		}
 
-		switch string(magic[:]) {
+		if magics == archiveMagic {
+			reader, err = arExportData(reader)
+			if err != nil {
+				return
+			}
+			magics, err = readMagic(reader)
+			if err != nil {
+				return
+			}
+		}
+
+		switch magics {
 		case gccgov1Magic, gccgov2Magic:
 			var p parser
 			p.init(fpath, reader, imports)
@@ -201,9 +228,22 @@ func GetImporter(searchpaths []string, initmap map[*types.Package]InitData) Impo
 		// 	}
 
 		default:
-			err = fmt.Errorf("unrecognized magic string: %q", string(magic[:]))
+			err = fmt.Errorf("unrecognized magic string: %q", magics)
 		}
 
 		return
 	}
 }
+
+// readMagic reads the four bytes at the start of a ReadSeeker and
+// returns them as a string.
+func readMagic(reader io.ReadSeeker) (string, error) {
+	var magic [4]byte
+	if _, err := reader.Read(magic[:]); err != nil {
+		return "", err
+	}
+	if _, err := reader.Seek(0, io.SeekStart); err != nil {
+		return "", err
+	}
+	return string(magic[:]), nil
+}

go/internal/gccgoimporter/importer_test.go

@@ -2,7 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// This is an almost verbatim copy of $GOROOT/src/go/internal/gccgoimporter/importer_test.go.
+// Except for this comment, this file is a verbatim copy of the file
+// with the same name in $GOROOT/src/go/internal/gccgoimporter without
+// the import of "testenv".
 
 package gccgoimporter
 
@@ -22,7 +24,7 @@ type importerTest struct {
 }
 
 func runImporterTest(t *testing.T, imp Importer, initmap map[*types.Package]InitData, test *importerTest) {
-	pkg, err := imp(make(map[string]*types.Package), test.pkgpath)
+	pkg, err := imp(make(map[string]*types.Package), test.pkgpath, ".", nil)
 	if err != nil {
 		t.Error(err)
 		return
@@ -90,7 +92,7 @@ func runImporterTest(t *testing.T, imp Importer, initmap map[*types.Package]Init
 	}
 }
 
-var importerTests = []importerTest{
+var importerTests = [...]importerTest{
 	{pkgpath: "pointer", name: "Int8Ptr", want: "type Int8Ptr *int8"},
 	{pkgpath: "complexnums", name: "NN", want: "const NN untyped complex", wantval: "(-1 + -1i)"},
 	{pkgpath: "complexnums", name: "NP", want: "const NP untyped complex", wantval: "(-1 + 1i)"},
@@ -102,6 +104,7 @@ var importerTests = []importerTest{
 	{pkgpath: "unicode", name: "IsUpper", want: "func IsUpper(r rune) bool"},
 	{pkgpath: "unicode", name: "MaxRune", want: "const MaxRune untyped rune", wantval: "1114111"},
 	{pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import", "math..import"}},
+	{pkgpath: "importsar", name: "Hello", want: "var Hello string"},
 	{pkgpath: "aliases", name: "A14", want: "type A14 = func(int, T0) chan T2"},
 	{pkgpath: "aliases", name: "C0", want: "type C0 struct{f1 C1; f2 C1}"},
 	{pkgpath: "escapeinfo", name: "NewT", want: "func NewT(data []byte) *T"},
@@ -126,7 +129,6 @@ func TestObjImporter(t *testing.T) {
 	// were compiled with gccgo.
 	if runtime.Compiler != "gccgo" {
 		t.Skip("This test needs gccgo")
-		return
 	}
 
 	tmpdir, err := ioutil.TempDir("", "")
@@ -145,13 +147,6 @@ func TestObjImporter(t *testing.T) {
 
 	for _, test := range importerTests {
 		gofile := filepath.Join("testdata", test.pkgpath+".go")
-
-		if _, err := os.Stat(gofile); err != nil {
-			// There is a .gox file but no .go file,
-			// so there is nothing to compile.
-			continue
-		}
-
 		ofile := filepath.Join(tmpdir, test.pkgpath+".o")
 		afile := filepath.Join(artmpdir, "lib"+test.pkgpath+".a")
 
@@ -162,10 +157,6 @@ func TestObjImporter(t *testing.T) {
 			t.Fatalf("gccgo %s failed: %s", gofile, err)
 		}
 
-		// The expected initializations are version dependent,
-		// so don't check for them.
-		test.wantinits = nil
-
 		runImporterTest(t, imp, initmap, &test)
 
 		cmd = exec.Command("ar", "cr", afile, ofile)

go/internal/gccgoimporter/parser.go

@@ -2,8 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// This is a verbatim copy of $GOROOT/src/go/internal/gccgoimporter/parser.go
+// Except for this comment, this file is a verbatim copy of the file
-// with a small modification in parseInterface to support older Go versions.
+// with the same name in $GOROOT/src/go/internal/gccgoimporter, with
+// a small modification in parseInterface to support older Go versions.
 
 package gccgoimporter
 

go/internal/gccgoimporter/parser_test.go

@@ -2,9 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package gccgoimporter
+// Except for this comment, this file is a verbatim copy of the file
+// with the same name in $GOROOT/src/go/internal/gccgoimporter.
 
-// This is a verbatim copy of $GOROOT/src/go/internal/gccgoimporter/parser_test.go.
+package gccgoimporter
 
 import (
 	"bytes"

go/internal/gccgoimporter/testenv_test.go

@@ -2,10 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package gccgoimporter
-
 // This file contains testing utilities copied from $GOROOT/src/internal/testenv/testenv.go.
 
+package gccgoimporter
+
 import (
 	"runtime"
 	"strings"