diff --git a/go/gcimporter15/bimport.go b/go/gcimporter15/bimport.go new file mode 100644 index 00000000..525e12b8 --- /dev/null +++ b/go/gcimporter15/bimport.go @@ -0,0 +1,685 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.5 + +// This file is a copy of $GOROOT/src/go/internal/gcimporter/bimport.go, tagged for go1.5. + +package gcimporter + +import ( + "encoding/binary" + "fmt" + "go/constant" + "go/token" + "go/types" + "sort" + "unicode" + "unicode/utf8" +) + +// BImportData imports a package from the serialized package data +// and returns the number of bytes consumed and a reference to the package. +// If data is obviously malformed, an error is returned but in +// general it is not recommended to call BImportData on untrusted data. +func BImportData(imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) { + p := importer{ + imports: imports, + data: data, + } + p.buf = p.bufarray[:] + + // read low-level encoding format + switch format := p.byte(); format { + case 'c': + // compact format - nothing to do + case 'd': + p.debugFormat = true + default: + return p.read, nil, fmt.Errorf("invalid encoding format in export data: got %q; want 'c' or 'd'", format) + } + + // --- generic export data --- + + if v := p.string(); v != "v0" { + return p.read, nil, fmt.Errorf("unknown version: %s", v) + } + + // populate typList with predeclared "known" types + p.typList = append(p.typList, predeclared...) + + // read package data + // TODO(gri) clean this up + i := p.tagOrIndex() + if i != packageTag { + panic(fmt.Sprintf("package tag expected, got %d", i)) + } + name := p.string() + if s := p.string(); s != "" { + panic(fmt.Sprintf("empty path expected, got %s", s)) + } + pkg := p.imports[path] + if pkg == nil { + pkg = types.NewPackage(path, name) + p.imports[path] = pkg + } + p.pkgList = append(p.pkgList, pkg) + + if debug && p.pkgList[0] != pkg { + panic("imported packaged not found in pkgList[0]") + } + + // read compiler-specific flags + p.string() // discard + + // read consts + for i := p.int(); i > 0; i-- { + name := p.string() + typ := p.typ(nil) + val := p.value() + p.declare(types.NewConst(token.NoPos, pkg, name, typ, val)) + } + + // read vars + for i := p.int(); i > 0; i-- { + name := p.string() + typ := p.typ(nil) + p.declare(types.NewVar(token.NoPos, pkg, name, typ)) + } + + // read funcs + for i := p.int(); i > 0; i-- { + name := p.string() + sig := p.typ(nil).(*types.Signature) + p.int() // read and discard index of inlined function body + p.declare(types.NewFunc(token.NoPos, pkg, name, sig)) + } + + // read types + for i := p.int(); i > 0; i-- { + // name is parsed as part of named type and the + // type object is added to scope via respective + // named type + _ = p.typ(nil).(*types.Named) + } + + // ignore compiler-specific import data + + // complete interfaces + for _, typ := range p.typList { + if it, ok := typ.(*types.Interface); ok { + it.Complete() + } + } + + // record all referenced packages as imports + list := append(([]*types.Package)(nil), p.pkgList[1:]...) + sort.Sort(byPath(list)) + pkg.SetImports(list) + + // package was imported completely and without errors + pkg.MarkComplete() + + return p.read, pkg, nil +} + +type importer struct { + imports map[string]*types.Package + data []byte + buf []byte // for reading strings + bufarray [64]byte // initial underlying array for buf, large enough to avoid allocation when compiling std lib + pkgList []*types.Package + typList []types.Type + + debugFormat bool + read int // bytes read +} + +func (p *importer) declare(obj types.Object) { + if alt := p.pkgList[0].Scope().Insert(obj); alt != nil { + // This can only happen if we import a package a second time. + panic(fmt.Sprintf("%s already declared", alt.Name())) + } +} + +func (p *importer) pkg() *types.Package { + // if the package was seen before, i is its index (>= 0) + i := p.tagOrIndex() + if i >= 0 { + return p.pkgList[i] + } + + // otherwise, i is the package tag (< 0) + if i != packageTag { + panic(fmt.Sprintf("unexpected package tag %d", i)) + } + + // read package data + name := p.string() + path := p.string() + + // we should never see an empty package name + if name == "" { + panic("empty package name in import") + } + + // we should never see an empty import path + if path == "" { + panic("empty import path") + } + + // if the package was imported before, use that one; otherwise create a new one + pkg := p.imports[path] + if pkg == nil { + pkg = types.NewPackage(path, name) + p.imports[path] = pkg + } + p.pkgList = append(p.pkgList, pkg) + + return pkg +} + +func (p *importer) record(t types.Type) { + p.typList = append(p.typList, t) +} + +// A dddSlice is a types.Type representing ...T parameters. +// It only appears for parameter types and does not escape +// the importer. +type dddSlice struct { + elem types.Type +} + +func (t *dddSlice) Underlying() types.Type { return t } +func (t *dddSlice) String() string { return "..." + t.elem.String() } + +// parent is the package which declared the type; parent == nil means +// the package currently imported. The parent package is needed for +// exported struct fields and interface methods which don't contain +// explicit package information in the export data. +func (p *importer) typ(parent *types.Package) types.Type { + // if the type was seen before, i is its index (>= 0) + i := p.tagOrIndex() + if i >= 0 { + return p.typList[i] + } + + // otherwise, i is the type tag (< 0) + switch i { + case namedTag: + // read type object + name := p.string() + parent = p.pkg() + scope := parent.Scope() + obj := scope.Lookup(name) + + // if the object doesn't exist yet, create and insert it + if obj == nil { + obj = types.NewTypeName(token.NoPos, parent, name, nil) + scope.Insert(obj) + } + + if _, ok := obj.(*types.TypeName); !ok { + panic(fmt.Sprintf("pkg = %s, name = %s => %s", parent, name, obj)) + } + + // associate new named type with obj if it doesn't exist yet + t0 := types.NewNamed(obj.(*types.TypeName), nil, nil) + + // but record the existing type, if any + t := obj.Type().(*types.Named) + p.record(t) + + // read underlying type + t0.SetUnderlying(p.typ(parent)) + + // interfaces don't have associated methods + if _, ok := t0.Underlying().(*types.Interface); ok { + return t + } + + // read associated methods + for i := p.int(); i > 0; i-- { + name := p.string() + recv, _ := p.paramList() // TODO(gri) do we need a full param list for the receiver? + params, isddd := p.paramList() + result, _ := p.paramList() + p.int() // read and discard index of inlined function body + sig := types.NewSignature(recv.At(0), params, result, isddd) + t0.AddMethod(types.NewFunc(token.NoPos, parent, name, sig)) + } + + return t + + case arrayTag: + t := new(types.Array) + p.record(t) + + n := p.int64() + *t = *types.NewArray(p.typ(parent), n) + return t + + case sliceTag: + t := new(types.Slice) + p.record(t) + + *t = *types.NewSlice(p.typ(parent)) + return t + + case dddTag: + t := new(dddSlice) + p.record(t) + + t.elem = p.typ(parent) + return t + + case structTag: + t := new(types.Struct) + p.record(t) + + n := p.int() + fields := make([]*types.Var, n) + tags := make([]string, n) + for i := range fields { + fields[i] = p.field(parent) + tags[i] = p.string() + } + *t = *types.NewStruct(fields, tags) + return t + + case pointerTag: + t := new(types.Pointer) + p.record(t) + + *t = *types.NewPointer(p.typ(parent)) + return t + + case signatureTag: + t := new(types.Signature) + p.record(t) + + params, isddd := p.paramList() + result, _ := p.paramList() + *t = *types.NewSignature(nil, params, result, isddd) + return t + + case interfaceTag: + // Create a dummy entry in the type list. This is safe because we + // cannot expect the interface type to appear in a cycle, as any + // such cycle must contain a named type which would have been + // first defined earlier. + n := len(p.typList) + p.record(nil) + + // no embedded interfaces with gc compiler + if p.int() != 0 { + panic("unexpected embedded interface") + } + + // read methods + methods := make([]*types.Func, p.int()) + for i := range methods { + pkg, name := p.fieldName(parent) + params, isddd := p.paramList() + result, _ := p.paramList() + sig := types.NewSignature(nil, params, result, isddd) + methods[i] = types.NewFunc(token.NoPos, pkg, name, sig) + } + + t := types.NewInterface(methods, nil) + p.typList[n] = t + return t + + case mapTag: + t := new(types.Map) + p.record(t) + + key := p.typ(parent) + val := p.typ(parent) + *t = *types.NewMap(key, val) + return t + + case chanTag: + t := new(types.Chan) + p.record(t) + + var dir types.ChanDir + // tag values must match the constants in cmd/compile/internal/gc/go.go + switch d := p.int(); d { + case 1 /* Crecv */ : + dir = types.RecvOnly + case 2 /* Csend */ : + dir = types.SendOnly + case 3 /* Cboth */ : + dir = types.SendRecv + default: + panic(fmt.Sprintf("unexpected channel dir %d", d)) + } + val := p.typ(parent) + *t = *types.NewChan(dir, val) + return t + + default: + panic(fmt.Sprintf("unexpected type tag %d", i)) + } +} + +func (p *importer) field(parent *types.Package) *types.Var { + pkg, name := p.fieldName(parent) + typ := p.typ(parent) + + anonymous := false + if name == "" { + // anonymous field - typ must be T or *T and T must be a type name + switch typ := deref(typ).(type) { + case *types.Basic: // basic types are named types + pkg = nil // // objects defined in Universe scope have no package + name = typ.Name() + case *types.Named: + name = typ.Obj().Name() + default: + panic("anonymous field expected") + } + anonymous = true + } + + return types.NewField(token.NoPos, pkg, name, typ, anonymous) +} + +func (p *importer) fieldName(parent *types.Package) (*types.Package, string) { + pkg := parent + if pkg == nil { + // use the imported package instead + pkg = p.pkgList[0] + } + name := p.string() + if name == "" { + return pkg, "" // anonymous + } + if name == "?" || name != "_" && !exported(name) { + // explicitly qualified field + if name == "?" { + name = "" // anonymous + } + pkg = p.pkg() + } + return pkg, name +} + +func (p *importer) paramList() (*types.Tuple, bool) { + n := p.int() + if n == 0 { + return nil, false + } + // negative length indicates unnamed parameters + named := true + if n < 0 { + n = -n + named = false + } + // n > 0 + params := make([]*types.Var, n) + isddd := false + for i := range params { + params[i], isddd = p.param(named) + } + return types.NewTuple(params...), isddd +} + +func (p *importer) param(named bool) (*types.Var, bool) { + t := p.typ(nil) + td, isddd := t.(*dddSlice) + if isddd { + t = types.NewSlice(td.elem) + } + + var name string + if named { + name = p.string() + if name == "" { + panic("expected named parameter") + } + } + + // read and discard compiler-specific info + p.string() + + return types.NewVar(token.NoPos, nil, name, t), isddd +} + +func exported(name string) bool { + ch, _ := utf8.DecodeRuneInString(name) + return unicode.IsUpper(ch) +} + +func (p *importer) value() constant.Value { + switch tag := p.tagOrIndex(); tag { + case falseTag: + return constant.MakeBool(false) + case trueTag: + return constant.MakeBool(true) + case int64Tag: + return constant.MakeInt64(p.int64()) + case floatTag: + return p.float() + case complexTag: + re := p.float() + im := p.float() + return constant.BinaryOp(re, token.ADD, constant.MakeImag(im)) + case stringTag: + return constant.MakeString(p.string()) + default: + panic(fmt.Sprintf("unexpected value tag %d", tag)) + } +} + +func (p *importer) float() constant.Value { + sign := p.int() + if sign == 0 { + return constant.MakeInt64(0) + } + + exp := p.int() + mant := []byte(p.string()) // big endian + + // remove leading 0's if any + for len(mant) > 0 && mant[0] == 0 { + mant = mant[1:] + } + + // convert to little endian + // TODO(gri) go/constant should have a more direct conversion function + // (e.g., once it supports a big.Float based implementation) + for i, j := 0, len(mant)-1; i < j; i, j = i+1, j-1 { + mant[i], mant[j] = mant[j], mant[i] + } + + // adjust exponent (constant.MakeFromBytes creates an integer value, + // but mant represents the mantissa bits such that 0.5 <= mant < 1.0) + exp -= len(mant) << 3 + if len(mant) > 0 { + for msd := mant[len(mant)-1]; msd&0x80 == 0; msd <<= 1 { + exp++ + } + } + + x := constant.MakeFromBytes(mant) + switch { + case exp < 0: + d := constant.Shift(constant.MakeInt64(1), token.SHL, uint(-exp)) + x = constant.BinaryOp(x, token.QUO, d) + case exp > 0: + x = constant.Shift(x, token.SHL, uint(exp)) + } + + if sign < 0 { + x = constant.UnaryOp(token.SUB, x, 0) + } + return x +} + +// ---------------------------------------------------------------------------- +// Low-level decoders + +func (p *importer) tagOrIndex() int { + if p.debugFormat { + p.marker('t') + } + + return int(p.rawInt64()) +} + +func (p *importer) int() int { + x := p.int64() + if int64(int(x)) != x { + panic("exported integer too large") + } + return int(x) +} + +func (p *importer) int64() int64 { + if p.debugFormat { + p.marker('i') + } + + return p.rawInt64() +} + +func (p *importer) string() string { + if p.debugFormat { + p.marker('s') + } + + if n := int(p.rawInt64()); n > 0 { + if cap(p.buf) < n { + p.buf = make([]byte, n) + } else { + p.buf = p.buf[:n] + } + for i := 0; i < n; i++ { + p.buf[i] = p.byte() + } + return string(p.buf) + } + + return "" +} + +func (p *importer) marker(want byte) { + if got := p.byte(); got != want { + panic(fmt.Sprintf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.read)) + } + + pos := p.read + if n := int(p.rawInt64()); n != pos { + panic(fmt.Sprintf("incorrect position: got %d; want %d", n, pos)) + } +} + +// rawInt64 should only be used by low-level decoders +func (p *importer) rawInt64() int64 { + i, err := binary.ReadVarint(p) + if err != nil { + panic(fmt.Sprintf("read error: %v", err)) + } + return i +} + +// needed for binary.ReadVarint in rawInt64 +func (p *importer) ReadByte() (byte, error) { + return p.byte(), nil +} + +// byte is the bottleneck interface for reading p.data. +// It unescapes '|' 'S' to '$' and '|' '|' to '|'. +func (p *importer) byte() byte { + b := p.data[0] + r := 1 + if b == '|' { + b = p.data[1] + r = 2 + switch b { + case 'S': + b = '$' + case '|': + // nothing to do + default: + panic("unexpected escape sequence in export data") + } + } + p.data = p.data[r:] + p.read += r + return b + +} + +// ---------------------------------------------------------------------------- +// Export format + +// Tags. Must be < 0. +const ( + // Packages + packageTag = -(iota + 1) + + // Types + namedTag + arrayTag + sliceTag + dddTag + structTag + pointerTag + signatureTag + interfaceTag + mapTag + chanTag + + // Values + falseTag + trueTag + int64Tag + floatTag + fractionTag // not used by gc + complexTag + stringTag +) + +var predeclared = []types.Type{ + // basic types + types.Typ[types.Bool], + types.Typ[types.Int], + types.Typ[types.Int8], + types.Typ[types.Int16], + types.Typ[types.Int32], + types.Typ[types.Int64], + types.Typ[types.Uint], + types.Typ[types.Uint8], + types.Typ[types.Uint16], + types.Typ[types.Uint32], + types.Typ[types.Uint64], + types.Typ[types.Uintptr], + types.Typ[types.Float32], + types.Typ[types.Float64], + types.Typ[types.Complex64], + types.Typ[types.Complex128], + types.Typ[types.String], + + // aliases + types.Universe.Lookup("byte").Type(), + types.Universe.Lookup("rune").Type(), + + // error + types.Universe.Lookup("error").Type(), + + // untyped types + types.Typ[types.UntypedBool], + types.Typ[types.UntypedInt], + types.Typ[types.UntypedRune], + types.Typ[types.UntypedFloat], + types.Typ[types.UntypedComplex], + types.Typ[types.UntypedString], + types.Typ[types.UntypedNil], + + // package unsafe + types.Typ[types.UnsafePointer], +} diff --git a/go/gcimporter15/exportdata.go b/go/gcimporter15/exportdata.go index 90cc8923..4e1d1e47 100644 --- a/go/gcimporter15/exportdata.go +++ b/go/gcimporter15/exportdata.go @@ -4,7 +4,7 @@ // +build go1.5 -// This file is a copy of go/gcimporter/exportdata.go, tagged for go1.5. +// This file is a copy of $GOROOT/src/go/internal/gcimporter/exportdata.go, tagged for go1.5. // This file implements FindExportData. @@ -43,14 +43,16 @@ func readGopackHeader(r *bufio.Reader) (name string, size int, err error) { // FindExportData positions the reader r at the beginning of the // export data section of an underlying GC-created object/archive // file by reading from it. The reader must be positioned at the -// start of the file before calling this function. +// start of the file before calling this function. The hdr result +// is the string before the export data, either "$$" or "$$B". // -func FindExportData(r *bufio.Reader) (err error) { +func FindExportData(r *bufio.Reader) (hdr string, err error) { // Read first line to make sure this is an object file. line, err := r.ReadSlice('\n') if err != nil { return } + if string(line) == "!\n" { // Archive file. Scan to __.PKGDEF. var name string @@ -75,7 +77,7 @@ func FindExportData(r *bufio.Reader) (err error) { size -= n } - if name, size, err = readGopackHeader(r); err != nil { + if name, _, err = readGopackHeader(r); err != nil { return } } @@ -101,12 +103,13 @@ func FindExportData(r *bufio.Reader) (err error) { } // Skip over object header to export data. - // Begins after first line with $$. + // Begins after first line starting with $$. for line[0] != '$' { if line, err = r.ReadSlice('\n'); err != nil { return } } + hdr = string(line) return } diff --git a/go/gcimporter15/gcimporter.go b/go/gcimporter15/gcimporter.go index 03741166..1c2a44ca 100644 --- a/go/gcimporter15/gcimporter.go +++ b/go/gcimporter15/gcimporter.go @@ -4,20 +4,17 @@ // +build go1.5 -// This file is: -// - a copy of go/gcimporter/gcimporter.go -// - tagged for Go 1.5 -// - changed to use the standard go/types package -// - without the assignment to DefaultImporter -// - with an extra parameter to calls to MakeFromLiteral. +// This file is a copy of $GOROOT/src/go/internal/gcimporter/gcimporter.go, tagged for go1.5, +// and minimally adjusted to make it build. // Package gcimporter15 provides various functions for reading -// gc-generated object files that can be used to implement the Importer -// interface defined by the Go 1.5 standard library package. +// gc-generated object files that can be used to implement the +// Importer interface defined by the Go 1.5 standard library package. // -// This package is a kludge: if the Go 1.5 standard go/importer -// package's Lookup feature had been fully implemented in Go 1.5, we -// would not need it. It will be deleted after Go 1.6. +// This package serves as a stop-gap for missing features in the +// standard library's go/importer package, specifically customizable +// package data lookup. This package should be deleted once that +// functionality becomes available in the standard library. package gcimporter // import "golang.org/x/tools/go/gcimporter15" import ( @@ -29,6 +26,7 @@ import ( "go/token" "go/types" "io" + "io/ioutil" "os" "path/filepath" "sort" @@ -40,7 +38,7 @@ import ( // debugging/development support const debug = false -var pkgExts = [...]string{".a", ".5", ".6", ".7", ".8", ".9"} +var pkgExts = [...]string{".a", ".o"} // FindPkg returns the filename and unique package id for an import // path based on package information provided by build.Import (using @@ -48,11 +46,10 @@ var pkgExts = [...]string{".a", ".5", ".6", ".7", ".8", ".9"} // If no file was found, an empty filename is returned. // func FindPkg(path, srcDir string) (filename, id string) { - if len(path) == 0 { + if path == "" { return } - id = path var noext string switch { default: @@ -63,6 +60,7 @@ func FindPkg(path, srcDir string) (filename, id string) { return } noext = strings.TrimSuffix(bp.PkgObj, ".a") + id = bp.ImportPath case build.IsLocalImport(path): // "./x" -> "/this/directory/x.ext", "/this/directory/x" @@ -74,6 +72,13 @@ func FindPkg(path, srcDir string) (filename, id string) { // does not support absolute imports // "/x" -> "/x.ext", "/x" noext = path + id = path + } + + if false { // for debugging + if path != id { + fmt.Printf("%s -> %s\n", path, id) + } } // try extensions @@ -120,26 +125,16 @@ func ImportData(packages map[string]*types.Package, filename, id string, data io return } -// Import imports a gc-generated package given its import path, adds the -// corresponding package object to the packages map, and returns the object. -// Local import paths are interpreted relative to the current working directory. -// The packages map must contains all packages already imported. +// Import imports a gc-generated package given its import path and srcDir, adds +// the corresponding package object to the packages map, and returns the object. +// The packages map must contain all packages already imported. // -func Import(packages map[string]*types.Package, path string) (pkg *types.Package, err error) { - if path == "unsafe" { - return types.Unsafe, nil - } - - srcDir := "." - if build.IsLocalImport(path) { - srcDir, err = os.Getwd() - if err != nil { - return - } - } - +func Import(packages map[string]*types.Package, path, srcDir string) (pkg *types.Package, err error) { filename, id := FindPkg(path, srcDir) if filename == "" { + if path == "unsafe" { + return types.Unsafe, nil + } err = fmt.Errorf("can't find import: %s", id) return } @@ -162,12 +157,25 @@ func Import(packages map[string]*types.Package, path string) (pkg *types.Package } }() + var hdr string buf := bufio.NewReader(f) - if err = FindExportData(buf); err != nil { + if hdr, err = FindExportData(buf); err != nil { return } - pkg, err = ImportData(packages, filename, id, buf) + switch hdr { + case "$$\n": + return ImportData(packages, filename, id, buf) + case "$$B\n": + var data []byte + data, err = ioutil.ReadAll(buf) + if err == nil { + _, pkg, err = BImportData(packages, data, path) + return + } + default: + err = fmt.Errorf("unknown export data header: %q", hdr) + } return } @@ -348,8 +356,10 @@ func (p *parser) parseQualifiedName() (id, name string) { } // getPkg returns the package for a given id. If the package is -// not found but we have a package name, create the package and -// add it to the p.localPkgs and p.sharedPkgs maps. +// not found, create the package and add it to the p.localPkgs +// and p.sharedPkgs maps. name is the (expected) name of the +// package. If name == "", the package name is expected to be +// set later via an import clause in the export data. // // id identifies a package, usually by a canonical package path like // "encoding/json" but possibly by a non-canonical import path like @@ -362,19 +372,28 @@ func (p *parser) getPkg(id, name string) *types.Package { } pkg := p.localPkgs[id] - if pkg == nil && name != "" { + if pkg == nil { // first import of id from this package pkg = p.sharedPkgs[id] if pkg == nil { - // first import of id by this importer + // first import of id by this importer; + // add (possibly unnamed) pkg to shared packages pkg = types.NewPackage(id, name) p.sharedPkgs[id] = pkg } - + // add (possibly unnamed) pkg to local packages if p.localPkgs == nil { p.localPkgs = make(map[string]*types.Package) } p.localPkgs[id] = pkg + } else if name != "" { + // package exists already and we have an expected package name; + // make sure names match or set package name if necessary + if pname := pkg.Name(); pname == "" { + pkg.SetName(name) + } else if pname != name { + p.errorf("%s package name mismatch: %s (given) vs %s (expected)", pname, name) + } } return pkg } @@ -385,9 +404,6 @@ func (p *parser) getPkg(id, name string) *types.Package { func (p *parser) parseExportedName() (pkg *types.Package, name string) { id, name := p.parseQualifiedName() pkg = p.getPkg(id, "") - if pkg == nil { - p.errorf("%s package not found", id) - } return } @@ -408,11 +424,11 @@ func (p *parser) parseBasicType() types.Type { // ArrayType = "[" int_lit "]" Type . // -func (p *parser) parseArrayType() types.Type { +func (p *parser) parseArrayType(parent *types.Package) types.Type { // "[" already consumed and lookahead known not to be "]" lit := p.expect(scanner.Int) p.expect(']') - elem := p.parseType() + elem := p.parseType(parent) n, err := strconv.ParseInt(lit, 10, 64) if err != nil { p.error(err) @@ -422,46 +438,47 @@ func (p *parser) parseArrayType() types.Type { // MapType = "map" "[" Type "]" Type . // -func (p *parser) parseMapType() types.Type { +func (p *parser) parseMapType(parent *types.Package) types.Type { p.expectKeyword("map") p.expect('[') - key := p.parseType() + key := p.parseType(parent) p.expect(']') - elem := p.parseType() + elem := p.parseType(parent) return types.NewMap(key, elem) } // Name = identifier | "?" | QualifiedName . // -// If materializePkg is set, the returned package is guaranteed to be set. -// For fully qualified names, the returned package may be a fake package -// (without name, scope, and not in the p.imports map), created for the -// sole purpose of providing a package path. Fake packages are created -// when the package id is not found in the p.imports map; in that case -// we cannot create a real package because we don't have a package name. -// For non-qualified names, the returned package is the imported package. +// For unqualified and anonymous names, the returned package is the parent +// package unless parent == nil, in which case the returned package is the +// package being imported. (The parent package is not nil if the the name +// is an unqualified struct field or interface method name belonging to a +// type declared in another package.) // -func (p *parser) parseName(materializePkg bool) (pkg *types.Package, name string) { +// For qualified names, the returned package is nil (and not created if +// it doesn't exist yet) unless materializePkg is set (which creates an +// unnamed package with valid package path). In the latter case, a +// subequent import clause is expected to provide a name for the package. +// +func (p *parser) parseName(parent *types.Package, materializePkg bool) (pkg *types.Package, name string) { + pkg = parent + if pkg == nil { + pkg = p.sharedPkgs[p.id] + } switch p.tok { case scanner.Ident: - pkg = p.sharedPkgs[p.id] name = p.lit p.next() case '?': // anonymous - pkg = p.sharedPkgs[p.id] p.next() case '@': // exported name prefixed with package path + pkg = nil var id string id, name = p.parseQualifiedName() if materializePkg { - // we don't have a package name - if the package - // doesn't exist yet, create a fake package instead pkg = p.getPkg(id, "") - if pkg == nil { - pkg = types.NewPackage(id, "") - } } default: p.error("name expected") @@ -478,15 +495,15 @@ func deref(typ types.Type) types.Type { // Field = Name Type [ string_lit ] . // -func (p *parser) parseField() (*types.Var, string) { - pkg, name := p.parseName(true) - typ := p.parseType() +func (p *parser) parseField(parent *types.Package) (*types.Var, string) { + pkg, name := p.parseName(parent, true) + typ := p.parseType(parent) anonymous := false if name == "" { // anonymous field - typ must be T or *T and T must be a type name switch typ := deref(typ).(type) { case *types.Basic: // basic types are named types - pkg = nil + pkg = nil // objects defined in Universe scope have no package name = typ.Name() case *types.Named: name = typ.Obj().Name() @@ -510,7 +527,7 @@ func (p *parser) parseField() (*types.Var, string) { // StructType = "struct" "{" [ FieldList ] "}" . // FieldList = Field { ";" Field } . // -func (p *parser) parseStructType() types.Type { +func (p *parser) parseStructType(parent *types.Package) types.Type { var fields []*types.Var var tags []string @@ -520,7 +537,7 @@ func (p *parser) parseStructType() types.Type { if i > 0 { p.expect(';') } - fld, tag := p.parseField() + fld, tag := p.parseField(parent) if tag != "" && tags == nil { tags = make([]string, i) } @@ -537,7 +554,7 @@ func (p *parser) parseStructType() types.Type { // Parameter = ( identifier | "?" ) [ "..." ] Type [ string_lit ] . // func (p *parser) parseParameter() (par *types.Var, isVariadic bool) { - _, name := p.parseName(false) + _, name := p.parseName(nil, false) // remove gc-specific parameter numbering if i := strings.Index(name, "ยท"); i >= 0 { name = name[:i] @@ -546,7 +563,7 @@ func (p *parser) parseParameter() (par *types.Var, isVariadic bool) { p.expectSpecial("...") isVariadic = true } - typ := p.parseType() + typ := p.parseType(nil) if isVariadic { typ = types.NewSlice(typ) } @@ -609,7 +626,7 @@ func (p *parser) parseSignature(recv *types.Var) *types.Signature { // by the compiler and thus embedded interfaces are never // visible in the export data. // -func (p *parser) parseInterfaceType() types.Type { +func (p *parser) parseInterfaceType(parent *types.Package) types.Type { var methods []*types.Func p.expectKeyword("interface") @@ -618,7 +635,7 @@ func (p *parser) parseInterfaceType() types.Type { if i > 0 { p.expect(';') } - pkg, name := p.parseName(true) + pkg, name := p.parseName(parent, true) sig := p.parseSignature(nil) methods = append(methods, types.NewFunc(token.NoPos, pkg, name, sig)) } @@ -631,7 +648,7 @@ func (p *parser) parseInterfaceType() types.Type { // ChanType = ( "chan" [ "<-" ] | "<-" "chan" ) Type . // -func (p *parser) parseChanType() types.Type { +func (p *parser) parseChanType(parent *types.Package) types.Type { dir := types.SendRecv if p.tok == scanner.Ident { p.expectKeyword("chan") @@ -644,7 +661,7 @@ func (p *parser) parseChanType() types.Type { p.expectKeyword("chan") dir = types.RecvOnly } - elem := p.parseType() + elem := p.parseType(parent) return types.NewChan(dir, elem) } @@ -659,24 +676,24 @@ func (p *parser) parseChanType() types.Type { // PointerType = "*" Type . // FuncType = "func" Signature . // -func (p *parser) parseType() types.Type { +func (p *parser) parseType(parent *types.Package) types.Type { switch p.tok { case scanner.Ident: switch p.lit { default: return p.parseBasicType() case "struct": - return p.parseStructType() + return p.parseStructType(parent) case "func": // FuncType p.next() return p.parseSignature(nil) case "interface": - return p.parseInterfaceType() + return p.parseInterfaceType(parent) case "map": - return p.parseMapType() + return p.parseMapType(parent) case "chan": - return p.parseChanType() + return p.parseChanType(parent) } case '@': // TypeName @@ -687,19 +704,19 @@ func (p *parser) parseType() types.Type { if p.tok == ']' { // SliceType p.next() - return types.NewSlice(p.parseType()) + return types.NewSlice(p.parseType(parent)) } - return p.parseArrayType() + return p.parseArrayType(parent) case '*': // PointerType p.next() - return types.NewPointer(p.parseType()) + return types.NewPointer(p.parseType(parent)) case '<': - return p.parseChanType() + return p.parseChanType(parent) case '(': // "(" Type ")" p.next() - typ := p.parseType() + typ := p.parseType(parent) p.expect(')') return typ } @@ -781,7 +798,8 @@ func (p *parser) parseConstDecl() { var typ0 types.Type if p.tok != '=' { - typ0 = p.parseType() + // constant types are never structured - no need for parent type + typ0 = p.parseType(nil) } p.expect('=') @@ -855,7 +873,7 @@ func (p *parser) parseTypeDecl() { // structure, but throw it away if the object already has a type. // This ensures that all imports refer to the same type object for // a given type declaration. - typ := p.parseType() + typ := p.parseType(pkg) if name := obj.Type().(*types.Named); name.Underlying() == nil { name.SetUnderlying(typ) @@ -867,7 +885,7 @@ func (p *parser) parseTypeDecl() { func (p *parser) parseVarDecl() { p.expectKeyword("var") pkg, name := p.parseExportedName() - typ := p.parseType() + typ := p.parseType(pkg) pkg.Scope().Insert(types.NewVar(token.NoPos, pkg, name, typ)) } @@ -903,7 +921,7 @@ func (p *parser) parseMethodDecl() { base := deref(recv.Type()).(*types.Named) // parse method name, signature, and possibly inlined body - _, name := p.parseName(true) + _, name := p.parseName(nil, false) sig := p.parseFunc(recv) // methods always belong to the same package as the base type object @@ -980,9 +998,12 @@ func (p *parser) parseExport() *types.Package { p.errorf("expected no scanner errors, got %d", n) } - // Record all referenced packages as imports. + // Record all locally referenced packages as imports. var imports []*types.Package for id, pkg2 := range p.localPkgs { + if pkg2.Name() == "" { + p.errorf("%s package has no name", id) + } if id == p.id { continue // avoid self-edge } diff --git a/go/gcimporter15/gcimporter_test.go b/go/gcimporter15/gcimporter_test.go index 6613b727..28402a8f 100644 --- a/go/gcimporter15/gcimporter_test.go +++ b/go/gcimporter15/gcimporter_test.go @@ -4,16 +4,13 @@ // +build go1.5 -// This file is: -// - a copy of go/gcimporter/gcimporter_test.go -// - tagged for Go 1.5 -// - changed to use the standard go/types package. +// This file is a copy of $GOROOT/src/go/internal/gcimporter/gcimporter_test.go, tagged for go1.5, +// and minimally adjusted to make it build with code from (std lib) internal/testenv copied. package gcimporter import ( "fmt" - "go/build" "go/types" "io/ioutil" "os" @@ -25,6 +22,44 @@ import ( "time" ) +// ---------------------------------------------------------------------------- +// The following three functions (Builder, HasGoBuild, MustHaveGoBuild) were +// copied from $GOROOT/src/internal/testenv since that package is not available +// in x/tools. + +// Builder reports the name of the builder running this test +// (for example, "linux-amd64" or "windows-386-gce"). +// If the test is not running on the build infrastructure, +// Builder returns the empty string. +func Builder() string { + return os.Getenv("GO_BUILDER_NAME") +} + +// HasGoBuild reports whether the current system can build programs with ``go build'' +// and then run them with os.StartProcess or exec.Command. +func HasGoBuild() bool { + switch runtime.GOOS { + case "android", "nacl": + return false + case "darwin": + if strings.HasPrefix(runtime.GOARCH, "arm") { + return false + } + } + return true +} + +// MustHaveGoBuild checks that the current system can build programs with ``go build'' +// and then run them with os.StartProcess or exec.Command. +// If not, MustHaveGoBuild calls t.Skip with an explanation. +func MustHaveGoBuild(t *testing.T) { + if !HasGoBuild() { + t.Skipf("skipping test: 'go build' not available on %s/%s", runtime.GOOS, runtime.GOARCH) + } +} + +// ---------------------------------------------------------------------------- + // skipSpecialPlatforms causes the test to be skipped for platforms where // builders (build.golang.org) don't have access to compiled packages for // import. @@ -39,36 +74,36 @@ func skipSpecialPlatforms(t *testing.T) { } } -var gcPath string // Go compiler path - -func init() { - if char, err := build.ArchChar(runtime.GOARCH); err == nil { - gcPath = filepath.Join(build.ToolDir, char+"g") - return - } - gcPath = "unknown-GOARCH-compiler" -} - func compile(t *testing.T, dirname, filename string) string { - cmd := exec.Command(gcPath, filename) + /* testenv. */ MustHaveGoBuild(t) + cmd := exec.Command("go", "tool", "compile", filename) cmd.Dir = dirname out, err := cmd.CombinedOutput() if err != nil { t.Logf("%s", out) - t.Fatalf("%s %s failed: %s", gcPath, filename, err) + t.Fatalf("go tool compile %s failed: %s", filename, err) } - archCh, _ := build.ArchChar(runtime.GOARCH) // filename should end with ".go" - return filepath.Join(dirname, filename[:len(filename)-2]+archCh) + return filepath.Join(dirname, filename[:len(filename)-2]+"o") } -// Use the same global imports map for all tests. The effect is -// as if all tested packages were imported into a single package. -var imports = make(map[string]*types.Package) +// TODO(gri) Remove this function once we switched to new export format by default. +func compileNewExport(t *testing.T, dirname, filename string) string { + /* testenv. */ MustHaveGoBuild(t) + cmd := exec.Command("go", "tool", "compile", "-newexport", filename) + cmd.Dir = dirname + out, err := cmd.CombinedOutput() + if err != nil { + t.Logf("%s", out) + t.Fatalf("go tool compile %s failed: %s", filename, err) + } + // filename should end with ".go" + return filepath.Join(dirname, filename[:len(filename)-2]+"o") +} -func testPath(t *testing.T, path string) *types.Package { +func testPath(t *testing.T, path, srcDir string) *types.Package { t0 := time.Now() - pkg, err := Import(imports, path) + pkg, err := Import(make(map[string]*types.Package), path, srcDir) if err != nil { t.Errorf("testPath(%s): %s", path, err) return nil @@ -96,7 +131,7 @@ func testDir(t *testing.T, dir string, endTime time.Time) (nimports int) { for _, ext := range pkgExts { if strings.HasSuffix(f.Name(), ext) { name := f.Name()[0 : len(f.Name())-len(ext)] // remove extension - if testPath(t, filepath.Join(dir, name)) != nil { + if testPath(t, filepath.Join(dir, name), dir) != nil { nimports++ } } @@ -108,37 +143,73 @@ func testDir(t *testing.T, dir string, endTime time.Time) (nimports int) { return } -func TestImport(t *testing.T) { +func TestImportTestdata(t *testing.T) { // This package only handles gc export data. if runtime.Compiler != "gc" { t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) return } - // On cross-compile builds, the path will not exist. - // Need to use GOHOSTOS, which is not available. - if _, err := os.Stat(gcPath); err != nil { - t.Skipf("skipping test: %v", err) - } - if outFn := compile(t, "testdata", "exports.go"); outFn != "" { defer os.Remove(outFn) } - nimports := 0 - if pkg := testPath(t, "./testdata/exports"); pkg != nil { - nimports++ - // The package's Imports should include all the types - // referenced by the exportdata, which may be more than - // the import statements in the package's source, but - // fewer than the transitive closure of dependencies. - want := `[package ast ("go/ast") package token ("go/token") package runtime ("runtime")]` + if pkg := testPath(t, "./testdata/exports", "."); pkg != nil { + // The package's Imports list must include all packages + // explicitly imported by exports.go, plus all packages + // referenced indirectly via exported objects in exports.go. + // With the textual export format, the list may also include + // additional packages that are not strictly required for + // import processing alone (they are exported to err "on + // the safe side"). + got := fmt.Sprint(pkg.Imports()) + for _, want := range []string{"go/ast", "go/token"} { + if !strings.Contains(got, want) { + t.Errorf(`Package("exports").Imports() = %s, does not contain %s`, got, want) + } + } + } +} + +// TODO(gri) Remove this function once we switched to new export format by default +// (and update the comment and want list in TestImportTestdata). +func TestImportTestdataNewExport(t *testing.T) { + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + return + } + + if outFn := compileNewExport(t, "testdata", "exports.go"); outFn != "" { + defer os.Remove(outFn) + } + + if pkg := testPath(t, "./testdata/exports", "."); pkg != nil { + // The package's Imports list must include all packages + // explicitly imported by exports.go, plus all packages + // referenced indirectly via exported objects in exports.go. + want := `[package ast ("go/ast") package token ("go/token")]` got := fmt.Sprint(pkg.Imports()) if got != want { t.Errorf(`Package("exports").Imports() = %s, want %s`, got, want) } } - nimports += testDir(t, "", time.Now().Add(maxTime)) // installed packages +} + +func TestImportStdLib(t *testing.T) { + skipSpecialPlatforms(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + return + } + + dt := maxTime + if testing.Short() && /* testenv. */ Builder() == "" { + dt = 10 * time.Millisecond + } + nimports := testDir(t, "", time.Now().Add(dt)) // installed packages t.Logf("tested %d imports", nimports) } @@ -146,7 +217,6 @@ var importedObjectTests = []struct { name string want string }{ - {"unsafe.Pointer", "type Pointer unsafe.Pointer"}, {"math.Pi", "const Pi untyped float"}, {"io.Reader", "type Reader interface{Read(p []byte) (n int, err error)}"}, {"io.ReadWriter", "type ReadWriter interface{Read(p []byte) (n int, err error); Write(p []byte) (n int, err error)}"}, @@ -171,7 +241,7 @@ func TestImportedTypes(t *testing.T) { importPath := s[0] objName := s[1] - pkg, err := Import(imports, importPath) + pkg, err := Import(make(map[string]*types.Package), importPath, ".") if err != nil { t.Error(err) continue @@ -199,7 +269,7 @@ func TestIssue5815(t *testing.T) { return } - pkg, err := Import(make(map[string]*types.Package), "strings") + pkg, err := Import(make(map[string]*types.Package), "strings", ".") if err != nil { t.Fatal(err) } @@ -233,7 +303,7 @@ func TestCorrectMethodPackage(t *testing.T) { } imports := make(map[string]*types.Package) - _, err := Import(imports, "net/http") + _, err := Import(imports, "net/http", ".") if err != nil { t.Fatal(err) } @@ -246,3 +316,89 @@ func TestCorrectMethodPackage(t *testing.T) { t.Errorf("got package path %q; want %q", got, want) } } + +func TestIssue13566(t *testing.T) { + skipSpecialPlatforms(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + return + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + if f := compile(t, "testdata", "a.go"); f != "" { + defer os.Remove(f) + } + if f := compile(t, "testdata", "b.go"); f != "" { + defer os.Remove(f) + } + + // import must succeed (test for issue at hand) + pkg, err := Import(make(map[string]*types.Package), "./testdata/b", ".") + if err != nil { + t.Fatal(err) + } + + // make sure all indirectly imported packages have names + for _, imp := range pkg.Imports() { + if imp.Name() == "" { + t.Errorf("no name for %s package", imp.Path()) + } + } +} + +func TestIssue13898(t *testing.T) { + skipSpecialPlatforms(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + return + } + + // import go/internal/gcimporter which imports go/types partially + imports := make(map[string]*types.Package) + _, err := Import(imports, "go/internal/gcimporter", ".") + if err != nil { + t.Fatal(err) + } + + // look for go/types package + var goTypesPkg *types.Package + for path, pkg := range imports { + if path == "go/types" { + goTypesPkg = pkg + break + } + } + if goTypesPkg == nil { + t.Fatal("go/types not found") + } + + // look for go/types.Object type + obj := goTypesPkg.Scope().Lookup("Object") + if obj == nil { + t.Fatal("go/types.Object not found") + } + typ, ok := obj.Type().(*types.Named) + if !ok { + t.Fatalf("go/types.Object type is %v; wanted named type", typ) + } + + // lookup go/types.Object.Pkg method + m, _, _ := types.LookupFieldOrMethod(typ, false, nil, "Pkg") + if m == nil { + t.Fatal("go/types.Object.Pkg not found") + } + + // the method must belong to go/types + if m.Pkg().Path() != "go/types" { + t.Fatalf("found %v; want go/types", m.Pkg()) + } +} diff --git a/go/gcimporter15/testdata/a.go b/go/gcimporter15/testdata/a.go new file mode 100644 index 00000000..56e4292c --- /dev/null +++ b/go/gcimporter15/testdata/a.go @@ -0,0 +1,14 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Input for TestIssue13566 + +package a + +import "encoding/json" + +type A struct { + a *A + json json.RawMessage +} diff --git a/go/gcimporter15/testdata/b.go b/go/gcimporter15/testdata/b.go new file mode 100644 index 00000000..41966782 --- /dev/null +++ b/go/gcimporter15/testdata/b.go @@ -0,0 +1,11 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Input for TestIssue13566 + +package b + +import "./a" + +type A a.A