diff --git a/go/importer/export.go b/go/importer/export.go new file mode 100644 index 00000000..c4d634fc --- /dev/null +++ b/go/importer/export.go @@ -0,0 +1,442 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package importer + +import ( + "bytes" + "encoding/binary" + "fmt" + "go/ast" + "strings" + + "code.google.com/p/go.tools/go/exact" + "code.google.com/p/go.tools/go/types" +) + +// debugging support +const ( + debug = false // emit debugging data + trace = false // print emitted data +) + +const ( + magic = "\n$$ exports $$\n" + version = 0 +) + +// Object and type tags. Must be < 0. +const ( + // Objects + _Package = -(iota + 1) + _Const + _Type + _Var + _Func + + // Types + _Basic + _Array + _Slice + _Struct + _Pointer + _Tuple + _Signature + _Interface + _Map + _Chan + _Named +) + +// ExportData serializes the interface (exported package objects) +// of package pkg and returns the corresponding data. The export +// format is described elsewhere (TODO). +func ExportData(pkg *types.Package) []byte { + p := exporter{ + data: []byte(magic), + // TODO(gri) If we can't have nil packages + // or types, remove nil entries at index 0. + pkgIndex: map[*types.Package]int{nil: 0}, + typIndex: map[types.Type]int{nil: 0}, + } + + // populate typIndex with predeclared types + for _, t := range types.Typ[1:] { + p.typIndex[t] = len(p.typIndex) + } + p.typIndex[types.Universe.Lookup("error").Type()] = len(p.typIndex) + + if trace { + p.tracef("export %s\n", pkg.Name()) + defer p.tracef("\n") + } + + p.int(version) + + p.pkg(pkg) + + // collect exported objects from package scope + var list []types.Object + scope := pkg.Scope() + for _, name := range scope.Names() { + if exported(name) { + list = append(list, scope.Lookup(name)) + } + } + + // write objects + p.int(len(list)) + for _, obj := range list { + p.obj(obj) + } + + return p.data +} + +type exporter struct { + data []byte + pkgIndex map[*types.Package]int + typIndex map[types.Type]int + + // tracing support + indent string +} + +func (p *exporter) pkg(pkg *types.Package) { + if trace { + p.tracef("package { ") + defer p.tracef("} ") + } + + // if the package was seen before, write its index (>= 0) + if i, ok := p.pkgIndex[pkg]; ok { + p.int(i) + return + } + p.pkgIndex[pkg] = len(p.pkgIndex) + + // otherwise, write the package tag (< 0) and package data + p.int(_Package) + p.string(pkg.Name()) + p.string(pkg.Path()) +} + +func (p *exporter) obj(obj types.Object) { + if trace { + p.tracef("object %s {\n", obj.Name()) + defer p.tracef("}\n") + } + + switch obj := obj.(type) { + case *types.Const: + p.int(_Const) + p.string(obj.Name()) + p.typ(obj.Type()) + p.val(obj.Val()) + case *types.TypeName: + p.int(_Type) + // name is written by corresponding named type + p.typ(obj.Type().(*types.Named)) + case *types.Var: + p.int(_Var) + p.string(obj.Name()) + p.typ(obj.Type()) + case *types.Func: + p.int(_Func) + p.string(obj.Name()) + p.signature(obj.Type().(*types.Signature)) + default: + panic(fmt.Sprintf("unexpected object type %T", obj)) + } +} + +func (p *exporter) val(x exact.Value) { + if trace { + p.tracef("value { ") + defer p.tracef("} ") + } + + kind := x.Kind() + p.int(int(kind)) + switch kind { + case exact.Bool: + p.bool(exact.BoolVal(x)) + case exact.String: + p.string(exact.StringVal(x)) + case exact.Int: + p.intVal(x) + case exact.Float: + p.floatVal(x) + case exact.Complex: + p.floatVal(exact.Real(x)) + p.floatVal(exact.Imag(x)) + default: + panic(fmt.Sprintf("unexpected value kind %d", kind)) + } +} + +func (p *exporter) intVal(x exact.Value) { + sign := exact.Sign(x) + p.int(sign) + if sign != 0 { + p.bytes(exact.Bytes(x)) + } +} + +func (p *exporter) floatVal(x exact.Value) { + p.intVal(exact.Num(x)) + if exact.Sign(x) != 0 { + // TODO(gri): For gc-generated constants, the denominator is + // often a large power of two. Use a more compact representation. + p.bytes(exact.Bytes(exact.Denom(x))) + } +} + +func (p *exporter) typ(typ types.Type) { + if trace { + p.tracef("type {\n") + defer p.tracef("}\n") + } + + // if the type was seen before, write its index (>= 0) + if i, ok := p.typIndex[typ]; ok { + p.int(i) + return + } + p.typIndex[typ] = len(p.typIndex) + + // otherwise, write the type tag (< 0) and type data + switch t := typ.(type) { + case *types.Basic: + // Basic types are pre-recorded and don't usually end up here. + // However, the alias types byte and rune are not in the types.Typ + // table and get emitted here (once per package, if they appear). + // This permits faithful reconstruction of the alias type (i.e., + // keeping the name). If we decide to eliminate the distinction + // between the alias types, this code can go. + p.int(_Basic) + p.string(t.Name()) + + case *types.Array: + p.int(_Array) + p.typ(t.Elem()) + p.int64(t.Len()) + + case *types.Slice: + p.int(_Slice) + p.typ(t.Elem()) + + case *types.Struct: + p.int(_Struct) + n := t.NumFields() + p.int(n) + for i := 0; i < n; i++ { + p.field(t.Field(i)) + p.string(t.Tag(i)) + } + + case *types.Pointer: + p.int(_Pointer) + p.typ(t.Elem()) + + case *types.Signature: + p.int(_Signature) + p.signature(t) + + case *types.Interface: + p.int(_Interface) + n := t.NumMethods() + p.int(n) + for i := 0; i < n; i++ { + m := t.Method(i) + p.qualifiedName(m.Pkg(), m.Name()) + p.signature(m.Type().(*types.Signature)) + } + + case *types.Map: + p.int(_Map) + p.typ(t.Key()) + p.typ(t.Elem()) + + case *types.Chan: + p.int(_Chan) + p.int(int(t.Dir())) + p.typ(t.Elem()) + + case *types.Named: + p.int(_Named) + + // write type object + obj := t.Obj() + p.string(obj.Name()) + p.pkg(obj.Pkg()) + + // write underlying type + p.typ(t.Underlying()) + + // write associated methods + n := t.NumMethods() + p.int(n) + for i := 0; i < n; i++ { + m := t.Method(i) + p.string(m.Name()) + p.signature(m.Type().(*types.Signature)) + } + + default: + panic("unreachable") + } +} + +func (p *exporter) field(f *types.Var) { + // anonymous fields have "" name + name := "" + if !f.Anonymous() { + name = f.Name() + } + + p.qualifiedName(f.Pkg(), name) + p.typ(f.Type()) +} + +func (p *exporter) qualifiedName(pkg *types.Package, name string) { + p.string(name) + // exported names don't write package + if !exported(name) { + if pkg == nil { + panic(fmt.Sprintf("nil package for unexported qualified name %s", name)) + } + p.pkg(pkg) + } +} + +func (p *exporter) signature(sig *types.Signature) { + // TODO(gri) We only need to record the receiver type + // for interface methods if we flatten them + // out. If we track embedded types instead, + // the information is already present. + if recv := sig.Recv(); recv != nil { + p.bool(true) + p.param(recv) + } else { + p.bool(false) + } + p.tuple(sig.Params()) + p.tuple(sig.Results()) + p.bool(sig.IsVariadic()) +} + +func (p *exporter) param(v *types.Var) { + p.string(v.Name()) + p.typ(v.Type()) +} + +func (p *exporter) tuple(t *types.Tuple) { + n := t.Len() + p.int(n) + for i := 0; i < n; i++ { + p.param(t.At(i)) + } +} + +// ---------------------------------------------------------------------------- +// encoders + +func (p *exporter) bool(b bool) { + var x int64 + if b { + x = 1 + } + p.int64(x) +} + +func (p *exporter) string(s string) { + // TODO(gri) consider inlining this to avoid an extra allocation + p.bytes([]byte(s)) +} + +func (p *exporter) int(x int) { + p.int64(int64(x)) +} + +func (p *exporter) int64(x int64) { + if debug { + p.marker('i') + } + + if trace { + p.tracef("%d ", x) + } + + p.rawInt64(x) +} + +func (p *exporter) bytes(b []byte) { + if debug { + p.marker('b') + } + + if trace { + p.tracef("%q ", b) + } + + p.rawInt64(int64(len(b))) + if len(b) > 0 { + p.data = append(p.data, b...) + } +} + +// marker emits a marker byte and position information which makes +// it easy for a reader to detect if it is "out of sync". Used in +// debug mode only. +func (p *exporter) marker(m byte) { + if debug { + p.data = append(p.data, m) + p.rawInt64(int64(len(p.data))) + } +} + +// rawInt64 should only be used by low-level encoders +func (p *exporter) rawInt64(x int64) { + var tmp [binary.MaxVarintLen64]byte + n := binary.PutVarint(tmp[:], x) + p.data = append(p.data, tmp[:n]...) +} + +// utility functions + +func (p *exporter) tracef(format string, args ...interface{}) { + // rewrite format string to take care of indentation + const indent = ". " + if strings.IndexAny(format, "{}\n") >= 0 { + var buf bytes.Buffer + for i := 0; i < len(format); i++ { + // no need to deal with runes + ch := format[i] + switch ch { + case '{': + p.indent += indent + case '}': + p.indent = p.indent[:len(p.indent)-len(indent)] + if i+1 < len(format) && format[i+1] == '\n' { + buf.WriteByte('\n') + buf.WriteString(p.indent) + buf.WriteString("} ") + i++ + continue + } + } + buf.WriteByte(ch) + if ch == '\n' { + buf.WriteString(p.indent) + } + } + format = buf.String() + } + fmt.Printf(format, args...) +} + +func exported(name string) bool { + return ast.IsExported(name) +} diff --git a/go/importer/import.go b/go/importer/import.go new file mode 100644 index 00000000..5aa01487 --- /dev/null +++ b/go/importer/import.go @@ -0,0 +1,417 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This implementation is loosely based on the algorithm described +// in: "On the linearization of graphs and writing symbol files", +// by R. Griesemer, Technical Report 156, ETH Zürich, 1991. + +// package importer implements an exporter and importer for Go export data. +package importer + +import ( + "encoding/binary" + "fmt" + "go/ast" + "go/token" + + "code.google.com/p/go.tools/go/exact" + "code.google.com/p/go.tools/go/types" +) + +// ImportData imports a package from the serialized package data. +// If data is obviously malformed, an error is returned but in +// general it is not recommended to call ImportData on untrusted +// data. +func ImportData(imports map[string]*types.Package, data []byte) (*types.Package, error) { + // check magic string + if n := len(magic); len(data) < n || string(data[:n]) != magic { + return nil, fmt.Errorf("incorrect magic string: got %q; want %q", data[:n], magic) + } + + p := importer{ + data: data[len(magic):], + imports: imports, + pkgList: []*types.Package{nil}, + typList: []types.Type{nil}, + consumed: len(magic), // for debugging only + } + + // populate typList with predeclared types + for _, t := range types.Typ[1:] { + p.typList = append(p.typList, t) + } + p.typList = append(p.typList, types.Universe.Lookup("error").Type()) + + if v := p.int(); v != version { + return nil, fmt.Errorf("unknown version: got %d; want %d", v, version) + } + + pkg := p.pkg() + if debug && p.pkgList[1] != pkg { + panic("imported packaged not found in pkgList[1]") + } + + // read objects + n := p.int() + for i := 0; i < n; i++ { + p.obj(pkg) + } + + if len(p.data) > 0 { + return nil, fmt.Errorf("not all input data consumed") + } + + // package was imported completely and without errors + pkg.MarkComplete() + + return pkg, nil +} + +type importer struct { + data []byte + imports map[string]*types.Package + pkgList []*types.Package + typList []types.Type + + // debugging support + consumed int +} + +func (p *importer) pkg() *types.Package { + // if the package was seen before, i is its index (>= 0) + i := p.int() + if i >= 0 { + return p.pkgList[i] + } + + // otherwise, i is the package tag (< 0) + if i != _Package { + panic(fmt.Sprintf("unexpected package tag %d", i)) + } + + // read package data + name := p.string() + path := p.string() + + // if the package was imported before, use that one; otherwise create a new one + pkg := p.imports[path] + if pkg == nil { + pkg = types.NewPackage(path, name, types.NewScope(nil)) + p.imports[path] = pkg + } + p.pkgList = append(p.pkgList, pkg) + + return pkg +} + +func (p *importer) obj(pkg *types.Package) { + var obj types.Object + switch tag := p.int(); tag { + case _Const: + obj = types.NewConst(token.NoPos, pkg, p.string(), p.typ(), p.val()) + case _Type: + // type object is added to scope via respective named type + _ = p.typ().(*types.Named) + return + case _Var: + obj = types.NewVar(token.NoPos, pkg, p.string(), p.typ()) + case _Func: + obj = types.NewFunc(token.NoPos, pkg, p.string(), p.signature()) + default: + panic(fmt.Sprintf("unexpected object tag %d", tag)) + } + + if alt := pkg.Scope().Insert(obj); alt != nil { + panic(fmt.Sprintf("%s already declared", alt.Name())) + } +} + +func (p *importer) val() exact.Value { + switch kind := exact.Kind(p.int()); kind { + case exact.Bool: + return exact.MakeBool(p.bool()) + case exact.String: + return exact.MakeString(p.string()) + case exact.Int: + return p.intVal() + case exact.Float: + return p.floatVal() + case exact.Complex: + re := p.floatVal() + im := p.floatVal() + return exact.BinaryOp(re, token.ADD, exact.MakeImag(im)) + default: + panic(fmt.Sprintf("unexpected value kind %d", kind)) + } +} + +func (p *importer) intVal() exact.Value { + sign := p.int() + var bytes []byte + if sign != 0 { + bytes = p.bytes() + } + x := exact.MakeFromBytes(bytes) + if sign < 0 { + x = exact.UnaryOp(token.SUB, x, 0) + } + return x +} + +func (p *importer) floatVal() exact.Value { + x := p.intVal() + if exact.Sign(x) != 0 { + y := exact.MakeFromBytes(p.bytes()) + x = exact.BinaryOp(x, token.QUO, y) + } + return x +} + +func (p *importer) record(t types.Type) { + p.typList = append(p.typList, t) +} + +func (p *importer) typ() types.Type { + // if the type was seen before, i is its index (>= 0) + i := p.int() + if i >= 0 { + return p.typList[i] + } + + // otherwise, i is the type tag (< 0) + switch i { + case _Basic: + t := types.Universe.Lookup(p.string()).(*types.TypeName).Type().(*types.Basic) + p.record(t) + return t + + case _Array: + t := new(types.Array) + p.record(t) + + *t = *types.NewArray(p.typ(), p.int64()) + return t + + case _Slice: + t := new(types.Slice) + p.record(t) + + *t = *types.NewSlice(p.typ()) + return t + + case _Struct: + t := new(types.Struct) + p.record(t) + + n := p.int() + fields := make([]*types.Var, n) + tags := make([]string, n) + for i := range fields { + fields[i] = p.field() + tags[i] = p.string() + } + *t = *types.NewStruct(fields, tags) + return t + + case _Pointer: + t := new(types.Pointer) + p.record(t) + + *t = *types.NewPointer(p.typ()) + return t + + case _Signature: + t := new(types.Signature) + p.record(t) + + *t = *p.signature() + return t + + case _Interface: + t := new(types.Interface) + p.record(t) + + methods := make([]*types.Func, p.int()) + for i := range methods { + pkg, name := p.qualifiedName() + methods[i] = types.NewFunc(token.NoPos, pkg, name, p.signature()) + } + *t = *types.NewInterface(methods, nil) + return t + + case _Map: + t := new(types.Map) + p.record(t) + + *t = *types.NewMap(p.typ(), p.typ()) + return t + + case _Chan: + t := new(types.Chan) + p.record(t) + + *t = *types.NewChan(ast.ChanDir(p.int()), p.typ()) + return t + + case _Named: + // import type object + name := p.string() + pkg := p.pkg() + scope := pkg.Scope() + obj := scope.Lookup(name) + if obj == nil { + new := types.NewTypeName(token.NoPos, pkg, name, nil) + types.NewNamed(new, nil, nil) + scope.Insert(new) + obj = new + } + t := obj.Type().(*types.Named) + p.record(t) + + // import underlying type + u := p.typ() + if t.Underlying() == nil { + t.SetUnderlying(u) + } + + // read associated methods + n := p.int() + for i := 0; i < n; i++ { + t.AddMethod(types.NewFunc(token.NoPos, pkg, p.string(), p.signature())) + } + + return t + + default: + panic(fmt.Sprintf("unexpected type tag %d", i)) + } +} + +func deref(typ types.Type) types.Type { + if p, _ := typ.(*types.Pointer); p != nil { + return p.Elem() + } + return typ +} + +func (p *importer) field() *types.Var { + pkg, name := p.qualifiedName() + typ := p.typ() + + anonymous := false + if name == "" { + // anonymous field - typ must be T or *T and T must be a type name + switch typ := deref(typ).(type) { + case *types.Basic: // basic types are named types + pkg = nil + name = typ.Name() + case *types.Named: + obj := typ.Obj() + pkg = obj.Pkg() // TODO(gri) is this still correct? + name = obj.Name() + default: + panic("anonymous field expected") + } + anonymous = true + } + + return types.NewField(token.NoPos, pkg, name, typ, anonymous) +} + +func (p *importer) qualifiedName() (*types.Package, string) { + name := p.string() + pkg := p.pkgList[1] // exported names assume current package + if !exported(name) { + pkg = p.pkg() + if pkg == nil { + panic(fmt.Sprintf("nil package for unexported qualified name %q", name)) + } + } + return pkg, name +} + +func (p *importer) signature() *types.Signature { + var recv *types.Var + if p.bool() { + recv = p.param() + } + return types.NewSignature(nil, recv, p.tuple(), p.tuple(), p.bool()) +} + +func (p *importer) param() *types.Var { + return types.NewVar(token.NoPos, nil, p.string(), p.typ()) +} + +func (p *importer) tuple() *types.Tuple { + vars := make([]*types.Var, p.int()) + for i := range vars { + vars[i] = p.param() + } + return types.NewTuple(vars...) +} + +// ---------------------------------------------------------------------------- +// decoders + +func (p *importer) bool() bool { + return p.int64() != 0 +} + +func (p *importer) string() string { + return string(p.bytes()) +} + +func (p *importer) int() int { + return int(p.int64()) +} + +func (p *importer) int64() int64 { + if debug { + p.marker('i') + } + + return p.rawInt64() +} + +// Note: bytes() returns the respective byte slice w/o copy. +func (p *importer) bytes() []byte { + if debug { + p.marker('b') + } + + var b []byte + if n := int(p.rawInt64()); n > 0 { + b = p.data[:n] + p.data = p.data[n:] + if debug { + p.consumed += n + } + } + return b +} + +func (p *importer) marker(want byte) { + if debug { + if got := p.data[0]; got != want { + panic(fmt.Sprintf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.consumed)) + } + p.data = p.data[1:] + p.consumed++ + + pos := p.consumed + if n := int(p.rawInt64()); n != pos { + panic(fmt.Sprintf("incorrect position: got %d; want %d", n, pos)) + } + } +} + +// rawInt64 should only be used by low-level decoders +func (p *importer) rawInt64() int64 { + i, n := binary.Varint(p.data) + p.data = p.data[n:] + if debug { + p.consumed += n + } + return i +} diff --git a/go/importer/import_test.go b/go/importer/import_test.go new file mode 100644 index 00000000..0d139ac4 --- /dev/null +++ b/go/importer/import_test.go @@ -0,0 +1,197 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package importer + +import ( + "bytes" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "path/filepath" + "testing" + + _ "code.google.com/p/go.tools/go/gcimporter" + "code.google.com/p/go.tools/go/types" +) + +var tests = []string{ + `package p`, + + // consts + `package p; const X = true`, + `package p; const X, y, Z = true, false, 0 != 0`, + `package p; const ( A float32 = 1< 0 { + buf.WriteString("\nmethods (\n") + for i := 0; i < n; i++ { + fmt.Fprintf(&buf, "\t%s\n", typ.Method(i)) + } + buf.WriteString(")") + } + } + } + buf.WriteByte('\n') + } + } + + return buf.String() +}