From 764c4ccf448e9568efd90fa837d6c38ee64252e9 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Thu, 14 Apr 2016 15:38:53 -0700 Subject: [PATCH] go/gcimporter15: match https://golang.org/cl/22096/ Read and write position info. Change-Id: Ibe4a914ff51911bbda656b08f1397132e495ab8a Reviewed-on: https://go-review.googlesource.com/22098 Reviewed-by: Alan Donovan --- go/gcimporter15/bexport.go | 71 ++++++++++++++--- go/gcimporter15/bexport_test.go | 2 +- go/gcimporter15/bimport.go | 132 ++++++++++++++++++++++---------- 3 files changed, 153 insertions(+), 52 deletions(-) diff --git a/go/gcimporter15/bexport.go b/go/gcimporter15/bexport.go index 7bdffe4e..e8ac1bab 100644 --- a/go/gcimporter15/bexport.go +++ b/go/gcimporter15/bexport.go @@ -16,6 +16,7 @@ import ( "fmt" "go/ast" "go/constant" + "go/token" "go/types" "log" "math" @@ -42,17 +43,29 @@ const trace = false // default: false const exportVersion = "v0" type exporter struct { - out bytes.Buffer + fset *token.FileSet + out bytes.Buffer + + // object -> index maps, indexed in order of serialization + strIndex map[string]int pkgIndex map[*types.Package]int typIndex map[types.Type]int + // position encoding + prevFile string + prevLine int + + // debugging support written int // bytes written indent int // for trace } // BExportData returns binary export data for pkg. -func BExportData(pkg *types.Package) []byte { +// If no file set is provided, position info will be missing. +func BExportData(fset *token.FileSet, pkg *types.Package) []byte { p := exporter{ + fset: fset, + strIndex: map[string]int{"": 0}, // empty string is mapped to 0 pkgIndex: make(map[*types.Package]int), typIndex: make(map[types.Type]int), } @@ -62,7 +75,7 @@ func BExportData(pkg *types.Package) []byte { if debugFormat { format = 'd' } - p.byte(format) + p.rawByte(format) // --- generic export data --- @@ -158,6 +171,7 @@ func (p *exporter) obj(obj types.Object) { switch obj := obj.(type) { case *types.Const: p.tag(constTag) + p.pos(obj) p.qualifiedName(obj) p.typ(obj.Type()) p.value(obj.Val()) @@ -168,11 +182,13 @@ func (p *exporter) obj(obj types.Object) { case *types.Var: p.tag(varTag) + p.pos(obj) p.qualifiedName(obj) p.typ(obj.Type()) case *types.Func: p.tag(funcTag) + p.pos(obj) p.qualifiedName(obj) sig := obj.Type().(*types.Signature) p.paramList(sig.Params(), sig.Variadic()) @@ -183,6 +199,28 @@ func (p *exporter) obj(obj types.Object) { } } +func (p *exporter) pos(obj types.Object) { + var file string + var line int + if p.fset != nil { + pos := p.fset.Position(obj.Pos()) + file = pos.Filename + line = pos.Line + } + + if file == p.prevFile && line != p.prevLine { + // common case: write delta-encoded line number + p.int(line - p.prevLine) // != 0 + } else { + // uncommon case: filename changed, or line didn't change + p.int(0) + p.string(file) + p.int(line) + p.prevFile = file + } + p.prevLine = line +} + func (p *exporter) qualifiedName(obj types.Object) { p.string(obj.Name()) p.pkg(obj.Pkg(), false) @@ -219,6 +257,7 @@ func (p *exporter) typ(t types.Type) { switch t := t.(type) { case *types.Named: p.tag(namedTag) + p.pos(t.Obj()) p.qualifiedName(t.Obj()) p.typ(t.Underlying()) if !types.IsInterface(t) { @@ -289,6 +328,7 @@ func (p *exporter) assocMethods(named *types.Named) { p.tracef("\n") } + p.pos(m) name := m.Name() p.string(name) if !exported(name) { @@ -333,6 +373,7 @@ func (p *exporter) field(f *types.Var) { log.Fatalf("gcimporter: field expected") } + p.pos(f) p.fieldName(f) p.typ(f.Type()) } @@ -362,6 +403,7 @@ func (p *exporter) method(m *types.Func) { log.Fatalf("gcimporter: method expected") } + p.pos(m) p.string(m.Name()) if m.Name() != "_" && !ast.IsExported(m.Name()) { p.pkg(m.Pkg(), false) @@ -571,9 +613,17 @@ func (p *exporter) string(s string) { if trace { p.tracef("%q ", s) } - p.rawInt64(int64(len(s))) + // if we saw the string before, write its index (>= 0) + // (the empty string is mapped to 0) + if i, ok := p.strIndex[s]; ok { + p.rawInt64(int64(i)) + return + } + // otherwise, remember string and write its negative length and bytes + p.strIndex[s] = len(p.strIndex) + p.rawInt64(-int64(len(s))) for i := 0; i < len(s); i++ { - p.byte(s[i]) + p.rawByte(s[i]) } } @@ -581,7 +631,7 @@ func (p *exporter) string(s string) { // it easy for a reader to detect if it is "out of sync". Used for // debugFormat format only. func (p *exporter) marker(m byte) { - p.byte(m) + p.rawByte(m) // Enable this for help tracking down the location // of an incorrect marker when running in debugFormat. if false && trace { @@ -595,12 +645,12 @@ func (p *exporter) rawInt64(x int64) { var tmp [binary.MaxVarintLen64]byte n := binary.PutVarint(tmp[:], x) for i := 0; i < n; i++ { - p.byte(tmp[i]) + p.rawByte(tmp[i]) } } -// byte is the bottleneck interface to write to p.out. -// byte escapes b as follows (any encoding does that +// rawByte is the bottleneck interface to write to p.out. +// rawByte escapes b as follows (any encoding does that // hides '$'): // // '$' => '|' 'S' @@ -608,7 +658,8 @@ func (p *exporter) rawInt64(x int64) { // // Necessary so other tools can find the end of the // export data by searching for "$$". -func (p *exporter) byte(b byte) { +// rawByte should only be used by low-level encoders. +func (p *exporter) rawByte(b byte) { switch b { case '$': // write '$' as '|' 'S' diff --git a/go/gcimporter15/bexport_test.go b/go/gcimporter15/bexport_test.go index 81a11549..35192558 100644 --- a/go/gcimporter15/bexport_test.go +++ b/go/gcimporter15/bexport_test.go @@ -63,7 +63,7 @@ type UnknownType undefined if info.Files == nil { continue // empty directory } - exportdata := gcimporter.BExportData(pkg) + exportdata := gcimporter.BExportData(conf.Fset, pkg) imports := make(map[string]*types.Package) n, pkg2, err := gcimporter.BImportData(imports, exportdata, pkg.Path()) diff --git a/go/gcimporter15/bimport.go b/go/gcimporter15/bimport.go index 31c34bc9..fd77daf9 100644 --- a/go/gcimporter15/bimport.go +++ b/go/gcimporter15/bimport.go @@ -6,6 +6,10 @@ // This file is a copy of $GOROOT/src/go/internal/gcimporter/bimport.go, tagged for go1.5. +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package gcimporter import ( @@ -23,13 +27,18 @@ type importer struct { imports map[string]*types.Package data []byte path string + buf []byte // for reading strings - buf []byte // for reading strings - bufarray [64]byte // initial underlying array for buf, large enough to avoid allocation when compiling std lib + // object lists + strList []string // in order of appearance + pkgList []*types.Package // in order of appearance + typList []types.Type // in order of appearance - pkgList []*types.Package - typList []types.Type + // position encoding + prevFile string + prevLine int + // debugging support debugFormat bool read int // bytes read } @@ -43,11 +52,11 @@ func BImportData(imports map[string]*types.Package, data []byte, path string) (i imports: imports, data: data, path: path, + strList: []string{""}, // empty string is mapped to 0 } - p.buf = p.bufarray[:] // read low-level encoding format - switch format := p.byte(); format { + switch format := p.rawByte(); format { case 'c': // compact format - nothing to do case 'd': @@ -164,6 +173,7 @@ func (p *importer) declare(obj types.Object) { func (p *importer) obj(tag int) { switch tag { case constTag: + p.pos() pkg, name := p.qualifiedName() typ := p.typ(nil) val := p.value() @@ -173,11 +183,13 @@ func (p *importer) obj(tag int) { _ = p.typ(nil) case varTag: + p.pos() pkg, name := p.qualifiedName() typ := p.typ(nil) p.declare(types.NewVar(token.NoPos, pkg, name, typ)) case funcTag: + p.pos() pkg, name := p.qualifiedName() params, isddd := p.paramList() result, _ := p.paramList() @@ -189,6 +201,22 @@ func (p *importer) obj(tag int) { } } +func (p *importer) pos() { + file := p.prevFile + line := p.prevLine + + if delta := p.int(); delta != 0 { + line += delta + } else { + file = p.string() + line = p.int() + p.prevFile = file + } + p.prevLine = line + + // TODO(gri) register new position +} + func (p *importer) qualifiedName() (pkg *types.Package, name string) { name = p.string() pkg = p.pkg() @@ -224,6 +252,7 @@ func (p *importer) typ(parent *types.Package) types.Type { switch i { case namedTag: // read type object + p.pos() parent, name := p.qualifiedName() scope := parent.Scope() obj := scope.Lookup(name) @@ -256,6 +285,7 @@ func (p *importer) typ(parent *types.Package) types.Type { // read associated methods for i := p.int(); i > 0; i-- { // TODO(gri) replace this with something closer to fieldName + p.pos() name := p.string() if !exported(name) { p.pkg() @@ -297,14 +327,7 @@ func (p *importer) typ(parent *types.Package) types.Type { t := new(types.Struct) p.record(t) - n := p.int() - fields := make([]*types.Var, n) - tags := make([]string, n) - for i := range fields { - fields[i] = p.field(parent) - tags[i] = p.string() - } - *t = *types.NewStruct(fields, tags) + *t = *types.NewStruct(p.fieldList(parent)) return t case pointerTag: @@ -336,17 +359,7 @@ func (p *importer) typ(parent *types.Package) types.Type { panic("unexpected embedded interface") } - // read methods - methods := make([]*types.Func, p.int()) - for i := range methods { - pkg, name := p.fieldName(parent) - params, isddd := p.paramList() - result, _ := p.paramList() - sig := types.NewSignature(nil, params, result, isddd) - methods[i] = types.NewFunc(token.NoPos, pkg, name, sig) - } - - t := types.NewInterface(methods, nil) + t := types.NewInterface(p.methodList(parent), nil) p.typList[n] = t return t @@ -384,7 +397,20 @@ func (p *importer) typ(parent *types.Package) types.Type { } } +func (p *importer) fieldList(parent *types.Package) (fields []*types.Var, tags []string) { + if n := p.int(); n > 0 { + fields = make([]*types.Var, n) + tags = make([]string, n) + for i := range fields { + fields[i] = p.field(parent) + tags[i] = p.string() + } + } + return +} + func (p *importer) field(parent *types.Package) *types.Var { + p.pos() pkg, name := p.fieldName(parent) typ := p.typ(parent) @@ -406,6 +432,25 @@ func (p *importer) field(parent *types.Package) *types.Var { return types.NewField(token.NoPos, pkg, name, typ, anonymous) } +func (p *importer) methodList(parent *types.Package) (methods []*types.Func) { + if n := p.int(); n > 0 { + methods = make([]*types.Func, n) + for i := range methods { + methods[i] = p.method(parent) + } + } + return +} + +func (p *importer) method(parent *types.Package) *types.Func { + p.pos() + pkg, name := p.fieldName(parent) + params, isddd := p.paramList() + result, _ := p.paramList() + sig := types.NewSignature(nil, params, result, isddd) + return types.NewFunc(token.NoPos, pkg, name, sig) +} + func (p *importer) fieldName(parent *types.Package) (*types.Package, string) { pkg := parent if pkg == nil { @@ -573,24 +618,28 @@ func (p *importer) string() string { if p.debugFormat { p.marker('s') } - - if n := int(p.rawInt64()); n > 0 { - if cap(p.buf) < n { - p.buf = make([]byte, n) - } else { - p.buf = p.buf[:n] - } - for i := 0; i < n; i++ { - p.buf[i] = p.byte() - } - return string(p.buf) + // if the string was seen before, i is its index (>= 0) + // (the empty string is at index 0) + i := p.rawInt64() + if i >= 0 { + return p.strList[i] } - - return "" + // otherwise, i is the negative string length (< 0) + if n := int(-i); n <= cap(p.buf) { + p.buf = p.buf[:n] + } else { + p.buf = make([]byte, n) + } + for i := range p.buf { + p.buf[i] = p.rawByte() + } + s := string(p.buf) + p.strList = append(p.strList, s) + return s } func (p *importer) marker(want byte) { - if got := p.byte(); got != want { + if got := p.rawByte(); got != want { panic(fmt.Sprintf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.read)) } @@ -611,12 +660,13 @@ func (p *importer) rawInt64() int64 { // needed for binary.ReadVarint in rawInt64 func (p *importer) ReadByte() (byte, error) { - return p.byte(), nil + return p.rawByte(), nil } // byte is the bottleneck interface for reading p.data. // It unescapes '|' 'S' to '$' and '|' '|' to '|'. -func (p *importer) byte() byte { +// rawByte should only be used by low-level decoders. +func (p *importer) rawByte() byte { b := p.data[0] r := 1 if b == '|' {