diff --git a/go/gcimporter15/bexport_test.go b/go/gcimporter15/bexport_test.go index 35192558..bc72bf64 100644 --- a/go/gcimporter15/bexport_test.go +++ b/go/gcimporter15/bexport_test.go @@ -11,10 +11,12 @@ import ( "go/ast" "go/build" "go/constant" + "go/parser" "go/token" "go/types" "reflect" "runtime" + "strings" "testing" "golang.org/x/tools/go/buildutil" @@ -66,7 +68,8 @@ type UnknownType undefined exportdata := gcimporter.BExportData(conf.Fset, pkg) imports := make(map[string]*types.Package) - n, pkg2, err := gcimporter.BImportData(imports, exportdata, pkg.Path()) + fset2 := token.NewFileSet() + n, pkg2, err := gcimporter.BImportData(fset2, imports, exportdata, pkg.Path()) if err != nil { t.Errorf("BImportData(%s): %v", pkg.Path(), err) continue @@ -87,6 +90,14 @@ type UnknownType undefined t.Errorf("%s.%s not found, want %s", pkg.Path(), name, obj1) continue } + + fl1 := fileLine(conf.Fset, obj1) + fl2 := fileLine(fset2, obj2) + if fl1 != fl2 { + t.Errorf("%s.%s: got posn %s, want %s", + pkg.Path(), name, fl2, fl1) + } + if err := equalObj(obj1, obj2); err != nil { t.Errorf("%s.%s: %s\ngot: %s\nwant: %s", pkg.Path(), name, err, obj2, obj1) @@ -95,6 +106,11 @@ type UnknownType undefined } } +func fileLine(fset *token.FileSet, obj types.Object) string { + posn := fset.Position(obj.Pos()) + return fmt.Sprintf("%s:%d", posn.Filename, posn.Line) +} + // equalObj reports how x and y differ. They are assumed to belong to // different universes so cannot be compared directly. func equalObj(x, y types.Object) error { @@ -273,3 +289,40 @@ func equalType(x, y types.Type) error { } return nil } + +// TestVeryLongFile tests the position of an import object declared in +// a very long input file. Line numbers greater than maxlines are +// reported as line 1, not garbage or token.NoPos. +func TestVeryLongFile(t *testing.T) { + // parse and typecheck + longFile := "package foo" + strings.Repeat("\n", 123456) + "var X int" + fset1 := token.NewFileSet() + f, err := parser.ParseFile(fset1, "foo.go", longFile, 0) + if err != nil { + t.Fatal(err) + } + var conf types.Config + pkg, err := conf.Check("foo", fset1, []*ast.File{f}, nil) + if err != nil { + t.Fatal(err) + } + + // export + exportdata := gcimporter.BExportData(fset1, pkg) + + // import + imports := make(map[string]*types.Package) + fset2 := token.NewFileSet() + _, pkg2, err := gcimporter.BImportData(fset2, imports, exportdata, pkg.Path()) + if err != nil { + t.Fatalf("BImportData(%s): %v", pkg.Path(), err) + } + + // compare + posn1 := fset1.Position(pkg.Scope().Lookup("X").Pos()) + posn2 := fset2.Position(pkg2.Scope().Lookup("X").Pos()) + if want := "foo.go:1:1"; posn2.String() != want { + t.Errorf("X position = %s, want %s (orig was %s)", + posn2, want, posn1) + } +} diff --git a/go/gcimporter15/bimport.go b/go/gcimporter15/bimport.go index a742dbfd..e6b4863e 100644 --- a/go/gcimporter15/bimport.go +++ b/go/gcimporter15/bimport.go @@ -16,6 +16,7 @@ import ( "go/types" "sort" "strings" + "sync" "unicode" "unicode/utf8" ) @@ -35,6 +36,8 @@ type importer struct { posInfoFormat bool prevFile string prevLine int + fset *token.FileSet + files map[string]*token.File // debugging support debugFormat bool @@ -45,12 +48,14 @@ type importer struct { // and returns the number of bytes consumed and a reference to the package. // If data is obviously malformed, an error is returned but in // general it is not recommended to call BImportData on untrusted data. -func BImportData(imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) { +func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) { p := importer{ imports: imports, data: data, path: path, strList: []string{""}, // empty string is mapped to 0 + fset: fset, + files: make(map[string]*token.File), } // read low-level encoding format @@ -173,37 +178,37 @@ func (p *importer) declare(obj types.Object) { func (p *importer) obj(tag int) { switch tag { case constTag: - p.pos() + pos := p.pos() pkg, name := p.qualifiedName() typ := p.typ(nil) val := p.value() - p.declare(types.NewConst(token.NoPos, pkg, name, typ, val)) + p.declare(types.NewConst(pos, pkg, name, typ, val)) case typeTag: _ = p.typ(nil) case varTag: - p.pos() + pos := p.pos() pkg, name := p.qualifiedName() typ := p.typ(nil) - p.declare(types.NewVar(token.NoPos, pkg, name, typ)) + p.declare(types.NewVar(pos, pkg, name, typ)) case funcTag: - p.pos() + pos := p.pos() pkg, name := p.qualifiedName() params, isddd := p.paramList() result, _ := p.paramList() sig := types.NewSignature(nil, params, result, isddd) - p.declare(types.NewFunc(token.NoPos, pkg, name, sig)) + p.declare(types.NewFunc(pos, pkg, name, sig)) default: panic(fmt.Sprintf("unexpected object tag %d", tag)) } } -func (p *importer) pos() { +func (p *importer) pos() token.Pos { if !p.posInfoFormat { - return + return token.NoPos } file := p.prevFile @@ -219,9 +224,40 @@ func (p *importer) pos() { } p.prevLine = line - // TODO(gri) register new position + // Synthesize a token.Pos + + // Since we don't know the set of needed file positions, we + // reserve maxlines positions per file. + const maxlines = 64 * 1024 + f := p.files[file] + if f == nil { + f = p.fset.AddFile(file, -1, maxlines) + p.files[file] = f + // Allocate the fake linebreak indices on first use. + // TODO(adonovan): opt: save ~512KB using a more complex scheme? + fakeLinesOnce.Do(func() { + fakeLines = make([]int, maxlines) + for i := range fakeLines { + fakeLines[i] = i + } + }) + f.SetLines(fakeLines) + } + + if line > maxlines { + line = 1 + } + + // Treat the file as if it contained only newlines + // and column=1: use the line number as the offset. + return f.Pos(line - 1) } +var ( + fakeLines []int + fakeLinesOnce sync.Once +) + func (p *importer) qualifiedName() (pkg *types.Package, name string) { name = p.string() pkg = p.pkg() @@ -257,14 +293,14 @@ func (p *importer) typ(parent *types.Package) types.Type { switch i { case namedTag: // read type object - p.pos() + pos := p.pos() parent, name := p.qualifiedName() scope := parent.Scope() obj := scope.Lookup(name) // if the object doesn't exist yet, create and insert it if obj == nil { - obj = types.NewTypeName(token.NoPos, parent, name, nil) + obj = types.NewTypeName(pos, parent, name, nil) scope.Insert(obj) } @@ -290,7 +326,7 @@ func (p *importer) typ(parent *types.Package) types.Type { // read associated methods for i := p.int(); i > 0; i-- { // TODO(gri) replace this with something closer to fieldName - p.pos() + pos := p.pos() name := p.string() if !exported(name) { p.pkg() @@ -301,7 +337,7 @@ func (p *importer) typ(parent *types.Package) types.Type { result, _ := p.paramList() sig := types.NewSignature(recv.At(0), params, result, isddd) - t0.AddMethod(types.NewFunc(token.NoPos, parent, name, sig)) + t0.AddMethod(types.NewFunc(pos, parent, name, sig)) } return t @@ -415,7 +451,7 @@ func (p *importer) fieldList(parent *types.Package) (fields []*types.Var, tags [ } func (p *importer) field(parent *types.Package) *types.Var { - p.pos() + pos := p.pos() pkg, name := p.fieldName(parent) typ := p.typ(parent) @@ -434,7 +470,7 @@ func (p *importer) field(parent *types.Package) *types.Var { anonymous = true } - return types.NewField(token.NoPos, pkg, name, typ, anonymous) + return types.NewField(pos, pkg, name, typ, anonymous) } func (p *importer) methodList(parent *types.Package) (methods []*types.Func) { @@ -448,12 +484,12 @@ func (p *importer) methodList(parent *types.Package) (methods []*types.Func) { } func (p *importer) method(parent *types.Package) *types.Func { - p.pos() + pos := p.pos() pkg, name := p.fieldName(parent) params, isddd := p.paramList() result, _ := p.paramList() sig := types.NewSignature(nil, params, result, isddd) - return types.NewFunc(token.NoPos, pkg, name, sig) + return types.NewFunc(pos, pkg, name, sig) } func (p *importer) fieldName(parent *types.Package) (*types.Package, string) { diff --git a/go/gcimporter15/gcimporter.go b/go/gcimporter15/gcimporter.go index e1022103..4416739b 100644 --- a/go/gcimporter15/gcimporter.go +++ b/go/gcimporter15/gcimporter.go @@ -174,7 +174,8 @@ func Import(packages map[string]*types.Package, path, srcDir string) (pkg *types var data []byte data, err = ioutil.ReadAll(buf) if err == nil { - _, pkg, err = BImportData(packages, data, path) + fset := token.NewFileSet() + _, pkg, err = BImportData(fset, packages, data, path) return } default: