go/gcimporter15: match https://golang.org/cl/22096/

Read and write position info.

Change-Id: Ibe4a914ff51911bbda656b08f1397132e495ab8a
Reviewed-on: https://go-review.googlesource.com/22098
Reviewed-by: Alan Donovan <adonovan@google.com>

go/gcimporter15/bexport.go

@@ -16,6 +16,7 @@ import (
 	"fmt"
 	"go/ast"
 	"go/constant"
+	"go/token"
 	"go/types"
 	"log"
 	"math"
@@ -42,17 +43,29 @@ const trace = false // default: false
 const exportVersion = "v0"
 
 type exporter struct {
-	out      bytes.Buffer
+	fset *token.FileSet
+	out  bytes.Buffer
+
+	// object -> index maps, indexed in order of serialization
+	strIndex map[string]int
 	pkgIndex map[*types.Package]int
 	typIndex map[types.Type]int
 
+	// position encoding
+	prevFile string
+	prevLine int
+
+	// debugging support
 	written int // bytes written
 	indent  int // for trace
 }
 
 // BExportData returns binary export data for pkg.
-func BExportData(pkg *types.Package) []byte {
+// If no file set is provided, position info will be missing.
+func BExportData(fset *token.FileSet, pkg *types.Package) []byte {
 	p := exporter{
+		fset:     fset,
+		strIndex: map[string]int{"": 0}, // empty string is mapped to 0
 		pkgIndex: make(map[*types.Package]int),
 		typIndex: make(map[types.Type]int),
 	}
@@ -62,7 +75,7 @@ func BExportData(pkg *types.Package) []byte {
 	if debugFormat {
 		format = 'd'
 	}
-	p.byte(format)
+	p.rawByte(format)
 
 	// --- generic export data ---
 
@@ -158,6 +171,7 @@ func (p *exporter) obj(obj types.Object) {
 	switch obj := obj.(type) {
 	case *types.Const:
 		p.tag(constTag)
+		p.pos(obj)
 		p.qualifiedName(obj)
 		p.typ(obj.Type())
 		p.value(obj.Val())
@@ -168,11 +182,13 @@ func (p *exporter) obj(obj types.Object) {
 
 	case *types.Var:
 		p.tag(varTag)
+		p.pos(obj)
 		p.qualifiedName(obj)
 		p.typ(obj.Type())
 
 	case *types.Func:
 		p.tag(funcTag)
+		p.pos(obj)
 		p.qualifiedName(obj)
 		sig := obj.Type().(*types.Signature)
 		p.paramList(sig.Params(), sig.Variadic())
@@ -183,6 +199,28 @@ func (p *exporter) obj(obj types.Object) {
 	}
 }
 
+func (p *exporter) pos(obj types.Object) {
+	var file string
+	var line int
+	if p.fset != nil {
+		pos := p.fset.Position(obj.Pos())
+		file = pos.Filename
+		line = pos.Line
+	}
+
+	if file == p.prevFile && line != p.prevLine {
+		// common case: write delta-encoded line number
+		p.int(line - p.prevLine) // != 0
+	} else {
+		// uncommon case: filename changed, or line didn't change
+		p.int(0)
+		p.string(file)
+		p.int(line)
+		p.prevFile = file
+	}
+	p.prevLine = line
+}
+
 func (p *exporter) qualifiedName(obj types.Object) {
 	p.string(obj.Name())
 	p.pkg(obj.Pkg(), false)
@@ -219,6 +257,7 @@ func (p *exporter) typ(t types.Type) {
 	switch t := t.(type) {
 	case *types.Named:
 		p.tag(namedTag)
+		p.pos(t.Obj())
 		p.qualifiedName(t.Obj())
 		p.typ(t.Underlying())
 		if !types.IsInterface(t) {
@@ -289,6 +328,7 @@ func (p *exporter) assocMethods(named *types.Named) {
 			p.tracef("\n")
 		}
 
+		p.pos(m)
 		name := m.Name()
 		p.string(name)
 		if !exported(name) {
@@ -333,6 +373,7 @@ func (p *exporter) field(f *types.Var) {
 		log.Fatalf("gcimporter: field expected")
 	}
 
+	p.pos(f)
 	p.fieldName(f)
 	p.typ(f.Type())
 }
@@ -362,6 +403,7 @@ func (p *exporter) method(m *types.Func) {
 		log.Fatalf("gcimporter: method expected")
 	}
 
+	p.pos(m)
 	p.string(m.Name())
 	if m.Name() != "_" && !ast.IsExported(m.Name()) {
 		p.pkg(m.Pkg(), false)
@@ -571,9 +613,17 @@ func (p *exporter) string(s string) {
 	if trace {
 		p.tracef("%q ", s)
 	}
-	p.rawInt64(int64(len(s)))
+	// if we saw the string before, write its index (>= 0)
+	// (the empty string is mapped to 0)
+	if i, ok := p.strIndex[s]; ok {
+		p.rawInt64(int64(i))
+		return
+	}
+	// otherwise, remember string and write its negative length and bytes
+	p.strIndex[s] = len(p.strIndex)
+	p.rawInt64(-int64(len(s)))
 	for i := 0; i < len(s); i++ {
-		p.byte(s[i])
+		p.rawByte(s[i])
 	}
 }
 
@@ -581,7 +631,7 @@ func (p *exporter) string(s string) {
 // it easy for a reader to detect if it is "out of sync". Used for
 // debugFormat format only.
 func (p *exporter) marker(m byte) {
-	p.byte(m)
+	p.rawByte(m)
 	// Enable this for help tracking down the location
 	// of an incorrect marker when running in debugFormat.
 	if false && trace {
@@ -595,12 +645,12 @@ func (p *exporter) rawInt64(x int64) {
 	var tmp [binary.MaxVarintLen64]byte
 	n := binary.PutVarint(tmp[:], x)
 	for i := 0; i < n; i++ {
-		p.byte(tmp[i])
+		p.rawByte(tmp[i])
 	}
 }
 
-// byte is the bottleneck interface to write to p.out.
+// rawByte is the bottleneck interface to write to p.out.
-// byte escapes b as follows (any encoding does that
+// rawByte escapes b as follows (any encoding does that
 // hides '$'):
 //
 //	'$'  => '|' 'S'
@@ -608,7 +658,8 @@ func (p *exporter) rawInt64(x int64) {
 //
 // Necessary so other tools can find the end of the
 // export data by searching for "$$".
-func (p *exporter) byte(b byte) {
+// rawByte should only be used by low-level encoders.
+func (p *exporter) rawByte(b byte) {
 	switch b {
 	case '$':
 		// write '$' as '|' 'S'

go/gcimporter15/bexport_test.go

@@ -63,7 +63,7 @@ type UnknownType undefined
 		if info.Files == nil {
 			continue // empty directory
 		}
-		exportdata := gcimporter.BExportData(pkg)
+		exportdata := gcimporter.BExportData(conf.Fset, pkg)
 
 		imports := make(map[string]*types.Package)
 		n, pkg2, err := gcimporter.BImportData(imports, exportdata, pkg.Path())

go/gcimporter15/bimport.go

@@ -6,6 +6,10 @@
 
 // This file is a copy of $GOROOT/src/go/internal/gcimporter/bimport.go, tagged for go1.5.
 
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package gcimporter
 
 import (
@@ -23,13 +27,18 @@ type importer struct {
 	imports map[string]*types.Package
 	data    []byte
 	path    string
+	buf     []byte // for reading strings
 
-	buf      []byte   // for reading strings
+	// object lists
-	bufarray [64]byte // initial underlying array for buf, large enough to avoid allocation when compiling std lib
+	strList []string         // in order of appearance
+	pkgList []*types.Package // in order of appearance
+	typList []types.Type     // in order of appearance
 
-	pkgList []*types.Package
+	// position encoding
-	typList []types.Type
+	prevFile string
+	prevLine int
 
+	// debugging support
 	debugFormat bool
 	read        int // bytes read
 }
@@ -43,11 +52,11 @@ func BImportData(imports map[string]*types.Package, data []byte, path string) (i
 		imports: imports,
 		data:    data,
 		path:    path,
+		strList: []string{""}, // empty string is mapped to 0
 	}
-	p.buf = p.bufarray[:]
 
 	// read low-level encoding format
-	switch format := p.byte(); format {
+	switch format := p.rawByte(); format {
 	case 'c':
 		// compact format - nothing to do
 	case 'd':
@@ -164,6 +173,7 @@ func (p *importer) declare(obj types.Object) {
 func (p *importer) obj(tag int) {
 	switch tag {
 	case constTag:
+		p.pos()
 		pkg, name := p.qualifiedName()
 		typ := p.typ(nil)
 		val := p.value()
@@ -173,11 +183,13 @@ func (p *importer) obj(tag int) {
 		_ = p.typ(nil)
 
 	case varTag:
+		p.pos()
 		pkg, name := p.qualifiedName()
 		typ := p.typ(nil)
 		p.declare(types.NewVar(token.NoPos, pkg, name, typ))
 
 	case funcTag:
+		p.pos()
 		pkg, name := p.qualifiedName()
 		params, isddd := p.paramList()
 		result, _ := p.paramList()
@@ -189,6 +201,22 @@ func (p *importer) obj(tag int) {
 	}
 }
 
+func (p *importer) pos() {
+	file := p.prevFile
+	line := p.prevLine
+
+	if delta := p.int(); delta != 0 {
+		line += delta
+	} else {
+		file = p.string()
+		line = p.int()
+		p.prevFile = file
+	}
+	p.prevLine = line
+
+	// TODO(gri) register new position
+}
+
 func (p *importer) qualifiedName() (pkg *types.Package, name string) {
 	name = p.string()
 	pkg = p.pkg()
@@ -224,6 +252,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
 	switch i {
 	case namedTag:
 		// read type object
+		p.pos()
 		parent, name := p.qualifiedName()
 		scope := parent.Scope()
 		obj := scope.Lookup(name)
@@ -256,6 +285,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
 		// read associated methods
 		for i := p.int(); i > 0; i-- {
 			// TODO(gri) replace this with something closer to fieldName
+			p.pos()
 			name := p.string()
 			if !exported(name) {
 				p.pkg()
@@ -297,14 +327,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
 		t := new(types.Struct)
 		p.record(t)
 
-		n := p.int()
+		*t = *types.NewStruct(p.fieldList(parent))
-		fields := make([]*types.Var, n)
-		tags := make([]string, n)
-		for i := range fields {
-			fields[i] = p.field(parent)
-			tags[i] = p.string()
-		}
-		*t = *types.NewStruct(fields, tags)
 		return t
 
 	case pointerTag:
@@ -336,17 +359,7 @@ func (p *importer) typ(parent *types.Package) types.Type {
 			panic("unexpected embedded interface")
 		}
 
-		// read methods
+		t := types.NewInterface(p.methodList(parent), nil)
-		methods := make([]*types.Func, p.int())
-		for i := range methods {
-			pkg, name := p.fieldName(parent)
-			params, isddd := p.paramList()
-			result, _ := p.paramList()
-			sig := types.NewSignature(nil, params, result, isddd)
-			methods[i] = types.NewFunc(token.NoPos, pkg, name, sig)
-		}
-
-		t := types.NewInterface(methods, nil)
 		p.typList[n] = t
 		return t
 
@@ -384,7 +397,20 @@ func (p *importer) typ(parent *types.Package) types.Type {
 	}
 }
 
+func (p *importer) fieldList(parent *types.Package) (fields []*types.Var, tags []string) {
+	if n := p.int(); n > 0 {
+		fields = make([]*types.Var, n)
+		tags = make([]string, n)
+		for i := range fields {
+			fields[i] = p.field(parent)
+			tags[i] = p.string()
+		}
+	}
+	return
+}
+
 func (p *importer) field(parent *types.Package) *types.Var {
+	p.pos()
 	pkg, name := p.fieldName(parent)
 	typ := p.typ(parent)
 
@@ -406,6 +432,25 @@ func (p *importer) field(parent *types.Package) *types.Var {
 	return types.NewField(token.NoPos, pkg, name, typ, anonymous)
 }
 
+func (p *importer) methodList(parent *types.Package) (methods []*types.Func) {
+	if n := p.int(); n > 0 {
+		methods = make([]*types.Func, n)
+		for i := range methods {
+			methods[i] = p.method(parent)
+		}
+	}
+	return
+}
+
+func (p *importer) method(parent *types.Package) *types.Func {
+	p.pos()
+	pkg, name := p.fieldName(parent)
+	params, isddd := p.paramList()
+	result, _ := p.paramList()
+	sig := types.NewSignature(nil, params, result, isddd)
+	return types.NewFunc(token.NoPos, pkg, name, sig)
+}
+
 func (p *importer) fieldName(parent *types.Package) (*types.Package, string) {
 	pkg := parent
 	if pkg == nil {
@@ -573,24 +618,28 @@ func (p *importer) string() string {
 	if p.debugFormat {
 		p.marker('s')
 	}
-
+	// if the string was seen before, i is its index (>= 0)
-	if n := int(p.rawInt64()); n > 0 {
+	// (the empty string is at index 0)
-		if cap(p.buf) < n {
+	i := p.rawInt64()
-			p.buf = make([]byte, n)
+	if i >= 0 {
-		} else {
+		return p.strList[i]
-			p.buf = p.buf[:n]
-		}
-		for i := 0; i < n; i++ {
-			p.buf[i] = p.byte()
-		}
-		return string(p.buf)
 	}
-
+	// otherwise, i is the negative string length (< 0)
-	return ""
+	if n := int(-i); n <= cap(p.buf) {
+		p.buf = p.buf[:n]
+	} else {
+		p.buf = make([]byte, n)
+	}
+	for i := range p.buf {
+		p.buf[i] = p.rawByte()
+	}
+	s := string(p.buf)
+	p.strList = append(p.strList, s)
+	return s
 }
 
 func (p *importer) marker(want byte) {
-	if got := p.byte(); got != want {
+	if got := p.rawByte(); got != want {
 		panic(fmt.Sprintf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.read))
 	}
 
@@ -611,12 +660,13 @@ func (p *importer) rawInt64() int64 {
 
 // needed for binary.ReadVarint in rawInt64
 func (p *importer) ReadByte() (byte, error) {
-	return p.byte(), nil
+	return p.rawByte(), nil
 }
 
 // byte is the bottleneck interface for reading p.data.
 // It unescapes '|' 'S' to '$' and '|' '|' to '|'.
-func (p *importer) byte() byte {
+// rawByte should only be used by low-level decoders.
+func (p *importer) rawByte() byte {
 	b := p.data[0]
 	r := 1
 	if b == '|' {