From 56a1b4d0b77cc9eab9a93ae9f415c00fa8c7f3ba Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 6 Nov 2013 15:00:26 -0500 Subject: [PATCH] godoc: index import counts, package name to path(s), and exported symbols R=golang-dev, crawshaw CC=golang-dev https://golang.org/cl/22190047 --- godoc/index.go | 173 ++++++++++++++++++++++++++++++++------------ godoc/index_test.go | 40 +++++++++- 2 files changed, 166 insertions(+), 47 deletions(-) diff --git a/godoc/index.go b/godoc/index.go index 34c4c352..c99b8e70 100644 --- a/godoc/index.go +++ b/godoc/index.go @@ -55,6 +55,7 @@ import ( "regexp" "runtime" "sort" + "strconv" "strings" "sync" "time" @@ -377,16 +378,29 @@ type Indexer struct { fset *token.FileSet // file set for all indexed files fsOpenGate chan bool // send pre fs.Open; receive on close - mu sync.Mutex // guards all the following - sources bytes.Buffer // concatenated sources - packages map[string]*Pak // map of canonicalized *Paks - words map[string]*IndexResult // RunLists of Spots - snippets []*Snippet // indices are stored in SpotInfos - current *token.File // last file added to file set - file *File // AST for current file - decl ast.Decl // AST for current decl - stats Statistics - throttle *util.Throttle + mu sync.Mutex // guards all the following + sources bytes.Buffer // concatenated sources + strings map[string]string // interned string + packages map[Pak]*Pak // interned *Paks + words map[string]*IndexResult // RunLists of Spots + snippets []*Snippet // indices are stored in SpotInfos + current *token.File // last file added to file set + file *File // AST for current file + decl ast.Decl // AST for current decl + stats Statistics + throttle *util.Throttle + importCount map[string]int // package path ("net/http") => count + packagePath map[string]map[string]bool // "template" => "text/template" => true + exports map[string]map[string]SpotKind // "net/http" => "ListenAndServe" => FuncDecl + curPkgExports map[string]SpotKind +} + +func (x *Indexer) intern(s string) string { + if s, ok := x.strings[s]; ok { + return s + } + x.strings[s] = s + return s } func (x *Indexer) lookupPackage(path, name string) *Pak { @@ -394,10 +408,10 @@ func (x *Indexer) lookupPackage(path, name string) *Pak { // live in the same directory. For the packages map, construct // a key that includes both the directory path and the package // name. - key := path + ":" + name + key := Pak{Path: x.intern(path), Name: x.intern(name)} pak := x.packages[key] if pak == nil { - pak = &Pak{path, name} + pak = &key x.packages[key] = pak } return pak @@ -410,26 +424,34 @@ func (x *Indexer) addSnippet(s *Snippet) int { } func (x *Indexer) visitIdent(kind SpotKind, id *ast.Ident) { - if id != nil { - lists, found := x.words[id.Name] - if !found { - lists = new(IndexResult) - x.words[id.Name] = lists - } - - if kind == Use || x.decl == nil { - // not a declaration or no snippet required - info := makeSpotInfo(kind, x.current.Line(id.Pos()), false) - lists.Others = append(lists.Others, Spot{x.file, info}) - } else { - // a declaration with snippet - index := x.addSnippet(NewSnippet(x.fset, x.decl, id)) - info := makeSpotInfo(kind, index, true) - lists.Decls = append(lists.Decls, Spot{x.file, info}) - } - - x.stats.Spots++ + if id == nil { + return } + name := x.intern(id.Name) + + switch kind { + case TypeDecl, FuncDecl: + x.curPkgExports[name] = kind + } + + lists, found := x.words[name] + if !found { + lists = new(IndexResult) + x.words[name] = lists + } + + if kind == Use || x.decl == nil { + // not a declaration or no snippet required + info := makeSpotInfo(kind, x.current.Line(id.Pos()), false) + lists.Others = append(lists.Others, Spot{x.file, info}) + } else { + // a declaration with snippet + index := x.addSnippet(NewSnippet(x.fset, x.decl, id)) + info := makeSpotInfo(kind, index, true) + lists.Decls = append(lists.Decls, Spot{x.file, info}) + } + + x.stats.Spots++ } func (x *Indexer) visitFieldList(kind SpotKind, flist *ast.FieldList) { @@ -447,7 +469,11 @@ func (x *Indexer) visitSpec(kind SpotKind, spec ast.Spec) { switch n := spec.(type) { case *ast.ImportSpec: x.visitIdent(ImportDecl, n.Name) - // ignore path - not indexed at the moment + if n.Path != nil { + if imp, err := strconv.Unquote(n.Path.Value); err == nil { + x.importCount[x.intern(imp)]++ + } + } case *ast.ValueSpec: for _, n := range n.Names { @@ -678,6 +704,7 @@ func (x *Indexer) visitFile(dirname string, fi os.FileInfo, fulltextIndex bool) x.throttle.Throttle() + x.curPkgExports = make(map[string]SpotKind) file, fast := x.addFile(f, filename, goFile) if file == nil { return // addFile failed @@ -689,6 +716,26 @@ func (x *Indexer) visitFile(dirname string, fi os.FileInfo, fulltextIndex bool) pak := x.lookupPackage(dirname, fast.Name.Name) x.file = &File{fi.Name(), pak} ast.Walk(x, fast) + + ppKey := x.intern(fast.Name.Name) + if _, ok := x.packagePath[ppKey]; !ok { + x.packagePath[ppKey] = make(map[string]bool) + } + pkgPath := x.intern(strings.TrimPrefix(dirname, "/src/pkg/")) + x.packagePath[ppKey][pkgPath] = true + + // Merge in exported symbols found walking this file into + // the map for that package. + if len(x.curPkgExports) > 0 { + dest, ok := x.exports[pkgPath] + if !ok { + dest = make(map[string]SpotKind) + x.exports[pkgPath] = dest + } + for k, v := range x.curPkgExports { + dest[k] = v + } + } } // update statistics @@ -706,12 +753,15 @@ type LookupResult struct { } type Index struct { - fset *token.FileSet // file set used during indexing; nil if no textindex - suffixes *suffixarray.Index // suffixes for concatenated sources; nil if no textindex - words map[string]*LookupResult // maps words to hit lists - alts map[string]*AltWords // maps canonical(words) to lists of alternative spellings - snippets []*Snippet // all snippets, indexed by snippet index - stats Statistics + fset *token.FileSet // file set used during indexing; nil if no textindex + suffixes *suffixarray.Index // suffixes for concatenated sources; nil if no textindex + words map[string]*LookupResult // maps words to hit lists + alts map[string]*AltWords // maps canonical(words) to lists of alternative spellings + snippets []*Snippet // all snippets, indexed by snippet index + stats Statistics + importCount map[string]int // package path ("net/http") => count + packagePath map[string]map[string]bool // "template" => "text/template" => true + exports map[string]map[string]SpotKind // "net/http" => "ListenAndServe" => FuncDecl } func canonical(w string) string { return strings.ToLower(w) } @@ -733,12 +783,16 @@ func NewIndex(c *Corpus, dirnames <-chan string, fulltextIndex bool, throttle fl // initialize Indexer // (use some reasonably sized maps to start) x := &Indexer{ - c: c, - fset: token.NewFileSet(), - fsOpenGate: make(chan bool, maxOpenFiles), - packages: make(map[string]*Pak, 256), - words: make(map[string]*IndexResult, 8192), - throttle: util.NewThrottle(throttle, 100*time.Millisecond), // run at least 0.1s at a time + c: c, + fset: token.NewFileSet(), + fsOpenGate: make(chan bool, maxOpenFiles), + strings: make(map[string]string), + packages: make(map[Pak]*Pak, 256), + words: make(map[string]*IndexResult, 8192), + throttle: util.NewThrottle(throttle, 100*time.Millisecond), // run at least 0.1s at a time + importCount: make(map[string]int), + packagePath: make(map[string]map[string]bool), + exports: make(map[string]map[string]SpotKind), } // index all files in the directories given by dirnames @@ -813,7 +867,17 @@ func NewIndex(c *Corpus, dirnames <-chan string, fulltextIndex bool, throttle fl suffixes = suffixarray.New(x.sources.Bytes()) } - return &Index{x.fset, suffixes, words, alts, x.snippets, x.stats} + return &Index{ + fset: x.fset, + suffixes: suffixes, + words: words, + alts: alts, + snippets: x.snippets, + stats: x.stats, + importCount: x.importCount, + packagePath: x.packagePath, + exports: x.exports, + } } type fileIndex struct { @@ -890,11 +954,28 @@ func (x *Index) Read(r io.Reader) error { return nil } -// Stats() returns index statistics. +// Stats returns index statistics. func (x *Index) Stats() Statistics { return x.stats } +// ImportCount returns a map from import paths to how many times they were seen. +func (x *Index) ImportCount() map[string]int { + return x.importCount +} + +// PackagePath returns a map from short package name to a set +// of full package path names that use that short package name. +func (x *Index) PackagePath() map[string]map[string]bool { + return x.packagePath +} + +// Exports returns a map from full package path to exported +// symbol name to its type. +func (x *Index) Exports() map[string]map[string]SpotKind { + return x.exports +} + func (x *Index) lookupWord(w string) (match *LookupResult, alt *AltWords) { match = x.words[w] alt = x.alts[canonical(w)] diff --git a/godoc/index_test.go b/godoc/index_test.go index 001d2a77..8a5a4737 100644 --- a/godoc/index_test.go +++ b/godoc/index_test.go @@ -17,6 +17,8 @@ func TestIndex(t *testing.T) { "src/pkg/foo/foo.go": `// Package foo is an example. package foo +import "bar" + // Foo is stuff. type Foo struct{} @@ -26,6 +28,10 @@ func New() *Foo { `, "src/pkg/bar/bar.go": `// Package bar is another example to test races. package bar +`, + "src/pkg/other/bar/bar.go": `// Package bar is another bar package. +package bar +func X() {} `, "src/pkg/skip/skip.go": `// Package skip should be skipped. package skip @@ -46,11 +52,43 @@ func Skip() {} t.Fatal("no index") } t.Logf("Got: %#v", ix) - wantStats := Statistics{Bytes: 179, Files: 2, Lines: 11, Words: 5, Spots: 7} + + wantStats := Statistics{Bytes: 256, Files: 3, Lines: 16, Words: 6, Spots: 9} if !reflect.DeepEqual(ix.Stats(), wantStats) { t.Errorf("Stats = %#v; want %#v", ix.Stats(), wantStats) } + if _, ok := ix.words["Skip"]; ok { t.Errorf("the word Skip was found; expected it to be skipped") } + + if got, want := ix.ImportCount(), map[string]int{ + "bar": 1, + }; !reflect.DeepEqual(got, want) { + t.Errorf("ImportCount = %v; want %v", got, want) + } + + if got, want := ix.PackagePath(), map[string]map[string]bool{ + "foo": map[string]bool{ + "foo": true, + }, + "bar": map[string]bool{ + "bar": true, + "other/bar": true, + }, + }; !reflect.DeepEqual(got, want) { + t.Errorf("PackagePath = %v; want %v", got, want) + } + + if got, want := ix.Exports(), map[string]map[string]SpotKind{ + "foo": map[string]SpotKind{ + "Foo": TypeDecl, + "New": FuncDecl, + }, + "other/bar": map[string]SpotKind{ + "X": FuncDecl, + }, + }; !reflect.DeepEqual(got, want) { + t.Errorf("Exports = %v; want %v", got, want) + } }