internal/lsp: add a file system abstraction

Change-Id: Ie42835b835ed22fddbba187ab10d8c31019ff008
Reviewed-on: https://go-review.googlesource.com/c/tools/+/178097
Run-TryBot: Ian Cottrell <iancottrell@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Rebecca Stambler <rstambler@golang.org>

internal/lsp/cache/cache.go

@@ -5,6 +5,8 @@
 package cache
 
 import (
+	"crypto/sha1"
+	"fmt"
 	"go/token"
 
 	"golang.org/x/tools/internal/lsp/source"
@@ -19,6 +21,8 @@ func New() source.Cache {
 }
 
 type cache struct {
+	nativeFileSystem
+
 	fset *token.FileSet
 }
 
@@ -26,10 +30,16 @@ func (c *cache) NewSession(log xlog.Logger) source.Session {
 	return &session{
 		cache:    c,
 		log:      log,
-		overlays: make(map[span.URI][]byte),
+		overlays: make(map[span.URI]*source.FileContent),
 	}
 }
 
 func (c *cache) FileSet() *token.FileSet {
 	return c.fset
 }
+
+func hashContents(contents []byte) string {
+	// TODO: consider whether sha1 is the best choice here
+	// This hash is used for internal identity detection only
+	return fmt.Sprintf("%x", sha1.Sum(contents))
+}

internal/lsp/cache/check.go

@@ -99,7 +99,10 @@ func (v *view) reparseImports(ctx context.Context, f *goFile, filename string) b
 	}
 	// Get file content in case we don't already have it.
 	f.read(ctx)
-	parsed, _ := parser.ParseFile(f.FileSet(), filename, f.content, parser.ImportsOnly)
+	if f.fc.Error != nil {
+		return true
+	}
+	parsed, _ := parser.ParseFile(f.FileSet(), filename, f.fc.Data, parser.ImportsOnly)
 	if parsed == nil {
 		return true
 	}

internal/lsp/cache/external.go

@@ -0,0 +1,29 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+	"io/ioutil"
+
+	"golang.org/x/tools/internal/lsp/source"
+	"golang.org/x/tools/internal/span"
+)
+
+// nativeFileSystem implements FileSystem reading from the normal os file system.
+type nativeFileSystem struct{}
+
+func (nativeFileSystem) ReadFile(uri span.URI) *source.FileContent {
+	r := &source.FileContent{URI: uri}
+	filename, err := uri.Filename()
+	if err != nil {
+		r.Error = err
+		return r
+	}
+	r.Data, r.Error = ioutil.ReadFile(filename)
+	if r.Error != nil {
+		r.Hash = hashContents(r.Data)
+	}
+	return r
+}

internal/lsp/cache/file.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"go/ast"
 	"go/token"
-	"io/ioutil"
 	"path/filepath"
 	"strings"
 
@@ -19,10 +18,9 @@ import (
 // viewFile extends source.File with helper methods for the view package.
 type viewFile interface {
 	source.File
-	setContent(content []byte)
+	invalidate()
 	filename() string
 	addURI(uri span.URI) int
-	isActive() bool
 }
 
 // fileBase holds the common functionality for all files.
@@ -32,8 +30,7 @@ type fileBase struct {
 	fname string
 
 	view  *view
-	active  bool
+	fc    *source.FileContent
-	content []byte
 	token *token.File
 }
 
@@ -59,25 +56,17 @@ func (f *fileBase) filename() string {
 	return f.fname
 }
 
-func (f *fileBase) isActive() bool {
-	return f.active
-}
-
 // View returns the view associated with the file.
 func (f *fileBase) View() source.View {
 	return f.view
 }
 
-// GetContent returns the contents of the file, reading it from file system if needed.
+// Content returns the contents of the file, reading it from file system if needed.
-func (f *fileBase) GetContent(ctx context.Context) []byte {
+func (f *fileBase) Content(ctx context.Context) *source.FileContent {
 	f.view.mu.Lock()
 	defer f.view.mu.Unlock()
-
-	if ctx.Err() == nil {
 	f.read(ctx)
-	}
+	return f.fc
-
-	return f.content
 }
 
 func (f *fileBase) FileSet() *token.FileSet {
@@ -127,10 +116,14 @@ func (f *goFile) GetPackage(ctx context.Context) source.Package {
 	return f.pkg
 }
 
-// read is the internal part of GetContent. It assumes that the caller is
+// read is the internal part of Content. It assumes that the caller is
 // holding the mutex of the file's view.
 func (f *fileBase) read(ctx context.Context) {
-	if f.content != nil {
+	if err := ctx.Err(); err != nil {
+		f.fc = &source.FileContent{Error: err}
+		return
+	}
+	if f.fc != nil {
 		if len(f.view.contentChanges) == 0 {
 			return
 		}
@@ -139,24 +132,13 @@ func (f *fileBase) read(ctx context.Context) {
 		err := f.view.applyContentChanges(ctx)
 		f.view.mcache.mu.Unlock()
 
-		if err == nil {
+		if err != nil {
+			f.fc = &source.FileContent{Error: err}
 			return
 		}
 	}
-	// We might have the content saved in an overlay.
-	f.view.session.overlayMu.Lock()
-	defer f.view.session.overlayMu.Unlock()
-	if content, ok := f.view.session.overlays[f.URI()]; ok {
-		f.content = content
-		return
-	}
 	// We don't know the content yet, so read it.
-	content, err := ioutil.ReadFile(f.filename())
+	f.fc = f.view.Session().ReadFile(f.URI())
-	if err != nil {
-		f.view.Session().Logger().Errorf(ctx, "unable to read file %s: %v", f.filename(), err)
-		return
-	}
-	f.content = content
 }
 
 // isPopulated returns true if all of the computed fields of the file are set.
@@ -204,7 +186,8 @@ func (v *view) reverseDeps(ctx context.Context, seen map[string]struct{}, result
 		return
 	}
 	for _, filename := range m.files {
-		if f, err := v.getFile(span.FileURI(filename)); err == nil && f.isActive() {
+		uri := span.FileURI(filename)
+		if f, err := v.getFile(uri); err == nil && v.session.IsOpen(uri) {
 			results[f.(*goFile)] = struct{}{}
 		}
 	}

internal/lsp/cache/parse.go

@@ -67,7 +67,10 @@ func (imp *importer) parseFiles(filenames []string) ([]*ast.File, []error) {
 			} else {
 				// We don't have a cached AST for this file.
 				gof.read(imp.ctx)
-				src := gof.content
+				if gof.fc.Error != nil {
+					return
+				}
+				src := gof.fc.Data
 				if src == nil {
 					parsed[i], errors[i] = nil, fmt.Errorf("No source for %v", filename)
 				} else {

internal/lsp/cache/session.go

@@ -25,7 +25,9 @@ type session struct {
 	viewMap map[span.URI]source.View
 
 	overlayMu sync.Mutex
-	overlays  map[span.URI][]byte
+	overlays  map[span.URI]*source.FileContent
+
+	openFiles sync.Map
 }
 
 func (s *session) Shutdown(ctx context.Context) {
@@ -153,25 +155,59 @@ func (s *session) Logger() xlog.Logger {
 	return s.log
 }
 
-func (s *session) setOverlay(uri span.URI, content []byte) {
+func (s *session) DidOpen(uri span.URI) {
+	s.openFiles.Store(uri, true)
+}
+
+func (s *session) DidSave(uri span.URI) {
+}
+
+func (s *session) DidClose(uri span.URI) {
+	s.openFiles.Delete(uri)
+}
+
+func (s *session) IsOpen(uri span.URI) bool {
+	_, open := s.openFiles.Load(uri)
+	return open
+}
+
+func (s *session) ReadFile(uri span.URI) *source.FileContent {
 	s.overlayMu.Lock()
 	defer s.overlayMu.Unlock()
-	//TODO: we also need to invalidate anything that depended on this "file"
+	// We might have the content saved in an overlay.
-	if content == nil {
+	if overlay, ok := s.overlays[uri]; ok {
+		return overlay
+	}
+	// fall back to the cache level file system
+	return s.Cache().ReadFile(uri)
+}
+
+func (s *session) SetOverlay(uri span.URI, data []byte) {
+	s.overlayMu.Lock()
+	defer s.overlayMu.Unlock()
+	//TODO: we also need to invoke and watchers in here
+	if data == nil {
 		delete(s.overlays, uri)
 		return
 	}
-	s.overlays[uri] = content
+	s.overlays[uri] = &source.FileContent{
+		URI:  uri,
+		Data: data,
+		Hash: hashContents(data),
+	}
 }
 
 func (s *session) buildOverlay() map[string][]byte {
 	s.overlayMu.Lock()
 	defer s.overlayMu.Unlock()
-	overlay := make(map[string][]byte)
+	overlays := make(map[string][]byte)
-	for uri, content := range s.overlays {
+	for uri, overlay := range s.overlays {
+		if overlay.Error != nil {
+			continue
+		}
 		if filename, err := uri.Filename(); err == nil {
-			overlay[filename] = content
+			overlays[filename] = overlay.Data
 		}
 	}
-	return overlay
+	return overlays
 }

internal/lsp/cache/view.go

@@ -232,19 +232,18 @@ func (v *view) applyContentChanges(ctx context.Context) error {
 	return nil
 }
 
-// setContent applies a content update for a given file. It assumes that the
+// applyContentChange applies a content update for a given file. It assumes that the
 // caller is holding the view's mutex.
 func (v *view) applyContentChange(uri span.URI, content []byte) {
+	v.session.SetOverlay(uri, content)
 	f, err := v.getFile(uri)
 	if err != nil {
 		return
 	}
-	f.setContent(content)
+	f.invalidate()
 }
 
-func (f *goFile) setContent(content []byte) {
+func (f *goFile) invalidate() {
-	f.content = content
-
 	// TODO(rstambler): Should we recompute these here?
 	f.ast = nil
 	f.token = nil
@@ -253,18 +252,7 @@ func (f *goFile) setContent(content []byte) {
 	if f.pkg != nil {
 		f.view.remove(f.pkg.pkgPath, map[string]struct{}{})
 	}
-
+	f.fc = nil
-	switch {
-	case f.active && content == nil:
-		// The file was active, so we need to forget its content.
-		f.active = false
-		f.view.session.setOverlay(f.URI(), nil)
-		f.content = nil
-	case content != nil:
-		// This is an active overlay, so we update the map.
-		f.active = true
-		f.view.session.setOverlay(f.URI(), f.content)
-	}
 }
 
 // remove invalidates a package and its reverse dependencies in the view's

internal/lsp/source/diagnostics.go

@@ -205,8 +205,8 @@ func pointToSpan(ctx context.Context, v View, spn span.Span) span.Span {
 		v.Session().Logger().Errorf(ctx, "Could not find tokens for diagnostic: %v", spn.URI())
 		return spn
 	}
-	content := diagFile.GetContent(ctx)
+	fc := diagFile.Content(ctx)
-	if content == nil {
+	if fc.Error != nil {
 		v.Session().Logger().Errorf(ctx, "Could not find content for diagnostic: %v", spn.URI())
 		return spn
 	}
@@ -219,7 +219,7 @@ func pointToSpan(ctx context.Context, v View, spn span.Span) span.Span {
 	}
 	start := s.Start()
 	offset := start.Offset()
-	width := bytes.IndexAny(content[offset:], " \n,():;[]")
+	width := bytes.IndexAny(fc.Data[offset:], " \n,():;[]")
 	if width <= 0 {
 		return spn
 	}

internal/lsp/source/format.go

@@ -56,11 +56,15 @@ func hasParseErrors(errors []packages.Error) bool {
 
 // Imports formats a file using the goimports tool.
 func Imports(ctx context.Context, f GoFile, rng span.Range) ([]TextEdit, error) {
+	fc := f.Content(ctx)
+	if fc.Error != nil {
+		return nil, fc.Error
+	}
 	tok := f.GetToken(ctx)
 	if tok == nil {
 		return nil, fmt.Errorf("no token file for %s", f.URI())
 	}
-	formatted, err := imports.Process(tok.Name(), f.GetContent(ctx), nil)
+	formatted, err := imports.Process(f.GetToken(ctx).Name(), fc.Data, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -68,7 +72,12 @@ func Imports(ctx context.Context, f GoFile, rng span.Range) ([]TextEdit, error)
 }
 
 func computeTextEdits(ctx context.Context, file File, formatted string) (edits []TextEdit) {
-	u := diff.SplitLines(string(file.GetContent(ctx)))
+	fc := file.Content(ctx)
+	if fc.Error != nil {
+		file.View().Session().Logger().Errorf(ctx, "Cannot compute text edits: %v", fc.Error)
+		return nil
+	}
+	u := diff.SplitLines(string(fc.Data))
 	f := diff.SplitLines(formatted)
 	return DiffToEdits(file.URI(), diff.Operations(u, f))
 }

internal/lsp/source/source_test.go

@@ -293,7 +293,12 @@ func (r *runner) Format(t *testing.T, data tests.Formats) {
 			continue
 		}
 		ops := source.EditsToDiff(edits)
-		got := strings.Join(diff.ApplyEdits(diff.SplitLines(string(f.GetContent(ctx))), ops), "")
+		fc := f.Content(ctx)
+		if fc.Error != nil {
+			t.Error(err)
+			continue
+		}
+		got := strings.Join(diff.ApplyEdits(diff.SplitLines(string(fc.Data)), ops), "")
 		if gofmted != got {
 			t.Errorf("format failed for %s, expected:\n%v\ngot:\n%v", filename, gofmted, got)
 		}

internal/lsp/source/view.go

@@ -18,6 +18,21 @@ import (
 	"golang.org/x/tools/internal/span"
 )
 
+// FileContents is returned from FileSystem implementation to represent the
+// contents of a file.
+type FileContent struct {
+	URI   span.URI
+	Data  []byte
+	Error error
+	Hash  string
+}
+
+// FileSystem is the interface to something that provides file contents.
+type FileSystem interface {
+	// ReadFile reads the contents of a file and returns it.
+	ReadFile(uri span.URI) *FileContent
+}
+
 // Cache abstracts the core logic of dealing with the environment from the
 // higher level logic that processes the information to produce results.
 // The cache provides access to files and their contents, so the source
@@ -26,6 +41,9 @@ import (
 // sharing between all consumers.
 // A cache may have many active sessions at any given time.
 type Cache interface {
+	// A FileSystem that reads file contents from external storage.
+	FileSystem
+
 	// NewSession creates a new Session manager and returns it.
 	NewSession(log xlog.Logger) Session
 
@@ -58,6 +76,25 @@ type Session interface {
 
 	// Shutdown the session and all views it has created.
 	Shutdown(ctx context.Context)
+
+	// A FileSystem prefers the contents from overlays, and falls back to the
+	// content from the underlying cache if no overlay is present.
+	FileSystem
+
+	// DidOpen is invoked each time a file is opened in the editor.
+	DidOpen(uri span.URI)
+
+	// DidSave is invoked each time an open file is saved in the editor.
+	DidSave(uri span.URI)
+
+	// DidClose is invoked each time an open file is closed in the editor.
+	DidClose(uri span.URI)
+
+	// IsOpen can be called to check if the editor has a file currently open.
+	IsOpen(uri span.URI) bool
+
+	// Called to set the effective contents of a file from this session.
+	SetOverlay(uri span.URI, data []byte)
 }
 
 // View represents a single workspace.
@@ -103,7 +140,7 @@ type View interface {
 type File interface {
 	URI() span.URI
 	View() View
-	GetContent(ctx context.Context) []byte
+	Content(ctx context.Context) *FileContent
 	FileSet() *token.FileSet
 	GetToken(ctx context.Context) *token.File
 }

internal/lsp/text_synchronization.go

@@ -14,7 +14,9 @@ import (
 )
 
 func (s *Server) didOpen(ctx context.Context, params *protocol.DidOpenTextDocumentParams) error {
-	return s.cacheAndDiagnose(ctx, span.NewURI(params.TextDocument.URI), []byte(params.TextDocument.Text))
+	uri := span.NewURI(params.TextDocument.URI)
+	s.session.DidOpen(uri)
+	return s.cacheAndDiagnose(ctx, uri, []byte(params.TextDocument.Text))
 }
 
 func (s *Server) didChange(ctx context.Context, params *protocol.DidChangeTextDocumentParams) error {
@@ -64,18 +66,20 @@ func (s *Server) applyChanges(ctx context.Context, params *protocol.DidChangeTex
 	}
 
 	uri := span.NewURI(params.TextDocument.URI)
-	view := s.session.ViewOf(uri)
+	fc := s.session.ReadFile(uri)
-	f, m, err := getSourceFile(ctx, view, uri)
+	if fc.Error != nil {
-	if err != nil {
 		return "", jsonrpc2.NewErrorf(jsonrpc2.CodeInternalError, "file not found")
 	}
-	fset := f.FileSet()
+	content := fc.Data
-	filename, err := f.URI().Filename()
+	fset := s.session.Cache().FileSet()
+	filename, err := uri.Filename()
 	if err != nil {
 		return "", jsonrpc2.NewErrorf(jsonrpc2.CodeInternalError, "no filename for %s", uri)
 	}
-	content := f.GetContent(ctx)
 	for _, change := range params.ContentChanges {
+		// Update column mapper along with the content.
+		m := protocol.NewColumnMapper(uri, filename, fset, nil, content)
+
 		spn, err := m.RangeSpan(*change.Range)
 		if err != nil {
 			return "", err
@@ -92,19 +96,19 @@ func (s *Server) applyChanges(ctx context.Context, params *protocol.DidChangeTex
 		buf.WriteString(change.Text)
 		buf.Write(content[end:])
 		content = buf.Bytes()
-
-		// Update column mapper along with the content.
-		m = protocol.NewColumnMapper(f.URI(), filename, fset, nil, content)
 	}
 	return string(content), nil
 }
 
 func (s *Server) didSave(ctx context.Context, params *protocol.DidSaveTextDocumentParams) error {
+	uri := span.NewURI(params.TextDocument.URI)
+	s.session.DidSave(uri)
 	return nil // ignore
 }
 
 func (s *Server) didClose(ctx context.Context, params *protocol.DidCloseTextDocumentParams) error {
 	uri := span.NewURI(params.TextDocument.URI)
+	s.session.DidClose(uri)
 	view := s.session.ViewOf(uri)
 	return view.SetContent(ctx, uri, nil)
 }

internal/lsp/util.go

@@ -55,7 +55,11 @@ func getSourceFile(ctx context.Context, v source.View, uri span.URI) (source.Fil
 	if err != nil {
 		return nil, nil, err
 	}
-	m := protocol.NewColumnMapper(f.URI(), filename, f.FileSet(), f.GetToken(ctx), f.GetContent(ctx))
+	fc := f.Content(ctx)
+	if fc.Error != nil {
+		return nil, nil, fc.Error
+	}
+	m := protocol.NewColumnMapper(f.URI(), filename, f.FileSet(), f.GetToken(ctx), fc.Data)
 
 	return f, m, nil
 }