cmd/mvpkg: a package moving tool

This change adds a command mvpkg that will move a given package and
update all its imports. It uses similar logic to gorename to update
the imports.

Change-Id: Iebbd0b4c93c2302b0a71c3b99c68f6778106012a
Reviewed-on: https://go-review.googlesource.com/1973
Reviewed-by: Alan Donovan <adonovan@google.com>

cmd/gomvpkg/main.go

@@ -0,0 +1,89 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// licence that can be found in the LICENSE file.
+
+// The gomvpkg command moves go packages, updating import declarations.
+// See the -help message or Usage constant for details.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"go/build"
+	"os"
+
+	"golang.org/x/tools/refactor/rename"
+)
+
+var (
+	fromFlag     = flag.String("from", "", "Import path of package to be moved")
+	toFlag       = flag.String("to", "", "Destination import path for package")
+	vcsMvCmdFlag = flag.String("vcs_mv_cmd", "", `A template for the version control system's "move directory" command, e.g. "git mv {{.Src}} {{.Dst}}`)
+	helpFlag     = flag.Bool("help", false, "show usage message")
+)
+
+const Usage = `gomvpkg: moves a package, updating import declarations
+
+Usage:
+
+ gomvpkg -from <path> -to <path> [-vcs_mv_cmd <template>]
+
+Flags:
+
+-from        specifies the import path of the package to be moved
+
+-to          specifies the destination import path
+
+-vcs_mv_cmd  specifies a shell command to inform the version control system of a
+             directory move.  The argument is a template using the syntax of the
+             text/template package. It has two fields: Src and Dst, the absolute
+             paths of the directories.
+
+             For example: "git mv {{.Src}} {{.Dst}}"
+
+gomvpkg determines the set of packages that might be affected, including all
+packages importing the 'from' package and any of its subpackages. It will move
+the 'from' package and all its subpackages to the destination path and update all
+imports of those packages to point to its new import path.
+
+gomvpkg rejects moves in which a package already exists at the destination import
+path, or in which a directory already exists at the location the package would be
+moved to.
+
+gomvpkg will not always be able to rename imports when a package's name is changed.
+Import statements may want further cleanup.
+
+gomvpkg's behavior is not defined if any of the packages to be moved are
+imported using dot imports.
+
+Examples:
+
+% gomvpkg -from myproject/foo -to myproject/bar
+
+  Move the package with import path "myproject/foo" to the new path
+  "myproject/bar".
+
+% gomvpkg -from myproject/foo -to myproject/bar -vcs_mv_cmd "git mv {{.Src}} {{.Dst}}"
+
+  Move the package with import path "myproject/foo" to the new path
+  "myproject/bar" using "git mv" to execute the directory move.
+`
+
+func main() {
+	flag.Parse()
+
+	if len(flag.Args()) > 0 {
+		fmt.Fprintln(os.Stderr, "gomvpkg: surplus arguments.")
+		os.Exit(1)
+	}
+
+	if *helpFlag || *fromFlag == "" || *toFlag == "" {
+		fmt.Println(Usage)
+		return
+	}
+
+	if err := rename.Move(&build.Default, *fromFlag, *toFlag, *vcsMvCmdFlag); err != nil {
+		fmt.Fprintf(os.Stderr, "gomvpkg: %s.\n", err)
+		os.Exit(1)
+	}
+}

refactor/rename/check.go

@@ -177,7 +177,7 @@ func (r *renamer) checkInLocalScope(from types.Object) {
 // same-, sub-, and super-block conflicts.  We will illustrate all three
 // using this example:
 //
-//      var x int
+//	var x int
 //	var z int
 //
 //	func f(y int) {

refactor/rename/mvpkg.go

@@ -0,0 +1,328 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// licence that can be found in the LICENSE file.
+
+// This file contains the implementation of the 'gomovepkg' command
+// whose main function is in golang.org/x/tools/cmd/gomovepkg.
+
+package rename
+
+// TODO(matloob):
+// - think about what happens if the package is moving across version control systems.
+// - think about windows, which uses "\" as its directory separator.
+// - dot imports are not supported. Make sure it's clearly documented.
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/build"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"text/template"
+
+	"golang.org/x/tools/go/buildutil"
+	"golang.org/x/tools/go/loader"
+	"golang.org/x/tools/refactor/importgraph"
+)
+
+// Move, given a package path and a destination package path, will try
+// to move the given package to the new path. The Move function will
+// first check for any conflicts preventing the move, such as a
+// package already existing at the destination package path. If the
+// move can proceed, it builds an import graph to find all imports of
+// the packages whose paths need to be renamed. This includes uses of
+// the subpackages of the package to be moved as those packages will
+// also need to be moved. It then renames all imports to point to the
+// new paths, and then moves the packages to their new paths.
+func Move(ctxt *build.Context, from, to, moveTmpl string) error {
+	srcDir, err := srcDir(ctxt, from)
+	if err != nil {
+		return err
+	}
+
+	// This should be the only place in the program that constructs
+	// file paths.
+	// TODO(matloob): test on Microsoft Windows.
+	fromDir := buildutil.JoinPath(ctxt, srcDir, filepath.FromSlash(from))
+	toDir := buildutil.JoinPath(ctxt, srcDir, filepath.FromSlash(to))
+	toParent := filepath.Dir(toDir)
+	if !buildutil.IsDir(ctxt, toParent) {
+		return fmt.Errorf("parent directory does not exist for path %s", toDir)
+	}
+
+	// Build the import graph and figure out which packages to update.
+	fwd, rev, errors := importgraph.Build(ctxt)
+	if len(errors) > 0 {
+		fmt.Fprintf(os.Stderr, "While scanning Go workspace:\n")
+		for path, err := range errors {
+			fmt.Fprintf(os.Stderr, "Package %q: %s.\n", path, err)
+		}
+		return fmt.Errorf("failed to construct import graph")
+	}
+
+	// Determine the affected packages---the set of packages whose import
+	// statements need updating.
+	affectedPackages := map[string]bool{from: true}
+	destinations := map[string]string{} // maps old dir to new dir
+	for pkg := range subpackages(ctxt, srcDir, from) {
+		for r := range rev[pkg] {
+			affectedPackages[r] = true
+		}
+		destinations[pkg] = strings.Replace(pkg,
+			// Ensure directories have a trailing "/".
+			filepath.Join(from, ""), filepath.Join(to, ""), 1)
+	}
+
+	// Load all the affected packages.
+	iprog, err := loadProgram(ctxt, affectedPackages)
+	if err != nil {
+		return err
+	}
+
+	// Prepare the move command, if one was supplied.
+	var cmd string
+	if moveTmpl != "" {
+		if cmd, err = moveCmd(moveTmpl, fromDir, toDir); err != nil {
+			return err
+		}
+	}
+
+	m := mover{
+		ctxt:             ctxt,
+		fwd:              fwd,
+		rev:              rev,
+		iprog:            iprog,
+		from:             from,
+		to:               to,
+		fromDir:          fromDir,
+		toDir:            toDir,
+		affectedPackages: affectedPackages,
+		destinations:     destinations,
+		cmd:              cmd,
+	}
+
+	if err := m.checkValid(); err != nil {
+		return err
+	}
+
+	m.move()
+
+	return nil
+}
+
+// srcDir returns the absolute path of the srcdir containing pkg.
+func srcDir(ctxt *build.Context, pkg string) (string, error) {
+	for _, srcDir := range ctxt.SrcDirs() {
+		path := buildutil.JoinPath(ctxt, srcDir, pkg)
+		if buildutil.IsDir(ctxt, path) {
+			return srcDir, nil
+		}
+	}
+	return "", fmt.Errorf("src dir not found for package: %s", pkg)
+}
+
+// subpackages returns the set of packages in the given srcDir whose
+// import paths start with dir.
+func subpackages(ctxt *build.Context, srcDir string, dir string) map[string]bool {
+	var mu sync.Mutex
+	subs := map[string]bool{dir: true}
+
+	// Find all packages under srcDir whose import paths start with dir.
+	buildutil.ForEachPackage(ctxt, func(pkg string, err error) {
+		if err != nil {
+			log.Fatalf("unexpected error in ForEackPackage: %v", err)
+		}
+
+		if !strings.HasPrefix(pkg, path.Join(dir, "")) {
+			return
+		}
+
+		p, err := ctxt.Import(pkg, "", build.FindOnly)
+		if err != nil {
+			log.Fatalf("unexpected: package %s can not be located by build context: %s", pkg, err)
+		}
+		if p.SrcRoot == "" {
+			log.Fatalf("unexpected: could not determine srcDir for package %s: %s", pkg, err)
+		}
+		if p.SrcRoot != srcDir {
+			return
+		}
+
+		mu.Lock()
+		subs[pkg] = true
+		mu.Unlock()
+	})
+
+	return subs
+}
+
+type mover struct {
+	// iprog contains all packages whose contents need to be updated
+	// with new package names or import paths.
+	iprog *loader.Program
+	ctxt  *build.Context
+	// fwd and rev are the forward and reverse import graphs
+	fwd, rev importgraph.Graph
+	// from and to are the source and destination import
+	// paths. fromDir and toDir are the source and destination
+	// absolute paths that package source files will be moved between.
+	from, to, fromDir, toDir string
+	// affectedPackages is the set of all packages whose contents need
+	// to be updated to reflect new package names or import paths.
+	affectedPackages map[string]bool
+	// destinations maps each subpackage to be moved to its
+	// destination path.
+	destinations map[string]string
+	// cmd, if not empty, will be executed to move fromDir to toDir.
+	cmd string
+}
+
+func (m *mover) checkValid() error {
+	const prefix = "invalid move destination"
+
+	match, err := regexp.MatchString("^[_\\pL][_\\pL\\p{Nd}]*$", path.Base(m.to))
+	if err != nil {
+		panic("regexp.MatchString failed")
+	}
+	if !match {
+		return fmt.Errorf("%s: %s; gomvpkg does not support move destinations "+
+			"whose base names are not valid go identifiers", prefix, m.to)
+	}
+
+	if buildutil.FileExists(m.ctxt, m.toDir) {
+		return fmt.Errorf("%s: %s conflicts with file %s", prefix, m.to, m.toDir)
+	}
+	if buildutil.IsDir(m.ctxt, m.toDir) {
+		return fmt.Errorf("%s: %s conflicts with directory %s", prefix, m.to, m.toDir)
+	}
+
+	for _, toSubPkg := range m.destinations {
+		if _, err := m.ctxt.Import(toSubPkg, "", build.FindOnly); err == nil {
+			return fmt.Errorf("%s: %s; package or subpackage %s already exists",
+				prefix, m.to, toSubPkg)
+		}
+	}
+
+	return nil
+}
+
+// moveCmd produces the version control move command used to move fromDir to toDir by
+// executing the given template.
+func moveCmd(moveTmpl, fromDir, toDir string) (string, error) {
+	tmpl, err := template.New("movecmd").Parse(moveTmpl)
+	if err != nil {
+		return "", err
+	}
+
+	var buf bytes.Buffer
+	err = tmpl.Execute(&buf, struct {
+		Src string
+		Dst string
+	}{fromDir, toDir})
+	return buf.String(), err
+}
+
+func (m *mover) move() error {
+	filesToUpdate := make(map[*ast.File]bool)
+
+	// Change the moved package's "package" declaration to its new base name.
+	pkg, ok := m.iprog.Imported[m.from]
+	if !ok {
+		log.Fatalf("unexpected: package %s is not in import map", m.from)
+	}
+	newName := filepath.Base(m.to)
+	for _, f := range pkg.Files {
+		f.Name.Name = newName // change package decl
+		filesToUpdate[f] = true
+	}
+	// Update imports of that package to use the new import name.
+	// None of the subpackages will change their name---only the from package
+	// itself will.
+	for p := range m.rev[m.from] {
+		_, err := importName(
+			m.iprog, m.iprog.Imported[p], m.from, path.Base(m.from), newName)
+		if err != nil {
+			return err
+		}
+	}
+
+	// For each affected package, rewrite all imports of the package to
+	// use the new import path.
+	for ap := range m.affectedPackages {
+		if ap == m.from {
+			continue
+		}
+		info, ok := m.iprog.Imported[ap]
+		if !ok {
+			log.Fatalf("unexpected: package %s is not in import map", ap)
+		}
+		for _, f := range info.Files {
+			for _, imp := range f.Imports {
+				importPath, _ := strconv.Unquote(imp.Path.Value)
+				if newPath, ok := m.destinations[importPath]; ok {
+					imp.Path.Value = strconv.Quote(newPath)
+
+					oldName := path.Base(importPath)
+					if imp.Name != nil {
+						oldName = imp.Name.Name
+					}
+
+					newName := path.Base(newPath)
+					if imp.Name == nil && oldName != newName {
+						imp.Name = ast.NewIdent(oldName)
+					} else if imp.Name == nil || imp.Name.Name == newName {
+						imp.Name = nil
+					}
+					filesToUpdate[f] = true
+				}
+			}
+		}
+	}
+
+	for f := range filesToUpdate {
+		tokenFile := m.iprog.Fset.File(f.Pos())
+		rewriteFile(m.iprog.Fset, f, tokenFile.Name())
+	}
+
+	// Move the directories.
+	// If either the fromDir or toDir are contained under version control it is
+	// the user's responsibility to provide a custom move command that updates
+	// version control to reflect the move.
+	// TODO(matloob): If the parent directory of toDir does not exist, create it.
+	//      For now, it's required that it does exist.
+
+	if m.cmd != "" {
+		// TODO(matloob): Verify that the windows and plan9 cases are correct.
+		var cmd *exec.Cmd
+		switch runtime.GOOS {
+		case "windows":
+			cmd = exec.Command("cmd", "/c", m.cmd)
+		case "plan9":
+			cmd = exec.Command("rc", "-c", m.cmd)
+		default:
+			cmd = exec.Command("sh", "-c", m.cmd)
+		}
+		cmd.Stderr = os.Stderr
+		cmd.Stdout = os.Stdout
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf("version control system's move command failed: %v", err)
+		}
+
+		return nil
+	}
+
+	return moveDirectory(m.fromDir, m.toDir)
+}
+
+var moveDirectory = func(from, to string) error {
+	return os.Rename(from, to)
+}

refactor/rename/mvpkg_test.go

@@ -0,0 +1,256 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// licence that can be found in the LICENSE file.
+
+package rename
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/build"
+	"go/format"
+	"go/token"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	"golang.org/x/tools/go/buildutil"
+)
+
+func TestErrors(t *testing.T) {
+	tests := []struct {
+		ctxt     *build.Context
+		from, to string
+		want     string // regexp to match error, or "OK"
+	}{
+		// Simple example.
+		{
+			ctxt: fakeContext(map[string][]string{
+				"foo": {`package foo; type T int`},
+				"bar": {`package bar`},
+				"main": {`package main
+
+import "foo"
+
+var _ foo.T
+`},
+			}),
+			from: "foo", to: "bar",
+			want: "invalid move destination: bar conflicts with directory /go/src/bar",
+		},
+		// Subpackage already exists.
+		{
+			ctxt: fakeContext(map[string][]string{
+				"foo":     {`package foo; type T int`},
+				"foo/sub": {`package sub`},
+				"bar/sub": {`package sub`},
+				"main": {`package main
+
+import "foo"
+
+var _ foo.T
+`},
+			}),
+			from: "foo", to: "bar",
+			want: "invalid move destination: bar; package or subpackage bar/sub already exists",
+		},
+		// Invalid base name.
+		{
+			ctxt: fakeContext(map[string][]string{
+				"foo": {`package foo; type T int`},
+				"main": {`package main
+
+import "foo"
+
+var _ foo.T
+`},
+			}),
+			from: "foo", to: "bar-v2.0",
+			want: "invalid move destination: bar-v2.0; gomvpkg does not " +
+				"support move destinations whose base names are not valid " +
+				"go identifiers",
+		},
+	}
+
+	for _, test := range tests {
+		ctxt := test.ctxt
+
+		got := make(map[string]string)
+		rewriteFile = func(fset *token.FileSet, f *ast.File, orig string) error {
+			var out bytes.Buffer
+			if err := format.Node(&out, fset, f); err != nil {
+				return err
+			}
+			got[orig] = out.String()
+			return nil
+		}
+		moveDirectory = func(from, to string) error {
+			for path, contents := range got {
+				if strings.HasPrefix(path, from) {
+					newPath := strings.Replace(path, from, to, 1)
+					delete(got, path)
+					got[newPath] = contents
+				}
+			}
+			return nil
+		}
+
+		err := Move(ctxt, test.from, test.to, "")
+		prefix := fmt.Sprintf("-from %q -to %q", test.from, test.to)
+		if err == nil {
+			t.Errorf("%s: nil error. Expected error: %s", prefix, test.want)
+			continue
+		}
+		if test.want != err.Error() {
+			t.Errorf("%s: conflict does not match expectation:\n"+
+				"Error: %q\n"+
+				"Pattern: %q",
+				prefix, err.Error(), test.want)
+		}
+	}
+}
+
+func TestMoves(t *testing.T) {
+	tests := []struct {
+		ctxt     *build.Context
+		from, to string
+		want     map[string]string
+	}{
+		// Simple example.
+		{
+			ctxt: fakeContext(map[string][]string{
+				"foo": {`package foo; type T int`},
+				"main": {`package main
+
+import "foo"
+
+var _ foo.T
+`},
+			}),
+			from: "foo", to: "bar",
+			want: map[string]string{
+				"/go/src/main/0.go": `package main
+
+import "bar"
+
+var _ bar.T
+`,
+				"/go/src/bar/0.go": `package bar
+
+type T int
+`,
+			},
+		},
+
+		// Example with subpackage.
+		{
+			ctxt: fakeContext(map[string][]string{
+				"foo":     {`package foo; type T int`},
+				"foo/sub": {`package sub; type T int`},
+				"main": {`package main
+
+import "foo"
+import "foo/sub"
+
+var _ foo.T
+var _ sub.T
+`},
+			}),
+			from: "foo", to: "bar",
+			want: map[string]string{
+				"/go/src/main/0.go": `package main
+
+import "bar"
+import "bar/sub"
+
+var _ bar.T
+var _ sub.T
+`,
+				"/go/src/bar/0.go": `package bar
+
+type T int
+`,
+				"/go/src/bar/sub/0.go": `package sub; type T int`,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		ctxt := test.ctxt
+
+		var mu sync.Mutex
+		got := make(map[string]string)
+		// Populate got with starting file set. rewriteFile and moveDirectory
+		// will mutate got to produce resulting file set.
+		buildutil.ForEachPackage(ctxt, func(importPath string, err error) {
+			if err != nil {
+				return
+			}
+			path := filepath.Join("/go/src", importPath, "0.go")
+			if !buildutil.FileExists(ctxt, path) {
+				return
+			}
+			f, err := ctxt.OpenFile(path)
+			defer f.Close()
+			if err != nil {
+				t.Errorf("unexpected error opening file: %s", err)
+				return
+			}
+			bytes, err := ioutil.ReadAll(f)
+			if err != nil {
+				t.Errorf("unexpected error reading file: %s", err)
+				return
+			}
+			mu.Lock()
+			got[path] = string(bytes)
+			defer mu.Unlock()
+		})
+		rewriteFile = func(fset *token.FileSet, f *ast.File, orig string) error {
+			var out bytes.Buffer
+			if err := format.Node(&out, fset, f); err != nil {
+				return err
+			}
+			got[orig] = out.String()
+			return nil
+		}
+		moveDirectory = func(from, to string) error {
+			for path, contents := range got {
+				if strings.HasPrefix(path, from) {
+					newPath := strings.Replace(path, from, to, 1)
+					delete(got, path)
+					got[newPath] = contents
+				}
+			}
+			return nil
+		}
+
+		err := Move(ctxt, test.from, test.to, "")
+		prefix := fmt.Sprintf("-from %q -to %q", test.from, test.to)
+		if err != nil {
+			t.Errorf("%s: unexpected error: %s", prefix, err)
+			continue
+		}
+
+		for file, wantContent := range test.want {
+			gotContent, ok := got[file]
+			delete(got, file)
+			if !ok {
+				// TODO(matloob): some testcases might have files that won't be
+				// rewritten
+				t.Errorf("%s: file %s not rewritten", prefix, file)
+				continue
+			}
+			if gotContent != wantContent {
+				t.Errorf("%s: rewritten file %s does not match expectation; got <<<%s>>>\n"+
+					"want <<<%s>>>", prefix, file, gotContent, wantContent)
+			}
+		}
+		// got should now be empty
+		for file := range got {
+			t.Errorf("%s: unexpected rewrite of file %s", prefix, file)
+		}
+	}
+}

refactor/rename/rename.go

@@ -16,8 +16,10 @@ import (
 	"go/parser"
 	"go/token"
 	"os"
+	"path"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 
 	"golang.org/x/tools/go/loader"
@@ -160,6 +162,46 @@ var reportError = func(posn token.Position, message string) {
 	fmt.Fprintf(os.Stderr, "%s: %s\n", posn, message)
 }
 
+// importName renames imports of the package with the given path in
+// the given package.  If fromName is not empty, only imports as
+// fromName will be renamed.  Even if renaming is successful, there
+// may be some files that are unchanged; they are reported in
+// unchangedFiles.
+func importName(iprog *loader.Program, info *loader.PackageInfo, fromPath, fromName, to string) (unchangedFiles []string, err error) {
+	for _, f := range info.Files {
+		var from types.Object
+		for _, imp := range f.Imports {
+			importPath, _ := strconv.Unquote(imp.Path.Value)
+			importName := path.Base(importPath)
+			if imp.Name != nil {
+				importName = imp.Name.Name
+			}
+			if importPath == fromPath && (fromName == "" || importName == fromName) {
+				from = info.Implicits[imp]
+				break
+			}
+		}
+		if from == nil {
+			continue
+		}
+		r := renamer{
+			iprog:        iprog,
+			objsToUpdate: make(map[types.Object]bool),
+			to:           to,
+			packages:     map[*types.Package]*loader.PackageInfo{info.Pkg: info},
+		}
+		r.check(from)
+		if r.hadConflicts {
+			continue // ignore errors; leave the existing name
+			unchangedFiles = append(unchangedFiles, f.Name.Name)
+		}
+		if err := r.update(); err != nil {
+			return nil, err
+		}
+	}
+	return unchangedFiles, nil
+}
+
 func Main(ctxt *build.Context, offsetFlag, fromFlag, to string) error {
 	// -- Parse the -from or -offset specifier ----------------------------
 

refactor/rename/rename_test.go

@@ -1067,6 +1067,9 @@ func fakeContext(pkgs map[string][]string) *build.Context {
 		dir, base := filepath.Split(path)
 		dir = filepath.Clean(dir)
 		index, _ := strconv.Atoi(strings.TrimSuffix(base, ".go"))
+		if _, ok := pkgs[dir]; !ok || index >= len(pkgs[dir]) {
+			return nil, fmt.Errorf("file does not exist")
+		}
 		return ioutil.NopCloser(bytes.NewBufferString(pkgs[dir][index])), nil
 	}
 	ctxt.IsAbsPath = func(path string) bool {