go.tools/dashboard: add option to build gccgo revisions

* Added --tool flag that allows user to specify "gccgo" as the tool to build.
* Added builderEnv struct to abstract away setting up the build.
* Made envv and envvWindows methods of the builderEnv struct.
* Modified Builder.envv() to wrap envv and envvWindows
* Added internal builderEnv in Builder.

R=adg
CC=golang-dev
https://golang.org/cl/13240044

dashboard/builder/env.go

@@ -0,0 +1,229 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"code.google.com/p/go.tools/go/vcs"
+)
+
+// These variables are copied from the gobuilder's environment
+// to the envv of its subprocesses.
+var extraEnv = []string{
+	"GOARM",
+
+	// For Unix derivatives.
+	"CC",
+	"PATH",
+	"TMPDIR",
+	"USER",
+
+	// For Plan 9.
+	"objtype",
+	"cputype",
+	"path",
+}
+
+// builderEnv represents the environment that a Builder will run tests in.
+type builderEnv interface {
+	// setup sets up the builder environment and returns the directory to run the buildCmd in.
+	setup(repo *Repo, workpath, hash string, envv []string) (string, error)
+}
+
+// goEnv represents the builderEnv for the main Go repo.
+type goEnv struct {
+	goos, goarch string
+}
+
+func (b *Builder) envv() []string {
+	if runtime.GOOS == "windows" {
+		return b.envvWindows()
+	}
+
+	var e []string
+	if *buildTool == "go" {
+		e = []string{
+			"GOOS=" + b.goos,
+			"GOHOSTOS=" + b.goos,
+			"GOARCH=" + b.goarch,
+			"GOHOSTARCH=" + b.goarch,
+			"GOROOT_FINAL=/usr/local/go",
+		}
+	}
+
+	for _, k := range extraEnv {
+		if s, ok := getenvOk(k); ok {
+			e = append(e, k+"="+s)
+		}
+	}
+	return e
+}
+
+func (b *Builder) envvWindows() []string {
+	var start map[string]string
+	if *buildTool == "go" {
+		start = map[string]string{
+			"GOOS":         b.goos,
+			"GOHOSTOS":     b.goos,
+			"GOARCH":       b.goarch,
+			"GOHOSTARCH":   b.goarch,
+			"GOROOT_FINAL": `c:\go`,
+			"GOBUILDEXIT":  "1", // exit all.bat with completion status.
+		}
+	}
+
+	for _, name := range extraEnv {
+		if s, ok := getenvOk(name); ok {
+			start[name] = s
+		}
+	}
+	skip := map[string]bool{
+		"GOBIN":   true,
+		"GOROOT":  true,
+		"INCLUDE": true,
+		"LIB":     true,
+	}
+	var e []string
+	for name, v := range start {
+		e = append(e, name+"="+v)
+		skip[name] = true
+	}
+	for _, kv := range os.Environ() {
+		s := strings.SplitN(kv, "=", 2)
+		name := strings.ToUpper(s[0])
+		switch {
+		case name == "":
+			// variables, like "=C:=C:\", just copy them
+			e = append(e, kv)
+		case !skip[name]:
+			e = append(e, kv)
+			skip[name] = true
+		}
+	}
+	return e
+}
+
+// setup for a goEnv clones the main go repo to workpath/go at the provided hash
+// and returns the path workpath/go/src, the location of all go build scripts.
+func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
+	goworkpath := filepath.Join(workpath, "go")
+	if _, err := repo.Clone(goworkpath, hash); err != nil {
+		return "", fmt.Errorf("error cloning repository: %s", err)
+	}
+	return filepath.Join(goworkpath, "src"), nil
+}
+
+// gccgoEnv represents the builderEnv for the gccgo compiler.
+type gccgoEnv struct{}
+
+// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
+// and clones the latest GCC branch to workpath/gcc. The gccgo sources are
+// replaced with the updated sources in the gofrontend repo and gcc gets
+// gets configured and built in workpath/gcc-objdir. The path to
+// workpath/gcc-objdir is returned.
+func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
+	gofrontendpath := filepath.Join(workpath, "gofrontend")
+	gccpath := filepath.Join(workpath, "gcc")
+
+	// get a handle to SVN vcs.Cmd for pulling down GCC.
+	svn := vcs.ByCmd("svn")
+
+	if err := timeout(*cmdTimeout, func() error {
+		// pull down a working copy of GCC.
+		return svn.Create(gccpath, *gccPath)
+	}); err != nil {
+		return "", err
+	}
+
+	// clone gofrontend repo at specified revision
+	if _, err := repo.Clone(gofrontendpath, hash); err != nil {
+		return "", err
+	}
+
+	// remove gccpath/gcc/go/gofrontend and gcc/libgo
+	gccgopath := filepath.Join(gccpath, "gcc", "go", "gofrontend")
+	gcclibgopath := filepath.Join(gccpath, "libgo")
+	if err := os.RemoveAll(gccgopath); err != nil {
+		return "", err
+	}
+	if err := os.RemoveAll(gcclibgopath); err != nil {
+		return "", err
+	}
+
+	// copy gofrontend and libgo to appropriate locations
+	if err := copyDir(filepath.Join(gofrontendpath, "go"), gccgopath); err != nil {
+		return "", fmt.Errorf("Failed to copy gofrontend/go to gcc/go/gofrontend: %s\n", err)
+	}
+	if err := copyDir(filepath.Join(gofrontendpath, "libgo"), gcclibgopath); err != nil {
+		return "", fmt.Errorf("Failed to copy gofrontend/libgo to gcc/libgo: %s\n", err)
+	}
+
+	// make objdir to work in
+	gccobjdir := filepath.Join(workpath, "gcc-objdir")
+	if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
+		return "", err
+	}
+
+	// configure GCC with substituted gofrontend and libgo
+	gccConfigCmd := []string{filepath.Join(gccpath, "configure"), "--enable-languages=c,c++,go", "--disable-bootstrap"}
+	if _, err := runOutput(*cmdTimeout, envv, ioutil.Discard, gccobjdir, gccConfigCmd...); err != nil {
+		return "", fmt.Errorf("Failed to configure GCC: %s", err)
+	}
+
+	// build gcc
+	if _, err := runOutput(*buildTimeout, envv, ioutil.Discard, gccobjdir, "make"); err != nil {
+		return "", fmt.Errorf("Failed to build GCC: %s", err)
+	}
+
+	return gccobjdir, nil
+
+}
+
+// copyDir copies the src directory into the dst
+func copyDir(src, dst string) error {
+	return filepath.Walk(src, func(path string, f os.FileInfo, err error) error {
+		dstPath := strings.Replace(path, src, dst, 1)
+		if f.IsDir() {
+			return os.Mkdir(dstPath, mkdirPerm)
+		}
+
+		srcFile, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer srcFile.Close()
+
+		dstFile, err := os.Create(dstPath)
+		if err != nil {
+			return err
+		}
+
+		if _, err := io.Copy(dstFile, srcFile); err != nil {
+			return err
+		}
+		return dstFile.Close()
+	})
+}
+
+func getenvOk(k string) (v string, ok bool) {
+	v = os.Getenv(k)
+	if v != "" {
+		return v, true
+	}
+	keq := k + "="
+	for _, kv := range os.Environ() {
+		if kv == keq {
+			return "", true
+		}
+	}
+	return "", false
+}

dashboard/builder/main.go

@@ -22,36 +22,21 @@ import (
 )
 
 const (
-	codeProject      = "go"
+	codeProject          = "go"
-	codePyScript     = "misc/dashboard/googlecode_upload.py"
+	codePyScript         = "misc/dashboard/googlecode_upload.py"
-	hgUrl            = "code.google.com/p/go"
+	goImportPath         = "code.google.com/p/go"
-	mkdirPerm        = 0750
+	gofrontendImportPath = "code.google.com/p/gofrontend"
-	waitInterval     = 30 * time.Second // time to wait before checking for new revs
+	mkdirPerm            = 0750
-	pkgBuildInterval = 24 * time.Hour   // rebuild packages every 24 hours
+	waitInterval         = 30 * time.Second // time to wait before checking for new revs
+	pkgBuildInterval     = 24 * time.Hour   // rebuild packages every 24 hours
 )
 
-// These variables are copied from the gobuilder's environment
-// to the envv of its subprocesses.
-var extraEnv = []string{
-	"GOARM",
-
-	// For Unix derivatives.
-	"CC",
-	"PATH",
-	"TMPDIR",
-	"USER",
-
-	// For Plan 9.
-	"objtype",
-	"cputype",
-	"path",
-}
-
 type Builder struct {
 	goroot       *Repo
 	name         string
 	goos, goarch string
 	key          string
+	env          builderEnv
 }
 
 var (
@@ -60,6 +45,8 @@ var (
 	buildRelease   = flag.Bool("release", false, "Build and upload binary release archives")
 	buildRevision  = flag.String("rev", "", "Build specified revision and exit")
 	buildCmd       = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)")
+	buildTool      = flag.String("tool", "go", "Tool to build.")
+	gccPath        = flag.String("gccpath", "svn://gcc.gnu.org/svn/gcc/trunk", "Path to download gcc from")
 	failAll        = flag.Bool("fail", false, "fail all builds")
 	parallel       = flag.Bool("parallel", false, "Build multiple targets in parallel")
 	buildTimeout   = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests")
@@ -91,7 +78,7 @@ func main() {
 	vcs.ShowCmd = *verbose
 	vcs.Verbose = *verbose
 
-	rr, err := vcs.RepoRootForImportPath(hgUrl, *verbose)
+	rr, err := repoForTool()
 	if err != nil {
 		log.Fatal("Error finding repository:", err)
 	}
@@ -112,9 +99,9 @@ func main() {
 			log.Fatalf("Error making build root (%s): %s", *buildroot, err)
 		}
 		var err error
-		goroot, err = RemoteRepo(hgUrl, rootPath)
+		goroot, err = RemoteRepo(goroot.Master.Root, rootPath)
 		if err != nil {
-			log.Fatalf("Error creating repository with url (%s): %s", hgUrl, err)
+			log.Fatalf("Error creating repository with url (%s): %s", goroot.Master.Root, err)
 		}
 
 		goroot, err = goroot.Clone(goroot.Path, "tip")
@@ -208,12 +195,10 @@ func NewBuilder(goroot *Repo, name string) (*Builder, error) {
 		name:   name,
 	}
 
-	// get goos/goarch from builder string
+	// get builderEnv for this tool
-	s := strings.SplitN(b.name, "-", 3)
+	var err error
-	if len(s) >= 2 {
+	if b.env, err = b.builderEnv(name); err != nil {
-		b.goos, b.goarch = s[0], s[1]
+		return nil, err
-	} else {
-		return nil, fmt.Errorf("unsupported builder form: %s", name)
 	}
 
 	// read keys from keyfile
@@ -235,6 +220,29 @@ func NewBuilder(goroot *Repo, name string) (*Builder, error) {
 	return b, nil
 }
 
+// builderEnv returns the builderEnv for this buildTool.
+func (b *Builder) builderEnv(name string) (builderEnv, error) {
+	// get goos/goarch from builder string
+	s := strings.SplitN(b.name, "-", 3)
+	if len(s) < 2 {
+		return nil, fmt.Errorf("unsupported builder form: %s", name)
+	}
+	b.goos = s[0]
+	b.goarch = s[1]
+
+	switch *buildTool {
+	case "go":
+		return &goEnv{
+			goos:   s[0],
+			goarch: s[1],
+		}, nil
+	case "gccgo":
+		return &gccgoEnv{}, nil
+	default:
+		return nil, fmt.Errorf("unsupported build tool: %s", *buildTool)
+	}
+}
+
 // buildCmd returns the build command to invoke.
 // Builders which contain the string '-race' in their
 // name will override *buildCmd and return raceCmd.
@@ -270,21 +278,20 @@ func (b *Builder) buildHash(hash string) error {
 	// create place in which to do work
 	workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
 	if err := os.Mkdir(workpath, mkdirPerm); err != nil {
-		//return err
+		return err
 	}
 	defer os.RemoveAll(workpath)
 
 	// pull before cloning to ensure we have the revision
 	if err := b.goroot.Pull(); err != nil {
-		// return err
-	}
-
-	// clone repo at specified revision
-	if _, err := b.goroot.Clone(filepath.Join(workpath, "go"), hash); err != nil {
 		return err
 	}
 
-	srcDir := filepath.Join(workpath, "go", "src")
+	// set up builder's environment.
+	srcDir, err := b.env.setup(b.goroot, workpath, hash, b.envv())
+	if err != nil {
+		return err
+	}
 
 	// build
 	var buildlog bytes.Buffer
@@ -297,11 +304,19 @@ func (b *Builder) buildHash(hash string) error {
 	w := io.MultiWriter(f, &buildlog)
 
 	cmd := b.buildCmd()
-	if !filepath.IsAbs(cmd) {
+
-		cmd = filepath.Join(srcDir, cmd)
+	// go's build command is a script relative to the srcDir, whereas
+	// gccgo's build command is usually "make check-go" in the srcDir.
+	if *buildTool == "go" {
+		if !filepath.IsAbs(cmd) {
+			cmd = filepath.Join(srcDir, cmd)
+		}
 	}
+
+	// make sure commands with extra arguments are handled properly
+	splitCmd := strings.Split(cmd, " ")
 	startTime := time.Now()
-	ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, cmd)
+	ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, splitCmd...)
 	runTime := time.Now().Sub(startTime)
 	errf := func() string {
 		if err != nil {
@@ -324,8 +339,8 @@ func (b *Builder) buildHash(hash string) error {
 		return fmt.Errorf("recordResult: %s", err)
 	}
 
-	// build Go sub-repositories
+	// build sub-repositories
-	goRoot := filepath.Join(workpath, "go")
+	goRoot := filepath.Join(workpath, *buildTool)
 	goPath := workpath
 	b.buildSubrepos(goRoot, goPath, hash)
 
@@ -424,65 +439,17 @@ func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error)
 	return log, err
 }
 
-// envv returns an environment for build/bench execution
+// repoForTool returns the correct RepoRoot for the buildTool, or an error if
-func (b *Builder) envv() []string {
+// the tool is unknown.
-	if runtime.GOOS == "windows" {
+func repoForTool() (*vcs.RepoRoot, error) {
-		return b.envvWindows()
+	switch *buildTool {
+	case "go":
+		return vcs.RepoRootForImportPath(goImportPath, *verbose)
+	case "gccgo":
+		return vcs.RepoRootForImportPath(gofrontendImportPath, *verbose)
+	default:
+		return nil, fmt.Errorf("unknown build tool: %s", *buildTool)
 	}
-	e := []string{
-		"GOOS=" + b.goos,
-		"GOHOSTOS=" + b.goos,
-		"GOARCH=" + b.goarch,
-		"GOHOSTARCH=" + b.goarch,
-		"GOROOT_FINAL=/usr/local/go",
-	}
-	for _, k := range extraEnv {
-		if s, ok := getenvOk(k); ok {
-			e = append(e, k+"="+s)
-		}
-	}
-	return e
-}
-
-// windows version of envv
-func (b *Builder) envvWindows() []string {
-	start := map[string]string{
-		"GOOS":         b.goos,
-		"GOHOSTOS":     b.goos,
-		"GOARCH":       b.goarch,
-		"GOHOSTARCH":   b.goarch,
-		"GOROOT_FINAL": `c:\go`,
-		"GOBUILDEXIT":  "1", // exit all.bat with completion status.
-	}
-	for _, name := range extraEnv {
-		if s, ok := getenvOk(name); ok {
-			start[name] = s
-		}
-	}
-	skip := map[string]bool{
-		"GOBIN":   true,
-		"GOROOT":  true,
-		"INCLUDE": true,
-		"LIB":     true,
-	}
-	var e []string
-	for name, v := range start {
-		e = append(e, name+"="+v)
-		skip[name] = true
-	}
-	for _, kv := range os.Environ() {
-		s := strings.SplitN(kv, "=", 2)
-		name := strings.ToUpper(s[0])
-		switch {
-		case name == "":
-			// variables, like "=C:=C:\", just copy them
-			e = append(e, kv)
-		case !skip[name]:
-			e = append(e, kv)
-			skip[name] = true
-		}
-	}
-	return e
 }
 
 func isDirectory(name string) bool {
@@ -653,17 +620,3 @@ func defaultBuildRoot() string {
 	}
 	return filepath.Join(d, "gobuilder")
 }
-
-func getenvOk(k string) (v string, ok bool) {
-	v = os.Getenv(k)
-	if v != "" {
-		return v, true
-	}
-	keq := k + "="
-	for _, kv := range os.Environ() {
-		if kv == keq {
-			return "", true
-		}
-	}
-	return "", false
-}