dashboard: consolidate upload code, update to new oauth2 libraries

Adds dashboard/upload/upload.go, instead of oddly shoving it as part
of the coordinator/buildongce tool.

And as part of that (in order to compile and test buildongce/create.go
without installing mercurial on this machine), I updated it from
goauth2 to oauth2.

Despite this "just" being a cleanup CL, it took forever because I hit
OAuth2+Cloud Storage+Web UI woes along the way, documented partially
in upload.go. The web UI misled me for a long time. Maybe I shouldn't
have used service accounts, but it does make configuration easier for
upload.go. The buildongce/create.go probably should use them too,
but I can do that later. I'm done cleaning for now.

Change-Id: Icb8e3decb682d3685edffecea2a10fcb4e385e10
Reviewed-on: https://go-review.googlesource.com/2731
Reviewed-by: Andrew Gerrand <adg@golang.org>

dashboard/README

@@ -18,6 +18,7 @@ env/:     configuration files describing the environment of builders and related
           environment.
 retrybuilds/: a Go client program to delete build results from the dashboard (app)
 types/:   a Go package contain common types used by other pieces.
+upload/:  a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere.
 watcher/: a daemon that watches for new commits to the Go repository and
           its sub-repositories, and notifies the dashboard of those commits.
 

dashboard/buildlet/Makefile

@@ -1,22 +1,26 @@
 buildlet: buildlet.go
 	go build --tags=buildlet
 
+buildlet.linux-amd64: buildlet.go
+	GOOS=linux GOARCH=amd64 go build -o $@ --tags=buildlet
+	cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@)
+
 buildlet.openbsd-amd64: buildlet.go
 	GOOS=openbsd GOARCH=amd64 go build -o $@ --tags=buildlet
-	cat $@ | (cd ../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@)
+	cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@)
 
 buildlet.plan9-386: buildlet.go
 	GOOS=plan9 GOARCH=386 go build -o $@ --tags=buildlet
-	cat $@ | (cd ../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@)
+	cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@)
 
 buildlet.windows-amd64: buildlet.go
 	GOOS=windows GOARCH=amd64 go build -o $@ --tags=buildlet
-	cat $@ | (cd ../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@)
+	cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@)
 
 buildlet.darwin-amd64: buildlet.go
 	GOOS=darwin GOARCH=amd64 go build -o $@ --tags=buildlet
-	cat $@ | (cd ../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@)
+	cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@)
 
 buildlet.netbsd-amd64: buildlet.go
 	GOOS=netbsd GOARCH=amd64 go build -o $@ --tags=buildlet
-	cat $@ | (cd ../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@)
+	cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@)

dashboard/buildlet/stage0/Makefile

@@ -1,3 +1,3 @@
 buildlet-stage0.windows-amd64: stage0.go
 	GOOS=windows GOARCH=amd64 go build -o $@ --tags=stage0
-	cat $@ | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@)
+	cat $@ | (cd ../../upload && go run upload.go --public go-builder-data/$@)

dashboard/coordinator/Makefile

@@ -6,4 +6,4 @@ coordinator: main.go
 # And watch its logs with:
 #   sudo journalctl -f -u gobuild.service
 upload: coordinator
-	cat coordinator | (cd buildongce && go run create.go --write_object=go-builder-data/coordinator)
+	cat coordinator | (cd ../upload && go run upload.go --public go-builder-data/coordinator)

dashboard/coordinator/buildongce/create.go

@@ -8,19 +8,17 @@ package main // import "golang.org/x/tools/dashboard/coordinator/buildongce"
 
 import (
 	"bufio"
-	"bytes"
 	"encoding/json"
 	"flag"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"log"
-	"net/http"
 	"os"
 	"strings"
 	"time"
 
-	"code.google.com/p/goauth2/oauth"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
 	compute "google.golang.org/api/compute/v1"
 )
 
@@ -33,8 +31,6 @@ var (
 	staticIP  = flag.String("static_ip", "", "Static IP to use. If empty, automatic.")
 	reuseDisk = flag.Bool("reuse_disk", true, "Whether disk images should be reused between shutdowns/restarts.")
 	ssd       = flag.Bool("ssd", false, "use a solid state disk (faster, more expensive)")
-
-	writeObject = flag.String("write_object", "", "If non-empty, a VM isn't created and the flag value is Google Cloud Storage bucket/object to write. The contents from stdin.")
 )
 
 func readFile(v string) string {
@@ -45,19 +41,18 @@ func readFile(v string) string {
 	return strings.TrimSpace(string(slurp))
 }
 
-var config = &oauth.Config{
+var config = &oauth2.Config{
 	// The client-id and secret should be for an "Installed Application" when using
 	// the CLI. Later we'll use a web application with a callback.
-	ClientId:     readFile("client-id.dat"),
+	ClientID:     readFile("client-id.dat"),
 	ClientSecret: readFile("client-secret.dat"),
-	Scope: strings.Join([]string{
+	Endpoint:     google.Endpoint,
+	Scopes: []string{
 		compute.DevstorageFull_controlScope,
 		compute.ComputeScope,
 		"https://www.googleapis.com/auth/sqlservice",
 		"https://www.googleapis.com/auth/sqlservice.admin",
-	}, " "),
+	},
-	AuthURL:     "https://accounts.google.com/o/oauth2/auth",
-	TokenURL:    "https://accounts.google.com/o/oauth2/token",
 	RedirectURL: "urn:ietf:wg:oauth:2.0:oob",
 }
 
@@ -94,35 +89,28 @@ func main() {
 	prefix := "https://www.googleapis.com/compute/v1/projects/" + *proj
 	machType := prefix + "/zones/" + *zone + "/machineTypes/" + *mach
 
-	tr := &oauth.Transport{
+	const tokenFileName = "token.dat"
-		Config: config,
+	tokenFile := tokenCacheFile(tokenFileName)
-	}
+	tokenSource := oauth2.ReuseTokenSource(nil, tokenFile)
-
+	token, err := tokenSource.Token()
-	tokenCache := oauth.CacheFile("token.dat")
-	token, err := tokenCache.Token()
 	if err != nil {
-		if *writeObject != "" {
+		log.Printf("Error getting token from %s: %v", tokenFileName, err)
-			log.Fatalf("Can't use --write_object without a valid token.dat file already cached.")
-		}
-		log.Printf("Error getting token from %s: %v", string(tokenCache), err)
 		log.Printf("Get auth code from %v", config.AuthCodeURL("my-state"))
 		fmt.Print("\nEnter auth code: ")
 		sc := bufio.NewScanner(os.Stdin)
 		sc.Scan()
 		authCode := strings.TrimSpace(sc.Text())
-		token, err = tr.Exchange(authCode)
+		token, err = config.Exchange(oauth2.NoContext, authCode)
 		if err != nil {
 			log.Fatalf("Error exchanging auth code for a token: %v", err)
 		}
-		tokenCache.PutToken(token)
+		if err := tokenFile.WriteToken(token); err != nil {
+			log.Fatalf("Error writing to %s: %v", tokenFileName, err)
+		}
+		tokenSource = oauth2.ReuseTokenSource(token, nil)
 	}
 
-	tr.Token = token
+	oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
-	oauthClient := &http.Client{Transport: tr}
-	if *writeObject != "" {
-		writeCloudStorageObject(oauthClient)
-		return
-	}
 
 	computeService, _ := compute.New(oauthClient)
 
@@ -288,31 +276,24 @@ func instanceDisk(svc *compute.Service) *compute.AttachedDisk {
 	}
 }
 
-func writeCloudStorageObject(httpClient *http.Client) {
+type tokenCacheFile string
-	content := os.Stdin
-	const maxSlurp = 1 << 20
-	var buf bytes.Buffer
-	n, err := io.CopyN(&buf, content, maxSlurp)
-	if err != nil && err != io.EOF {
-		log.Fatalf("Error reading from stdin: %v, %v", n, err)
-	}
-	contentType := http.DetectContentType(buf.Bytes())
 
-	req, err := http.NewRequest("PUT", "https://storage.googleapis.com/"+*writeObject, io.MultiReader(&buf, content))
+func (f tokenCacheFile) Token() (*oauth2.Token, error) {
+	slurp, err := ioutil.ReadFile(string(f))
 	if err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
-	req.Header.Set("x-goog-api-version", "2")
+	t := new(oauth2.Token)
-	req.Header.Set("x-goog-acl", "public-read")
+	if err := json.Unmarshal(slurp, t); err != nil {
-	req.Header.Set("Content-Type", contentType)
+		return nil, err
-	res, err := httpClient.Do(req)
+	}
+	return t, nil
+}
+
+func (f tokenCacheFile) WriteToken(t *oauth2.Token) error {
+	jt, err := json.Marshal(t)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
-	if res.StatusCode != 200 {
+	return ioutil.WriteFile(string(f), jt, 0600)
-		res.Write(os.Stderr)
-		log.Fatalf("Failed.")
-	}
-	log.Printf("Success.")
-	os.Exit(0)
 }

dashboard/env/commit-watcher/Makefile

@@ -6,4 +6,4 @@ docker: Dockerfile
 	docker build -t go-commit-watcher .
 
 docker-commit-watcher.tar.gz: docker
-	docker save go-commit-watcher | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-commit-watcher.tar.gz)
+	docker save go-commit-watcher | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-commit-watcher.tar.gz)

dashboard/env/linux-x86-base/Makefile

@@ -6,7 +6,7 @@ docker: Dockerfile
 	docker build -t gobuilders/linux-x86-base .
 
 docker-linux.base.tar.gz: docker
-	docker save gobuilders/linux-x86-base | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-linux.base.tar.gz)
+	docker save gobuilders/linux-x86-base | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.base.tar.gz)
 
 check: docker
 	docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-base /usr/local/bin/builder -rev=20a10e7ddd1 -buildroot=/ -v -report=false linux-amd64-temp

dashboard/env/linux-x86-clang/Makefile

@@ -6,7 +6,7 @@ docker: Dockerfile
 	docker build -t gobuilders/linux-x86-clang .
 
 docker-linux.clang.tar.gz: docker
-	docker save gobuilders/linux-x86-clang | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-linux.clang.tar.gz)
+	docker save gobuilders/linux-x86-clang | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz)
 
 check: docker
 	docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-temp

dashboard/env/linux-x86-gccgo/Makefile

@@ -6,7 +6,7 @@ docker: Dockerfile
 	docker build -t gobuilders/linux-x86-gccgo .
 
 docker-linux.gccgo.tar.gz: docker
-	docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-linux.gccgo.tar.gz)
+	docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.gccgo.tar.gz)
 
 check: docker
 	docker run gobuilders/linux-x86-gccgo /usr/local/bin/builder -tool="gccgo" -rev=b9151e911a54 -v -cmd='make RUNTESTFLAGS="--target_board=unix/-m64" check-go' -report=false linux-amd64-gccgo-temp

dashboard/env/linux-x86-nacl/Makefile

@@ -6,7 +6,7 @@ docker: Dockerfile
 	docker build -t gobuilders/linux-x86-nacl .
 
 upload: docker
-	docker save gobuilders/linux-x86-nacl | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-linux.nacl.tar.gz)
+	docker save gobuilders/linux-x86-nacl | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.nacl.tar.gz)
 
 check: docker
 	docker run gobuilders/linux-x86-nacl /usr/local/bin/builder -rev=77e96c9208d0 -buildroot=/ -v -cmd=/usr/local/bin/build-command.pl -report=false nacl-amd64p32

dashboard/env/linux-x86-sid/Makefile

@@ -6,7 +6,7 @@ docker: Dockerfile
 	docker build -t gobuilders/linux-x86-sid .
 
 docker-linux.sid.tar.gz: docker
-	docker save gobuilders/linux-x86-sid | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-linux.sid.tar.gz)
+	docker save gobuilders/linux-x86-sid | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.sid.tar.gz)
 
 check: docker
 	docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-sid /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-sid

dashboard/upload/upload.go

@@ -0,0 +1,132 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build upload
+
+//       ^ this is so we don't break the build of x/tools/...  build due
+//         to missing depenencies on the builders. We don't want full builds
+//         needing to pull in dependencies outside of the x/tools repo.
+
+// The upload command writes a file to Google Cloud Storage. It's used
+// exclusively by the Makefiles in the Go project repos. Think of it
+// as a very light version of gsutil or gcloud, but with some
+// Go-specific configuration knowledge baked in.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/cloud"
+	"google.golang.org/cloud/storage"
+)
+
+var (
+	public  = flag.Bool("public", false, "object should be world-readable")
+	file    = flag.String("file", "-", "Filename to read object from, or '-' for stdin.")
+	verbose = flag.Bool("verbose", false, "verbose logging")
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage: upload [--public] [--file=...] <bucket/object>\n")
+		flag.PrintDefaults()
+	}
+	flag.Parse()
+	if flag.NArg() != 1 {
+		flag.Usage()
+		os.Exit(1)
+	}
+	args := strings.SplitN(flag.Arg(0), "/", 2)
+	if len(args) != 2 {
+		flag.Usage()
+		os.Exit(1)
+	}
+	bucket, object := args[0], args[1]
+
+	proj, ok := bucketProject[bucket]
+	if !ok {
+		log.Fatalf("bucket %q doesn't have an associated project in upload.go")
+	}
+
+	ts, err := tokenSource(bucket)
+	if err != nil {
+		log.Fatalf("Failed to get an OAuth2 token source: %v", err)
+	}
+	httpClient := oauth2.NewClient(oauth2.NoContext, ts)
+
+	ctx := cloud.NewContext(proj, httpClient)
+	w := storage.NewWriter(ctx, bucket, object)
+	// If you don't give the owners access, the web UI seems to
+	// have a bug and doesn't have access to see that it's public, so
+	// won't render the "Shared Publicly" link. So we do that, even
+	// though it's dumb and unnecessary otherwise:
+	w.ACL = append(w.ACL, storage.ACLRule{Entity: storage.ACLEntity("project-owners-" + proj), Role: storage.RoleOwner})
+	if *public {
+		w.ACL = append(w.ACL, storage.ACLRule{Entity: storage.AllUsers, Role: storage.RoleReader})
+	}
+	var content io.Reader
+	if *file == "-" {
+		content = os.Stdin
+	} else {
+		content, err = os.Open(*file)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	const maxSlurp = 1 << 20
+	var buf bytes.Buffer
+	n, err := io.CopyN(&buf, content, maxSlurp)
+	if err != nil && err != io.EOF {
+		log.Fatalf("Error reading from stdin: %v, %v", n, err)
+	}
+	w.ContentType = http.DetectContentType(buf.Bytes())
+
+	_, err = io.Copy(w, io.MultiReader(&buf, content))
+	if cerr := w.Close(); cerr != nil && err == nil {
+		err = cerr
+	}
+	if err != nil {
+		log.Fatalf("Write error: %v", err)
+	}
+	if *verbose {
+		log.Printf("Wrote %v", object)
+	}
+	os.Exit(0)
+}
+
+var bucketProject = map[string]string{
+	"go-builder-data":       "symbolic-datum-552",
+	"http2-demo-server-tls": "symbolic-datum-552",
+	"winstrap":              "999119582588",
+	"gobuilder":             "999119582588", // deprecated
+}
+
+func tokenSource(bucket string) (oauth2.TokenSource, error) {
+	proj := bucketProject[bucket]
+	fileName := filepath.Join(os.Getenv("HOME"), "keys", proj+".key.json")
+	jsonConf, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("Missing JSON key configuration. Download the Service Account JSON key from https://console.developers.google.com/project/%s/apiui/credential and place it at %s", proj, fileName)
+		}
+		return nil, err
+	}
+	conf, err := google.JWTConfigFromJSON(jsonConf, storage.ScopeReadWrite)
+	if err != nil {
+		return nil, fmt.Errorf("reading JSON config from %s: %v", fileName, err)
+	}
+	return conf.TokenSource(oauth2.NoContext), nil
+}