From 84afeba47153a634589b0a7249cb1e8c17b03d29 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 13 Jan 2015 15:06:48 -0800 Subject: [PATCH] dashboard: consolidate upload code, update to new oauth2 libraries Adds dashboard/upload/upload.go, instead of oddly shoving it as part of the coordinator/buildongce tool. And as part of that (in order to compile and test buildongce/create.go without installing mercurial on this machine), I updated it from goauth2 to oauth2. Despite this "just" being a cleanup CL, it took forever because I hit OAuth2+Cloud Storage+Web UI woes along the way, documented partially in upload.go. The web UI misled me for a long time. Maybe I shouldn't have used service accounts, but it does make configuration easier for upload.go. The buildongce/create.go probably should use them too, but I can do that later. I'm done cleaning for now. Change-Id: Icb8e3decb682d3685edffecea2a10fcb4e385e10 Reviewed-on: https://go-review.googlesource.com/2731 Reviewed-by: Andrew Gerrand --- dashboard/README | 1 + dashboard/buildlet/Makefile | 14 ++- dashboard/buildlet/stage0/Makefile | 2 +- dashboard/coordinator/Makefile | 2 +- dashboard/coordinator/buildongce/create.go | 87 ++++++-------- dashboard/env/commit-watcher/Makefile | 2 +- dashboard/env/linux-x86-base/Makefile | 2 +- dashboard/env/linux-x86-clang/Makefile | 2 +- dashboard/env/linux-x86-gccgo/Makefile | 2 +- dashboard/env/linux-x86-nacl/Makefile | 2 +- dashboard/env/linux-x86-sid/Makefile | 2 +- dashboard/upload/upload.go | 132 +++++++++++++++++++++ 12 files changed, 184 insertions(+), 66 deletions(-) create mode 100644 dashboard/upload/upload.go diff --git a/dashboard/README b/dashboard/README index d9f62bef..4e596a06 100644 --- a/dashboard/README +++ b/dashboard/README @@ -18,6 +18,7 @@ env/: configuration files describing the environment of builders and related environment. retrybuilds/: a Go client program to delete build results from the dashboard (app) types/: a Go package contain common types used by other pieces. +upload/: a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere. watcher/: a daemon that watches for new commits to the Go repository and its sub-repositories, and notifies the dashboard of those commits. diff --git a/dashboard/buildlet/Makefile b/dashboard/buildlet/Makefile index 47adac93..b30ce59b 100644 --- a/dashboard/buildlet/Makefile +++ b/dashboard/buildlet/Makefile @@ -1,22 +1,26 @@ buildlet: buildlet.go go build --tags=buildlet +buildlet.linux-amd64: buildlet.go + GOOS=linux GOARCH=amd64 go build -o $@ --tags=buildlet + cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@) + buildlet.openbsd-amd64: buildlet.go GOOS=openbsd GOARCH=amd64 go build -o $@ --tags=buildlet - cat $@ | (cd ../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@) + cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@) buildlet.plan9-386: buildlet.go GOOS=plan9 GOARCH=386 go build -o $@ --tags=buildlet - cat $@ | (cd ../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@) + cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@) buildlet.windows-amd64: buildlet.go GOOS=windows GOARCH=amd64 go build -o $@ --tags=buildlet - cat $@ | (cd ../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@) + cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@) buildlet.darwin-amd64: buildlet.go GOOS=darwin GOARCH=amd64 go build -o $@ --tags=buildlet - cat $@ | (cd ../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@) + cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@) buildlet.netbsd-amd64: buildlet.go GOOS=netbsd GOARCH=amd64 go build -o $@ --tags=buildlet - cat $@ | (cd ../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@) + cat $@ | (cd ../upload && go run upload.go --public go-builder-data/$@) diff --git a/dashboard/buildlet/stage0/Makefile b/dashboard/buildlet/stage0/Makefile index 99abc63a..f9a3c19a 100644 --- a/dashboard/buildlet/stage0/Makefile +++ b/dashboard/buildlet/stage0/Makefile @@ -1,3 +1,3 @@ buildlet-stage0.windows-amd64: stage0.go GOOS=windows GOARCH=amd64 go build -o $@ --tags=stage0 - cat $@ | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/$@) + cat $@ | (cd ../../upload && go run upload.go --public go-builder-data/$@) diff --git a/dashboard/coordinator/Makefile b/dashboard/coordinator/Makefile index 7bbcaf0f..166086fe 100644 --- a/dashboard/coordinator/Makefile +++ b/dashboard/coordinator/Makefile @@ -6,4 +6,4 @@ coordinator: main.go # And watch its logs with: # sudo journalctl -f -u gobuild.service upload: coordinator - cat coordinator | (cd buildongce && go run create.go --write_object=go-builder-data/coordinator) + cat coordinator | (cd ../upload && go run upload.go --public go-builder-data/coordinator) diff --git a/dashboard/coordinator/buildongce/create.go b/dashboard/coordinator/buildongce/create.go index 95dc2781..2148ebb8 100644 --- a/dashboard/coordinator/buildongce/create.go +++ b/dashboard/coordinator/buildongce/create.go @@ -8,19 +8,17 @@ package main // import "golang.org/x/tools/dashboard/coordinator/buildongce" import ( "bufio" - "bytes" "encoding/json" "flag" "fmt" - "io" "io/ioutil" "log" - "net/http" "os" "strings" "time" - "code.google.com/p/goauth2/oauth" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" compute "google.golang.org/api/compute/v1" ) @@ -33,8 +31,6 @@ var ( staticIP = flag.String("static_ip", "", "Static IP to use. If empty, automatic.") reuseDisk = flag.Bool("reuse_disk", true, "Whether disk images should be reused between shutdowns/restarts.") ssd = flag.Bool("ssd", false, "use a solid state disk (faster, more expensive)") - - writeObject = flag.String("write_object", "", "If non-empty, a VM isn't created and the flag value is Google Cloud Storage bucket/object to write. The contents from stdin.") ) func readFile(v string) string { @@ -45,19 +41,18 @@ func readFile(v string) string { return strings.TrimSpace(string(slurp)) } -var config = &oauth.Config{ +var config = &oauth2.Config{ // The client-id and secret should be for an "Installed Application" when using // the CLI. Later we'll use a web application with a callback. - ClientId: readFile("client-id.dat"), + ClientID: readFile("client-id.dat"), ClientSecret: readFile("client-secret.dat"), - Scope: strings.Join([]string{ + Endpoint: google.Endpoint, + Scopes: []string{ compute.DevstorageFull_controlScope, compute.ComputeScope, "https://www.googleapis.com/auth/sqlservice", "https://www.googleapis.com/auth/sqlservice.admin", - }, " "), - AuthURL: "https://accounts.google.com/o/oauth2/auth", - TokenURL: "https://accounts.google.com/o/oauth2/token", + }, RedirectURL: "urn:ietf:wg:oauth:2.0:oob", } @@ -94,35 +89,28 @@ func main() { prefix := "https://www.googleapis.com/compute/v1/projects/" + *proj machType := prefix + "/zones/" + *zone + "/machineTypes/" + *mach - tr := &oauth.Transport{ - Config: config, - } - - tokenCache := oauth.CacheFile("token.dat") - token, err := tokenCache.Token() + const tokenFileName = "token.dat" + tokenFile := tokenCacheFile(tokenFileName) + tokenSource := oauth2.ReuseTokenSource(nil, tokenFile) + token, err := tokenSource.Token() if err != nil { - if *writeObject != "" { - log.Fatalf("Can't use --write_object without a valid token.dat file already cached.") - } - log.Printf("Error getting token from %s: %v", string(tokenCache), err) + log.Printf("Error getting token from %s: %v", tokenFileName, err) log.Printf("Get auth code from %v", config.AuthCodeURL("my-state")) fmt.Print("\nEnter auth code: ") sc := bufio.NewScanner(os.Stdin) sc.Scan() authCode := strings.TrimSpace(sc.Text()) - token, err = tr.Exchange(authCode) + token, err = config.Exchange(oauth2.NoContext, authCode) if err != nil { log.Fatalf("Error exchanging auth code for a token: %v", err) } - tokenCache.PutToken(token) + if err := tokenFile.WriteToken(token); err != nil { + log.Fatalf("Error writing to %s: %v", tokenFileName, err) + } + tokenSource = oauth2.ReuseTokenSource(token, nil) } - tr.Token = token - oauthClient := &http.Client{Transport: tr} - if *writeObject != "" { - writeCloudStorageObject(oauthClient) - return - } + oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource) computeService, _ := compute.New(oauthClient) @@ -288,31 +276,24 @@ func instanceDisk(svc *compute.Service) *compute.AttachedDisk { } } -func writeCloudStorageObject(httpClient *http.Client) { - content := os.Stdin - const maxSlurp = 1 << 20 - var buf bytes.Buffer - n, err := io.CopyN(&buf, content, maxSlurp) - if err != nil && err != io.EOF { - log.Fatalf("Error reading from stdin: %v, %v", n, err) - } - contentType := http.DetectContentType(buf.Bytes()) +type tokenCacheFile string - req, err := http.NewRequest("PUT", "https://storage.googleapis.com/"+*writeObject, io.MultiReader(&buf, content)) +func (f tokenCacheFile) Token() (*oauth2.Token, error) { + slurp, err := ioutil.ReadFile(string(f)) if err != nil { - log.Fatal(err) + return nil, err } - req.Header.Set("x-goog-api-version", "2") - req.Header.Set("x-goog-acl", "public-read") - req.Header.Set("Content-Type", contentType) - res, err := httpClient.Do(req) - if err != nil { - log.Fatal(err) + t := new(oauth2.Token) + if err := json.Unmarshal(slurp, t); err != nil { + return nil, err } - if res.StatusCode != 200 { - res.Write(os.Stderr) - log.Fatalf("Failed.") - } - log.Printf("Success.") - os.Exit(0) + return t, nil +} + +func (f tokenCacheFile) WriteToken(t *oauth2.Token) error { + jt, err := json.Marshal(t) + if err != nil { + return err + } + return ioutil.WriteFile(string(f), jt, 0600) } diff --git a/dashboard/env/commit-watcher/Makefile b/dashboard/env/commit-watcher/Makefile index eb87757a..7ca67b8e 100644 --- a/dashboard/env/commit-watcher/Makefile +++ b/dashboard/env/commit-watcher/Makefile @@ -6,4 +6,4 @@ docker: Dockerfile docker build -t go-commit-watcher . docker-commit-watcher.tar.gz: docker - docker save go-commit-watcher | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-commit-watcher.tar.gz) + docker save go-commit-watcher | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-commit-watcher.tar.gz) diff --git a/dashboard/env/linux-x86-base/Makefile b/dashboard/env/linux-x86-base/Makefile index 347f48fd..464bac31 100644 --- a/dashboard/env/linux-x86-base/Makefile +++ b/dashboard/env/linux-x86-base/Makefile @@ -6,7 +6,7 @@ docker: Dockerfile docker build -t gobuilders/linux-x86-base . docker-linux.base.tar.gz: docker - docker save gobuilders/linux-x86-base | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-linux.base.tar.gz) + docker save gobuilders/linux-x86-base | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.base.tar.gz) check: docker docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-base /usr/local/bin/builder -rev=20a10e7ddd1 -buildroot=/ -v -report=false linux-amd64-temp diff --git a/dashboard/env/linux-x86-clang/Makefile b/dashboard/env/linux-x86-clang/Makefile index fa56b137..22867624 100644 --- a/dashboard/env/linux-x86-clang/Makefile +++ b/dashboard/env/linux-x86-clang/Makefile @@ -6,7 +6,7 @@ docker: Dockerfile docker build -t gobuilders/linux-x86-clang . docker-linux.clang.tar.gz: docker - docker save gobuilders/linux-x86-clang | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-linux.clang.tar.gz) + docker save gobuilders/linux-x86-clang | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz) check: docker docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-temp diff --git a/dashboard/env/linux-x86-gccgo/Makefile b/dashboard/env/linux-x86-gccgo/Makefile index 9d5143fe..a12f6e1b 100644 --- a/dashboard/env/linux-x86-gccgo/Makefile +++ b/dashboard/env/linux-x86-gccgo/Makefile @@ -6,7 +6,7 @@ docker: Dockerfile docker build -t gobuilders/linux-x86-gccgo . docker-linux.gccgo.tar.gz: docker - docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-linux.gccgo.tar.gz) + docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.gccgo.tar.gz) check: docker docker run gobuilders/linux-x86-gccgo /usr/local/bin/builder -tool="gccgo" -rev=b9151e911a54 -v -cmd='make RUNTESTFLAGS="--target_board=unix/-m64" check-go' -report=false linux-amd64-gccgo-temp diff --git a/dashboard/env/linux-x86-nacl/Makefile b/dashboard/env/linux-x86-nacl/Makefile index db62b071..adb0c842 100644 --- a/dashboard/env/linux-x86-nacl/Makefile +++ b/dashboard/env/linux-x86-nacl/Makefile @@ -6,7 +6,7 @@ docker: Dockerfile docker build -t gobuilders/linux-x86-nacl . upload: docker - docker save gobuilders/linux-x86-nacl | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-linux.nacl.tar.gz) + docker save gobuilders/linux-x86-nacl | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.nacl.tar.gz) check: docker docker run gobuilders/linux-x86-nacl /usr/local/bin/builder -rev=77e96c9208d0 -buildroot=/ -v -cmd=/usr/local/bin/build-command.pl -report=false nacl-amd64p32 diff --git a/dashboard/env/linux-x86-sid/Makefile b/dashboard/env/linux-x86-sid/Makefile index 3fcd0d09..eac489cc 100644 --- a/dashboard/env/linux-x86-sid/Makefile +++ b/dashboard/env/linux-x86-sid/Makefile @@ -6,7 +6,7 @@ docker: Dockerfile docker build -t gobuilders/linux-x86-sid . docker-linux.sid.tar.gz: docker - docker save gobuilders/linux-x86-sid | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-linux.sid.tar.gz) + docker save gobuilders/linux-x86-sid | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.sid.tar.gz) check: docker docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-sid /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-sid diff --git a/dashboard/upload/upload.go b/dashboard/upload/upload.go new file mode 100644 index 00000000..44f5a727 --- /dev/null +++ b/dashboard/upload/upload.go @@ -0,0 +1,132 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build upload + +// ^ this is so we don't break the build of x/tools/... build due +// to missing depenencies on the builders. We don't want full builds +// needing to pull in dependencies outside of the x/tools repo. + +// The upload command writes a file to Google Cloud Storage. It's used +// exclusively by the Makefiles in the Go project repos. Think of it +// as a very light version of gsutil or gcloud, but with some +// Go-specific configuration knowledge baked in. +package main + +import ( + "bytes" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strings" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/cloud" + "google.golang.org/cloud/storage" +) + +var ( + public = flag.Bool("public", false, "object should be world-readable") + file = flag.String("file", "-", "Filename to read object from, or '-' for stdin.") + verbose = flag.Bool("verbose", false, "verbose logging") +) + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: upload [--public] [--file=...] \n") + flag.PrintDefaults() + } + flag.Parse() + if flag.NArg() != 1 { + flag.Usage() + os.Exit(1) + } + args := strings.SplitN(flag.Arg(0), "/", 2) + if len(args) != 2 { + flag.Usage() + os.Exit(1) + } + bucket, object := args[0], args[1] + + proj, ok := bucketProject[bucket] + if !ok { + log.Fatalf("bucket %q doesn't have an associated project in upload.go") + } + + ts, err := tokenSource(bucket) + if err != nil { + log.Fatalf("Failed to get an OAuth2 token source: %v", err) + } + httpClient := oauth2.NewClient(oauth2.NoContext, ts) + + ctx := cloud.NewContext(proj, httpClient) + w := storage.NewWriter(ctx, bucket, object) + // If you don't give the owners access, the web UI seems to + // have a bug and doesn't have access to see that it's public, so + // won't render the "Shared Publicly" link. So we do that, even + // though it's dumb and unnecessary otherwise: + w.ACL = append(w.ACL, storage.ACLRule{Entity: storage.ACLEntity("project-owners-" + proj), Role: storage.RoleOwner}) + if *public { + w.ACL = append(w.ACL, storage.ACLRule{Entity: storage.AllUsers, Role: storage.RoleReader}) + } + var content io.Reader + if *file == "-" { + content = os.Stdin + } else { + content, err = os.Open(*file) + if err != nil { + log.Fatal(err) + } + } + + const maxSlurp = 1 << 20 + var buf bytes.Buffer + n, err := io.CopyN(&buf, content, maxSlurp) + if err != nil && err != io.EOF { + log.Fatalf("Error reading from stdin: %v, %v", n, err) + } + w.ContentType = http.DetectContentType(buf.Bytes()) + + _, err = io.Copy(w, io.MultiReader(&buf, content)) + if cerr := w.Close(); cerr != nil && err == nil { + err = cerr + } + if err != nil { + log.Fatalf("Write error: %v", err) + } + if *verbose { + log.Printf("Wrote %v", object) + } + os.Exit(0) +} + +var bucketProject = map[string]string{ + "go-builder-data": "symbolic-datum-552", + "http2-demo-server-tls": "symbolic-datum-552", + "winstrap": "999119582588", + "gobuilder": "999119582588", // deprecated +} + +func tokenSource(bucket string) (oauth2.TokenSource, error) { + proj := bucketProject[bucket] + fileName := filepath.Join(os.Getenv("HOME"), "keys", proj+".key.json") + jsonConf, err := ioutil.ReadFile(fileName) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("Missing JSON key configuration. Download the Service Account JSON key from https://console.developers.google.com/project/%s/apiui/credential and place it at %s", proj, fileName) + } + return nil, err + } + conf, err := google.JWTConfigFromJSON(jsonConf, storage.ScopeReadWrite) + if err != nil { + return nil, fmt.Errorf("reading JSON config from %s: %v", fileName, err) + } + return conf.TokenSource(oauth2.NoContext), nil +}