From e83451b9c2f0bffdd8ae5e5889d982ad960a81f4 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 14 Jan 2015 17:24:45 -0800 Subject: [PATCH] dashboard: rearrange tree, in prep for packagification Package main binaries now go under cmd, leaving the top level for similarly-named packages. I'll be moving a lot of data structures and functions out of the commands and into common packages. A future CL will also unify all the "+build whatever" restrictions into one common build tag, since there will be so much package sharing coming up. Probably they will become "+build extdep", to indicate that they have dependencies outside of x/tools. Change-Id: Idc198e0dfa933b5f5de8f2b581533e8d299d2293 Reviewed-on: https://go-review.googlesource.com/2738 Reviewed-by: Andrew Gerrand --- .gitignore | 5 +- dashboard/README | 29 ++-- dashboard/{ => cmd}/builder/bench.go | 0 dashboard/{ => cmd}/builder/doc.go | 2 +- dashboard/{ => cmd}/builder/env.go | 0 dashboard/{ => cmd}/builder/exec.go | 0 .../{ => cmd}/builder/filemutex_flock.go | 0 .../{ => cmd}/builder/filemutex_local.go | 0 .../{ => cmd}/builder/filemutex_windows.go | 0 dashboard/{ => cmd}/builder/http.go | 0 dashboard/{ => cmd}/builder/main.go | 0 dashboard/{ => cmd}/builder/vcs.go | 0 dashboard/{ => cmd}/buildlet/.gitignore | 0 dashboard/{ => cmd}/buildlet/Makefile | 0 dashboard/{ => cmd}/buildlet/README | 0 dashboard/{ => cmd}/buildlet/buildlet.go | 0 dashboard/{ => cmd}/buildlet/stage0/Makefile | 0 dashboard/{ => cmd}/buildlet/stage0/stage0.go | 0 dashboard/cmd/coordinator/.gitignore | 3 + dashboard/{ => cmd}/coordinator/Makefile | 0 .../coordinator/buildongce/create.go | 0 .../coordinator/coordinator.go} | 0 .../{ => cmd}/retrybuilds/retrybuilds.go | 0 dashboard/{ => cmd}/upload/upload.go | 0 dashboard/env/commit-watcher/Makefile | 2 +- dashboard/env/linux-x86-base/Makefile | 2 +- dashboard/env/linux-x86-clang/Makefile | 2 +- dashboard/env/linux-x86-gccgo/Makefile | 2 +- dashboard/env/linux-x86-nacl/Makefile | 2 +- dashboard/env/linux-x86-sid/Makefile | 2 +- dashboard/updater/updater.go | 128 ------------------ 31 files changed, 30 insertions(+), 149 deletions(-) rename dashboard/{ => cmd}/builder/bench.go (100%) rename dashboard/{ => cmd}/builder/doc.go (96%) rename dashboard/{ => cmd}/builder/env.go (100%) rename dashboard/{ => cmd}/builder/exec.go (100%) rename dashboard/{ => cmd}/builder/filemutex_flock.go (100%) rename dashboard/{ => cmd}/builder/filemutex_local.go (100%) rename dashboard/{ => cmd}/builder/filemutex_windows.go (100%) rename dashboard/{ => cmd}/builder/http.go (100%) rename dashboard/{ => cmd}/builder/main.go (100%) rename dashboard/{ => cmd}/builder/vcs.go (100%) rename dashboard/{ => cmd}/buildlet/.gitignore (100%) rename dashboard/{ => cmd}/buildlet/Makefile (100%) rename dashboard/{ => cmd}/buildlet/README (100%) rename dashboard/{ => cmd}/buildlet/buildlet.go (100%) rename dashboard/{ => cmd}/buildlet/stage0/Makefile (100%) rename dashboard/{ => cmd}/buildlet/stage0/stage0.go (100%) create mode 100644 dashboard/cmd/coordinator/.gitignore rename dashboard/{ => cmd}/coordinator/Makefile (100%) rename dashboard/{ => cmd}/coordinator/buildongce/create.go (100%) rename dashboard/{coordinator/main.go => cmd/coordinator/coordinator.go} (100%) rename dashboard/{ => cmd}/retrybuilds/retrybuilds.go (100%) rename dashboard/{ => cmd}/upload/upload.go (100%) delete mode 100644 dashboard/updater/updater.go diff --git a/.gitignore b/.gitignore index 90e56498..5a9d62ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,2 @@ -# Add no patterns to .hgignore except for files generated by the build. +# Add no patterns to .gitignore except for files generated by the build. last-change -dashboard/coordinator/buildongce/client-*.dat -dashboard/coordinator/buildongce/token.dat -dashboard/coordinator/coordinator diff --git a/dashboard/README b/dashboard/README index 4e596a06..77319479 100644 --- a/dashboard/README +++ b/dashboard/README @@ -4,28 +4,37 @@ The files in these directories constitute the continuous builder: -app/: a.k.a the "dashboard"; the App Engine code that runs http://build.golang.org/ -buildlet/: HTTP server that runs on a VM and is told what to write to disk +app/: a.k.a the "dashboard"; the App Engine code that runs http://build.golang.org/ + +cmd/: + + buildlet/: HTTP server that runs on a VM and is told what to write to disk and what command to run. This is cross-compiled to different architectures and is the first program run when a builder VM comes up. It then is contacted by the coordinator to do a build. Not all builders use the buildlet (at least not yet). -builder/: gobuilder, a Go continuous build client. The original Go builder program. -coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages + + builder/: gobuilder, a Go continuous build client. The original Go builder program. + + coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages builds using Docker containers and/or VMs as needed. + + retrybuilds/: a Go client program to delete build results from the dashboard (app) + + upload/: a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere. + + watcher/: a daemon that watches for new commits to the Go repository and + its sub-repositories, and notifies the dashboard of those commits. + env/: configuration files describing the environment of builders and related binaries. Many builders are still configured ad-hoc, without a hermetic environment. -retrybuilds/: a Go client program to delete build results from the dashboard (app) + types/: a Go package contain common types used by other pieces. -upload/: a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere. -watcher/: a daemon that watches for new commits to the Go repository and - its sub-repositories, and notifies the dashboard of those commits. + If you wish to run a Go builder, please email golang-dev@googlegroups.com first. There is documentation at https://golang.org/wiki/DashboardBuilders but depending on the type of builder, we may want to run it ourselves, after you prepare an environment description (resulting in a VM image) of it. See the env directory. - - diff --git a/dashboard/builder/bench.go b/dashboard/cmd/builder/bench.go similarity index 100% rename from dashboard/builder/bench.go rename to dashboard/cmd/builder/bench.go diff --git a/dashboard/builder/doc.go b/dashboard/cmd/builder/doc.go similarity index 96% rename from dashboard/builder/doc.go rename to dashboard/cmd/builder/doc.go index e958e899..15b7252a 100644 --- a/dashboard/builder/doc.go +++ b/dashboard/cmd/builder/doc.go @@ -55,4 +55,4 @@ If the Google Code credentials are not provided the archival step will be skipped. */ -package main // import "golang.org/x/tools/dashboard/builder" +package main // import "golang.org/x/tools/dashboard/cmd/builder" diff --git a/dashboard/builder/env.go b/dashboard/cmd/builder/env.go similarity index 100% rename from dashboard/builder/env.go rename to dashboard/cmd/builder/env.go diff --git a/dashboard/builder/exec.go b/dashboard/cmd/builder/exec.go similarity index 100% rename from dashboard/builder/exec.go rename to dashboard/cmd/builder/exec.go diff --git a/dashboard/builder/filemutex_flock.go b/dashboard/cmd/builder/filemutex_flock.go similarity index 100% rename from dashboard/builder/filemutex_flock.go rename to dashboard/cmd/builder/filemutex_flock.go diff --git a/dashboard/builder/filemutex_local.go b/dashboard/cmd/builder/filemutex_local.go similarity index 100% rename from dashboard/builder/filemutex_local.go rename to dashboard/cmd/builder/filemutex_local.go diff --git a/dashboard/builder/filemutex_windows.go b/dashboard/cmd/builder/filemutex_windows.go similarity index 100% rename from dashboard/builder/filemutex_windows.go rename to dashboard/cmd/builder/filemutex_windows.go diff --git a/dashboard/builder/http.go b/dashboard/cmd/builder/http.go similarity index 100% rename from dashboard/builder/http.go rename to dashboard/cmd/builder/http.go diff --git a/dashboard/builder/main.go b/dashboard/cmd/builder/main.go similarity index 100% rename from dashboard/builder/main.go rename to dashboard/cmd/builder/main.go diff --git a/dashboard/builder/vcs.go b/dashboard/cmd/builder/vcs.go similarity index 100% rename from dashboard/builder/vcs.go rename to dashboard/cmd/builder/vcs.go diff --git a/dashboard/buildlet/.gitignore b/dashboard/cmd/buildlet/.gitignore similarity index 100% rename from dashboard/buildlet/.gitignore rename to dashboard/cmd/buildlet/.gitignore diff --git a/dashboard/buildlet/Makefile b/dashboard/cmd/buildlet/Makefile similarity index 100% rename from dashboard/buildlet/Makefile rename to dashboard/cmd/buildlet/Makefile diff --git a/dashboard/buildlet/README b/dashboard/cmd/buildlet/README similarity index 100% rename from dashboard/buildlet/README rename to dashboard/cmd/buildlet/README diff --git a/dashboard/buildlet/buildlet.go b/dashboard/cmd/buildlet/buildlet.go similarity index 100% rename from dashboard/buildlet/buildlet.go rename to dashboard/cmd/buildlet/buildlet.go diff --git a/dashboard/buildlet/stage0/Makefile b/dashboard/cmd/buildlet/stage0/Makefile similarity index 100% rename from dashboard/buildlet/stage0/Makefile rename to dashboard/cmd/buildlet/stage0/Makefile diff --git a/dashboard/buildlet/stage0/stage0.go b/dashboard/cmd/buildlet/stage0/stage0.go similarity index 100% rename from dashboard/buildlet/stage0/stage0.go rename to dashboard/cmd/buildlet/stage0/stage0.go diff --git a/dashboard/cmd/coordinator/.gitignore b/dashboard/cmd/coordinator/.gitignore new file mode 100644 index 00000000..91d0fb06 --- /dev/null +++ b/dashboard/cmd/coordinator/.gitignore @@ -0,0 +1,3 @@ +buildongce/client-*.dat +buildongce/token.dat +coordinator diff --git a/dashboard/coordinator/Makefile b/dashboard/cmd/coordinator/Makefile similarity index 100% rename from dashboard/coordinator/Makefile rename to dashboard/cmd/coordinator/Makefile diff --git a/dashboard/coordinator/buildongce/create.go b/dashboard/cmd/coordinator/buildongce/create.go similarity index 100% rename from dashboard/coordinator/buildongce/create.go rename to dashboard/cmd/coordinator/buildongce/create.go diff --git a/dashboard/coordinator/main.go b/dashboard/cmd/coordinator/coordinator.go similarity index 100% rename from dashboard/coordinator/main.go rename to dashboard/cmd/coordinator/coordinator.go diff --git a/dashboard/retrybuilds/retrybuilds.go b/dashboard/cmd/retrybuilds/retrybuilds.go similarity index 100% rename from dashboard/retrybuilds/retrybuilds.go rename to dashboard/cmd/retrybuilds/retrybuilds.go diff --git a/dashboard/upload/upload.go b/dashboard/cmd/upload/upload.go similarity index 100% rename from dashboard/upload/upload.go rename to dashboard/cmd/upload/upload.go diff --git a/dashboard/env/commit-watcher/Makefile b/dashboard/env/commit-watcher/Makefile index 7ca67b8e..d154c519 100644 --- a/dashboard/env/commit-watcher/Makefile +++ b/dashboard/env/commit-watcher/Makefile @@ -6,4 +6,4 @@ docker: Dockerfile docker build -t go-commit-watcher . docker-commit-watcher.tar.gz: docker - docker save go-commit-watcher | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-commit-watcher.tar.gz) + docker save go-commit-watcher | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-commit-watcher.tar.gz) diff --git a/dashboard/env/linux-x86-base/Makefile b/dashboard/env/linux-x86-base/Makefile index 464bac31..981675cc 100644 --- a/dashboard/env/linux-x86-base/Makefile +++ b/dashboard/env/linux-x86-base/Makefile @@ -6,7 +6,7 @@ docker: Dockerfile docker build -t gobuilders/linux-x86-base . docker-linux.base.tar.gz: docker - docker save gobuilders/linux-x86-base | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.base.tar.gz) + docker save gobuilders/linux-x86-base | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.base.tar.gz) check: docker docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-base /usr/local/bin/builder -rev=20a10e7ddd1 -buildroot=/ -v -report=false linux-amd64-temp diff --git a/dashboard/env/linux-x86-clang/Makefile b/dashboard/env/linux-x86-clang/Makefile index 22867624..2c945bb1 100644 --- a/dashboard/env/linux-x86-clang/Makefile +++ b/dashboard/env/linux-x86-clang/Makefile @@ -6,7 +6,7 @@ docker: Dockerfile docker build -t gobuilders/linux-x86-clang . docker-linux.clang.tar.gz: docker - docker save gobuilders/linux-x86-clang | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz) + docker save gobuilders/linux-x86-clang | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz) check: docker docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-temp diff --git a/dashboard/env/linux-x86-gccgo/Makefile b/dashboard/env/linux-x86-gccgo/Makefile index a12f6e1b..e114b3dd 100644 --- a/dashboard/env/linux-x86-gccgo/Makefile +++ b/dashboard/env/linux-x86-gccgo/Makefile @@ -6,7 +6,7 @@ docker: Dockerfile docker build -t gobuilders/linux-x86-gccgo . docker-linux.gccgo.tar.gz: docker - docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.gccgo.tar.gz) + docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.gccgo.tar.gz) check: docker docker run gobuilders/linux-x86-gccgo /usr/local/bin/builder -tool="gccgo" -rev=b9151e911a54 -v -cmd='make RUNTESTFLAGS="--target_board=unix/-m64" check-go' -report=false linux-amd64-gccgo-temp diff --git a/dashboard/env/linux-x86-nacl/Makefile b/dashboard/env/linux-x86-nacl/Makefile index adb0c842..d2f76ed6 100644 --- a/dashboard/env/linux-x86-nacl/Makefile +++ b/dashboard/env/linux-x86-nacl/Makefile @@ -6,7 +6,7 @@ docker: Dockerfile docker build -t gobuilders/linux-x86-nacl . upload: docker - docker save gobuilders/linux-x86-nacl | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.nacl.tar.gz) + docker save gobuilders/linux-x86-nacl | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.nacl.tar.gz) check: docker docker run gobuilders/linux-x86-nacl /usr/local/bin/builder -rev=77e96c9208d0 -buildroot=/ -v -cmd=/usr/local/bin/build-command.pl -report=false nacl-amd64p32 diff --git a/dashboard/env/linux-x86-sid/Makefile b/dashboard/env/linux-x86-sid/Makefile index eac489cc..df7b2cfb 100644 --- a/dashboard/env/linux-x86-sid/Makefile +++ b/dashboard/env/linux-x86-sid/Makefile @@ -6,7 +6,7 @@ docker: Dockerfile docker build -t gobuilders/linux-x86-sid . docker-linux.sid.tar.gz: docker - docker save gobuilders/linux-x86-sid | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.sid.tar.gz) + docker save gobuilders/linux-x86-sid | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.sid.tar.gz) check: docker docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-sid /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-sid diff --git a/dashboard/updater/updater.go b/dashboard/updater/updater.go deleted file mode 100644 index 81919f6f..00000000 --- a/dashboard/updater/updater.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main // import "golang.org/x/tools/dashboard/updater" - -import ( - "bytes" - "encoding/json" - "encoding/xml" - "flag" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "os" - "os/exec" - "strings" -) - -var ( - builder = flag.String("builder", "", "builder name") - key = flag.String("key", "", "builder key") - gopath = flag.String("gopath", "", "path to go repo") - dashboard = flag.String("dashboard", "build.golang.org", "Go Dashboard Host") - batch = flag.Int("batch", 100, "upload batch size") -) - -// Do not benchmark beyond this commit. -// There is little sense in benchmarking till first commit, -// and the benchmark won't build anyway. -const Go1Commit = "0051c7442fed" // test/bench/shootout: update timing.log to Go 1. - -// HgLog represents a single Mercurial revision. -type HgLog struct { - Hash string - Branch string - Files string -} - -func main() { - flag.Parse() - logs := hgLog() - var hashes []string - ngo1 := 0 - for i := range logs { - if strings.HasPrefix(logs[i].Hash, Go1Commit) { - break - } - if needsBenchmarking(&logs[i]) { - hashes = append(hashes, logs[i].Hash) - } - ngo1++ - } - fmt.Printf("found %v commits, %v after Go1, %v need benchmarking\n", len(logs), ngo1, len(hashes)) - for i := 0; i < len(hashes); i += *batch { - j := i + *batch - if j > len(hashes) { - j = len(hashes) - } - fmt.Printf("sending %v-%v... ", i, j) - res := postCommits(hashes[i:j]) - fmt.Printf("%s\n", res) - } -} - -func hgLog() []HgLog { - var out bytes.Buffer - cmd := exec.Command("hg", "log", "--encoding=utf-8", "--template", xmlLogTemplate) - cmd.Dir = *gopath - cmd.Stdout = &out - cmd.Stderr = os.Stderr - err := cmd.Run() - if err != nil { - fmt.Printf("failed to execute 'hg log': %v\n", err) - os.Exit(1) - } - var top struct{ Log []HgLog } - err = xml.Unmarshal([]byte(""+out.String()+""), &top) - if err != nil { - fmt.Printf("failed to parse log: %v\n", err) - os.Exit(1) - } - return top.Log -} - -func needsBenchmarking(log *HgLog) bool { - if log.Branch != "" { - return false - } - for _, f := range strings.Split(log.Files, " ") { - if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) && - !strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") { - return true - } - } - return false -} - -func postCommits(hashes []string) string { - args := url.Values{"builder": {*builder}, "key": {*key}} - cmd := fmt.Sprintf("http://%v/updatebenchmark?%v", *dashboard, args.Encode()) - b, err := json.Marshal(hashes) - if err != nil { - return fmt.Sprintf("failed to encode request: %v\n", err) - } - r, err := http.Post(cmd, "text/json", bytes.NewReader(b)) - if err != nil { - return fmt.Sprintf("failed to send http request: %v\n", err) - } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - return fmt.Sprintf("http request failed: %v\n", r.Status) - } - resp, err := ioutil.ReadAll(r.Body) - if err != nil { - return fmt.Sprintf("failed to read http response: %v\n", err) - } - return string(resp) -} - -const xmlLogTemplate = ` - - {node|escape} - {branches} - {files} - -`