From 9393b170806def447d4de9dbe299bc5445f48f77 Mon Sep 17 00:00:00 2001 From: Chris Manghane Date: Mon, 1 Dec 2014 09:59:02 -0800 Subject: [PATCH] dashboard/env: add go-commit-watcher image LGTM=bradfitz R=bradfitz, adg CC=adg, golang-codereviews https://golang.org/cl/179370043 --- dashboard/README | 2 +- dashboard/coordinator/main.go | 82 ++++++++++++++++++- dashboard/env/commit-watcher/Dockerfile | 17 ++++ dashboard/env/commit-watcher/Makefile | 9 ++ .../scripts/build-commit-watcher.sh | 20 +++++ .../scripts/install-apt-deps.sh | 12 +++ 6 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 dashboard/env/commit-watcher/Dockerfile create mode 100644 dashboard/env/commit-watcher/Makefile create mode 100755 dashboard/env/commit-watcher/scripts/build-commit-watcher.sh create mode 100755 dashboard/env/commit-watcher/scripts/install-apt-deps.sh diff --git a/dashboard/README b/dashboard/README index 12244509..94193cd8 100644 --- a/dashboard/README +++ b/dashboard/README @@ -8,7 +8,7 @@ app/: an AppEngine server. The code that runs http://build.golang.org/ builder/: gobuilder, a Go continuous build client coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages builds (using the builder in single-shot mode) in Docker containers. -env/: configuration files describing the environment of builders. +env/: configuration files describing the environment of builders and related binaries. Many builders are still configured ad-hoc. watcher/: a daemon that watches for new commits to the Go repository and its sub-repositories, and notifies the dashboard of those commits. diff --git a/dashboard/coordinator/main.go b/dashboard/coordinator/main.go index a8550001..07565d07 100644 --- a/dashboard/coordinator/main.go +++ b/dashboard/coordinator/main.go @@ -37,6 +37,7 @@ var ( var ( startTime = time.Now() builders = map[string]buildConfig{} // populated once at startup + watchers = map[string]watchConfig{} // populated once at startup donec = make(chan builderRev) // reports of finished builders statusMu sync.Mutex @@ -51,6 +52,7 @@ type imageInfo struct { } var images = map[string]*imageInfo{ + "go-commit-watcher": {url: "https://storage.googleapis.com/go-builder-data/docker-commit-watcher.tar.gz"}, "gobuilders/linux-x86-base": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.base.tar.gz"}, "gobuilders/linux-x86-clang": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.clang.tar.gz"}, "gobuilders/linux-x86-gccgo": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.gccgo.tar.gz"}, @@ -67,6 +69,12 @@ type buildConfig struct { tool string // the tool this configuration is for } +type watchConfig struct { + repo string // "https://code.google.com/p/go" + dash string // "https://build.golang.org/" (must end in /) + interval time.Duration // Polling interval +} + func main() { flag.Parse() addBuilder(buildConfig{name: "linux-386"}) @@ -96,6 +104,9 @@ func main() { addBuilder(buildConfig{name: "linux-386-clang", image: "gobuilders/linux-x86-clang"}) addBuilder(buildConfig{name: "linux-amd64-clang", image: "gobuilders/linux-x86-clang"}) + addWatcher(watchConfig{repo: "https://code.google.com/p/go", dash: "https://build.golang.org/"}) + addWatcher(watchConfig{repo: "https://code.google.com/p/gofrontend", dash: "https://build.golang.org/gccgo/"}) + if (*just != "") != (*rev != "") { log.Fatalf("--just and --rev must be used together") } @@ -117,6 +128,12 @@ func main() { http.HandleFunc("/logs", handleLogs) go http.ListenAndServe(":80", nil) + for _, watcher := range watchers { + if err := startWatching(watchers[watcher.repo]); err != nil { + log.Printf("Error starting watcher for %s: %v", watcher.repo, err) + } + } + workc := make(chan builderRev) for name, builder := range builders { go findWorkLoop(name, builder.dashURL, workc) @@ -129,10 +146,7 @@ func main() { log.Printf("workc received %+v; len(status) = %v, maxBuilds = %v; cur = %p", work, len(status), *maxBuilds, status[work]) mayBuild := mayBuildRev(work) if mayBuild { - out, _ := exec.Command("docker", "ps").Output() - numBuilds := bytes.Count(out, []byte("\n")) - 1 - log.Printf("num current docker builds: %d", numBuilds) - if numBuilds > *maxBuilds { + if numBuilds() > *maxBuilds { mayBuild = false } } @@ -333,6 +347,40 @@ func addBuilder(c buildConfig) { builders[c.name] = c } +// returns the part after "docker run" +func (conf watchConfig) dockerRunArgs() (args []string) { + if key := builderKey("watcher"); key != "" { + tmpKey := "/tmp/watcher.buildkey" + if _, err := os.Stat(tmpKey); err != nil { + if err := ioutil.WriteFile(tmpKey, []byte(key), 0600); err != nil { + log.Fatal(err) + } + } + args = append(args, "-v", tmpKey+":/.gobuildkey") + } + args = append(args, + "go-commit-watcher", + "/usr/local/bin/watcher", + "-repo="+conf.repo, + "-dash="+conf.dash, + "-poll="+conf.interval.String(), + ) + return +} + +func addWatcher(c watchConfig) { + if c.repo == "" { + c.repo = "https://code.google.com/p/go" + } + if c.dash == "" { + c.dash = "https://build.golang.org/" + } + if c.interval == 0 { + c.interval = 10 * time.Second + } + watchers[c.repo] = c +} + func condUpdateImage(img string) error { ii := images[img] if ii == nil { @@ -373,6 +421,20 @@ func condUpdateImage(img string) error { return nil } +// numBuilds finds the number of go builder instances currently running. +func numBuilds() int { + out, _ := exec.Command("docker", "ps").Output() + numBuilds := 0 + ps := bytes.Split(out, []byte("\n")) + for _, p := range ps { + if bytes.HasPrefix(p, []byte("gobuilders/")) { + numBuilds++ + } + } + log.Printf("num current docker builds: %d", numBuilds) + return numBuilds +} + func startBuilding(conf buildConfig, rev string) (*buildStatus, error) { if err := condUpdateImage(conf.image); err != nil { log.Printf("Failed to setup container for %v %v: %v", conf.name, rev, err) @@ -411,6 +473,18 @@ type buildStatus struct { // ... } +func startWatching(conf watchConfig) error { + if err := condUpdateImage("go-commit-watcher"); err != nil { + log.Printf("Failed to setup container for commit watcher: %v", err) + return err + } + + cmd := exec.Command("docker", append([]string{"run", "-d"}, conf.dockerRunArgs()...)...) + all, err := cmd.CombinedOutput() + log.Printf("Docker run for commit watcher = err:%v, output: %s", err, all) + return err +} + func builderKey(builder string) string { master := masterKey() if len(master) == 0 { diff --git a/dashboard/env/commit-watcher/Dockerfile b/dashboard/env/commit-watcher/Dockerfile new file mode 100644 index 00000000..6fffbb44 --- /dev/null +++ b/dashboard/env/commit-watcher/Dockerfile @@ -0,0 +1,17 @@ +# Copyright 2014 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Commit watcher for Go repos. + +# Wheezy has Mercurial 2.2.2 and we need Mercurial >= 2.8, sid has 3.1. +FROM debian:sid +MAINTAINER golang-dev + +ENV DEBIAN_FRONTEND noninteractive + +ADD /scripts/install-apt-deps.sh /scripts/ +RUN /scripts/install-apt-deps.sh + +ADD /scripts/build-commit-watcher.sh /scripts/ +RUN GO_REV=8c27884843c3 WATCHER_REV=ae08a5291439 /scripts/build-commit-watcher.sh && test -f /usr/local/bin/watcher diff --git a/dashboard/env/commit-watcher/Makefile b/dashboard/env/commit-watcher/Makefile new file mode 100644 index 00000000..eb87757a --- /dev/null +++ b/dashboard/env/commit-watcher/Makefile @@ -0,0 +1,9 @@ +# Copyright 2014 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +docker: Dockerfile + docker build -t go-commit-watcher . + +docker-commit-watcher.tar.gz: docker + docker save go-commit-watcher | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-commit-watcher.tar.gz) diff --git a/dashboard/env/commit-watcher/scripts/build-commit-watcher.sh b/dashboard/env/commit-watcher/scripts/build-commit-watcher.sh new file mode 100755 index 00000000..7bbfa31c --- /dev/null +++ b/dashboard/env/commit-watcher/scripts/build-commit-watcher.sh @@ -0,0 +1,20 @@ +set -ex + +export GOPATH=/gopath +export GOROOT=/goroot +PREFIX=/usr/local +: ${GO_REV:?"need to be set to the golang repo revision used to build the commit watcher."} +: ${WATCHER_REV:?"need to be set to the go.tools repo revision for the commit watcher."} + +mkdir -p $GOROOT +curl -s https://storage.googleapis.com/gobuilder/go-snap.tar.gz | tar x --no-same-owner -zv -C $GOROOT +(cd $GOROOT/src && hg pull -r $GO_REV -u && find && ./make.bash) + +GO_TOOLS=$GOPATH/src/golang.org/x/tools +mkdir -p $GO_TOOLS +curl -s https://storage.googleapis.com/gobuilder/go.tools-snap.tar.gz | tar x --no-same-owner -zv -C $GO_TOOLS + +mkdir -p $PREFIX/bin +(cd $GO_TOOLS && hg pull -r $WATCHER_REV -u && GOBIN=$PREFIX/bin /goroot/bin/go install golang.org/x/tools/dashboard/watcher) + +rm -fR $GOROOT/bin $GOROOT/pkg $GOPATH diff --git a/dashboard/env/commit-watcher/scripts/install-apt-deps.sh b/dashboard/env/commit-watcher/scripts/install-apt-deps.sh new file mode 100755 index 00000000..a4e5fc4b --- /dev/null +++ b/dashboard/env/commit-watcher/scripts/install-apt-deps.sh @@ -0,0 +1,12 @@ +set -ex + +apt-get update +# For running curl to get the hg starter tarballs (faster than hg clone). +apt-get install -y --no-install-recommends curl ca-certificates +# For building Go's bootstrap 'dist' prog +apt-get install -y --no-install-recommends gcc libc6-dev +# For interacting with the Go source & subrepos: +apt-get install -y --no-install-recommends mercurial git-core + +apt-get clean +rm -fr /var/lib/apt/lists