From a705311956be8b6f407d08438c9be24b0bd595f8 Mon Sep 17 00:00:00 2001 From: Chris Manghane Date: Thu, 1 Aug 2013 13:23:51 +1000 Subject: [PATCH] go.tools: add dashboard Moving misc/dashboard to its new home. It will be deleted (except for misc/dashboard/codereview) from its current home if this is approved. R=golang-dev, bradfitz, cmang, adg CC=golang-dev https://golang.org/cl/12180043 --- dashboard/README | 26 + dashboard/app/app.yaml | 20 + dashboard/app/build/build.go | 332 +++++++++++++ dashboard/app/build/handler.go | 449 ++++++++++++++++++ dashboard/app/build/init.go | 68 +++ dashboard/app/build/key.go | 64 +++ dashboard/app/build/notify.go | 168 +++++++ dashboard/app/build/notify.txt | 9 + dashboard/app/build/test.go | 258 ++++++++++ dashboard/app/build/ui.go | 321 +++++++++++++ dashboard/app/build/ui.html | 209 +++++++++ dashboard/app/cache/cache.go | 84 ++++ dashboard/app/static/status_alert.gif | Bin 0 -> 570 bytes dashboard/app/static/status_good.gif | Bin 0 -> 328 bytes dashboard/builder/Makefile | 9 + dashboard/builder/doc.go | 58 +++ dashboard/builder/exec.go | 79 ++++ dashboard/builder/http.go | 167 +++++++ dashboard/builder/main.go | 653 ++++++++++++++++++++++++++ dashboard/builder/vcs.go | 148 ++++++ 20 files changed, 3122 insertions(+) create mode 100644 dashboard/README create mode 100644 dashboard/app/app.yaml create mode 100644 dashboard/app/build/build.go create mode 100644 dashboard/app/build/handler.go create mode 100644 dashboard/app/build/init.go create mode 100644 dashboard/app/build/key.go create mode 100644 dashboard/app/build/notify.go create mode 100644 dashboard/app/build/notify.txt create mode 100644 dashboard/app/build/test.go create mode 100644 dashboard/app/build/ui.go create mode 100644 dashboard/app/build/ui.html create mode 100644 dashboard/app/cache/cache.go create mode 100644 dashboard/app/static/status_alert.gif create mode 100644 dashboard/app/static/status_good.gif create mode 100644 dashboard/builder/Makefile create mode 100644 dashboard/builder/doc.go create mode 100644 dashboard/builder/exec.go create mode 100644 dashboard/builder/http.go create mode 100644 dashboard/builder/main.go create mode 100644 dashboard/builder/vcs.go diff --git a/dashboard/README b/dashboard/README new file mode 100644 index 00000000..d599f3d0 --- /dev/null +++ b/dashboard/README @@ -0,0 +1,26 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +The files in this directory constitute the continuous builder: + +app/: an AppEngine server +builder/: gobuilder, a Go continuous build client + +If you wish to run a Go builder, please email golang-dev@googlegroups.com + +To run a builder: + +* Write the key ~gobuild/.gobuildkey + You need to get it from someone who knows the key. + You may also use a filename of the form .gobuildkey-$BUILDER if you + wish to run builders for multiple targets. + +* Append your username and password googlecode.com credentials from + https://code.google.com/hosting/settings + to the buildkey file in the format "Username\nPassword\n". + (This is for uploading tarballs to the project downloads section, + and is an optional step.) + +* Build and run gobuilder (see its documentation for command-line options). + diff --git a/dashboard/app/app.yaml b/dashboard/app/app.yaml new file mode 100644 index 00000000..c5a1f6cb --- /dev/null +++ b/dashboard/app/app.yaml @@ -0,0 +1,20 @@ +# Update with +# google_appengine/appcfg.py [-V test-build] update . +# +# Using -V test-build will run as test-build.golang.org. + +application: golang-org +version: build +runtime: go +api_version: go1 + +handlers: +- url: /static + static_dir: static +- url: /log/.+ + script: _go_app +- url: /(|commit|packages|result|tag|todo) + script: _go_app +- url: /(init|buildtest|key|_ah/queue/go/delay) + script: _go_app + login: admin diff --git a/dashboard/app/build/build.go b/dashboard/app/build/build.go new file mode 100644 index 00000000..03dc173b --- /dev/null +++ b/dashboard/app/build/build.go @@ -0,0 +1,332 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package build + +import ( + "bytes" + "compress/gzip" + "crypto/sha1" + "errors" + "fmt" + "io" + "io/ioutil" + "strings" + "time" + + "appengine" + "appengine/datastore" +) + +const maxDatastoreStringLen = 500 + +// A Package describes a package that is listed on the dashboard. +type Package struct { + Kind string // "subrepo", "external", or empty for the main Go tree + Name string + Path string // (empty for the main Go tree) + NextNum int // Num of the next head Commit +} + +func (p *Package) String() string { + return fmt.Sprintf("%s: %q", p.Path, p.Name) +} + +func (p *Package) Key(c appengine.Context) *datastore.Key { + key := p.Path + if key == "" { + key = "go" + } + return datastore.NewKey(c, "Package", key, 0, nil) +} + +// LastCommit returns the most recent Commit for this Package. +func (p *Package) LastCommit(c appengine.Context) (*Commit, error) { + var commits []*Commit + _, err := datastore.NewQuery("Commit"). + Ancestor(p.Key(c)). + Order("-Time"). + Limit(1). + GetAll(c, &commits) + if _, ok := err.(*datastore.ErrFieldMismatch); ok { + // Some fields have been removed, so it's okay to ignore this error. + err = nil + } + if err != nil { + return nil, err + } + if len(commits) != 1 { + return nil, datastore.ErrNoSuchEntity + } + return commits[0], nil +} + +// GetPackage fetches a Package by path from the datastore. +func GetPackage(c appengine.Context, path string) (*Package, error) { + p := &Package{Path: path} + err := datastore.Get(c, p.Key(c), p) + if err == datastore.ErrNoSuchEntity { + return nil, fmt.Errorf("package %q not found", path) + } + if _, ok := err.(*datastore.ErrFieldMismatch); ok { + // Some fields have been removed, so it's okay to ignore this error. + err = nil + } + return p, err +} + +// A Commit describes an individual commit in a package. +// +// Each Commit entity is a descendant of its associated Package entity. +// In other words, all Commits with the same PackagePath belong to the same +// datastore entity group. +type Commit struct { + PackagePath string // (empty for Go commits) + Hash string + ParentHash string + Num int // Internal monotonic counter unique to this package. + + User string + Desc string `datastore:",noindex"` + Time time.Time + + // ResultData is the Data string of each build Result for this Commit. + // For non-Go commits, only the Results for the current Go tip, weekly, + // and release Tags are stored here. This is purely de-normalized data. + // The complete data set is stored in Result entities. + ResultData []string `datastore:",noindex"` + + FailNotificationSent bool +} + +func (com *Commit) Key(c appengine.Context) *datastore.Key { + if com.Hash == "" { + panic("tried Key on Commit with empty Hash") + } + p := Package{Path: com.PackagePath} + key := com.PackagePath + "|" + com.Hash + return datastore.NewKey(c, "Commit", key, 0, p.Key(c)) +} + +func (c *Commit) Valid() error { + if !validHash(c.Hash) { + return errors.New("invalid Hash") + } + if c.ParentHash != "" && !validHash(c.ParentHash) { // empty is OK + return errors.New("invalid ParentHash") + } + return nil +} + +// each result line is approx 105 bytes. This constant is a tradeoff between +// build history and the AppEngine datastore limit of 1mb. +const maxResults = 1000 + +// AddResult adds the denormalized Result data to the Commit's Result field. +// It must be called from inside a datastore transaction. +func (com *Commit) AddResult(c appengine.Context, r *Result) error { + if err := datastore.Get(c, com.Key(c), com); err != nil { + return fmt.Errorf("getting Commit: %v", err) + } + com.ResultData = trim(append(com.ResultData, r.Data()), maxResults) + if _, err := datastore.Put(c, com.Key(c), com); err != nil { + return fmt.Errorf("putting Commit: %v", err) + } + return nil +} + +func trim(s []string, n int) []string { + l := min(len(s), n) + return s[len(s)-l:] +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// Result returns the build Result for this Commit for the given builder/goHash. +func (c *Commit) Result(builder, goHash string) *Result { + for _, r := range c.ResultData { + p := strings.SplitN(r, "|", 4) + if len(p) != 4 || p[0] != builder || p[3] != goHash { + continue + } + return partsToHash(c, p) + } + return nil +} + +// Results returns the build Results for this Commit for the given goHash. +func (c *Commit) Results(goHash string) (results []*Result) { + for _, r := range c.ResultData { + p := strings.SplitN(r, "|", 4) + if len(p) != 4 || p[3] != goHash { + continue + } + results = append(results, partsToHash(c, p)) + } + return +} + +// partsToHash converts a Commit and ResultData substrings to a Result. +func partsToHash(c *Commit, p []string) *Result { + return &Result{ + Builder: p[0], + Hash: c.Hash, + PackagePath: c.PackagePath, + GoHash: p[3], + OK: p[1] == "true", + LogHash: p[2], + } +} + +// A Result describes a build result for a Commit on an OS/architecture. +// +// Each Result entity is a descendant of its associated Commit entity. +type Result struct { + Builder string // "os-arch[-note]" + Hash string + PackagePath string // (empty for Go commits) + + // The Go Commit this was built against (empty for Go commits). + GoHash string + + OK bool + Log string `datastore:"-"` // for JSON unmarshaling only + LogHash string `datastore:",noindex"` // Key to the Log record. + + RunTime int64 // time to build+test in nanoseconds +} + +func (r *Result) Key(c appengine.Context) *datastore.Key { + p := Package{Path: r.PackagePath} + key := r.Builder + "|" + r.PackagePath + "|" + r.Hash + "|" + r.GoHash + return datastore.NewKey(c, "Result", key, 0, p.Key(c)) +} + +func (r *Result) Valid() error { + if !validHash(r.Hash) { + return errors.New("invalid Hash") + } + if r.PackagePath != "" && !validHash(r.GoHash) { + return errors.New("invalid GoHash") + } + return nil +} + +// Data returns the Result in string format +// to be stored in Commit's ResultData field. +func (r *Result) Data() string { + return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash) +} + +// A Log is a gzip-compressed log file stored under the SHA1 hash of the +// uncompressed log text. +type Log struct { + CompressedLog []byte +} + +func (l *Log) Text() ([]byte, error) { + d, err := gzip.NewReader(bytes.NewBuffer(l.CompressedLog)) + if err != nil { + return nil, fmt.Errorf("reading log data: %v", err) + } + b, err := ioutil.ReadAll(d) + if err != nil { + return nil, fmt.Errorf("reading log data: %v", err) + } + return b, nil +} + +func PutLog(c appengine.Context, text string) (hash string, err error) { + h := sha1.New() + io.WriteString(h, text) + b := new(bytes.Buffer) + z, _ := gzip.NewWriterLevel(b, gzip.BestCompression) + io.WriteString(z, text) + z.Close() + hash = fmt.Sprintf("%x", h.Sum(nil)) + key := datastore.NewKey(c, "Log", hash, 0, nil) + _, err = datastore.Put(c, key, &Log{b.Bytes()}) + return +} + +// A Tag is used to keep track of the most recent Go weekly and release tags. +// Typically there will be one Tag entity for each kind of hg tag. +type Tag struct { + Kind string // "weekly", "release", or "tip" + Name string // the tag itself (for example: "release.r60") + Hash string +} + +func (t *Tag) Key(c appengine.Context) *datastore.Key { + p := &Package{} + return datastore.NewKey(c, "Tag", t.Kind, 0, p.Key(c)) +} + +func (t *Tag) Valid() error { + if t.Kind != "weekly" && t.Kind != "release" && t.Kind != "tip" { + return errors.New("invalid Kind") + } + if !validHash(t.Hash) { + return errors.New("invalid Hash") + } + return nil +} + +// Commit returns the Commit that corresponds with this Tag. +func (t *Tag) Commit(c appengine.Context) (*Commit, error) { + com := &Commit{Hash: t.Hash} + err := datastore.Get(c, com.Key(c), com) + return com, err +} + +// GetTag fetches a Tag by name from the datastore. +func GetTag(c appengine.Context, tag string) (*Tag, error) { + t := &Tag{Kind: tag} + if err := datastore.Get(c, t.Key(c), t); err != nil { + if err == datastore.ErrNoSuchEntity { + return nil, errors.New("tag not found: " + tag) + } + return nil, err + } + if err := t.Valid(); err != nil { + return nil, err + } + return t, nil +} + +// Packages returns packages of the specified kind. +// Kind must be one of "external" or "subrepo". +func Packages(c appengine.Context, kind string) ([]*Package, error) { + switch kind { + case "external", "subrepo": + default: + return nil, errors.New(`kind must be one of "external" or "subrepo"`) + } + var pkgs []*Package + q := datastore.NewQuery("Package").Filter("Kind=", kind) + for t := q.Run(c); ; { + pkg := new(Package) + _, err := t.Next(pkg) + if _, ok := err.(*datastore.ErrFieldMismatch); ok { + // Some fields have been removed, so it's okay to ignore this error. + err = nil + } + if err == datastore.Done { + break + } else if err != nil { + return nil, err + } + if pkg.Path != "" { + pkgs = append(pkgs, pkg) + } + } + return pkgs, nil +} diff --git a/dashboard/app/build/handler.go b/dashboard/app/build/handler.go new file mode 100644 index 00000000..01bf95e6 --- /dev/null +++ b/dashboard/app/build/handler.go @@ -0,0 +1,449 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package build + +import ( + "crypto/hmac" + "crypto/md5" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "appengine" + "appengine/datastore" + "cache" +) + +const commitsPerPage = 30 + +// commitHandler retrieves commit data or records a new commit. +// +// For GET requests it returns a Commit value for the specified +// packagePath and hash. +// +// For POST requests it reads a JSON-encoded Commit value from the request +// body and creates a new Commit entity. It also updates the "tip" Tag for +// each new commit at tip. +// +// This handler is used by a gobuilder process in -commit mode. +func commitHandler(r *http.Request) (interface{}, error) { + c := appengine.NewContext(r) + com := new(Commit) + + if r.Method == "GET" { + com.PackagePath = r.FormValue("packagePath") + com.Hash = r.FormValue("hash") + if err := datastore.Get(c, com.Key(c), com); err != nil { + return nil, fmt.Errorf("getting Commit: %v", err) + } + return com, nil + } + if r.Method != "POST" { + return nil, errBadMethod(r.Method) + } + + // POST request + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(com); err != nil { + return nil, fmt.Errorf("decoding Body: %v", err) + } + if len(com.Desc) > maxDatastoreStringLen { + com.Desc = com.Desc[:maxDatastoreStringLen] + } + if err := com.Valid(); err != nil { + return nil, fmt.Errorf("validating Commit: %v", err) + } + defer cache.Tick(c) + tx := func(c appengine.Context) error { + return addCommit(c, com) + } + return nil, datastore.RunInTransaction(c, tx, nil) +} + +// addCommit adds the Commit entity to the datastore and updates the tip Tag. +// It must be run inside a datastore transaction. +func addCommit(c appengine.Context, com *Commit) error { + var tc Commit // temp value so we don't clobber com + err := datastore.Get(c, com.Key(c), &tc) + if err != datastore.ErrNoSuchEntity { + // if this commit is already in the datastore, do nothing + if err == nil { + return nil + } + return fmt.Errorf("getting Commit: %v", err) + } + // get the next commit number + p, err := GetPackage(c, com.PackagePath) + if err != nil { + return fmt.Errorf("GetPackage: %v", err) + } + com.Num = p.NextNum + p.NextNum++ + if _, err := datastore.Put(c, p.Key(c), p); err != nil { + return fmt.Errorf("putting Package: %v", err) + } + // if this isn't the first Commit test the parent commit exists + if com.Num > 0 { + n, err := datastore.NewQuery("Commit"). + Filter("Hash =", com.ParentHash). + Ancestor(p.Key(c)). + Count(c) + if err != nil { + return fmt.Errorf("testing for parent Commit: %v", err) + } + if n == 0 { + return errors.New("parent commit not found") + } + } + // update the tip Tag if this is the Go repo and this isn't on a release branch + if p.Path == "" && !strings.HasPrefix(com.Desc, "[release-branch") { + t := &Tag{Kind: "tip", Hash: com.Hash} + if _, err = datastore.Put(c, t.Key(c), t); err != nil { + return fmt.Errorf("putting Tag: %v", err) + } + } + // put the Commit + if _, err = datastore.Put(c, com.Key(c), com); err != nil { + return fmt.Errorf("putting Commit: %v", err) + } + return nil +} + +// tagHandler records a new tag. It reads a JSON-encoded Tag value from the +// request body and updates the Tag entity for the Kind of tag provided. +// +// This handler is used by a gobuilder process in -commit mode. +func tagHandler(r *http.Request) (interface{}, error) { + if r.Method != "POST" { + return nil, errBadMethod(r.Method) + } + + t := new(Tag) + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(t); err != nil { + return nil, err + } + if err := t.Valid(); err != nil { + return nil, err + } + c := appengine.NewContext(r) + defer cache.Tick(c) + _, err := datastore.Put(c, t.Key(c), t) + return nil, err +} + +// Todo is a todoHandler response. +type Todo struct { + Kind string // "build-go-commit" or "build-package" + Data interface{} +} + +// todoHandler returns the next action to be performed by a builder. +// It expects "builder" and "kind" query parameters and returns a *Todo value. +// Multiple "kind" parameters may be specified. +func todoHandler(r *http.Request) (interface{}, error) { + c := appengine.NewContext(r) + now := cache.Now(c) + key := "build-todo-" + r.Form.Encode() + var todo *Todo + if cache.Get(r, now, key, &todo) { + return todo, nil + } + var err error + builder := r.FormValue("builder") + for _, kind := range r.Form["kind"] { + var data interface{} + switch kind { + case "build-go-commit": + data, err = buildTodo(c, builder, "", "") + case "build-package": + packagePath := r.FormValue("packagePath") + goHash := r.FormValue("goHash") + data, err = buildTodo(c, builder, packagePath, goHash) + } + if data != nil || err != nil { + todo = &Todo{Kind: kind, Data: data} + break + } + } + if err == nil { + cache.Set(r, now, key, todo) + } + return todo, err +} + +// buildTodo returns the next Commit to be built (or nil if none available). +// +// If packagePath and goHash are empty, it scans the first 20 Go Commits in +// Num-descending order and returns the first one it finds that doesn't have a +// Result for this builder. +// +// If provided with non-empty packagePath and goHash args, it scans the first +// 20 Commits in Num-descending order for the specified packagePath and +// returns the first that doesn't have a Result for this builder and goHash. +func buildTodo(c appengine.Context, builder, packagePath, goHash string) (interface{}, error) { + p, err := GetPackage(c, packagePath) + if err != nil { + return nil, err + } + + t := datastore.NewQuery("Commit"). + Ancestor(p.Key(c)). + Limit(commitsPerPage). + Order("-Num"). + Run(c) + for { + com := new(Commit) + if _, err := t.Next(com); err == datastore.Done { + break + } else if err != nil { + return nil, err + } + if com.Result(builder, goHash) == nil { + return com, nil + } + } + + // Nothing left to do if this is a package (not the Go tree). + if packagePath != "" { + return nil, nil + } + + // If there are no Go tree commits left to build, + // see if there are any subrepo commits that need to be built at tip. + // If so, ask the builder to build a go tree at the tip commit. + // TODO(adg): do the same for "weekly" and "release" tags. + + tag, err := GetTag(c, "tip") + if err != nil { + return nil, err + } + + // Check that this Go commit builds OK for this builder. + // If not, don't re-build as the subrepos will never get built anyway. + com, err := tag.Commit(c) + if err != nil { + return nil, err + } + if r := com.Result(builder, ""); r != nil && !r.OK { + return nil, nil + } + + pkgs, err := Packages(c, "subrepo") + if err != nil { + return nil, err + } + for _, pkg := range pkgs { + com, err := pkg.LastCommit(c) + if err != nil { + c.Warningf("%v: no Commit found: %v", pkg, err) + continue + } + if com.Result(builder, tag.Hash) == nil { + return tag.Commit(c) + } + } + + return nil, nil +} + +// packagesHandler returns a list of the non-Go Packages monitored +// by the dashboard. +func packagesHandler(r *http.Request) (interface{}, error) { + kind := r.FormValue("kind") + c := appengine.NewContext(r) + now := cache.Now(c) + key := "build-packages-" + kind + var p []*Package + if cache.Get(r, now, key, &p) { + return p, nil + } + p, err := Packages(c, kind) + if err != nil { + return nil, err + } + cache.Set(r, now, key, p) + return p, nil +} + +// resultHandler records a build result. +// It reads a JSON-encoded Result value from the request body, +// creates a new Result entity, and updates the relevant Commit entity. +// If the Log field is not empty, resultHandler creates a new Log entity +// and updates the LogHash field before putting the Commit entity. +func resultHandler(r *http.Request) (interface{}, error) { + if r.Method != "POST" { + return nil, errBadMethod(r.Method) + } + + c := appengine.NewContext(r) + res := new(Result) + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(res); err != nil { + return nil, fmt.Errorf("decoding Body: %v", err) + } + if err := res.Valid(); err != nil { + return nil, fmt.Errorf("validating Result: %v", err) + } + defer cache.Tick(c) + // store the Log text if supplied + if len(res.Log) > 0 { + hash, err := PutLog(c, res.Log) + if err != nil { + return nil, fmt.Errorf("putting Log: %v", err) + } + res.LogHash = hash + } + tx := func(c appengine.Context) error { + // check Package exists + if _, err := GetPackage(c, res.PackagePath); err != nil { + return fmt.Errorf("GetPackage: %v", err) + } + // put Result + if _, err := datastore.Put(c, res.Key(c), res); err != nil { + return fmt.Errorf("putting Result: %v", err) + } + // add Result to Commit + com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash} + if err := com.AddResult(c, res); err != nil { + return fmt.Errorf("AddResult: %v", err) + } + // Send build failure notifications, if necessary. + // Note this must run after the call AddResult, which + // populates the Commit's ResultData field. + return notifyOnFailure(c, com, res.Builder) + } + return nil, datastore.RunInTransaction(c, tx, nil) +} + +// logHandler displays log text for a given hash. +// It handles paths like "/log/hash". +func logHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-type", "text/plain; charset=utf-8") + c := appengine.NewContext(r) + hash := r.URL.Path[len("/log/"):] + key := datastore.NewKey(c, "Log", hash, 0, nil) + l := new(Log) + if err := datastore.Get(c, key, l); err != nil { + logErr(w, r, err) + return + } + b, err := l.Text() + if err != nil { + logErr(w, r, err) + return + } + w.Write(b) +} + +type dashHandler func(*http.Request) (interface{}, error) + +type dashResponse struct { + Response interface{} + Error string +} + +// errBadMethod is returned by a dashHandler when +// the request has an unsuitable method. +type errBadMethod string + +func (e errBadMethod) Error() string { + return "bad method: " + string(e) +} + +// AuthHandler wraps a http.HandlerFunc with a handler that validates the +// supplied key and builder query parameters. +func AuthHandler(h dashHandler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + + // Put the URL Query values into r.Form to avoid parsing the + // request body when calling r.FormValue. + r.Form = r.URL.Query() + + var err error + var resp interface{} + + // Validate key query parameter for POST requests only. + key := r.FormValue("key") + builder := r.FormValue("builder") + if r.Method == "POST" && !validKey(c, key, builder) { + err = errors.New("invalid key: " + key) + } + + // Call the original HandlerFunc and return the response. + if err == nil { + resp, err = h(r) + } + + // Write JSON response. + dashResp := &dashResponse{Response: resp} + if err != nil { + c.Errorf("%v", err) + dashResp.Error = err.Error() + } + w.Header().Set("Content-Type", "application/json") + if err = json.NewEncoder(w).Encode(dashResp); err != nil { + c.Criticalf("encoding response: %v", err) + } + } +} + +func keyHandler(w http.ResponseWriter, r *http.Request) { + builder := r.FormValue("builder") + if builder == "" { + logErr(w, r, errors.New("must supply builder in query string")) + return + } + c := appengine.NewContext(r) + fmt.Fprint(w, builderKey(c, builder)) +} + +func init() { + // admin handlers + http.HandleFunc("/init", initHandler) + http.HandleFunc("/key", keyHandler) + + // authenticated handlers + http.HandleFunc("/commit", AuthHandler(commitHandler)) + http.HandleFunc("/packages", AuthHandler(packagesHandler)) + http.HandleFunc("/result", AuthHandler(resultHandler)) + http.HandleFunc("/tag", AuthHandler(tagHandler)) + http.HandleFunc("/todo", AuthHandler(todoHandler)) + + // public handlers + http.HandleFunc("/log/", logHandler) +} + +func validHash(hash string) bool { + // TODO(adg): correctly validate a hash + return hash != "" +} + +func validKey(c appengine.Context, key, builder string) bool { + if appengine.IsDevAppServer() { + return true + } + if key == secretKey(c) { + return true + } + return key == builderKey(c, builder) +} + +func builderKey(c appengine.Context, builder string) string { + h := hmac.New(md5.New, []byte(secretKey(c))) + h.Write([]byte(builder)) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func logErr(w http.ResponseWriter, r *http.Request, err error) { + appengine.NewContext(r).Errorf("Error: %v", err) + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, "Error: ", err) +} diff --git a/dashboard/app/build/init.go b/dashboard/app/build/init.go new file mode 100644 index 00000000..da1f78b0 --- /dev/null +++ b/dashboard/app/build/init.go @@ -0,0 +1,68 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package build + +import ( + "fmt" + "net/http" + + "appengine" + "appengine/datastore" + "cache" +) + +// defaultPackages specifies the Package records to be created by initHandler. +var defaultPackages = []*Package{ + {Name: "Go", Kind: "go"}, +} + +// subRepos specifies the Go project sub-repositories. +var subRepos = []string{ + "blog", + "codereview", + "crypto", + "exp", + "image", + "net", + "talks", + "tools", +} + +// Put subRepos into defaultPackages. +func init() { + for _, name := range subRepos { + p := &Package{ + Kind: "subrepo", + Name: "go." + name, + Path: "code.google.com/p/go." + name, + } + defaultPackages = append(defaultPackages, p) + } +} + +func initHandler(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + defer cache.Tick(c) + for _, p := range defaultPackages { + err := datastore.Get(c, p.Key(c), new(Package)) + if _, ok := err.(*datastore.ErrFieldMismatch); ok { + // Some fields have been removed, so it's okay to ignore this error. + err = nil + } + if err == nil { + continue + } else if err != datastore.ErrNoSuchEntity { + logErr(w, r, err) + return + } + if _, err := datastore.Put(c, p.Key(c), p); err != nil { + logErr(w, r, err) + return + } + } + fmt.Fprint(w, "OK") +} diff --git a/dashboard/app/build/key.go b/dashboard/app/build/key.go new file mode 100644 index 00000000..91882734 --- /dev/null +++ b/dashboard/app/build/key.go @@ -0,0 +1,64 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package build + +import ( + "sync" + + "appengine" + "appengine/datastore" +) + +var theKey struct { + sync.RWMutex + BuilderKey +} + +type BuilderKey struct { + Secret string +} + +func (k *BuilderKey) Key(c appengine.Context) *datastore.Key { + return datastore.NewKey(c, "BuilderKey", "root", 0, nil) +} + +func secretKey(c appengine.Context) string { + // check with rlock + theKey.RLock() + k := theKey.Secret + theKey.RUnlock() + if k != "" { + return k + } + + // prepare to fill; check with lock and keep lock + theKey.Lock() + defer theKey.Unlock() + if theKey.Secret != "" { + return theKey.Secret + } + + // fill + if err := datastore.Get(c, theKey.Key(c), &theKey.BuilderKey); err != nil { + if err == datastore.ErrNoSuchEntity { + // If the key is not stored in datastore, write it. + // This only happens at the beginning of a new deployment. + // The code is left here for SDK use and in case a fresh + // deployment is ever needed. "gophers rule" is not the + // real key. + if !appengine.IsDevAppServer() { + panic("lost key from datastore") + } + theKey.Secret = "gophers rule" + datastore.Put(c, theKey.Key(c), &theKey.BuilderKey) + return theKey.Secret + } + panic("cannot load builder key: " + err.Error()) + } + + return theKey.Secret +} diff --git a/dashboard/app/build/notify.go b/dashboard/app/build/notify.go new file mode 100644 index 00000000..c0fc4a6a --- /dev/null +++ b/dashboard/app/build/notify.go @@ -0,0 +1,168 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package build + +import ( + "appengine" + "appengine/datastore" + "appengine/delay" + "appengine/mail" + "bytes" + "encoding/gob" + "fmt" + "text/template" +) + +const ( + mailFrom = "builder@golang.org" // use this for sending any mail + failMailTo = "golang-dev@googlegroups.com" + domain = "build.golang.org" +) + +// failIgnore is a set of builders that we don't email about because +// they're too flaky. +var failIgnore = map[string]bool{ + "netbsd-386-bsiegert": true, + "netbsd-amd64-bsiegert": true, +} + +// notifyOnFailure checks whether the supplied Commit or the subsequent +// Commit (if present) breaks the build for this builder. +// If either of those commits break the build an email notification is sent +// from a delayed task. (We use a task because this way the mail won't be +// sent if the enclosing datastore transaction fails.) +// +// This must be run in a datastore transaction, and the provided *Commit must +// have been retrieved from the datastore within that transaction. +func notifyOnFailure(c appengine.Context, com *Commit, builder string) error { + if failIgnore[builder] { + return nil + } + + // TODO(adg): implement notifications for packages + if com.PackagePath != "" { + return nil + } + + p := &Package{Path: com.PackagePath} + var broken *Commit + cr := com.Result(builder, "") + if cr == nil { + return fmt.Errorf("no result for %s/%s", com.Hash, builder) + } + q := datastore.NewQuery("Commit").Ancestor(p.Key(c)) + if cr.OK { + // This commit is OK. Notify if next Commit is broken. + next := new(Commit) + q = q.Filter("ParentHash=", com.Hash) + if err := firstMatch(c, q, next); err != nil { + if err == datastore.ErrNoSuchEntity { + // OK at tip, no notification necessary. + return nil + } + return err + } + if nr := next.Result(builder, ""); nr != nil && !nr.OK { + c.Debugf("commit ok: %#v\nresult: %#v", com, cr) + c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr) + broken = next + } + } else { + // This commit is broken. Notify if the previous Commit is OK. + prev := new(Commit) + q = q.Filter("Hash=", com.ParentHash) + if err := firstMatch(c, q, prev); err != nil { + if err == datastore.ErrNoSuchEntity { + // No previous result, let the backfill of + // this result trigger the notification. + return nil + } + return err + } + if pr := prev.Result(builder, ""); pr != nil && pr.OK { + c.Debugf("commit broken: %#v\nresult: %#v", com, cr) + c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr) + broken = com + } + } + var err error + if broken != nil && !broken.FailNotificationSent { + c.Infof("%s is broken commit; notifying", broken.Hash) + sendFailMailLater.Call(c, broken, builder) // add task to queue + broken.FailNotificationSent = true + _, err = datastore.Put(c, broken.Key(c), broken) + } + return err +} + +// firstMatch executes the query q and loads the first entity into v. +func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error { + t := q.Limit(1).Run(c) + _, err := t.Next(v) + if err == datastore.Done { + err = datastore.ErrNoSuchEntity + } + return err +} + +var ( + sendFailMailLater = delay.Func("sendFailMail", sendFailMail) + sendFailMailTmpl = template.Must( + template.New("notify.txt"). + Funcs(template.FuncMap(tmplFuncs)). + ParseFiles("build/notify.txt"), + ) +) + +func init() { + gob.Register(&Commit{}) // for delay +} + +// sendFailMail sends a mail notification that the build failed on the +// provided commit and builder. +func sendFailMail(c appengine.Context, com *Commit, builder string) { + // TODO(adg): handle packages + + // get Result + r := com.Result(builder, "") + if r == nil { + c.Errorf("finding result for %q: %+v", builder, com) + return + } + + // get Log + k := datastore.NewKey(c, "Log", r.LogHash, 0, nil) + l := new(Log) + if err := datastore.Get(c, k, l); err != nil { + c.Errorf("finding Log record %v: %v", r.LogHash, err) + return + } + + // prepare mail message + var body bytes.Buffer + err := sendFailMailTmpl.Execute(&body, map[string]interface{}{ + "Builder": builder, "Commit": com, "Result": r, "Log": l, + "Hostname": domain, + }) + if err != nil { + c.Errorf("rendering mail template: %v", err) + return + } + subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc)) + msg := &mail.Message{ + Sender: mailFrom, + To: []string{failMailTo}, + ReplyTo: failMailTo, + Subject: subject, + Body: body.String(), + } + + // send mail + if err := mail.Send(c, msg); err != nil { + c.Errorf("sending mail: %v", err) + } +} diff --git a/dashboard/app/build/notify.txt b/dashboard/app/build/notify.txt new file mode 100644 index 00000000..6c900670 --- /dev/null +++ b/dashboard/app/build/notify.txt @@ -0,0 +1,9 @@ +Change {{shortHash .Commit.Hash}} broke the {{.Builder}} build: +http://{{.Hostname}}/log/{{.Result.LogHash}} + +{{.Commit.Desc}} + +http://code.google.com/p/go/source/detail?r={{shortHash .Commit.Hash}} + +$ tail -200 < log +{{printf "%s" .Log.Text | tail 200}} diff --git a/dashboard/app/build/test.go b/dashboard/app/build/test.go new file mode 100644 index 00000000..d78ba573 --- /dev/null +++ b/dashboard/app/build/test.go @@ -0,0 +1,258 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package build + +// TODO(adg): test authentication + +import ( + "appengine" + "appengine/datastore" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "time" +) + +func init() { + http.HandleFunc("/buildtest", testHandler) +} + +var testEntityKinds = []string{ + "Package", + "Commit", + "Result", + "Log", +} + +const testPkg = "code.google.com/p/go.test" + +var testPackage = &Package{Name: "Test", Kind: "subrepo", Path: testPkg} + +var testPackages = []*Package{ + {Name: "Go", Path: ""}, + testPackage, +} + +var tCommitTime = time.Now().Add(-time.Hour * 24 * 7) + +func tCommit(hash, parentHash, path string) *Commit { + tCommitTime.Add(time.Hour) // each commit should have a different time + return &Commit{ + PackagePath: path, + Hash: hash, + ParentHash: parentHash, + Time: tCommitTime, + User: "adg", + Desc: "change description " + hash, + } +} + +var testRequests = []struct { + path string + vals url.Values + req interface{} + res interface{} +}{ + // Packages + {"/packages?kind=subrepo", nil, nil, []*Package{testPackage}}, + + // Go repo + {"/commit", nil, tCommit("0001", "0000", ""), nil}, + {"/commit", nil, tCommit("0002", "0001", ""), nil}, + {"/commit", nil, tCommit("0003", "0002", ""), nil}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, + {"/result", nil, &Result{Builder: "linux-386", Hash: "0001", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, + {"/result", nil, &Result{Builder: "linux-386", Hash: "0002", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, + + // multiple builders + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, + {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0003", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}}, + + // branches + {"/commit", nil, tCommit("0004", "0003", ""), nil}, + {"/commit", nil, tCommit("0005", "0002", ""), nil}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}}, + {"/result", nil, &Result{Builder: "linux-386", Hash: "0005", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}}, + {"/result", nil, &Result{Builder: "linux-386", Hash: "0004", OK: false}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, + + // logs + {"/result", nil, &Result{Builder: "linux-386", Hash: "0003", OK: false, Log: "test"}, nil}, + {"/log/a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", nil, nil, "test"}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, nil}, + + // repeat failure (shouldn't re-send mail) + {"/result", nil, &Result{Builder: "linux-386", Hash: "0003", OK: false, Log: "test"}, nil}, + + // non-Go repos + {"/commit", nil, tCommit("1001", "1000", testPkg), nil}, + {"/commit", nil, tCommit("1002", "1001", testPkg), nil}, + {"/commit", nil, tCommit("1003", "1002", testPkg), nil}, + {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1003"}}}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0001", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1002"}}}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1002", GoHash: "0001", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1001"}}}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0001", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, nil}, + {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0002"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1003"}}}, + + // re-build Go revision for stale subrepos + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0005", OK: false, Log: "boo"}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, nil}, +} + +func testHandler(w http.ResponseWriter, r *http.Request) { + if !appengine.IsDevAppServer() { + fmt.Fprint(w, "These tests must be run under the dev_appserver.") + return + } + c := appengine.NewContext(r) + if err := nukeEntities(c, testEntityKinds); err != nil { + logErr(w, r, err) + return + } + if r.FormValue("nukeonly") != "" { + fmt.Fprint(w, "OK") + return + } + + for _, p := range testPackages { + if _, err := datastore.Put(c, p.Key(c), p); err != nil { + logErr(w, r, err) + return + } + } + + for i, t := range testRequests { + c.Infof("running test %d %s", i, t.path) + errorf := func(format string, args ...interface{}) { + fmt.Fprintf(w, "%d %s: ", i, t.path) + fmt.Fprintf(w, format, args...) + fmt.Fprintln(w) + } + var body io.ReadWriter + if t.req != nil { + body = new(bytes.Buffer) + json.NewEncoder(body).Encode(t.req) + } + url := "http://" + domain + t.path + if t.vals != nil { + url += "?" + t.vals.Encode() + } + req, err := http.NewRequest("POST", url, body) + if err != nil { + logErr(w, r, err) + return + } + if t.req != nil { + req.Method = "POST" + } + req.Header = r.Header + rec := httptest.NewRecorder() + + // Make the request + http.DefaultServeMux.ServeHTTP(rec, req) + + if rec.Code != 0 && rec.Code != 200 { + errorf(rec.Body.String()) + return + } + resp := new(dashResponse) + + // If we're expecting a *Todo value, + // prime the Response field with a Todo and a Commit inside it. + if _, ok := t.res.(*Todo); ok { + resp.Response = &Todo{Data: &Commit{}} + } + + if strings.HasPrefix(t.path, "/log/") { + resp.Response = rec.Body.String() + } else { + err := json.NewDecoder(rec.Body).Decode(resp) + if err != nil { + errorf("decoding response: %v", err) + return + } + } + if e, ok := t.res.(string); ok { + g, ok := resp.Response.(string) + if !ok { + errorf("Response not string: %T", resp.Response) + return + } + if g != e { + errorf("response mismatch: got %q want %q", g, e) + return + } + } + if e, ok := t.res.(*Todo); ok { + g, ok := resp.Response.(*Todo) + if !ok { + errorf("Response not *Todo: %T", resp.Response) + return + } + if e.Data == nil && g.Data != nil { + errorf("Response.Data should be nil, got: %v", g.Data) + return + } + if g.Data == nil { + errorf("Response.Data is nil, want: %v", e.Data) + return + } + gd, ok := g.Data.(*Commit) + if !ok { + errorf("Response.Data not *Commit: %T", g.Data) + return + } + if eh := e.Data.(*Commit).Hash; eh != gd.Hash { + errorf("hashes don't match: got %q, want %q", gd.Hash, eh) + return + } + } + if t.res == nil && resp.Response != nil { + errorf("response mismatch: got %q expected ", + resp.Response) + return + } + } + fmt.Fprint(w, "PASS\nYou should see only one mail notification (for 0003/linux-386) in the dev_appserver logs.") +} + +func nukeEntities(c appengine.Context, kinds []string) error { + if !appengine.IsDevAppServer() { + return errors.New("can't nuke production data") + } + var keys []*datastore.Key + for _, kind := range kinds { + q := datastore.NewQuery(kind).KeysOnly() + for t := q.Run(c); ; { + k, err := t.Next(nil) + if err == datastore.Done { + break + } + if err != nil { + return err + } + keys = append(keys, k) + } + } + return datastore.DeleteMulti(c, keys) +} diff --git a/dashboard/app/build/ui.go b/dashboard/app/build/ui.go new file mode 100644 index 00000000..9d286136 --- /dev/null +++ b/dashboard/app/build/ui.go @@ -0,0 +1,321 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO(adg): packages at weekly/release +// TODO(adg): some means to register new packages + +// +build appengine + +package build + +import ( + "bytes" + "errors" + "html/template" + "net/http" + "regexp" + "sort" + "strconv" + "strings" + + "appengine" + "appengine/datastore" + "cache" +) + +func init() { + http.HandleFunc("/", uiHandler) +} + +// uiHandler draws the build status page. +func uiHandler(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + now := cache.Now(c) + const key = "build-ui" + + page, _ := strconv.Atoi(r.FormValue("page")) + if page < 0 { + page = 0 + } + + // Used cached version of front page, if available. + if page == 0 { + var b []byte + if cache.Get(r, now, key, &b) { + w.Write(b) + return + } + } + + commits, err := goCommits(c, page) + if err != nil { + logErr(w, r, err) + return + } + builders := commitBuilders(commits, "") + + var tipState *TagState + if page == 0 { + // only show sub-repo state on first page + tipState, err = TagStateByName(c, "tip") + if err != nil { + logErr(w, r, err) + return + } + } + + p := &Pagination{} + if len(commits) == commitsPerPage { + p.Next = page + 1 + } + if page > 0 { + p.Prev = page - 1 + p.HasPrev = true + } + data := &uiTemplateData{commits, builders, tipState, p} + + var buf bytes.Buffer + if err := uiTemplate.Execute(&buf, data); err != nil { + logErr(w, r, err) + return + } + + // Cache the front page. + if page == 0 { + cache.Set(r, now, key, buf.Bytes()) + } + + buf.WriteTo(w) +} + +type Pagination struct { + Next, Prev int + HasPrev bool +} + +// goCommits gets a slice of the latest Commits to the Go repository. +// If page > 0 it paginates by commitsPerPage. +func goCommits(c appengine.Context, page int) ([]*Commit, error) { + q := datastore.NewQuery("Commit"). + Ancestor((&Package{}).Key(c)). + Order("-Num"). + Limit(commitsPerPage). + Offset(page * commitsPerPage) + var commits []*Commit + _, err := q.GetAll(c, &commits) + return commits, err +} + +// commitBuilders returns the names of the builders that provided +// Results for the provided commits. +func commitBuilders(commits []*Commit, goHash string) []string { + builders := make(map[string]bool) + for _, commit := range commits { + for _, r := range commit.Results(goHash) { + builders[r.Builder] = true + } + } + return keys(builders) +} + +func keys(m map[string]bool) (s []string) { + for k := range m { + s = append(s, k) + } + sort.Strings(s) + return +} + +// TagState represents the state of all Packages at a Tag. +type TagState struct { + Tag *Commit + Packages []*PackageState +} + +// PackageState represents the state of a Package at a Tag. +type PackageState struct { + Package *Package + Commit *Commit +} + +// TagStateByName fetches the results for all Go subrepos at the specified Tag. +func TagStateByName(c appengine.Context, name string) (*TagState, error) { + tag, err := GetTag(c, name) + if err != nil { + return nil, err + } + pkgs, err := Packages(c, "subrepo") + if err != nil { + return nil, err + } + var st TagState + for _, pkg := range pkgs { + com, err := pkg.LastCommit(c) + if err != nil { + c.Warningf("%v: no Commit found: %v", pkg, err) + continue + } + st.Packages = append(st.Packages, &PackageState{pkg, com}) + } + st.Tag, err = tag.Commit(c) + if err != nil { + return nil, err + } + return &st, nil +} + +type uiTemplateData struct { + Commits []*Commit + Builders []string + TipState *TagState + Pagination *Pagination +} + +var uiTemplate = template.Must( + template.New("ui.html").Funcs(tmplFuncs).ParseFiles("build/ui.html"), +) + +var tmplFuncs = template.FuncMap{ + "builderOS": builderOS, + "builderArch": builderArch, + "builderArchShort": builderArchShort, + "builderArchChar": builderArchChar, + "builderTitle": builderTitle, + "builderSpans": builderSpans, + "repoURL": repoURL, + "shortDesc": shortDesc, + "shortHash": shortHash, + "shortUser": shortUser, + "tail": tail, +} + +func splitDash(s string) (string, string) { + i := strings.Index(s, "-") + if i >= 0 { + return s[:i], s[i+1:] + } + return s, "" +} + +// builderOS returns the os tag for a builder string +func builderOS(s string) string { + os, _ := splitDash(s) + return os +} + +// builderArch returns the arch tag for a builder string +func builderArch(s string) string { + _, arch := splitDash(s) + arch, _ = splitDash(arch) // chop third part + return arch +} + +// builderArchShort returns a short arch tag for a builder string +func builderArchShort(s string) string { + if strings.Contains(s+"-", "-race-") { + return "race" + } + arch := builderArch(s) + switch arch { + case "amd64": + return "x64" + } + return arch +} + +// builderArchChar returns the architecture letter for a builder string +func builderArchChar(s string) string { + arch := builderArch(s) + switch arch { + case "386": + return "8" + case "amd64": + return "6" + case "arm": + return "5" + } + return arch +} + +type builderSpan struct { + N int + OS string +} + +// builderSpans creates a list of tags showing +// the builder's operating system names, spanning +// the appropriate number of columns. +func builderSpans(s []string) []builderSpan { + var sp []builderSpan + for len(s) > 0 { + i := 1 + os := builderOS(s[0]) + for i < len(s) && builderOS(s[i]) == os { + i++ + } + sp = append(sp, builderSpan{i, os}) + s = s[i:] + } + return sp +} + +// builderTitle formats "linux-amd64-foo" as "linux amd64 foo". +func builderTitle(s string) string { + return strings.Replace(s, "-", " ", -1) +} + +// shortDesc returns the first line of a description. +func shortDesc(desc string) string { + if i := strings.Index(desc, "\n"); i != -1 { + desc = desc[:i] + } + return desc +} + +// shortHash returns a short version of a hash. +func shortHash(hash string) string { + if len(hash) > 12 { + hash = hash[:12] + } + return hash +} + +// shortUser returns a shortened version of a user string. +func shortUser(user string) string { + if i, j := strings.Index(user, "<"), strings.Index(user, ">"); 0 <= i && i < j { + user = user[i+1 : j] + } + if i := strings.Index(user, "@"); i >= 0 { + return user[:i] + } + return user +} + +// repoRe matches Google Code repositories and subrepositories (without paths). +var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+)(\.[a-z0-9\-]+)?$`) + +// repoURL returns the URL of a change at a Google Code repository or subrepo. +func repoURL(hash, packagePath string) (string, error) { + if packagePath == "" { + return "https://code.google.com/p/go/source/detail?r=" + hash, nil + } + m := repoRe.FindStringSubmatch(packagePath) + if m == nil { + return "", errors.New("unrecognized package: " + packagePath) + } + url := "https://code.google.com/p/" + m[1] + "/source/detail?r=" + hash + if len(m) > 2 { + url += "&repo=" + m[2][1:] + } + return url, nil +} + +// tail returns the trailing n lines of s. +func tail(n int, s string) string { + lines := strings.Split(s, "\n") + if len(lines) < n { + return s + } + return strings.Join(lines[len(lines)-n:], "\n") +} diff --git a/dashboard/app/build/ui.html b/dashboard/app/build/ui.html new file mode 100644 index 00000000..5b5f4ebe --- /dev/null +++ b/dashboard/app/build/ui.html @@ -0,0 +1,209 @@ + + + + Go Build Dashboard + + + + +

Go Build Status

+ + {{if $.Commits}} + + + + {{range $.Builders | builderSpans}} + + {{end}} + + + + + + + + + {{range $.Builders | builderSpans}} + + {{end}} + + + + + + + {{range $.Builders}} + + {{end}} + + {{range $c := $.Commits}} + + + {{range $.Builders}} + + {{end}} + + + + + {{end}} +
 {{.OS}}
 {{builderArchShort .}}
{{shortHash .Hash}} + {{with $c.Result . ""}} + {{if .OK}} + ok + {{else}} + fail + {{end}} + {{else}} +   + {{end}} + {{shortUser .User}}{{.Time.Format "Mon 02 Jan 15:04"}}{{shortDesc .Desc}}
+ + {{with $.Pagination}} +
+ newer + older + latest +
+ {{end}} + + {{else}} +

No commits to display. Hm.

+ {{end}} + + {{with $.TipState}} + {{$goHash := .Tag.Hash}} +

+ Sub-repositories at tip + ({{shortHash .Tag.Hash}}) +

+ + + + + {{range $.Builders | builderSpans}} + + {{end}} + + + + + + + + + + {{range $.Builders | builderSpans}} + + {{end}} + + + + + + + + {{range $.Builders}} + + {{end}} + + + + + {{range $pkg := .Packages}} + + + + {{range $.Builders}} + + {{end}} + {{with $pkg.Commit}} + + + + {{end}} + + {{end}} +
{{.OS}}
{{builderArchShort .}}
{{.Package.Name}} + {{$h := $pkg.Commit.Hash}} + {{shortHash $h}} + + {{with $pkg.Commit.Result . $goHash}} + {{if .OK}} + ok + {{else}} + fail + {{end}} + {{else}} +   + {{end}} + {{shortUser .User}}{{.Time.Format "Mon 02 Jan 15:04"}}{{shortDesc .Desc}}
+ {{end}} + + + diff --git a/dashboard/app/cache/cache.go b/dashboard/app/cache/cache.go new file mode 100644 index 00000000..3c6a5c63 --- /dev/null +++ b/dashboard/app/cache/cache.go @@ -0,0 +1,84 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package cache + +import ( + "fmt" + "net/http" + "time" + + "appengine" + "appengine/memcache" +) + +const ( + nocache = "nocache" + timeKey = "cachetime" + expiry = 600 // 10 minutes +) + +func newTime() uint64 { return uint64(time.Now().Unix()) << 32 } + +// Now returns the current logical datastore time to use for cache lookups. +func Now(c appengine.Context) uint64 { + t, err := memcache.Increment(c, timeKey, 0, newTime()) + if err != nil { + c.Errorf("cache.Now: %v", err) + return 0 + } + return t +} + +// Tick sets the current logical datastore time to a never-before-used time +// and returns that time. It should be called to invalidate the cache. +func Tick(c appengine.Context) uint64 { + t, err := memcache.Increment(c, timeKey, 1, newTime()) + if err != nil { + c.Errorf("cache.Tick: %v", err) + return 0 + } + return t +} + +// Get fetches data for name at time now from memcache and unmarshals it into +// value. It reports whether it found the cache record and logs any errors to +// the admin console. +func Get(r *http.Request, now uint64, name string, value interface{}) bool { + if now == 0 || r.FormValue(nocache) != "" { + return false + } + c := appengine.NewContext(r) + key := fmt.Sprintf("%s.%d", name, now) + _, err := memcache.JSON.Get(c, key, value) + if err == nil { + c.Debugf("cache hit %q", key) + return true + } + c.Debugf("cache miss %q", key) + if err != memcache.ErrCacheMiss { + c.Errorf("get cache %q: %v", key, err) + } + return false +} + +// Set puts value into memcache under name at time now. +// It logs any errors to the admin console. +func Set(r *http.Request, now uint64, name string, value interface{}) { + if now == 0 || r.FormValue(nocache) != "" { + return + } + c := appengine.NewContext(r) + key := fmt.Sprintf("%s.%d", name, now) + err := memcache.JSON.Set(c, &memcache.Item{ + Key: key, + Object: value, + Expiration: expiry, + }) + if err != nil { + c.Errorf("set cache %q: %v", key, err) + } +} diff --git a/dashboard/app/static/status_alert.gif b/dashboard/app/static/status_alert.gif new file mode 100644 index 0000000000000000000000000000000000000000..495d9d2e0c7a7570c22c9a1ad3d673a789636962 GIT binary patch literal 570 zcmZ?wbhEHb6kyuAo;Y{m?NP^f$DCj8HGa9*MAyab&qL;?Tjic^ReiBbYHq9gzsHPUF7iIx zuKE7B(7U6;Pq(W5y2JK-r_|3|>|ZZ)pIRz=ZiVpgd#o>Z=|9=5`1+vr@B1tpXGx!4 zuJ_@j;MXhM#W||CHc36%BL8HQ;?pe(zwWZWIc)pkw7|b7j8C_ye7nYVVxIJu3%o^{ z>aX^hKHaAN_c7D^Q=aoXfkuhF*scF?pYZDg7BBaiygq2PVTR1pE%I*;ifUVHTe@g5 zFfe@k_6QHc*@}uupIBZE9|5ZS&>w^5SZ5;q_+K@#gJqV$?Dd zac7*$&8}*vDaFp+E9@-k-Q;D+CT1(m d)Z(Gw#N5Q7+Qgs{d1%8zXVo2AvRo_-)&MQZtHJ;P literal 0 HcmV?d00001 diff --git a/dashboard/app/static/status_good.gif b/dashboard/app/static/status_good.gif new file mode 100644 index 0000000000000000000000000000000000000000..ef9c5a8f6458b03f0ae08209ad17ed2ec3a563e2 GIT binary patch literal 328 zcmZ?wbhEHb6kyde{eH*bbU#TwgMWzPz(X=qAH&+P2&fBNj1f4|-0Wy@Dy zty*=VCS{Iq`-ZfD1yv;Ne0#yyYIKp-_(BU`JCVc-=j|_+j=;B`t&JjmbZzW zb>*bQ)&mLtb&eK2T1TEt+;Fq5e0h{L(416bD@T_No44qin1B2BjR6BF{$ycfV6bG+ z0m*>;#K4w%U`~ODj#M`XGhey|U;Z|!HO 0 { + cmd += "?" + args.Encode() + } + switch meth { + case "GET": + if req != nil { + log.Panicf("%s to %s with req", meth, cmd) + } + r, err = http.Get(cmd) + case "POST": + var body io.Reader + if req != nil { + b, err := json.Marshal(req) + if err != nil { + return err + } + body = bytes.NewBuffer(b) + } + r, err = http.Post(cmd, "text/json", body) + default: + log.Panicf("%s: invalid method %q", cmd, meth) + panic("invalid method: " + meth) + } + if err != nil { + return err + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return fmt.Errorf("bad http response: %v", r.Status) + } + body := new(bytes.Buffer) + if _, err := body.ReadFrom(r.Body); err != nil { + return err + } + + // Read JSON-encoded Response into provided resp + // and return an error if present. + var result = struct { + Response interface{} + Error string + }{ + // Put the provided resp in here as it can be a pointer to + // some value we should unmarshal into. + Response: resp, + } + if err = json.Unmarshal(body.Bytes(), &result); err != nil { + log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err) + return err + } + if result.Error != "" { + return errors.New(result.Error) + } + + return nil +} + +// todo returns the next hash to build. +func (b *Builder) todo(kind, pkg, goHash string) (rev string, err error) { + args := url.Values{ + "kind": {kind}, + "builder": {b.name}, + "packagePath": {pkg}, + "goHash": {goHash}, + } + var resp *struct { + Kind string + Data struct { + Hash string + } + } + if err = dash("GET", "todo", args, nil, &resp); err != nil { + return "", err + } + if resp == nil { + return "", nil + } + if kind != resp.Kind { + return "", fmt.Errorf("expecting Kind %q, got %q", kind, resp.Kind) + } + return resp.Data.Hash, nil +} + +// recordResult sends build results to the dashboard +func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runTime time.Duration) error { + req := obj{ + "Builder": b.name, + "PackagePath": pkg, + "Hash": hash, + "GoHash": goHash, + "OK": ok, + "Log": buildLog, + "RunTime": runTime, + } + args := url.Values{"key": {b.key}, "builder": {b.name}} + return dash("POST", "result", args, req, nil) +} + +func postCommit(key, pkg string, l *HgLog) error { + t, err := time.Parse(time.RFC3339, l.Date) + if err != nil { + return fmt.Errorf("parsing %q: %v", l.Date, t) + } + return dash("POST", "commit", url.Values{"key": {key}}, obj{ + "PackagePath": pkg, + "Hash": l.Hash, + "ParentHash": l.Parent, + "Time": t.Format(time.RFC3339), + "User": l.Author, + "Desc": l.Desc, + }, nil) +} + +func dashboardCommit(pkg, hash string) bool { + err := dash("GET", "commit", url.Values{ + "packagePath": {pkg}, + "hash": {hash}, + }, nil, nil) + return err == nil +} + +func dashboardPackages(kind string) []string { + args := url.Values{"kind": []string{kind}} + var resp []struct { + Path string + } + if err := dash("GET", "packages", args, nil, &resp); err != nil { + log.Println("dashboardPackages:", err) + return nil + } + var pkgs []string + for _, r := range resp { + pkgs = append(pkgs, r.Path) + } + return pkgs +} diff --git a/dashboard/builder/main.go b/dashboard/builder/main.go new file mode 100644 index 00000000..6ef357ad --- /dev/null +++ b/dashboard/builder/main.go @@ -0,0 +1,653 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" +) + +const ( + codeProject = "go" + codePyScript = "misc/dashboard/googlecode_upload.py" + hgUrl = "https://code.google.com/p/go/" + mkdirPerm = 0750 + waitInterval = 30 * time.Second // time to wait before checking for new revs + pkgBuildInterval = 24 * time.Hour // rebuild packages every 24 hours +) + +// These variables are copied from the gobuilder's environment +// to the envv of its subprocesses. +var extraEnv = []string{ + "GOARM", + + // For Unix derivatives. + "CC", + "PATH", + "TMPDIR", + "USER", + + // For Plan 9. + "objtype", + "cputype", + "path", +} + +type Builder struct { + goroot *Repo + name string + goos, goarch string + key string +} + +var ( + buildroot = flag.String("buildroot", defaultBuildRoot(), "Directory under which to build") + dashboard = flag.String("dashboard", "build.golang.org", "Go Dashboard Host") + buildRelease = flag.Bool("release", false, "Build and upload binary release archives") + buildRevision = flag.String("rev", "", "Build specified revision and exit") + buildCmd = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)") + failAll = flag.Bool("fail", false, "fail all builds") + parallel = flag.Bool("parallel", false, "Build multiple targets in parallel") + buildTimeout = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests") + cmdTimeout = flag.Duration("cmdTimeout", 10*time.Minute, "Maximum time to wait for an external command") + commitInterval = flag.Duration("commitInterval", 1*time.Minute, "Time to wait between polling for new commits (0 disables commit poller)") + verbose = flag.Bool("v", false, "verbose") +) + +var ( + binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`) + releaseRe = regexp.MustCompile(`^release\.r[0-9\-.]+`) + allCmd = "all" + suffix + raceCmd = "race" + suffix + cleanCmd = "clean" + suffix + suffix = defaultSuffix() +) + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0]) + flag.PrintDefaults() + os.Exit(2) + } + flag.Parse() + if len(flag.Args()) == 0 { + flag.Usage() + } + goroot := &Repo{ + Path: filepath.Join(*buildroot, "goroot"), + } + + // set up work environment, use existing enviroment if possible + if goroot.Exists() || *failAll { + log.Print("Found old workspace, will use it") + } else { + if err := os.RemoveAll(*buildroot); err != nil { + log.Fatalf("Error removing build root (%s): %s", *buildroot, err) + } + if err := os.Mkdir(*buildroot, mkdirPerm); err != nil { + log.Fatalf("Error making build root (%s): %s", *buildroot, err) + } + var err error + goroot, err = RemoteRepo(hgUrl).Clone(goroot.Path, "tip") + if err != nil { + log.Fatal("Error cloning repository:", err) + } + } + + // set up builders + builders := make([]*Builder, len(flag.Args())) + for i, name := range flag.Args() { + b, err := NewBuilder(goroot, name) + if err != nil { + log.Fatal(err) + } + builders[i] = b + } + + if *failAll { + failMode(builders) + return + } + + // if specified, build revision and return + if *buildRevision != "" { + hash, err := goroot.FullHash(*buildRevision) + if err != nil { + log.Fatal("Error finding revision: ", err) + } + for _, b := range builders { + if err := b.buildHash(hash); err != nil { + log.Println(err) + } + } + return + } + + // Start commit watcher + go commitWatcher(goroot) + + // go continuous build mode + // check for new commits and build them + for { + built := false + t := time.Now() + if *parallel { + done := make(chan bool) + for _, b := range builders { + go func(b *Builder) { + done <- b.build() + }(b) + } + for _ = range builders { + built = <-done || built + } + } else { + for _, b := range builders { + built = b.build() || built + } + } + // sleep if there was nothing to build + if !built { + time.Sleep(waitInterval) + } + // sleep if we're looping too fast. + dt := time.Now().Sub(t) + if dt < waitInterval { + time.Sleep(waitInterval - dt) + } + } +} + +// go continuous fail mode +// check for new commits and FAIL them +func failMode(builders []*Builder) { + for { + built := false + for _, b := range builders { + built = b.failBuild() || built + } + // stop if there was nothing to fail + if !built { + break + } + } +} + +func NewBuilder(goroot *Repo, name string) (*Builder, error) { + b := &Builder{ + goroot: goroot, + name: name, + } + + // get goos/goarch from builder string + s := strings.SplitN(b.name, "-", 3) + if len(s) >= 2 { + b.goos, b.goarch = s[0], s[1] + } else { + return nil, fmt.Errorf("unsupported builder form: %s", name) + } + + // read keys from keyfile + fn := "" + if runtime.GOOS == "windows" { + fn = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") + } else { + fn = os.Getenv("HOME") + } + fn = filepath.Join(fn, ".gobuildkey") + if s := fn + "-" + b.name; isFile(s) { // builder-specific file + fn = s + } + c, err := ioutil.ReadFile(fn) + if err != nil { + return nil, fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err) + } + b.key = string(bytes.TrimSpace(bytes.SplitN(c, []byte("\n"), 2)[0])) + return b, nil +} + +// buildCmd returns the build command to invoke. +// Builders which contain the string '-race' in their +// name will override *buildCmd and return raceCmd. +func (b *Builder) buildCmd() string { + if strings.Contains(b.name, "-race") { + return raceCmd + } + return *buildCmd +} + +// build checks for a new commit for this builder +// and builds it if one is found. +// It returns true if a build was attempted. +func (b *Builder) build() bool { + hash, err := b.todo("build-go-commit", "", "") + if err != nil { + log.Println(err) + return false + } + if hash == "" { + return false + } + + if err := b.buildHash(hash); err != nil { + log.Println(err) + } + return true +} + +func (b *Builder) buildHash(hash string) error { + log.Println(b.name, "building", hash) + + // create place in which to do work + workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12]) + if err := os.Mkdir(workpath, mkdirPerm); err != nil { + return err + } + defer os.RemoveAll(workpath) + + // pull before cloning to ensure we have the revision + if err := b.goroot.Pull(); err != nil { + return err + } + + // clone repo at specified revision + if _, err := b.goroot.Clone(filepath.Join(workpath, "go"), hash); err != nil { + return err + } + + srcDir := filepath.Join(workpath, "go", "src") + + // build + var buildlog bytes.Buffer + logfile := filepath.Join(workpath, "build.log") + f, err := os.Create(logfile) + if err != nil { + return err + } + defer f.Close() + w := io.MultiWriter(f, &buildlog) + + cmd := b.buildCmd() + if !filepath.IsAbs(cmd) { + cmd = filepath.Join(srcDir, cmd) + } + startTime := time.Now() + ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, cmd) + runTime := time.Now().Sub(startTime) + errf := func() string { + if err != nil { + return fmt.Sprintf("error: %v", err) + } + if !ok { + return "failed" + } + return "success" + } + fmt.Fprintf(w, "Build complete, duration %v. Result: %v\n", runTime, errf()) + + if err != nil || !ok { + // record failure + return b.recordResult(false, "", hash, "", buildlog.String(), runTime) + } + + // record success + if err = b.recordResult(true, "", hash, "", "", runTime); err != nil { + return fmt.Errorf("recordResult: %s", err) + } + + // build Go sub-repositories + goRoot := filepath.Join(workpath, "go") + goPath := workpath + b.buildSubrepos(goRoot, goPath, hash) + + return nil +} + +// failBuild checks for a new commit for this builder +// and fails it if one is found. +// It returns true if a build was "attempted". +func (b *Builder) failBuild() bool { + hash, err := b.todo("build-go-commit", "", "") + if err != nil { + log.Println(err) + return false + } + if hash == "" { + return false + } + + log.Printf("fail %s %s\n", b.name, hash) + + if err := b.recordResult(false, "", hash, "", "auto-fail mode run by "+os.Getenv("USER"), 0); err != nil { + log.Print(err) + } + return true +} + +func (b *Builder) buildSubrepos(goRoot, goPath, goHash string) { + for _, pkg := range dashboardPackages("subrepo") { + // get the latest todo for this package + hash, err := b.todo("build-package", pkg, goHash) + if err != nil { + log.Printf("buildSubrepos %s: %v", pkg, err) + continue + } + if hash == "" { + continue + } + + // build the package + if *verbose { + log.Printf("buildSubrepos %s: building %q", pkg, hash) + } + buildLog, err := b.buildSubrepo(goRoot, goPath, pkg, hash) + if err != nil { + if buildLog == "" { + buildLog = err.Error() + } + log.Printf("buildSubrepos %s: %v", pkg, err) + } + + // record the result + err = b.recordResult(err == nil, pkg, hash, goHash, buildLog, 0) + if err != nil { + log.Printf("buildSubrepos %s: %v", pkg, err) + } + } +} + +// buildSubrepo fetches the given package, updates it to the specified hash, +// and runs 'go test -short pkg/...'. It returns the build log and any error. +func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error) { + goTool := filepath.Join(goRoot, "bin", "go") + env := append(b.envv(), "GOROOT="+goRoot, "GOPATH="+goPath) + + // add $GOROOT/bin and $GOPATH/bin to PATH + for i, e := range env { + const p = "PATH=" + if !strings.HasPrefix(e, p) { + continue + } + sep := string(os.PathListSeparator) + env[i] = p + filepath.Join(goRoot, "bin") + sep + filepath.Join(goPath, "bin") + sep + e[len(p):] + } + + // fetch package and dependencies + log, ok, err := runLog(*cmdTimeout, env, goPath, goTool, "get", "-d", pkg+"/...") + if err == nil && !ok { + err = fmt.Errorf("go exited with status 1") + } + if err != nil { + return log, err + } + + // hg update to the specified hash + repo := Repo{Path: filepath.Join(goPath, "src", pkg)} + if err := repo.UpdateTo(hash); err != nil { + return "", err + } + + // test the package + log, ok, err = runLog(*buildTimeout, env, goPath, goTool, "test", "-short", pkg+"/...") + if err == nil && !ok { + err = fmt.Errorf("go exited with status 1") + } + return log, err +} + +// envv returns an environment for build/bench execution +func (b *Builder) envv() []string { + if runtime.GOOS == "windows" { + return b.envvWindows() + } + e := []string{ + "GOOS=" + b.goos, + "GOHOSTOS=" + b.goos, + "GOARCH=" + b.goarch, + "GOHOSTARCH=" + b.goarch, + "GOROOT_FINAL=/usr/local/go", + } + for _, k := range extraEnv { + if s, ok := getenvOk(k); ok { + e = append(e, k+"="+s) + } + } + return e +} + +// windows version of envv +func (b *Builder) envvWindows() []string { + start := map[string]string{ + "GOOS": b.goos, + "GOHOSTOS": b.goos, + "GOARCH": b.goarch, + "GOHOSTARCH": b.goarch, + "GOROOT_FINAL": `c:\go`, + "GOBUILDEXIT": "1", // exit all.bat with completion status. + } + for _, name := range extraEnv { + if s, ok := getenvOk(name); ok { + start[name] = s + } + } + skip := map[string]bool{ + "GOBIN": true, + "GOROOT": true, + "INCLUDE": true, + "LIB": true, + } + var e []string + for name, v := range start { + e = append(e, name+"="+v) + skip[name] = true + } + for _, kv := range os.Environ() { + s := strings.SplitN(kv, "=", 2) + name := strings.ToUpper(s[0]) + switch { + case name == "": + // variables, like "=C:=C:\", just copy them + e = append(e, kv) + case !skip[name]: + e = append(e, kv) + skip[name] = true + } + } + return e +} + +func isDirectory(name string) bool { + s, err := os.Stat(name) + return err == nil && s.IsDir() +} + +func isFile(name string) bool { + s, err := os.Stat(name) + return err == nil && !s.IsDir() +} + +// commitWatcher polls hg for new commits and tells the dashboard about them. +func commitWatcher(goroot *Repo) { + if *commitInterval == 0 { + log.Printf("commitInterval is %s, disabling commitWatcher", *commitInterval) + return + } + // Create builder just to get master key. + b, err := NewBuilder(goroot, "mercurial-commit") + if err != nil { + log.Fatal(err) + } + key := b.key + + for { + if *verbose { + log.Printf("poll...") + } + // Main Go repository. + commitPoll(goroot, "", key) + // Go sub-repositories. + for _, pkg := range dashboardPackages("subrepo") { + pkgroot := &Repo{ + Path: filepath.Join(*buildroot, pkg), + } + commitPoll(pkgroot, pkg, key) + } + if *verbose { + log.Printf("sleep...") + } + time.Sleep(*commitInterval) + } +} + +// logByHash is a cache of all Mercurial revisions we know about, +// indexed by full hash. +var logByHash = map[string]*HgLog{} + +// commitPoll pulls any new revisions from the hg server +// and tells the server about them. +func commitPoll(repo *Repo, pkg, key string) { + if !repo.Exists() { + var err error + repo, err = RemoteRepo(repoURL(pkg)).Clone(repo.Path, "tip") + if err != nil { + log.Printf("%s: hg clone failed: %v", pkg, err) + if err := os.RemoveAll(repo.Path); err != nil { + log.Printf("%s: %v", pkg, err) + } + } + return + } + + logs, err := repo.Log() // repo.Log calls repo.Pull internally + if err != nil { + log.Printf("hg log: %v", err) + return + } + + // Pass 1. Fill in parents and add new log entries to logsByHash. + // Empty parent means take parent from next log entry. + // Non-empty parent has form 1234:hashhashhash; we want full hash. + for i := range logs { + l := &logs[i] + if l.Parent == "" && i+1 < len(logs) { + l.Parent = logs[i+1].Hash + } else if l.Parent != "" { + l.Parent, _ = repo.FullHash(l.Parent) + } + if *verbose { + log.Printf("hg log %s: %s < %s\n", pkg, l.Hash, l.Parent) + } + if logByHash[l.Hash] == nil { + // Make copy to avoid pinning entire slice when only one entry is new. + t := *l + logByHash[t.Hash] = &t + } + } + + for _, l := range logs { + addCommit(pkg, l.Hash, key) + } +} + +// addCommit adds the commit with the named hash to the dashboard. +// key is the secret key for authentication to the dashboard. +// It avoids duplicate effort. +func addCommit(pkg, hash, key string) bool { + l := logByHash[hash] + if l == nil { + return false + } + if l.added { + return true + } + + // Check for already added, perhaps in an earlier run. + if dashboardCommit(pkg, hash) { + log.Printf("%s already on dashboard\n", hash) + // Record that this hash is on the dashboard, + // as must be all its parents. + for l != nil { + l.added = true + l = logByHash[l.Parent] + } + return true + } + + // Create parent first, to maintain some semblance of order. + if l.Parent != "" { + if !addCommit(pkg, l.Parent, key) { + return false + } + } + + // Create commit. + if err := postCommit(key, pkg, l); err != nil { + log.Printf("failed to add %s to dashboard: %v", key, err) + return false + } + return true +} + +var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`) + +// repoURL returns the repository URL for the supplied import path. +func repoURL(importPath string) string { + m := repoRe.FindStringSubmatch(importPath) + if len(m) < 2 { + log.Printf("repoURL: couldn't decipher %q", importPath) + return "" + } + return "https://code.google.com/p/" + m[1] +} + +// defaultSuffix returns file extension used for command files in +// current os environment. +func defaultSuffix() string { + switch runtime.GOOS { + case "windows": + return ".bat" + case "plan9": + return ".rc" + default: + return ".bash" + } +} + +// defaultBuildRoot returns default buildroot directory. +func defaultBuildRoot() string { + var d string + if runtime.GOOS == "windows" { + // will use c:\, otherwise absolute paths become too long + // during builder run, see http://golang.org/issue/3358. + d = `c:\` + } else { + d = os.TempDir() + } + return filepath.Join(d, "gobuilder") +} + +func getenvOk(k string) (v string, ok bool) { + v = os.Getenv(k) + if v != "" { + return v, true + } + keq := k + "=" + for _, kv := range os.Environ() { + if kv == keq { + return "", true + } + } + return "", false +} diff --git a/dashboard/builder/vcs.go b/dashboard/builder/vcs.go new file mode 100644 index 00000000..63198a34 --- /dev/null +++ b/dashboard/builder/vcs.go @@ -0,0 +1,148 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/xml" + "fmt" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "sync" +) + +// Repo represents a mercurial repository. +type Repo struct { + Path string + sync.Mutex +} + +// RemoteRepo constructs a *Repo representing a remote repository. +func RemoteRepo(url string) *Repo { + return &Repo{ + Path: url, + } +} + +// Clone clones the current Repo to a new destination +// returning a new *Repo if successful. +func (r *Repo) Clone(path, rev string) (*Repo, error) { + r.Lock() + defer r.Unlock() + if err := run(*cmdTimeout, nil, *buildroot, r.hgCmd("clone", "-r", rev, r.Path, path)...); err != nil { + return nil, err + } + return &Repo{ + Path: path, + }, nil +} + +// UpdateTo updates the working copy of this Repo to the +// supplied revision. +func (r *Repo) UpdateTo(hash string) error { + r.Lock() + defer r.Unlock() + return run(*cmdTimeout, nil, r.Path, r.hgCmd("update", hash)...) +} + +// Exists reports whether this Repo represents a valid Mecurial repository. +func (r *Repo) Exists() bool { + fi, err := os.Stat(filepath.Join(r.Path, ".hg")) + if err != nil { + return false + } + return fi.IsDir() +} + +// Pull pulls changes from the default path, that is, the path +// this Repo was cloned from. +func (r *Repo) Pull() error { + r.Lock() + defer r.Unlock() + return run(*cmdTimeout, nil, r.Path, r.hgCmd("pull")...) +} + +// Log returns the changelog for this repository. +func (r *Repo) Log() ([]HgLog, error) { + if err := r.Pull(); err != nil { + return nil, err + } + const N = 50 // how many revisions to grab + + r.Lock() + defer r.Unlock() + data, _, err := runLog(*cmdTimeout, nil, r.Path, r.hgCmd("log", + "--encoding=utf-8", + "--limit="+strconv.Itoa(N), + "--template="+xmlLogTemplate)..., + ) + if err != nil { + return nil, err + } + + var logStruct struct { + Log []HgLog + } + err = xml.Unmarshal([]byte(""+data+""), &logStruct) + if err != nil { + log.Printf("unmarshal hg log: %v", err) + return nil, err + } + return logStruct.Log, nil +} + +// FullHash returns the full hash for the given Mercurial revision. +func (r *Repo) FullHash(rev string) (string, error) { + r.Lock() + defer r.Unlock() + s, _, err := runLog(*cmdTimeout, nil, r.Path, + r.hgCmd("log", + "--encoding=utf-8", + "--rev="+rev, + "--limit=1", + "--template={node}")..., + ) + if err != nil { + return "", nil + } + s = strings.TrimSpace(s) + if s == "" { + return "", fmt.Errorf("cannot find revision") + } + if len(s) != 40 { + return "", fmt.Errorf("hg returned invalid hash " + s) + } + return s, nil +} + +func (r *Repo) hgCmd(args ...string) []string { + return append([]string{"hg", "--config", "extensions.codereview=!"}, args...) +} + +// HgLog represents a single Mercurial revision. +type HgLog struct { + Hash string + Author string + Date string + Desc string + Parent string + + // Internal metadata + added bool +} + +// xmlLogTemplate is a template to pass to Mercurial to make +// hg log print the log in valid XML for parsing with xml.Unmarshal. +const xmlLogTemplate = ` + + {node|escape} + {parent|escape} + {author|escape} + {date|rfc3339date} + {desc|escape} + +`