dashboard: server app changes for performance dashboard

This CL moves code from code.google.com/p/dvyukov-go-perf-dashboard,
which was previously reviewed.
UI part will be submitted separately.

LGTM=adg
R=adg
CC=golang-codereviews
https://golang.org/cl/97260043

dashboard/app/build/build.go

@@ -14,14 +14,22 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 
 	"appengine"
 	"appengine/datastore"
+
+	"cache"
 )
 
-const maxDatastoreStringLen = 500
+const (
+	maxDatastoreStringLen = 500
+	PerfRunLength         = 1024
+)
 
 // A Package describes a package that is listed on the dashboard.
 type Package struct {
@@ -92,6 +100,7 @@ type Commit struct {
 	User              string
 	Desc              string `datastore:",noindex"`
 	Time              time.Time
+	NeedsBenchmarking bool
 
 	// ResultData is the Data string of each build Result for this Commit.
 	// For non-Go commits, only the Results for the current Go tip, weekly,
@@ -99,6 +108,10 @@ type Commit struct {
 	// The complete data set is stored in Result entities.
 	ResultData []string `datastore:",noindex"`
 
+	// PerfResults holds a set of “builder|benchmark” tuples denoting
+	// what benchmarks have been executed on the commit.
+	PerfResults []string `datastore:",noindex"`
+
 	FailNotificationSent bool
 }
 
@@ -138,6 +151,28 @@ func (com *Commit) AddResult(c appengine.Context, r *Result) error {
 	return nil
 }
 
+// AddPerfResult remembers that the builder has run the benchmark on the commit.
+// It must be called from inside a datastore transaction.
+func (com *Commit) AddPerfResult(c appengine.Context, builder, benchmark string) error {
+	if err := datastore.Get(c, com.Key(c), com); err != nil {
+		return fmt.Errorf("getting Commit: %v", err)
+	}
+	if !com.NeedsBenchmarking {
+		return fmt.Errorf("trying to add perf result to Commit(%v) that does not require benchmarking", com.Hash)
+	}
+	s := builder + "|" + benchmark
+	for _, v := range com.PerfResults {
+		if v == s {
+			return nil
+		}
+	}
+	com.PerfResults = append(com.PerfResults, s)
+	if _, err := datastore.Put(c, com.Key(c), com); err != nil {
+		return fmt.Errorf("putting Commit: %v", err)
+	}
+	return nil
+}
+
 func trim(s []string, n int) []string {
 	l := min(len(s), n)
 	return s[len(s)-l:]
@@ -207,6 +242,94 @@ func reverse(s []string) {
 	}
 }
 
+// A CommitRun provides summary information for commits [StartCommitNum, StartCommitNum + PerfRunLength).
+// Descendant of Package.
+type CommitRun struct {
+	PackagePath       string // (empty for main repo commits)
+	StartCommitNum    int
+	Hash              []string    `datastore:",noindex"`
+	User              []string    `datastore:",noindex"`
+	Desc              []string    `datastore:",noindex"` // Only first line.
+	Time              []time.Time `datastore:",noindex"`
+	NeedsBenchmarking []bool      `datastore:",noindex"`
+}
+
+func (cr *CommitRun) Key(c appengine.Context) *datastore.Key {
+	p := Package{Path: cr.PackagePath}
+	key := strconv.Itoa(cr.StartCommitNum)
+	return datastore.NewKey(c, "CommitRun", key, 0, p.Key(c))
+}
+
+// GetCommitRun loads and returns CommitRun that contains information
+// for commit commitNum.
+func GetCommitRun(c appengine.Context, commitNum int) (*CommitRun, error) {
+	cr := &CommitRun{StartCommitNum: commitNum / PerfRunLength * PerfRunLength}
+	err := datastore.Get(c, cr.Key(c), cr)
+	if err != nil && err != datastore.ErrNoSuchEntity {
+		return nil, fmt.Errorf("getting CommitRun: %v", err)
+	}
+	if len(cr.Hash) != PerfRunLength {
+		cr.Hash = make([]string, PerfRunLength)
+		cr.User = make([]string, PerfRunLength)
+		cr.Desc = make([]string, PerfRunLength)
+		cr.Time = make([]time.Time, PerfRunLength)
+		cr.NeedsBenchmarking = make([]bool, PerfRunLength)
+	}
+	return cr, nil
+}
+
+func (cr *CommitRun) AddCommit(c appengine.Context, com *Commit) error {
+	if com.Num < cr.StartCommitNum || com.Num >= cr.StartCommitNum+PerfRunLength {
+		return fmt.Errorf("AddCommit: commit num %v out of range [%v, %v)",
+			com.Num, cr.StartCommitNum, cr.StartCommitNum+PerfRunLength)
+	}
+	i := com.Num - cr.StartCommitNum
+	// Be careful with string lengths,
+	// we need to fit 1024 commits into 1 MB.
+	cr.Hash[i] = com.Hash
+	cr.User[i] = shortDesc(com.User)
+	cr.Desc[i] = shortDesc(com.Desc)
+	cr.Time[i] = com.Time
+	cr.NeedsBenchmarking[i] = com.NeedsBenchmarking
+	if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
+		return fmt.Errorf("putting CommitRun: %v", err)
+	}
+	return nil
+}
+
+// GetCommits returns [startCommitNum, startCommitNum+n) commits.
+// Commits information is partial (obtained from CommitRun),
+// do not store them back into datastore.
+func GetCommits(c appengine.Context, startCommitNum, n int) ([]*Commit, error) {
+	if startCommitNum < 0 || n <= 0 {
+		return nil, fmt.Errorf("GetCommits: invalid args (%v, %v)", startCommitNum, n)
+	}
+	var res []*Commit
+	for n > 0 {
+		cr, err := GetCommitRun(c, startCommitNum)
+		if err != nil {
+			return nil, err
+		}
+		idx := startCommitNum - cr.StartCommitNum
+		cnt := PerfRunLength - idx
+		if cnt > n {
+			cnt = n
+		}
+		for i := idx; i < idx+cnt; i++ {
+			com := new(Commit)
+			com.Hash = cr.Hash[i]
+			com.User = cr.User[i]
+			com.Desc = cr.Desc[i]
+			com.Time = cr.Time[i]
+			com.NeedsBenchmarking = cr.NeedsBenchmarking[i]
+			res = append(res, com)
+		}
+		startCommitNum += cnt
+		n -= cnt
+	}
+	return res, nil
+}
+
 // partsToHash converts a Commit and ResultData substrings to a Result.
 func partsToHash(c *Commit, p []string) *Result {
 	return &Result{
@@ -223,9 +346,9 @@ func partsToHash(c *Commit, p []string) *Result {
 //
 // Each Result entity is a descendant of its associated Package entity.
 type Result struct {
+	PackagePath string // (empty for Go commits)
 	Builder     string // "os-arch[-note]"
 	Hash        string
-	PackagePath string // (empty for Go commits)
 
 	// The Go Commit this was built against (empty for Go commits).
 	GoHash string
@@ -259,6 +382,349 @@ func (r *Result) Data() string {
 	return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash)
 }
 
+// A PerfResult describes all benchmarking result for a Commit.
+// Descendant of Package.
+type PerfResult struct {
+	PackagePath string
+	CommitHash  string
+	CommitNum   int
+	Data        []string `datastore:",noindex"` // "builder|benchmark|ok|metric1=val1|metric2=val2|file:log=hash|file:cpuprof=hash"
+
+	// Local cache with parsed Data.
+	// Maps builder->benchmark->ParsedPerfResult.
+	parsedData map[string]map[string]*ParsedPerfResult
+}
+
+type ParsedPerfResult struct {
+	OK        bool
+	Metrics   map[string]uint64
+	Artifacts map[string]string
+}
+
+func (r *PerfResult) Key(c appengine.Context) *datastore.Key {
+	p := Package{Path: r.PackagePath}
+	key := r.CommitHash
+	return datastore.NewKey(c, "PerfResult", key, 0, p.Key(c))
+}
+
+// AddResult add the benchmarking result to r.
+// Existing result for the same builder/benchmark is replaced if already exists.
+// Returns whether the result was already present.
+func (r *PerfResult) AddResult(req *PerfRequest) bool {
+	present := false
+	str := fmt.Sprintf("%v|%v|", req.Builder, req.Benchmark)
+	for i, s := range r.Data {
+		if strings.HasPrefix(s, str) {
+			present = true
+			last := len(r.Data) - 1
+			r.Data[i] = r.Data[last]
+			r.Data = r.Data[:last]
+			break
+		}
+	}
+	ok := "ok"
+	if !req.OK {
+		ok = "false"
+	}
+	str += ok
+	for _, m := range req.Metrics {
+		str += fmt.Sprintf("|%v=%v", m.Type, m.Val)
+	}
+	for _, a := range req.Artifacts {
+		str += fmt.Sprintf("|file:%v=%v", a.Type, a.Body)
+	}
+	r.Data = append(r.Data, str)
+	r.parsedData = nil
+	return present
+}
+
+func (r *PerfResult) ParseData() map[string]map[string]*ParsedPerfResult {
+	if r.parsedData != nil {
+		return r.parsedData
+	}
+	res := make(map[string]map[string]*ParsedPerfResult)
+	for _, str := range r.Data {
+		ss := strings.Split(str, "|")
+		builder := ss[0]
+		bench := ss[1]
+		ok := ss[2]
+		m := res[builder]
+		if m == nil {
+			m = make(map[string]*ParsedPerfResult)
+			res[builder] = m
+		}
+		var p ParsedPerfResult
+		p.OK = ok == "ok"
+		p.Metrics = make(map[string]uint64)
+		p.Artifacts = make(map[string]string)
+		for _, entry := range ss[3:] {
+			if strings.HasPrefix(entry, "file:") {
+				ss1 := strings.Split(entry[len("file:"):], "=")
+				p.Artifacts[ss1[0]] = ss1[1]
+			} else {
+				ss1 := strings.Split(entry, "=")
+				val, _ := strconv.ParseUint(ss1[1], 10, 64)
+				p.Metrics[ss1[0]] = val
+			}
+		}
+		m[bench] = &p
+	}
+	r.parsedData = res
+	return res
+}
+
+// A PerfMetricRun entity holds a set of metric values for builder/benchmark/metric
+// for commits [StartCommitNum, StartCommitNum + PerfRunLength).
+// Descendant of Package.
+type PerfMetricRun struct {
+	PackagePath    string
+	Builder        string
+	Benchmark      string
+	Metric         string // e.g. realtime, cputime, gc-pause
+	StartCommitNum int
+	Vals           []int64 `datastore:",noindex"`
+}
+
+func (m *PerfMetricRun) Key(c appengine.Context) *datastore.Key {
+	p := Package{Path: m.PackagePath}
+	key := m.Builder + "|" + m.Benchmark + "|" + m.Metric + "|" + strconv.Itoa(m.StartCommitNum)
+	return datastore.NewKey(c, "PerfMetricRun", key, 0, p.Key(c))
+}
+
+// GetPerfMetricRun loads and returns PerfMetricRun that contains information
+// for commit commitNum.
+func GetPerfMetricRun(c appengine.Context, builder, benchmark, metric string, commitNum int) (*PerfMetricRun, error) {
+	startCommitNum := commitNum / PerfRunLength * PerfRunLength
+	m := &PerfMetricRun{Builder: builder, Benchmark: benchmark, Metric: metric, StartCommitNum: startCommitNum}
+	err := datastore.Get(c, m.Key(c), m)
+	if err != nil && err != datastore.ErrNoSuchEntity {
+		return nil, fmt.Errorf("getting PerfMetricRun: %v", err)
+	}
+	if len(m.Vals) != PerfRunLength {
+		m.Vals = make([]int64, PerfRunLength)
+	}
+	return m, nil
+}
+
+func (m *PerfMetricRun) AddMetric(c appengine.Context, commitNum int, v uint64) error {
+	if commitNum < m.StartCommitNum || commitNum >= m.StartCommitNum+PerfRunLength {
+		return fmt.Errorf("AddMetric: CommitNum %v out of range [%v, %v)",
+			commitNum, m.StartCommitNum, m.StartCommitNum+PerfRunLength)
+	}
+	m.Vals[commitNum-m.StartCommitNum] = int64(v)
+	if _, err := datastore.Put(c, m.Key(c), m); err != nil {
+		return fmt.Errorf("putting PerfMetricRun: %v", err)
+	}
+	return nil
+}
+
+// GetPerfMetricsForCommits returns perf metrics for builder/benchmark/metric
+// and commits [startCommitNum, startCommitNum+n).
+func GetPerfMetricsForCommits(c appengine.Context, builder, benchmark, metric string, startCommitNum, n int) ([]uint64, error) {
+	if startCommitNum < 0 || n <= 0 {
+		return nil, fmt.Errorf("GetPerfMetricsForCommits: invalid args (%v, %v)", startCommitNum, n)
+	}
+	var res []uint64
+	for n > 0 {
+		metrics, err := GetPerfMetricRun(c, builder, benchmark, metric, startCommitNum)
+		if err != nil {
+			return nil, err
+		}
+		idx := startCommitNum - metrics.StartCommitNum
+		cnt := PerfRunLength - idx
+		if cnt > n {
+			cnt = n
+		}
+		for _, v := range metrics.Vals[idx : idx+cnt] {
+			res = append(res, uint64(v))
+		}
+		startCommitNum += cnt
+		n -= cnt
+	}
+	return res, nil
+}
+
+// PerfConfig holds read-mostly configuration related to benchmarking.
+// There is only one PerfConfig entity.
+type PerfConfig struct {
+	BuilderBench []string `datastore:",noindex"` // "builder|benchmark" pairs
+	BuilderProcs []string `datastore:",noindex"` // "builder|proc" pairs
+	BenchMetric  []string `datastore:",noindex"` // "benchmark|metric" pairs
+	NoiseLevels  []string `datastore:",noindex"` // "builder|benchmark|metric1=noise1|metric2=noise2"
+
+	// Local cache of "builder|benchmark|metric" -> noise.
+	noise map[string]float64
+}
+
+func PerfConfigKey(c appengine.Context) *datastore.Key {
+	p := Package{}
+	return datastore.NewKey(c, "PerfConfig", "PerfConfig", 0, p.Key(c))
+}
+
+const perfConfigCacheKey = "perf-config"
+
+func GetPerfConfig(c appengine.Context, r *http.Request) (*PerfConfig, error) {
+	pc := new(PerfConfig)
+	now := cache.Now(c)
+	if cache.Get(r, now, perfConfigCacheKey, pc) {
+		return pc, nil
+	}
+	err := datastore.Get(c, PerfConfigKey(c), pc)
+	if err != nil && err != datastore.ErrNoSuchEntity {
+		return nil, fmt.Errorf("GetPerfConfig: %v", err)
+	}
+	cache.Set(r, now, perfConfigCacheKey, pc)
+	return pc, nil
+}
+
+func (pc *PerfConfig) NoiseLevel(builder, benchmark, metric string) float64 {
+	if pc.noise == nil {
+		pc.noise = make(map[string]float64)
+		for _, str := range pc.NoiseLevels {
+			split := strings.Split(str, "|")
+			builderBench := split[0] + "|" + split[1]
+			for _, entry := range split[2:] {
+				metricValue := strings.Split(entry, "=")
+				noise, _ := strconv.ParseFloat(metricValue[1], 64)
+				pc.noise[builderBench+"|"+metricValue[0]] = noise
+			}
+		}
+	}
+	me := fmt.Sprintf("%v|%v|%v", builder, benchmark, metric)
+	n := pc.noise[me]
+	if n == 0 {
+		// Use a very conservative value
+		// until we have learned the real noise level.
+		n = 200
+	}
+	return n
+}
+
+// UpdatePerfConfig updates the PerfConfig entity with results of benchmarking.
+// Returns whether it's a benchmark that we have not yet seem on the builder.
+func UpdatePerfConfig(c appengine.Context, r *http.Request, req *PerfRequest) (newBenchmark bool, err error) {
+	pc, err := GetPerfConfig(c, r)
+	if err != nil {
+		return false, err
+	}
+
+	modified := false
+	add := func(arr *[]string, str string) {
+		for _, s := range *arr {
+			if s == str {
+				return
+			}
+		}
+		*arr = append(*arr, str)
+		modified = true
+		return
+	}
+
+	BenchProcs := strings.Split(req.Benchmark, "-")
+	benchmark := BenchProcs[0]
+	procs := "1"
+	if len(BenchProcs) > 1 {
+		procs = BenchProcs[1]
+	}
+
+	add(&pc.BuilderBench, req.Builder+"|"+benchmark)
+	newBenchmark = modified
+	add(&pc.BuilderProcs, req.Builder+"|"+procs)
+	for _, m := range req.Metrics {
+		add(&pc.BenchMetric, benchmark+"|"+m.Type)
+	}
+
+	if modified {
+		if _, err := datastore.Put(c, PerfConfigKey(c), pc); err != nil {
+			return false, fmt.Errorf("putting PerfConfig: %v", err)
+		}
+		cache.Tick(c)
+	}
+	return newBenchmark, nil
+}
+
+func collectList(all []string, idx int, second string) (res []string) {
+	m := make(map[string]bool)
+	for _, str := range all {
+		ss := strings.Split(str, "|")
+		v := ss[idx]
+		v2 := ss[1-idx]
+		if (second == "" || second == v2) && !m[v] {
+			m[v] = true
+			res = append(res, v)
+		}
+	}
+	sort.Strings(res)
+	return res
+}
+
+func (pc *PerfConfig) BuildersForBenchmark(bench string) []string {
+	return collectList(pc.BuilderBench, 0, bench)
+}
+
+func (pc *PerfConfig) BenchmarksForBuilder(builder string) []string {
+	return collectList(pc.BuilderBench, 1, builder)
+}
+
+func (pc *PerfConfig) MetricsForBenchmark(bench string) []string {
+	return collectList(pc.BenchMetric, 1, bench)
+}
+
+func (pc *PerfConfig) BenchmarkProcList() (res []string) {
+	bl := pc.BenchmarksForBuilder("")
+	pl := pc.ProcList("")
+	for _, b := range bl {
+		for _, p := range pl {
+			res = append(res, fmt.Sprintf("%v-%v", b, p))
+		}
+	}
+	return res
+}
+
+func (pc *PerfConfig) ProcList(builder string) []int {
+	ss := collectList(pc.BuilderProcs, 1, builder)
+	var procs []int
+	for _, s := range ss {
+		p, _ := strconv.ParseInt(s, 10, 32)
+		procs = append(procs, int(p))
+	}
+	sort.Ints(procs)
+	return procs
+}
+
+// A PerfTodo contains outstanding commits for benchmarking for a builder.
+// Descendant of Package.
+type PerfTodo struct {
+	PackagePath string // (empty for main repo commits)
+	Builder     string
+	CommitNums  []int `datastore:",noindex"` // LIFO queue of commits to benchmark.
+}
+
+func (todo *PerfTodo) Key(c appengine.Context) *datastore.Key {
+	p := Package{Path: todo.PackagePath}
+	key := todo.Builder
+	return datastore.NewKey(c, "PerfTodo", key, 0, p.Key(c))
+}
+
+// AddCommitToPerfTodo adds the commit to all existing PerfTodo entities.
+func AddCommitToPerfTodo(c appengine.Context, com *Commit) error {
+	var todos []*PerfTodo
+	_, err := datastore.NewQuery("PerfTodo").
+		Ancestor((&Package{}).Key(c)).
+		GetAll(c, &todos)
+	if err != nil {
+		return fmt.Errorf("fetching PerfTodo's: %v", err)
+	}
+	for _, todo := range todos {
+		todo.CommitNums = append(todo.CommitNums, com.Num)
+		_, err = datastore.Put(c, todo.Key(c), todo)
+		if err != nil {
+			return fmt.Errorf("updating PerfTodo: %v", err)
+		}
+	}
+	return nil
+}
+
 // A Log is a gzip-compressed log file stored under the SHA1 hash of the
 // uncompressed log text.
 type Log struct {

dashboard/app/build/handler.go

@@ -44,6 +44,9 @@ func commitHandler(r *http.Request) (interface{}, error) {
 		if err := datastore.Get(c, com.Key(c), com); err != nil {
 			return nil, fmt.Errorf("getting Commit: %v", err)
 		}
+		// Strip potentially large and unnecessary fields.
+		com.ResultData = nil
+		com.PerfResults = nil
 		return com, nil
 	}
 	if r.Method != "POST" {
@@ -115,6 +118,25 @@ func addCommit(c appengine.Context, com *Commit) error {
 	if _, err = datastore.Put(c, com.Key(c), com); err != nil {
 		return fmt.Errorf("putting Commit: %v", err)
 	}
+	if com.NeedsBenchmarking {
+		// add to CommitRun
+		cr, err := GetCommitRun(c, com.Num)
+		if err != nil {
+			return err
+		}
+		if err = cr.AddCommit(c, com); err != nil {
+			return err
+		}
+		// create PerfResult
+		res := &PerfResult{CommitHash: com.Hash, CommitNum: com.Num}
+		if _, err := datastore.Put(c, res.Key(c), res); err != nil {
+			return fmt.Errorf("putting PerfResult: %v", err)
+		}
+		// Update perf todo if necessary.
+		if err = AddCommitToPerfTodo(c, com); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -165,10 +187,18 @@ func todoHandler(r *http.Request) (interface{}, error) {
 		switch kind {
 		case "build-go-commit":
 			com, err = buildTodo(c, builder, "", "")
+			if com != nil {
+				com.PerfResults = []string{}
+			}
 		case "build-package":
 			packagePath := r.FormValue("packagePath")
 			goHash := r.FormValue("goHash")
 			com, err = buildTodo(c, builder, packagePath, goHash)
+			if com != nil {
+				com.PerfResults = []string{}
+			}
+		case "benchmark-go-commit":
+			com, err = perfTodo(c, builder)
 		}
 		if com != nil || err != nil {
 			if com != nil {
@@ -260,6 +290,129 @@ func buildTodo(c appengine.Context, builder, packagePath, goHash string) (*Commi
 	return nil, nil
 }
 
+// perfTodo returns the next Commit to be benchmarked (or nil if none available).
+func perfTodo(c appengine.Context, builder string) (*Commit, error) {
+	p := &Package{}
+	todo := &PerfTodo{Builder: builder}
+	err := datastore.Get(c, todo.Key(c), todo)
+	if err != nil && err != datastore.ErrNoSuchEntity {
+		return nil, fmt.Errorf("fetching PerfTodo: %v", err)
+	}
+	if err == datastore.ErrNoSuchEntity {
+		todo, err = buildPerfTodo(c, builder)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if len(todo.CommitNums) == 0 {
+		return nil, nil
+	}
+
+	// Have commit to benchmark, fetch it.
+	num := todo.CommitNums[len(todo.CommitNums)-1]
+	t := datastore.NewQuery("Commit").
+		Ancestor(p.Key(c)).
+		Filter("Num =", num).
+		Limit(1).
+		Run(c)
+	com := new(Commit)
+	if _, err := t.Next(com); err != nil {
+		return nil, err
+	}
+	if !com.NeedsBenchmarking {
+		return nil, fmt.Errorf("commit from perf todo queue is not intended for benchmarking")
+	}
+
+	// Remove benchmarks from other builders.
+	var benchs []string
+	for _, b := range com.PerfResults {
+		bb := strings.Split(b, "|")
+		if bb[0] == builder && bb[1] != "meta-done" {
+			benchs = append(benchs, bb[1])
+		}
+	}
+	com.PerfResults = benchs
+
+	return com, nil
+}
+
+// buildPerfTodo creates PerfTodo for the builder with all commits. In a transaction.
+func buildPerfTodo(c appengine.Context, builder string) (*PerfTodo, error) {
+	todo := &PerfTodo{Builder: builder}
+	tx := func(c appengine.Context) error {
+		err := datastore.Get(c, todo.Key(c), todo)
+		if err != nil && err != datastore.ErrNoSuchEntity {
+			return fmt.Errorf("fetching PerfTodo: %v", err)
+		}
+		if err == nil {
+			return nil
+		}
+		t := datastore.NewQuery("CommitRun").
+			Ancestor((&Package{}).Key(c)).
+			Order("-StartCommitNum").
+			Run(c)
+		var nums []int
+		var releaseNums []int
+	loop:
+		for {
+			cr := new(CommitRun)
+			if _, err := t.Next(cr); err == datastore.Done {
+				break
+			} else if err != nil {
+				return fmt.Errorf("scanning commit runs for perf todo: %v", err)
+			}
+			for i := len(cr.Hash) - 1; i >= 0; i-- {
+				if !cr.NeedsBenchmarking[i] || cr.Hash[i] == "" {
+					continue // There's nothing to see here. Move along.
+				}
+				num := cr.StartCommitNum + i
+				for k, v := range knownTags {
+					// Releases are benchmarked first, because they are important (and there are few of them).
+					if cr.Hash[i] == v {
+						releaseNums = append(releaseNums, num)
+						if k == "go1" {
+							break loop // Point of no benchmark: test/bench/shootout: update timing.log to Go 1.
+						}
+					}
+				}
+				nums = append(nums, num)
+			}
+		}
+		todo.CommitNums = orderPrefTodo(nums)
+		todo.CommitNums = append(todo.CommitNums, releaseNums...)
+		if _, err = datastore.Put(c, todo.Key(c), todo); err != nil {
+			return fmt.Errorf("putting PerfTodo: %v", err)
+		}
+		return nil
+	}
+	return todo, datastore.RunInTransaction(c, tx, nil)
+}
+
+func removeCommitFromPerfTodo(c appengine.Context, builder string, num int) error {
+	todo := &PerfTodo{Builder: builder}
+	err := datastore.Get(c, todo.Key(c), todo)
+	if err != nil && err != datastore.ErrNoSuchEntity {
+		return fmt.Errorf("fetching PerfTodo: %v", err)
+	}
+	if err == datastore.ErrNoSuchEntity {
+		return nil
+	}
+	for i := len(todo.CommitNums) - 1; i >= 0; i-- {
+		if todo.CommitNums[i] == num {
+			for ; i < len(todo.CommitNums)-1; i++ {
+				todo.CommitNums[i] = todo.CommitNums[i+1]
+			}
+			todo.CommitNums = todo.CommitNums[:i]
+			_, err = datastore.Put(c, todo.Key(c), todo)
+			if err != nil {
+				return fmt.Errorf("putting PerfTodo: %v", err)
+			}
+			break
+		}
+	}
+	return nil
+}
+
 // packagesHandler returns a list of the non-Go Packages monitored
 // by the dashboard.
 func packagesHandler(r *http.Request) (interface{}, error) {
@@ -329,6 +482,202 @@ func resultHandler(r *http.Request) (interface{}, error) {
 	return nil, datastore.RunInTransaction(c, tx, nil)
 }
 
+// perf-result request payload
+type PerfRequest struct {
+	Builder   string
+	Benchmark string
+	Hash      string
+	OK        bool
+	Metrics   []PerfMetric
+	Artifacts []PerfArtifact
+}
+
+type PerfMetric struct {
+	Type string
+	Val  uint64
+}
+
+type PerfArtifact struct {
+	Type string
+	Body string
+}
+
+// perfResultHandler records a becnhmarking result.
+func perfResultHandler(r *http.Request) (interface{}, error) {
+	defer r.Body.Close()
+	if r.Method != "POST" {
+		return nil, errBadMethod(r.Method)
+	}
+
+	req := new(PerfRequest)
+	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
+		return nil, fmt.Errorf("decoding Body: %v", err)
+	}
+
+	c := contextForRequest(r)
+	defer cache.Tick(c)
+
+	// store the text files if supplied
+	for i, a := range req.Artifacts {
+		hash, err := PutLog(c, a.Body)
+		if err != nil {
+			return nil, fmt.Errorf("putting Log: %v", err)
+		}
+		req.Artifacts[i].Body = hash
+	}
+	tx := func(c appengine.Context) error {
+		return addPerfResult(c, r, req)
+	}
+	return nil, datastore.RunInTransaction(c, tx, nil)
+}
+
+// addPerfResult creates PerfResult and updates Commit, PerfTodo,
+// PerfMetricRun and PerfConfig. Must be executed within a transaction.
+func addPerfResult(c appengine.Context, r *http.Request, req *PerfRequest) error {
+	// check Package exists
+	p, err := GetPackage(c, "")
+	if err != nil {
+		return fmt.Errorf("GetPackage: %v", err)
+	}
+	// add result to Commit
+	com := &Commit{Hash: req.Hash}
+	if err := com.AddPerfResult(c, req.Builder, req.Benchmark); err != nil {
+		return fmt.Errorf("AddPerfResult: %v", err)
+	}
+
+	// add the result to PerfResult
+	res := &PerfResult{CommitHash: req.Hash}
+	if err := datastore.Get(c, res.Key(c), res); err != nil {
+		return fmt.Errorf("getting PerfResult: %v", err)
+	}
+	present := res.AddResult(req)
+	if _, err := datastore.Put(c, res.Key(c), res); err != nil {
+		return fmt.Errorf("putting PerfResult: %v", err)
+	}
+
+	// Meta-done denotes that there are no benchmarks left.
+	if req.Benchmark == "meta-done" {
+		// Don't send duplicate emails for the same commit/builder.
+		// And don't send emails about too old commits.
+		if !present && com.Num >= p.NextNum-commitsPerPage {
+			if err := checkPerfChanges(c, r, com, req.Builder, res); err != nil {
+				return err
+			}
+		}
+		if err := removeCommitFromPerfTodo(c, req.Builder, com.Num); err != nil {
+			return nil
+		}
+		return nil
+	}
+
+	// update PerfConfig
+	newBenchmark, err := UpdatePerfConfig(c, r, req)
+	if err != nil {
+		return fmt.Errorf("updating PerfConfig: %v", err)
+	}
+	if newBenchmark {
+		// If this is a new benchmark on the builder, delete PerfTodo.
+		// It will be recreated later with all commits again.
+		todo := &PerfTodo{Builder: req.Builder}
+		err = datastore.Delete(c, todo.Key(c))
+		if err != nil && err != datastore.ErrNoSuchEntity {
+			return fmt.Errorf("deleting PerfTodo: %v", err)
+		}
+	}
+
+	// add perf metrics
+	for _, metric := range req.Metrics {
+		m, err := GetPerfMetricRun(c, req.Builder, req.Benchmark, metric.Type, com.Num)
+		if err != nil {
+			return fmt.Errorf("GetPerfMetrics: %v", err)
+		}
+		if err = m.AddMetric(c, com.Num, metric.Val); err != nil {
+			return fmt.Errorf("AddMetric: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func checkPerfChanges(c appengine.Context, r *http.Request, com *Commit, builder string, res *PerfResult) error {
+	pc, err := GetPerfConfig(c, r)
+	if err != nil {
+		return err
+	}
+
+	results := res.ParseData()[builder]
+	rcNewer := MakePerfResultCache(c, com, true)
+	rcOlder := MakePerfResultCache(c, com, false)
+
+	// Check whether we need to send failure notification email.
+	if results["meta-done"].OK {
+		// This one is successful, see if the next is failed.
+		nextRes, err := rcNewer.Next(com.Num)
+		if err != nil {
+			return err
+		}
+		if nextRes != nil && isPerfFailed(nextRes, builder) {
+			sendPerfFailMail(c, builder, nextRes)
+		}
+	} else {
+		// This one is failed, see if the previous is successful.
+		prevRes, err := rcOlder.Next(com.Num)
+		if err != nil {
+			return err
+		}
+		if prevRes != nil && !isPerfFailed(prevRes, builder) {
+			sendPerfFailMail(c, builder, res)
+		}
+	}
+
+	// Now see if there are any performance changes.
+	// Find the previous and the next results for performance comparison.
+	prevRes, err := rcOlder.NextForComparison(com.Num, builder)
+	if err != nil {
+		return err
+	}
+	nextRes, err := rcNewer.NextForComparison(com.Num, builder)
+	if err != nil {
+		return err
+	}
+	if results["meta-done"].OK {
+		// This one is successful, compare with a previous one.
+		if prevRes != nil {
+			if err := comparePerfResults(c, pc, builder, prevRes, res); err != nil {
+				return err
+			}
+		}
+		// Compare a next one with the current.
+		if nextRes != nil {
+			if err := comparePerfResults(c, pc, builder, res, nextRes); err != nil {
+				return err
+			}
+		}
+	} else {
+		// This one is failed, compare a previous one with a next one.
+		if prevRes != nil && nextRes != nil {
+			if err := comparePerfResults(c, pc, builder, prevRes, nextRes); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func comparePerfResults(c appengine.Context, pc *PerfConfig, builder string, prevRes, res *PerfResult) error {
+	changes := significantPerfChanges(pc, builder, prevRes, res)
+	if len(changes) == 0 {
+		return nil
+	}
+	com := &Commit{Hash: res.CommitHash}
+	if err := datastore.Get(c, com.Key(c), com); err != nil {
+		return fmt.Errorf("getting commit %v: %v", com.Hash, err)
+	}
+	sendPerfMailLater.Call(c, com, prevRes.CommitHash, builder, changes) // add task to queue
+	return nil
+}
+
 // logHandler displays log text for a given hash.
 // It handles paths like "/log/hash".
 func logHandler(w http.ResponseWriter, r *http.Request) {
@@ -422,6 +771,7 @@ func init() {
 		http.HandleFunc(d.RelPath+"commit", AuthHandler(commitHandler))
 		http.HandleFunc(d.RelPath+"packages", AuthHandler(packagesHandler))
 		http.HandleFunc(d.RelPath+"result", AuthHandler(resultHandler))
+		http.HandleFunc(d.RelPath+"perf-result", AuthHandler(perfResultHandler))
 		http.HandleFunc(d.RelPath+"tag", AuthHandler(tagHandler))
 		http.HandleFunc(d.RelPath+"todo", AuthHandler(todoHandler))
 

dashboard/app/build/init.go

@@ -36,5 +36,9 @@ func initHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
+
+	// Create secret key.
+	secretKey(c)
+
 	fmt.Fprint(w, "OK")
 }

dashboard/app/build/notify.go

@@ -14,6 +14,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"regexp"
+	"sort"
 	"text/template"
 
 	"appengine"
@@ -99,14 +100,14 @@ func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
 			broken = com
 		}
 	}
-	var err error
+	if broken == nil {
-	if broken != nil && !broken.FailNotificationSent {
+		return nil
-		c.Infof("%s is broken commit; notifying", broken.Hash)
-		notifyLater.Call(c, broken, builder) // add task to queue
-		broken.FailNotificationSent = true
-		_, err = datastore.Put(c, broken.Key(c), broken)
 	}
-	return err
+	r := broken.Result(builder, "")
+	if r == nil {
+		return fmt.Errorf("finding result for %q: %+v", builder, com)
+	}
+	return commonNotify(c, broken, builder, r.LogHash)
 }
 
 // firstMatch executes the query q and loads the first entity into v.
@@ -123,27 +124,22 @@ var notifyLater = delay.Func("notify", notify)
 
 // notify tries to update the CL for the given Commit with a failure message.
 // If it doesn't succeed, it sends a failure email to golang-dev.
-func notify(c appengine.Context, com *Commit, builder string) {
+func notify(c appengine.Context, com *Commit, builder, logHash string) {
-	if !updateCL(c, com, builder) {
+	if !updateCL(c, com, builder, logHash) {
 		// Send a mail notification if the CL can't be found.
-		sendFailMail(c, com, builder)
+		sendFailMail(c, com, builder, logHash)
 	}
 }
 
 // updateCL updates the CL for the given Commit with a failure message
 // for the given builder.
-func updateCL(c appengine.Context, com *Commit, builder string) bool {
+func updateCL(c appengine.Context, com *Commit, builder, logHash string) bool {
 	cl, err := lookupCL(c, com)
 	if err != nil {
 		c.Errorf("could not find CL for %v: %v", com.Hash, err)
 		return false
 	}
-	res := com.Result(builder, "")
+	url := fmt.Sprintf("%v?cl=%v&brokebuild=%v&log=%v", gobotBase, cl, builder, logHash)
-	if res == nil {
-		c.Errorf("finding result for %q: %+v", builder, com)
-		return false
-	}
-	url := fmt.Sprintf("%v?cl=%v&brokebuild=%v&log=%v", gobotBase, cl, builder, res.LogHash)
 	r, err := urlfetch.Client(c).Post(url, "text/plain", nil)
 	if err != nil {
 		c.Errorf("could not update CL %v: %v", cl, err)
@@ -192,30 +188,65 @@ func init() {
 	gob.Register(&Commit{}) // for delay
 }
 
-// sendFailMail sends a mail notification that the build failed on the
+var (
-// provided commit and builder.
+	sendPerfMailLater = delay.Func("sendPerfMail", sendPerfMailFunc)
-func sendFailMail(c appengine.Context, com *Commit, builder string) {
+	sendPerfMailTmpl  = template.Must(
-	// TODO(adg): handle packages
+		template.New("perf_notify.txt").
+			Funcs(template.FuncMap(tmplFuncs)).
+			ParseFiles("build/perf_notify.txt"),
+	)
+)
 
-	// get Result
+func sendPerfFailMail(c appengine.Context, builder string, res *PerfResult) error {
-	r := com.Result(builder, "")
+	com := &Commit{Hash: res.CommitHash}
-	if r == nil {
+	if err := datastore.Get(c, com.Key(c), com); err != nil {
-		c.Errorf("finding result for %q: %+v", builder, com)
+		return fmt.Errorf("getting commit %v: %v", com.Hash, err)
-		return
+	}
+	logHash := ""
+	parsed := res.ParseData()
+	for _, data := range parsed[builder] {
+		if !data.OK {
+			logHash = data.Artifacts["log"]
+			break
+		}
+	}
+	if logHash == "" {
+		return fmt.Errorf("can not find failed result for commit %v on builder %v", com.Hash, builder)
+	}
+	return commonNotify(c, com, builder, logHash)
 }
 
+func commonNotify(c appengine.Context, com *Commit, builder, logHash string) error {
+	if com.FailNotificationSent {
+		return nil
+	}
+	c.Infof("%s is broken commit; notifying", com.Hash)
+	notifyLater.Call(c, com, builder, logHash) // add task to queue
+	com.FailNotificationSent = true
+	_, err := datastore.Put(c, com.Key(c), com)
+	return err
+}
+
+// sendFailMail sends a mail notification that the build failed on the
+// provided commit and builder.
+func sendFailMail(c appengine.Context, com *Commit, builder, logHash string) {
 	// get Log
-	k := datastore.NewKey(c, "Log", r.LogHash, 0, nil)
+	k := datastore.NewKey(c, "Log", logHash, 0, nil)
 	l := new(Log)
 	if err := datastore.Get(c, k, l); err != nil {
-		c.Errorf("finding Log record %v: %v", r.LogHash, err)
+		c.Errorf("finding Log record %v: %v", logHash, err)
+		return
+	}
+	logText, err := l.Text()
+	if err != nil {
+		c.Errorf("unpacking Log record %v: %v", logHash, err)
 		return
 	}
 
 	// prepare mail message
 	var body bytes.Buffer
-	err := sendFailMailTmpl.Execute(&body, map[string]interface{}{
+	err = sendFailMailTmpl.Execute(&body, map[string]interface{}{
-		"Builder": builder, "Commit": com, "Result": r, "Log": l,
+		"Builder": builder, "Commit": com, "LogHash": logHash, "LogText": logText,
 		"Hostname": domain,
 	})
 	if err != nil {
@@ -236,3 +267,83 @@ func sendFailMail(c appengine.Context, com *Commit, builder string) {
 		c.Errorf("sending mail: %v", err)
 	}
 }
+
+type PerfChangeBenchmark struct {
+	Name    string
+	Metrics []*PerfChangeMetric
+}
+
+type PerfChangeMetric struct {
+	Name  string
+	Old   uint64
+	New   uint64
+	Delta float64
+}
+
+type PerfChangeBenchmarkSlice []*PerfChangeBenchmark
+
+func (l PerfChangeBenchmarkSlice) Len() int      { return len(l) }
+func (l PerfChangeBenchmarkSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
+func (l PerfChangeBenchmarkSlice) Less(i, j int) bool {
+	b1, p1 := splitBench(l[i].Name)
+	b2, p2 := splitBench(l[j].Name)
+	if b1 != b2 {
+		return b1 < b2
+	}
+	return p1 < p2
+}
+
+type PerfChangeMetricSlice []*PerfChangeMetric
+
+func (l PerfChangeMetricSlice) Len() int           { return len(l) }
+func (l PerfChangeMetricSlice) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
+func (l PerfChangeMetricSlice) Less(i, j int) bool { return l[i].Name < l[j].Name }
+
+func sendPerfMailFunc(c appengine.Context, com *Commit, prevCommitHash, builder string, changes []*PerfChange) {
+	// Sort the changes into the right order.
+	var benchmarks []*PerfChangeBenchmark
+	for _, ch := range changes {
+		// Find the benchmark.
+		var b *PerfChangeBenchmark
+		for _, b1 := range benchmarks {
+			if b1.Name == ch.bench {
+				b = b1
+				break
+			}
+		}
+		if b == nil {
+			b = &PerfChangeBenchmark{Name: ch.bench}
+			benchmarks = append(benchmarks, b)
+		}
+		b.Metrics = append(b.Metrics, &PerfChangeMetric{Name: ch.metric, Old: ch.old, New: ch.new, Delta: ch.diff})
+	}
+	for _, b := range benchmarks {
+		sort.Sort(PerfChangeMetricSlice(b.Metrics))
+	}
+	sort.Sort(PerfChangeBenchmarkSlice(benchmarks))
+
+	url := fmt.Sprintf("http://%v/perfdetail?commit=%v&commit0=%v&kind=builder&builder=%v", domain, com.Hash, prevCommitHash, builder)
+
+	// prepare mail message
+	var body bytes.Buffer
+	err := sendPerfMailTmpl.Execute(&body, map[string]interface{}{
+		"Builder": builder, "Commit": com, "Hostname": domain, "Url": url, "Benchmarks": benchmarks,
+	})
+	if err != nil {
+		c.Errorf("rendering perf mail template: %v", err)
+		return
+	}
+	subject := fmt.Sprintf("Perf changes on %s by %s", builder, shortDesc(com.Desc))
+	msg := &mail.Message{
+		Sender:  mailFrom,
+		To:      []string{failMailTo},
+		ReplyTo: failMailTo,
+		Subject: subject,
+		Body:    body.String(),
+	}
+
+	// send mail
+	if err := mail.Send(c, msg); err != nil {
+		c.Errorf("sending mail: %v", err)
+	}
+}

dashboard/app/build/notify.txt

@@ -1,9 +1,9 @@
 Change {{shortHash .Commit.Hash}} broke the {{.Builder}} build:
-http://{{.Hostname}}/log/{{.Result.LogHash}}
+http://{{.Hostname}}/log/{{.LogHash}}
 
 {{.Commit.Desc}}
 
 http://code.google.com/p/go/source/detail?r={{shortHash .Commit.Hash}}
 
 $ tail -200 < log
-{{printf "%s" .Log.Text | tail 200}}
+{{printf "%s" .LogText | tail 200}}

dashboard/app/build/perf.go

@@ -0,0 +1,310 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine
+
+package build
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	"appengine"
+	"appengine/datastore"
+)
+
+var knownTags = map[string]string{
+	"go1":   "0051c7442fed9c888de6617fa9239a913904d96e",
+	"go1.1": "d29da2ced72ba2cf48ed6a8f1ec4abc01e4c5bf1",
+	"go1.2": "b1edf8faa5d6cbc50c6515785df9df9c19296564",
+}
+
+var lastRelease = "go1.2"
+
+func splitBench(benchProcs string) (string, int) {
+	ss := strings.Split(benchProcs, "-")
+	procs, _ := strconv.Atoi(ss[1])
+	return ss[0], procs
+}
+
+func dashPerfCommits(c appengine.Context, page int) ([]*Commit, error) {
+	q := datastore.NewQuery("Commit").
+		Ancestor((&Package{}).Key(c)).
+		Order("-Num").
+		Filter("NeedsBenchmarking =", true).
+		Limit(commitsPerPage).
+		Offset(page * commitsPerPage)
+	var commits []*Commit
+	_, err := q.GetAll(c, &commits)
+	if err == nil && len(commits) == 0 {
+		err = fmt.Errorf("no commits")
+	}
+	return commits, err
+}
+
+func perfChangeStyle(pc *PerfConfig, v float64, builder, benchmark, metric string) string {
+	noise := pc.NoiseLevel(builder, benchmark, metric)
+	if isNoise(v, noise) {
+		return "noise"
+	}
+	if v > 0 {
+		return "bad"
+	}
+	return "good"
+}
+
+func isNoise(diff, noise float64) bool {
+	rnoise := -100 * noise / (noise + 100)
+	return diff < noise && diff > rnoise
+}
+
+func perfDiff(old, new uint64) float64 {
+	return 100*float64(new)/float64(old) - 100
+}
+
+func isPerfFailed(res *PerfResult, builder string) bool {
+	data := res.ParseData()[builder]
+	return data != nil && data["meta-done"] != nil && !data["meta-done"].OK
+}
+
+// PerfResultCache caches a set of PerfResults so that it's easy to access them
+// without lots of duplicate accesses to datastore.
+// It allows to iterate over newer or older results for some base commit.
+type PerfResultCache struct {
+	c       appengine.Context
+	newer   bool
+	iter    *datastore.Iterator
+	results map[int]*PerfResult
+}
+
+func MakePerfResultCache(c appengine.Context, com *Commit, newer bool) *PerfResultCache {
+	p := &Package{}
+	q := datastore.NewQuery("PerfResult").Ancestor(p.Key(c)).Limit(100)
+	if newer {
+		q = q.Filter("CommitNum >=", com.Num).Order("CommitNum")
+	} else {
+		q = q.Filter("CommitNum <=", com.Num).Order("-CommitNum")
+	}
+	rc := &PerfResultCache{c: c, newer: newer, iter: q.Run(c), results: make(map[int]*PerfResult)}
+	return rc
+}
+
+func (rc *PerfResultCache) Get(commitNum int) *PerfResult {
+	rc.Next(commitNum) // fetch the commit, if necessary
+	return rc.results[commitNum]
+}
+
+// Next returns the next PerfResult for the commit commitNum.
+// It does not care whether the result has any data, failed or whatever.
+func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) {
+	// See if we have next result in the cache.
+	next := -1
+	for ci := range rc.results {
+		if rc.newer {
+			if ci > commitNum && (next == -1 || ci < next) {
+				next = ci
+			}
+		} else {
+			if ci < commitNum && (next == -1 || ci > next) {
+				next = ci
+			}
+		}
+	}
+	//rc.c.Errorf("PerfResultCache.Next: num=%v next=%v", commitNum, next)
+	if next != -1 {
+		return rc.results[next], nil
+	}
+	// Fetch next result from datastore.
+	res := new(PerfResult)
+	_, err := rc.iter.Next(res)
+	//rc.c.Errorf("PerfResultCache.Next: fetched %v %+v", err, res)
+	if err == datastore.Done {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("fetching perf results: %v", err)
+	}
+	if (rc.newer && res.CommitNum < commitNum) || (!rc.newer && res.CommitNum > commitNum) {
+		rc.c.Errorf("PerfResultCache.Next: bad commit num")
+	}
+	rc.results[res.CommitNum] = res
+	return res, nil
+}
+
+// NextForComparison returns PerfResult which we need to use for performance comprison.
+// It skips failed results, but does not skip results with no data.
+func (rc *PerfResultCache) NextForComparison(commitNum int, builder string) (*PerfResult, error) {
+	for {
+		res, err := rc.Next(commitNum)
+		if err != nil {
+			return nil, err
+		}
+		if res == nil {
+			return nil, nil
+		}
+		if res.CommitNum == commitNum {
+			continue
+		}
+		parsed := res.ParseData()
+		if builder != "" {
+			// Comparing for a particular builder.
+			// This is used in perf_changes and in email notifications.
+			b := parsed[builder]
+			if b == nil || b["meta-done"] == nil {
+				// No results yet, must not do the comparison.
+				return nil, nil
+			}
+			if b["meta-done"].OK {
+				// Have complete results, compare.
+				return res, nil
+			}
+		} else {
+			// Comparing for all builders, find a result with at least
+			// one successful meta-done.
+			// This is used in perf_detail.
+			for _, benchs := range parsed {
+				if data := benchs["meta-done"]; data != nil && data.OK {
+					return res, nil
+				}
+			}
+		}
+		// Failed, try next result.
+		commitNum = res.CommitNum
+	}
+}
+
+type PerfChange struct {
+	builder string
+	bench   string
+	metric  string
+	old     uint64
+	new     uint64
+	diff    float64
+}
+
+func significantPerfChanges(pc *PerfConfig, builder string, prevRes, res *PerfResult) (changes []*PerfChange) {
+	// First, collect all significant changes.
+	for builder1, benchmarks1 := range res.ParseData() {
+		if builder != "" && builder != builder1 {
+			// This is not the builder you're looking for, Luke.
+			continue
+		}
+		benchmarks0 := prevRes.ParseData()[builder1]
+		if benchmarks0 == nil {
+			continue
+		}
+		for benchmark, data1 := range benchmarks1 {
+			data0 := benchmarks0[benchmark]
+			if data0 == nil {
+				continue
+			}
+			for metric, val := range data1.Metrics {
+				val0 := data0.Metrics[metric]
+				if val0 == 0 {
+					continue
+				}
+				diff := perfDiff(val0, val)
+				noise := pc.NoiseLevel(builder, benchmark, metric)
+				if isNoise(diff, noise) {
+					continue
+				}
+				ch := &PerfChange{builder: builder, bench: benchmark, metric: metric, old: val0, new: val, diff: diff}
+				changes = append(changes, ch)
+			}
+		}
+	}
+	// Then, strip non-repeatable changes (flakes).
+	// The hypothesis is that a real change must show up with at least
+	// 2 different values of GOMAXPROCS.
+	cnt := make(map[string]int)
+	for _, ch := range changes {
+		b, _ := splitBench(ch.bench)
+		name := b + "|" + ch.metric
+		inc := 1
+		if ch.diff < 0 {
+			inc = -1
+		}
+		cnt[name] = cnt[name] + inc
+	}
+	for i := 0; i < len(changes); i++ {
+		ch := changes[i]
+		b, _ := splitBench(ch.bench)
+		name := b + "|" + ch.metric
+		if n := cnt[name]; n <= -2 || n >= 2 {
+			continue
+		}
+		last := len(changes) - 1
+		changes[i] = changes[last]
+		changes = changes[:last]
+		i--
+	}
+	return changes
+}
+
+// orderPrefTodo reorders commit nums for benchmarking todo.
+// The resulting order is somewhat tricky. We want 2 things:
+// 1. benchmark sequentially backwards (this provides information about most
+// recent changes, and allows to estimate noise levels)
+// 2. benchmark old commits in "scatter" order (this allows to quickly gather
+// brief information about thousands of old commits)
+// So this function interleaves the two orders.
+func orderPrefTodo(nums []int) []int {
+	sort.Ints(nums)
+	n := len(nums)
+	pow2 := uint32(0) // next power-of-two that is >= n
+	npow2 := 0
+	for npow2 <= n {
+		pow2++
+		npow2 = 1 << pow2
+	}
+	res := make([]int, n)
+	resPos := n - 1            // result array is filled backwards
+	present := make([]bool, n) // denotes values that already present in result array
+	for i0, i1 := n-1, 0; i0 >= 0 || i1 < npow2; {
+		// i0 represents "benchmark sequentially backwards" sequence
+		// find the next commit that is not yet present and add it
+		for cnt := 0; cnt < 2; cnt++ {
+			for ; i0 >= 0; i0-- {
+				if !present[i0] {
+					present[i0] = true
+					res[resPos] = nums[i0]
+					resPos--
+					i0--
+					break
+				}
+			}
+		}
+		// i1 represents "scatter order" sequence
+		// find the next commit that is not yet present and add it
+		for ; i1 < npow2; i1++ {
+			// do the "recursive split-ordering" trick
+			idx := 0 // bitwise reverse of i1
+			for j := uint32(0); j <= pow2; j++ {
+				if (i1 & (1 << j)) != 0 {
+					idx = idx | (1 << (pow2 - j - 1))
+				}
+			}
+			if idx < n && !present[idx] {
+				present[idx] = true
+				res[resPos] = nums[idx]
+				resPos--
+				i1++
+				break
+			}
+		}
+	}
+	// The above can't possibly be correct. Do dump check.
+	res2 := make([]int, n)
+	copy(res2, res)
+	sort.Ints(res2)
+	for i := range res2 {
+		if res2[i] != nums[i] {
+			panic(fmt.Sprintf("diff at %v: expect %v, want %v\nwas: %v\n become: %v",
+				i, nums[i], res2[i], nums, res2))
+		}
+	}
+	return res
+}

dashboard/app/build/test.go

@@ -32,7 +32,12 @@ func init() {
 var testEntityKinds = []string{
 	"Package",
 	"Commit",
+	"CommitRun",
 	"Result",
+	"PerfResult",
+	"PerfMetricRun",
+	"PerfConfig",
+	"PerfTodo",
 	"Log",
 }
 
@@ -47,7 +52,7 @@ var testPackages = []*Package{
 
 var tCommitTime = time.Now().Add(-time.Hour * 24 * 7)
 
-func tCommit(hash, parentHash, path string) *Commit {
+func tCommit(hash, parentHash, path string, bench bool) *Commit {
 	tCommitTime.Add(time.Hour) // each commit should have a different time
 	return &Commit{
 		PackagePath:       path,
@@ -56,6 +61,7 @@ func tCommit(hash, parentHash, path string) *Commit {
 		Time:              tCommitTime,
 		User:              "adg",
 		Desc:              "change description " + hash,
+		NeedsBenchmarking: bench,
 	}
 }
 
@@ -69,9 +75,9 @@ var testRequests = []struct {
 	{"/packages?kind=subrepo", nil, nil, []*Package{testPackage}},
 
 	// Go repo
-	{"/commit", nil, tCommit("0001", "0000", ""), nil},
+	{"/commit", nil, tCommit("0001", "0000", "", true), nil},
-	{"/commit", nil, tCommit("0002", "0001", ""), nil},
+	{"/commit", nil, tCommit("0002", "0001", "", false), nil},
-	{"/commit", nil, tCommit("0003", "0002", ""), nil},
+	{"/commit", nil, tCommit("0003", "0002", "", true), nil},
 	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
 	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
 	{"/result", nil, &Result{Builder: "linux-386", Hash: "0001", OK: true}, nil},
@@ -95,8 +101,8 @@ var testRequests = []struct {
 	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}},
 
 	// branches
-	{"/commit", nil, tCommit("0004", "0003", ""), nil},
+	{"/commit", nil, tCommit("0004", "0003", "", false), nil},
-	{"/commit", nil, tCommit("0005", "0002", ""), nil},
+	{"/commit", nil, tCommit("0005", "0002", "", false), nil},
 	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
 	{"/result", nil, &Result{Builder: "linux-386", Hash: "0005", OK: true}, nil},
 	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}},
@@ -112,9 +118,9 @@ var testRequests = []struct {
 	{"/result", nil, &Result{Builder: "linux-386", Hash: "0003", OK: false, Log: "test"}, nil},
 
 	// non-Go repos
-	{"/commit", nil, tCommit("1001", "1000", testPkg), nil},
+	{"/commit", nil, tCommit("1001", "1000", testPkg, false), nil},
-	{"/commit", nil, tCommit("1002", "1001", testPkg), nil},
+	{"/commit", nil, tCommit("1002", "1001", testPkg, false), nil},
-	{"/commit", nil, tCommit("1003", "1002", testPkg), nil},
+	{"/commit", nil, tCommit("1003", "1002", testPkg, false), nil},
 	{"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1003"}}},
 	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0001", OK: true}, nil},
 	{"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1002"}}},
@@ -128,6 +134,84 @@ var testRequests = []struct {
 	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
 	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0005", OK: false, Log: "boo"}, nil},
 	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, nil},
+
+	// benchmarks
+	// build-go-commit must have precedence over benchmark-go-commit
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
+	// drain build-go-commit todo
+	{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0005", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}},
+	{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0004", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}},
+	{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0002", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0001"}}},
+	{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0001", OK: true}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0005", OK: false}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0005", OK: false}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0005", OK: false}, nil},
+	// now we must get benchmark todo
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{}}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0003", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http"}}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "json", Hash: "0003", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http", "json"}}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0003", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001", PerfResults: []string{}}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0001", OK: true}, nil},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0001", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil},
+	// create new commit, it must appear in todo
+	{"/commit", nil, tCommit("0006", "0005", "", true), nil},
+	// drain build-go-commit todo
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0006"}}},
+	{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0006", OK: true}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0006", OK: false}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0006", OK: false}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0006", OK: false}, nil},
+	// now we must get benchmark todo
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006", PerfResults: []string{}}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0006", OK: true}, nil},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0006", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil},
+	// create new benchmark, all commits must re-appear in todo
+	{"/commit", nil, tCommit("0007", "0006", "", true), nil},
+	// drain build-go-commit todo
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0007"}}},
+	{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0007", OK: true}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0007", OK: false}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0007", OK: false}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0007", OK: false}, nil},
+	// now we must get benchmark todo
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007", PerfResults: []string{}}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "bson", Hash: "0007", OK: true}, nil},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0007", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007", PerfResults: []string{"bson"}}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0007", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006", PerfResults: []string{"http"}}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0006", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http", "json"}}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0003", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001", PerfResults: []string{"http"}}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0001", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil},
+	// attach second builder
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0007"}}},
+	// drain build-go-commit todo
+	{"/result", nil, &Result{Builder: "linux-386", Hash: "0007", OK: true}, nil},
+	{"/result", nil, &Result{Builder: "linux-386", Hash: "0006", OK: true}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0007", OK: false}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1002", GoHash: "0007", OK: false}, nil},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0007", OK: false}, nil},
+	// now we must get benchmark todo
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007"}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0007", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006"}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0006", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003"}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0003", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001"}}},
+	{"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0001", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, nil},
 }
 
 func testHandler(w http.ResponseWriter, r *http.Request) {
@@ -159,7 +243,7 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
 		*r = origReq
 	}()
 	for i, t := range testRequests {
-		c.Infof("running test %d %s", i, t.path)
+		c.Infof("running test %d %s vals='%q' req='%q' res='%q'", i, t.path, t.vals, t.req, t.res)
 		errorf := func(format string, args ...interface{}) {
 			fmt.Fprintf(w, "%d %s: ", i, t.path)
 			fmt.Fprintf(w, format, args...)
@@ -194,11 +278,12 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
 			errorf(rec.Body.String())
 			return
 		}
+		c.Infof("response='%v'", rec.Body.String())
 		resp := new(dashResponse)
 
 		// If we're expecting a *Todo value,
 		// prime the Response field with a Todo and a Commit inside it.
-		if _, ok := t.res.(*Todo); ok {
+		if t.path == "/todo" {
 			resp.Response = &Todo{Data: &Commit{}}
 		}
 
@@ -241,14 +326,28 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
 				errorf("Response.Data not *Commit: %T", g.Data)
 				return
 			}
-			if eh := e.Data.(*Commit).Hash; eh != gd.Hash {
+			if g.Kind != e.Kind {
-				errorf("hashes don't match: got %q, want %q", gd.Hash, eh)
+				errorf("kind don't match: got %q, want %q", g.Kind, e.Kind)
+				return
+			}
+			ed := e.Data.(*Commit)
+			if ed.Hash != gd.Hash {
+				errorf("hashes don't match: got %q, want %q", gd.Hash, ed.Hash)
+				return
+			}
+			if len(gd.PerfResults) != len(ed.PerfResults) {
+				errorf("result data len don't match: got %v, want %v", len(gd.PerfResults), len(ed.PerfResults))
+				return
+			}
+			for i := range gd.PerfResults {
+				if gd.PerfResults[i] != ed.PerfResults[i] {
+					errorf("result data %v don't match: got %v, want %v", i, gd.PerfResults[i], ed.PerfResults[i])
 					return
 				}
 			}
+		}
 		if t.res == nil && resp.Response != nil {
-			errorf("response mismatch: got %q expected <nil>",
+			errorf("response mismatch: got %q expected <nil>", resp.Response)
-				resp.Response)
 			return
 		}
 	}

dashboard/app/cron.yaml

@@ -0,0 +1,5 @@
+cron:
+- description: updates noise level for benchmarking results
+  url: /perflearn
+  schedule: every 24 hours
+  

dashboard/app/index.yaml

@@ -11,3 +11,29 @@ indexes:
   properties:
   - name: Time
     direction: desc
+    
+- kind: Commit
+  ancestor: yes
+  properties:
+  - name: NeedsBenchmarking
+  - name: Num
+    direction: desc
+
+- kind: CommitRun
+  ancestor: yes
+  properties:
+  - name: StartCommitNum
+    direction: desc
+
+- kind: PerfResult
+  ancestor: yes
+  properties:
+  - name: CommitNum
+    direction: desc
+
+- kind: PerfResult   
+  ancestor: yes
+  properties:
+  - name: CommitNum
+    direction: asc
+