314 lines
8.3 KiB
Go
314 lines
8.3 KiB
Go
// Copyright 2014 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// +build appengine
|
|
|
|
package build
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"appengine"
|
|
"appengine/datastore"
|
|
)
|
|
|
|
var knownTags = map[string]string{
|
|
"go1": "0051c7442fed9c888de6617fa9239a913904d96e",
|
|
"go1.1": "d29da2ced72ba2cf48ed6a8f1ec4abc01e4c5bf1",
|
|
"go1.2": "b1edf8faa5d6cbc50c6515785df9df9c19296564",
|
|
"go1.3": "f153208c0a0e306bfca14f71ef11f09859ccabc8",
|
|
"go1.4": "faa3ed1dc30e42771a68b6337dcf8be9518d5c07",
|
|
}
|
|
|
|
var lastRelease = "go1.4"
|
|
|
|
func splitBench(benchProcs string) (string, int) {
|
|
ss := strings.Split(benchProcs, "-")
|
|
procs, _ := strconv.Atoi(ss[1])
|
|
return ss[0], procs
|
|
}
|
|
|
|
func dashPerfCommits(c appengine.Context, page int) ([]*Commit, error) {
|
|
q := datastore.NewQuery("Commit").
|
|
Ancestor((&Package{}).Key(c)).
|
|
Order("-Num").
|
|
Filter("NeedsBenchmarking =", true).
|
|
Limit(commitsPerPage).
|
|
Offset(page * commitsPerPage)
|
|
var commits []*Commit
|
|
_, err := q.GetAll(c, &commits)
|
|
if err == nil && len(commits) == 0 {
|
|
err = fmt.Errorf("no commits")
|
|
}
|
|
return commits, err
|
|
}
|
|
|
|
func perfChangeStyle(pc *PerfConfig, v float64, builder, benchmark, metric string) string {
|
|
noise := pc.NoiseLevel(builder, benchmark, metric)
|
|
if isNoise(v, noise) {
|
|
return "noise"
|
|
}
|
|
if v > 0 {
|
|
return "bad"
|
|
}
|
|
return "good"
|
|
}
|
|
|
|
func isNoise(diff, noise float64) bool {
|
|
rnoise := -100 * noise / (noise + 100)
|
|
return diff < noise && diff > rnoise
|
|
}
|
|
|
|
func perfDiff(old, new uint64) float64 {
|
|
return 100*float64(new)/float64(old) - 100
|
|
}
|
|
|
|
func isPerfFailed(res *PerfResult, builder string) bool {
|
|
data := res.ParseData()[builder]
|
|
return data != nil && data["meta-done"] != nil && !data["meta-done"].OK
|
|
}
|
|
|
|
// PerfResultCache caches a set of PerfResults so that it's easy to access them
|
|
// without lots of duplicate accesses to datastore.
|
|
// It allows to iterate over newer or older results for some base commit.
|
|
type PerfResultCache struct {
|
|
c appengine.Context
|
|
newer bool
|
|
iter *datastore.Iterator
|
|
results map[int]*PerfResult
|
|
}
|
|
|
|
func MakePerfResultCache(c appengine.Context, com *Commit, newer bool) *PerfResultCache {
|
|
p := &Package{}
|
|
q := datastore.NewQuery("PerfResult").Ancestor(p.Key(c)).Limit(100)
|
|
if newer {
|
|
q = q.Filter("CommitNum >=", com.Num).Order("CommitNum")
|
|
} else {
|
|
q = q.Filter("CommitNum <=", com.Num).Order("-CommitNum")
|
|
}
|
|
rc := &PerfResultCache{c: c, newer: newer, iter: q.Run(c), results: make(map[int]*PerfResult)}
|
|
return rc
|
|
}
|
|
|
|
func (rc *PerfResultCache) Get(commitNum int) *PerfResult {
|
|
rc.Next(commitNum) // fetch the commit, if necessary
|
|
return rc.results[commitNum]
|
|
}
|
|
|
|
// Next returns the next PerfResult for the commit commitNum.
|
|
// It does not care whether the result has any data, failed or whatever.
|
|
func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) {
|
|
// See if we have next result in the cache.
|
|
next := -1
|
|
for ci := range rc.results {
|
|
if rc.newer {
|
|
if ci > commitNum && (next == -1 || ci < next) {
|
|
next = ci
|
|
}
|
|
} else {
|
|
if ci < commitNum && (next == -1 || ci > next) {
|
|
next = ci
|
|
}
|
|
}
|
|
}
|
|
if next != -1 {
|
|
return rc.results[next], nil
|
|
}
|
|
// Fetch next result from datastore.
|
|
res := new(PerfResult)
|
|
_, err := rc.iter.Next(res)
|
|
if err == datastore.Done {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetching perf results: %v", err)
|
|
}
|
|
if (rc.newer && res.CommitNum < commitNum) || (!rc.newer && res.CommitNum > commitNum) {
|
|
rc.c.Errorf("PerfResultCache.Next: bad commit num")
|
|
}
|
|
rc.results[res.CommitNum] = res
|
|
return res, nil
|
|
}
|
|
|
|
// NextForComparison returns PerfResult which we need to use for performance comprison.
|
|
// It skips failed results, but does not skip results with no data.
|
|
func (rc *PerfResultCache) NextForComparison(commitNum int, builder string) (*PerfResult, error) {
|
|
for {
|
|
res, err := rc.Next(commitNum)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if res == nil {
|
|
return nil, nil
|
|
}
|
|
if res.CommitNum == commitNum {
|
|
continue
|
|
}
|
|
parsed := res.ParseData()
|
|
if builder != "" {
|
|
// Comparing for a particular builder.
|
|
// This is used in perf_changes and in email notifications.
|
|
b := parsed[builder]
|
|
if b == nil || b["meta-done"] == nil {
|
|
// No results yet, must not do the comparison.
|
|
return nil, nil
|
|
}
|
|
if b["meta-done"].OK {
|
|
// Have complete results, compare.
|
|
return res, nil
|
|
}
|
|
} else {
|
|
// Comparing for all builders, find a result with at least
|
|
// one successful meta-done.
|
|
// This is used in perf_detail.
|
|
for _, benchs := range parsed {
|
|
if data := benchs["meta-done"]; data != nil && data.OK {
|
|
return res, nil
|
|
}
|
|
}
|
|
}
|
|
// Failed, try next result.
|
|
commitNum = res.CommitNum
|
|
}
|
|
}
|
|
|
|
type PerfChange struct {
|
|
Builder string
|
|
Bench string
|
|
Metric string
|
|
Old uint64
|
|
New uint64
|
|
Diff float64
|
|
}
|
|
|
|
func significantPerfChanges(pc *PerfConfig, builder string, prevRes, res *PerfResult) (changes []*PerfChange) {
|
|
// First, collect all significant changes.
|
|
for builder1, benchmarks1 := range res.ParseData() {
|
|
if builder != "" && builder != builder1 {
|
|
// This is not the builder you're looking for, Luke.
|
|
continue
|
|
}
|
|
benchmarks0 := prevRes.ParseData()[builder1]
|
|
if benchmarks0 == nil {
|
|
continue
|
|
}
|
|
for benchmark, data1 := range benchmarks1 {
|
|
data0 := benchmarks0[benchmark]
|
|
if data0 == nil {
|
|
continue
|
|
}
|
|
for metric, val := range data1.Metrics {
|
|
val0 := data0.Metrics[metric]
|
|
if val0 == 0 {
|
|
continue
|
|
}
|
|
diff := perfDiff(val0, val)
|
|
noise := pc.NoiseLevel(builder, benchmark, metric)
|
|
if isNoise(diff, noise) {
|
|
continue
|
|
}
|
|
ch := &PerfChange{Builder: builder, Bench: benchmark, Metric: metric, Old: val0, New: val, Diff: diff}
|
|
changes = append(changes, ch)
|
|
}
|
|
}
|
|
}
|
|
// Then, strip non-repeatable changes (flakes).
|
|
// The hypothesis is that a real change must show up with the majority of GOMAXPROCS values.
|
|
majority := len(pc.ProcList(builder))/2 + 1
|
|
cnt := make(map[string]int)
|
|
for _, ch := range changes {
|
|
b, _ := splitBench(ch.Bench)
|
|
name := b + "|" + ch.Metric
|
|
if ch.Diff < 0 {
|
|
name += "--"
|
|
}
|
|
cnt[name] = cnt[name] + 1
|
|
}
|
|
for i := 0; i < len(changes); i++ {
|
|
ch := changes[i]
|
|
b, _ := splitBench(ch.Bench)
|
|
name := b + "|" + ch.Metric
|
|
if cnt[name] >= majority {
|
|
continue
|
|
}
|
|
if cnt[name+"--"] >= majority {
|
|
continue
|
|
}
|
|
// Remove flake.
|
|
last := len(changes) - 1
|
|
changes[i] = changes[last]
|
|
changes = changes[:last]
|
|
i--
|
|
}
|
|
return changes
|
|
}
|
|
|
|
// orderPerfTodo reorders commit nums for benchmarking todo.
|
|
// The resulting order is somewhat tricky. We want 2 things:
|
|
// 1. benchmark sequentially backwards (this provides information about most
|
|
// recent changes, and allows to estimate noise levels)
|
|
// 2. benchmark old commits in "scatter" order (this allows to quickly gather
|
|
// brief information about thousands of old commits)
|
|
// So this function interleaves the two orders.
|
|
func orderPerfTodo(nums []int) []int {
|
|
sort.Ints(nums)
|
|
n := len(nums)
|
|
pow2 := uint32(0) // next power-of-two that is >= n
|
|
npow2 := 0
|
|
for npow2 <= n {
|
|
pow2++
|
|
npow2 = 1 << pow2
|
|
}
|
|
res := make([]int, n)
|
|
resPos := n - 1 // result array is filled backwards
|
|
present := make([]bool, n) // denotes values that already present in result array
|
|
for i0, i1 := n-1, 0; i0 >= 0 || i1 < npow2; {
|
|
// i0 represents "benchmark sequentially backwards" sequence
|
|
// find the next commit that is not yet present and add it
|
|
for cnt := 0; cnt < 2; cnt++ {
|
|
for ; i0 >= 0; i0-- {
|
|
if !present[i0] {
|
|
present[i0] = true
|
|
res[resPos] = nums[i0]
|
|
resPos--
|
|
i0--
|
|
break
|
|
}
|
|
}
|
|
}
|
|
// i1 represents "scatter order" sequence
|
|
// find the next commit that is not yet present and add it
|
|
for ; i1 < npow2; i1++ {
|
|
// do the "recursive split-ordering" trick
|
|
idx := 0 // bitwise reverse of i1
|
|
for j := uint32(0); j <= pow2; j++ {
|
|
if (i1 & (1 << j)) != 0 {
|
|
idx = idx | (1 << (pow2 - j - 1))
|
|
}
|
|
}
|
|
if idx < n && !present[idx] {
|
|
present[idx] = true
|
|
res[resPos] = nums[idx]
|
|
resPos--
|
|
i1++
|
|
break
|
|
}
|
|
}
|
|
}
|
|
// The above can't possibly be correct. Do dump check.
|
|
res2 := make([]int, n)
|
|
copy(res2, res)
|
|
sort.Ints(res2)
|
|
for i := range res2 {
|
|
if res2[i] != nums[i] {
|
|
panic(fmt.Sprintf("diff at %v: expect %v, want %v\nwas: %v\n become: %v",
|
|
i, nums[i], res2[i], nums, res2))
|
|
}
|
|
}
|
|
return res
|
|
}
|