diff --git a/dashboard/app/build/build.go b/dashboard/app/build/build.go index 2fa6e7b4..87a5befa 100644 --- a/dashboard/app/build/build.go +++ b/dashboard/app/build/build.go @@ -14,14 +14,22 @@ import ( "fmt" "io" "io/ioutil" + "net/http" + "sort" + "strconv" "strings" "time" "appengine" "appengine/datastore" + + "cache" ) -const maxDatastoreStringLen = 500 +const ( + maxDatastoreStringLen = 500 + PerfRunLength = 1024 +) // A Package describes a package that is listed on the dashboard. type Package struct { @@ -89,9 +97,10 @@ type Commit struct { ParentHash string Num int // Internal monotonic counter unique to this package. - User string - Desc string `datastore:",noindex"` - Time time.Time + User string + Desc string `datastore:",noindex"` + Time time.Time + NeedsBenchmarking bool // ResultData is the Data string of each build Result for this Commit. // For non-Go commits, only the Results for the current Go tip, weekly, @@ -99,6 +108,10 @@ type Commit struct { // The complete data set is stored in Result entities. ResultData []string `datastore:",noindex"` + // PerfResults holds a set of “builder|benchmark” tuples denoting + // what benchmarks have been executed on the commit. + PerfResults []string `datastore:",noindex"` + FailNotificationSent bool } @@ -138,6 +151,28 @@ func (com *Commit) AddResult(c appengine.Context, r *Result) error { return nil } +// AddPerfResult remembers that the builder has run the benchmark on the commit. +// It must be called from inside a datastore transaction. +func (com *Commit) AddPerfResult(c appengine.Context, builder, benchmark string) error { + if err := datastore.Get(c, com.Key(c), com); err != nil { + return fmt.Errorf("getting Commit: %v", err) + } + if !com.NeedsBenchmarking { + return fmt.Errorf("trying to add perf result to Commit(%v) that does not require benchmarking", com.Hash) + } + s := builder + "|" + benchmark + for _, v := range com.PerfResults { + if v == s { + return nil + } + } + com.PerfResults = append(com.PerfResults, s) + if _, err := datastore.Put(c, com.Key(c), com); err != nil { + return fmt.Errorf("putting Commit: %v", err) + } + return nil +} + func trim(s []string, n int) []string { l := min(len(s), n) return s[len(s)-l:] @@ -207,6 +242,94 @@ func reverse(s []string) { } } +// A CommitRun provides summary information for commits [StartCommitNum, StartCommitNum + PerfRunLength). +// Descendant of Package. +type CommitRun struct { + PackagePath string // (empty for main repo commits) + StartCommitNum int + Hash []string `datastore:",noindex"` + User []string `datastore:",noindex"` + Desc []string `datastore:",noindex"` // Only first line. + Time []time.Time `datastore:",noindex"` + NeedsBenchmarking []bool `datastore:",noindex"` +} + +func (cr *CommitRun) Key(c appengine.Context) *datastore.Key { + p := Package{Path: cr.PackagePath} + key := strconv.Itoa(cr.StartCommitNum) + return datastore.NewKey(c, "CommitRun", key, 0, p.Key(c)) +} + +// GetCommitRun loads and returns CommitRun that contains information +// for commit commitNum. +func GetCommitRun(c appengine.Context, commitNum int) (*CommitRun, error) { + cr := &CommitRun{StartCommitNum: commitNum / PerfRunLength * PerfRunLength} + err := datastore.Get(c, cr.Key(c), cr) + if err != nil && err != datastore.ErrNoSuchEntity { + return nil, fmt.Errorf("getting CommitRun: %v", err) + } + if len(cr.Hash) != PerfRunLength { + cr.Hash = make([]string, PerfRunLength) + cr.User = make([]string, PerfRunLength) + cr.Desc = make([]string, PerfRunLength) + cr.Time = make([]time.Time, PerfRunLength) + cr.NeedsBenchmarking = make([]bool, PerfRunLength) + } + return cr, nil +} + +func (cr *CommitRun) AddCommit(c appengine.Context, com *Commit) error { + if com.Num < cr.StartCommitNum || com.Num >= cr.StartCommitNum+PerfRunLength { + return fmt.Errorf("AddCommit: commit num %v out of range [%v, %v)", + com.Num, cr.StartCommitNum, cr.StartCommitNum+PerfRunLength) + } + i := com.Num - cr.StartCommitNum + // Be careful with string lengths, + // we need to fit 1024 commits into 1 MB. + cr.Hash[i] = com.Hash + cr.User[i] = shortDesc(com.User) + cr.Desc[i] = shortDesc(com.Desc) + cr.Time[i] = com.Time + cr.NeedsBenchmarking[i] = com.NeedsBenchmarking + if _, err := datastore.Put(c, cr.Key(c), cr); err != nil { + return fmt.Errorf("putting CommitRun: %v", err) + } + return nil +} + +// GetCommits returns [startCommitNum, startCommitNum+n) commits. +// Commits information is partial (obtained from CommitRun), +// do not store them back into datastore. +func GetCommits(c appengine.Context, startCommitNum, n int) ([]*Commit, error) { + if startCommitNum < 0 || n <= 0 { + return nil, fmt.Errorf("GetCommits: invalid args (%v, %v)", startCommitNum, n) + } + var res []*Commit + for n > 0 { + cr, err := GetCommitRun(c, startCommitNum) + if err != nil { + return nil, err + } + idx := startCommitNum - cr.StartCommitNum + cnt := PerfRunLength - idx + if cnt > n { + cnt = n + } + for i := idx; i < idx+cnt; i++ { + com := new(Commit) + com.Hash = cr.Hash[i] + com.User = cr.User[i] + com.Desc = cr.Desc[i] + com.Time = cr.Time[i] + com.NeedsBenchmarking = cr.NeedsBenchmarking[i] + res = append(res, com) + } + startCommitNum += cnt + n -= cnt + } + return res, nil +} + // partsToHash converts a Commit and ResultData substrings to a Result. func partsToHash(c *Commit, p []string) *Result { return &Result{ @@ -223,9 +346,9 @@ func partsToHash(c *Commit, p []string) *Result { // // Each Result entity is a descendant of its associated Package entity. type Result struct { + PackagePath string // (empty for Go commits) Builder string // "os-arch[-note]" Hash string - PackagePath string // (empty for Go commits) // The Go Commit this was built against (empty for Go commits). GoHash string @@ -259,6 +382,349 @@ func (r *Result) Data() string { return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash) } +// A PerfResult describes all benchmarking result for a Commit. +// Descendant of Package. +type PerfResult struct { + PackagePath string + CommitHash string + CommitNum int + Data []string `datastore:",noindex"` // "builder|benchmark|ok|metric1=val1|metric2=val2|file:log=hash|file:cpuprof=hash" + + // Local cache with parsed Data. + // Maps builder->benchmark->ParsedPerfResult. + parsedData map[string]map[string]*ParsedPerfResult +} + +type ParsedPerfResult struct { + OK bool + Metrics map[string]uint64 + Artifacts map[string]string +} + +func (r *PerfResult) Key(c appengine.Context) *datastore.Key { + p := Package{Path: r.PackagePath} + key := r.CommitHash + return datastore.NewKey(c, "PerfResult", key, 0, p.Key(c)) +} + +// AddResult add the benchmarking result to r. +// Existing result for the same builder/benchmark is replaced if already exists. +// Returns whether the result was already present. +func (r *PerfResult) AddResult(req *PerfRequest) bool { + present := false + str := fmt.Sprintf("%v|%v|", req.Builder, req.Benchmark) + for i, s := range r.Data { + if strings.HasPrefix(s, str) { + present = true + last := len(r.Data) - 1 + r.Data[i] = r.Data[last] + r.Data = r.Data[:last] + break + } + } + ok := "ok" + if !req.OK { + ok = "false" + } + str += ok + for _, m := range req.Metrics { + str += fmt.Sprintf("|%v=%v", m.Type, m.Val) + } + for _, a := range req.Artifacts { + str += fmt.Sprintf("|file:%v=%v", a.Type, a.Body) + } + r.Data = append(r.Data, str) + r.parsedData = nil + return present +} + +func (r *PerfResult) ParseData() map[string]map[string]*ParsedPerfResult { + if r.parsedData != nil { + return r.parsedData + } + res := make(map[string]map[string]*ParsedPerfResult) + for _, str := range r.Data { + ss := strings.Split(str, "|") + builder := ss[0] + bench := ss[1] + ok := ss[2] + m := res[builder] + if m == nil { + m = make(map[string]*ParsedPerfResult) + res[builder] = m + } + var p ParsedPerfResult + p.OK = ok == "ok" + p.Metrics = make(map[string]uint64) + p.Artifacts = make(map[string]string) + for _, entry := range ss[3:] { + if strings.HasPrefix(entry, "file:") { + ss1 := strings.Split(entry[len("file:"):], "=") + p.Artifacts[ss1[0]] = ss1[1] + } else { + ss1 := strings.Split(entry, "=") + val, _ := strconv.ParseUint(ss1[1], 10, 64) + p.Metrics[ss1[0]] = val + } + } + m[bench] = &p + } + r.parsedData = res + return res +} + +// A PerfMetricRun entity holds a set of metric values for builder/benchmark/metric +// for commits [StartCommitNum, StartCommitNum + PerfRunLength). +// Descendant of Package. +type PerfMetricRun struct { + PackagePath string + Builder string + Benchmark string + Metric string // e.g. realtime, cputime, gc-pause + StartCommitNum int + Vals []int64 `datastore:",noindex"` +} + +func (m *PerfMetricRun) Key(c appengine.Context) *datastore.Key { + p := Package{Path: m.PackagePath} + key := m.Builder + "|" + m.Benchmark + "|" + m.Metric + "|" + strconv.Itoa(m.StartCommitNum) + return datastore.NewKey(c, "PerfMetricRun", key, 0, p.Key(c)) +} + +// GetPerfMetricRun loads and returns PerfMetricRun that contains information +// for commit commitNum. +func GetPerfMetricRun(c appengine.Context, builder, benchmark, metric string, commitNum int) (*PerfMetricRun, error) { + startCommitNum := commitNum / PerfRunLength * PerfRunLength + m := &PerfMetricRun{Builder: builder, Benchmark: benchmark, Metric: metric, StartCommitNum: startCommitNum} + err := datastore.Get(c, m.Key(c), m) + if err != nil && err != datastore.ErrNoSuchEntity { + return nil, fmt.Errorf("getting PerfMetricRun: %v", err) + } + if len(m.Vals) != PerfRunLength { + m.Vals = make([]int64, PerfRunLength) + } + return m, nil +} + +func (m *PerfMetricRun) AddMetric(c appengine.Context, commitNum int, v uint64) error { + if commitNum < m.StartCommitNum || commitNum >= m.StartCommitNum+PerfRunLength { + return fmt.Errorf("AddMetric: CommitNum %v out of range [%v, %v)", + commitNum, m.StartCommitNum, m.StartCommitNum+PerfRunLength) + } + m.Vals[commitNum-m.StartCommitNum] = int64(v) + if _, err := datastore.Put(c, m.Key(c), m); err != nil { + return fmt.Errorf("putting PerfMetricRun: %v", err) + } + return nil +} + +// GetPerfMetricsForCommits returns perf metrics for builder/benchmark/metric +// and commits [startCommitNum, startCommitNum+n). +func GetPerfMetricsForCommits(c appengine.Context, builder, benchmark, metric string, startCommitNum, n int) ([]uint64, error) { + if startCommitNum < 0 || n <= 0 { + return nil, fmt.Errorf("GetPerfMetricsForCommits: invalid args (%v, %v)", startCommitNum, n) + } + var res []uint64 + for n > 0 { + metrics, err := GetPerfMetricRun(c, builder, benchmark, metric, startCommitNum) + if err != nil { + return nil, err + } + idx := startCommitNum - metrics.StartCommitNum + cnt := PerfRunLength - idx + if cnt > n { + cnt = n + } + for _, v := range metrics.Vals[idx : idx+cnt] { + res = append(res, uint64(v)) + } + startCommitNum += cnt + n -= cnt + } + return res, nil +} + +// PerfConfig holds read-mostly configuration related to benchmarking. +// There is only one PerfConfig entity. +type PerfConfig struct { + BuilderBench []string `datastore:",noindex"` // "builder|benchmark" pairs + BuilderProcs []string `datastore:",noindex"` // "builder|proc" pairs + BenchMetric []string `datastore:",noindex"` // "benchmark|metric" pairs + NoiseLevels []string `datastore:",noindex"` // "builder|benchmark|metric1=noise1|metric2=noise2" + + // Local cache of "builder|benchmark|metric" -> noise. + noise map[string]float64 +} + +func PerfConfigKey(c appengine.Context) *datastore.Key { + p := Package{} + return datastore.NewKey(c, "PerfConfig", "PerfConfig", 0, p.Key(c)) +} + +const perfConfigCacheKey = "perf-config" + +func GetPerfConfig(c appengine.Context, r *http.Request) (*PerfConfig, error) { + pc := new(PerfConfig) + now := cache.Now(c) + if cache.Get(r, now, perfConfigCacheKey, pc) { + return pc, nil + } + err := datastore.Get(c, PerfConfigKey(c), pc) + if err != nil && err != datastore.ErrNoSuchEntity { + return nil, fmt.Errorf("GetPerfConfig: %v", err) + } + cache.Set(r, now, perfConfigCacheKey, pc) + return pc, nil +} + +func (pc *PerfConfig) NoiseLevel(builder, benchmark, metric string) float64 { + if pc.noise == nil { + pc.noise = make(map[string]float64) + for _, str := range pc.NoiseLevels { + split := strings.Split(str, "|") + builderBench := split[0] + "|" + split[1] + for _, entry := range split[2:] { + metricValue := strings.Split(entry, "=") + noise, _ := strconv.ParseFloat(metricValue[1], 64) + pc.noise[builderBench+"|"+metricValue[0]] = noise + } + } + } + me := fmt.Sprintf("%v|%v|%v", builder, benchmark, metric) + n := pc.noise[me] + if n == 0 { + // Use a very conservative value + // until we have learned the real noise level. + n = 200 + } + return n +} + +// UpdatePerfConfig updates the PerfConfig entity with results of benchmarking. +// Returns whether it's a benchmark that we have not yet seem on the builder. +func UpdatePerfConfig(c appengine.Context, r *http.Request, req *PerfRequest) (newBenchmark bool, err error) { + pc, err := GetPerfConfig(c, r) + if err != nil { + return false, err + } + + modified := false + add := func(arr *[]string, str string) { + for _, s := range *arr { + if s == str { + return + } + } + *arr = append(*arr, str) + modified = true + return + } + + BenchProcs := strings.Split(req.Benchmark, "-") + benchmark := BenchProcs[0] + procs := "1" + if len(BenchProcs) > 1 { + procs = BenchProcs[1] + } + + add(&pc.BuilderBench, req.Builder+"|"+benchmark) + newBenchmark = modified + add(&pc.BuilderProcs, req.Builder+"|"+procs) + for _, m := range req.Metrics { + add(&pc.BenchMetric, benchmark+"|"+m.Type) + } + + if modified { + if _, err := datastore.Put(c, PerfConfigKey(c), pc); err != nil { + return false, fmt.Errorf("putting PerfConfig: %v", err) + } + cache.Tick(c) + } + return newBenchmark, nil +} + +func collectList(all []string, idx int, second string) (res []string) { + m := make(map[string]bool) + for _, str := range all { + ss := strings.Split(str, "|") + v := ss[idx] + v2 := ss[1-idx] + if (second == "" || second == v2) && !m[v] { + m[v] = true + res = append(res, v) + } + } + sort.Strings(res) + return res +} + +func (pc *PerfConfig) BuildersForBenchmark(bench string) []string { + return collectList(pc.BuilderBench, 0, bench) +} + +func (pc *PerfConfig) BenchmarksForBuilder(builder string) []string { + return collectList(pc.BuilderBench, 1, builder) +} + +func (pc *PerfConfig) MetricsForBenchmark(bench string) []string { + return collectList(pc.BenchMetric, 1, bench) +} + +func (pc *PerfConfig) BenchmarkProcList() (res []string) { + bl := pc.BenchmarksForBuilder("") + pl := pc.ProcList("") + for _, b := range bl { + for _, p := range pl { + res = append(res, fmt.Sprintf("%v-%v", b, p)) + } + } + return res +} + +func (pc *PerfConfig) ProcList(builder string) []int { + ss := collectList(pc.BuilderProcs, 1, builder) + var procs []int + for _, s := range ss { + p, _ := strconv.ParseInt(s, 10, 32) + procs = append(procs, int(p)) + } + sort.Ints(procs) + return procs +} + +// A PerfTodo contains outstanding commits for benchmarking for a builder. +// Descendant of Package. +type PerfTodo struct { + PackagePath string // (empty for main repo commits) + Builder string + CommitNums []int `datastore:",noindex"` // LIFO queue of commits to benchmark. +} + +func (todo *PerfTodo) Key(c appengine.Context) *datastore.Key { + p := Package{Path: todo.PackagePath} + key := todo.Builder + return datastore.NewKey(c, "PerfTodo", key, 0, p.Key(c)) +} + +// AddCommitToPerfTodo adds the commit to all existing PerfTodo entities. +func AddCommitToPerfTodo(c appengine.Context, com *Commit) error { + var todos []*PerfTodo + _, err := datastore.NewQuery("PerfTodo"). + Ancestor((&Package{}).Key(c)). + GetAll(c, &todos) + if err != nil { + return fmt.Errorf("fetching PerfTodo's: %v", err) + } + for _, todo := range todos { + todo.CommitNums = append(todo.CommitNums, com.Num) + _, err = datastore.Put(c, todo.Key(c), todo) + if err != nil { + return fmt.Errorf("updating PerfTodo: %v", err) + } + } + return nil +} + // A Log is a gzip-compressed log file stored under the SHA1 hash of the // uncompressed log text. type Log struct { diff --git a/dashboard/app/build/handler.go b/dashboard/app/build/handler.go index bdef89e0..59aa7842 100644 --- a/dashboard/app/build/handler.go +++ b/dashboard/app/build/handler.go @@ -44,6 +44,9 @@ func commitHandler(r *http.Request) (interface{}, error) { if err := datastore.Get(c, com.Key(c), com); err != nil { return nil, fmt.Errorf("getting Commit: %v", err) } + // Strip potentially large and unnecessary fields. + com.ResultData = nil + com.PerfResults = nil return com, nil } if r.Method != "POST" { @@ -115,6 +118,25 @@ func addCommit(c appengine.Context, com *Commit) error { if _, err = datastore.Put(c, com.Key(c), com); err != nil { return fmt.Errorf("putting Commit: %v", err) } + if com.NeedsBenchmarking { + // add to CommitRun + cr, err := GetCommitRun(c, com.Num) + if err != nil { + return err + } + if err = cr.AddCommit(c, com); err != nil { + return err + } + // create PerfResult + res := &PerfResult{CommitHash: com.Hash, CommitNum: com.Num} + if _, err := datastore.Put(c, res.Key(c), res); err != nil { + return fmt.Errorf("putting PerfResult: %v", err) + } + // Update perf todo if necessary. + if err = AddCommitToPerfTodo(c, com); err != nil { + return err + } + } return nil } @@ -165,10 +187,18 @@ func todoHandler(r *http.Request) (interface{}, error) { switch kind { case "build-go-commit": com, err = buildTodo(c, builder, "", "") + if com != nil { + com.PerfResults = []string{} + } case "build-package": packagePath := r.FormValue("packagePath") goHash := r.FormValue("goHash") com, err = buildTodo(c, builder, packagePath, goHash) + if com != nil { + com.PerfResults = []string{} + } + case "benchmark-go-commit": + com, err = perfTodo(c, builder) } if com != nil || err != nil { if com != nil { @@ -260,6 +290,129 @@ func buildTodo(c appengine.Context, builder, packagePath, goHash string) (*Commi return nil, nil } +// perfTodo returns the next Commit to be benchmarked (or nil if none available). +func perfTodo(c appengine.Context, builder string) (*Commit, error) { + p := &Package{} + todo := &PerfTodo{Builder: builder} + err := datastore.Get(c, todo.Key(c), todo) + if err != nil && err != datastore.ErrNoSuchEntity { + return nil, fmt.Errorf("fetching PerfTodo: %v", err) + } + if err == datastore.ErrNoSuchEntity { + todo, err = buildPerfTodo(c, builder) + if err != nil { + return nil, err + } + } + if len(todo.CommitNums) == 0 { + return nil, nil + } + + // Have commit to benchmark, fetch it. + num := todo.CommitNums[len(todo.CommitNums)-1] + t := datastore.NewQuery("Commit"). + Ancestor(p.Key(c)). + Filter("Num =", num). + Limit(1). + Run(c) + com := new(Commit) + if _, err := t.Next(com); err != nil { + return nil, err + } + if !com.NeedsBenchmarking { + return nil, fmt.Errorf("commit from perf todo queue is not intended for benchmarking") + } + + // Remove benchmarks from other builders. + var benchs []string + for _, b := range com.PerfResults { + bb := strings.Split(b, "|") + if bb[0] == builder && bb[1] != "meta-done" { + benchs = append(benchs, bb[1]) + } + } + com.PerfResults = benchs + + return com, nil +} + +// buildPerfTodo creates PerfTodo for the builder with all commits. In a transaction. +func buildPerfTodo(c appengine.Context, builder string) (*PerfTodo, error) { + todo := &PerfTodo{Builder: builder} + tx := func(c appengine.Context) error { + err := datastore.Get(c, todo.Key(c), todo) + if err != nil && err != datastore.ErrNoSuchEntity { + return fmt.Errorf("fetching PerfTodo: %v", err) + } + if err == nil { + return nil + } + t := datastore.NewQuery("CommitRun"). + Ancestor((&Package{}).Key(c)). + Order("-StartCommitNum"). + Run(c) + var nums []int + var releaseNums []int + loop: + for { + cr := new(CommitRun) + if _, err := t.Next(cr); err == datastore.Done { + break + } else if err != nil { + return fmt.Errorf("scanning commit runs for perf todo: %v", err) + } + for i := len(cr.Hash) - 1; i >= 0; i-- { + if !cr.NeedsBenchmarking[i] || cr.Hash[i] == "" { + continue // There's nothing to see here. Move along. + } + num := cr.StartCommitNum + i + for k, v := range knownTags { + // Releases are benchmarked first, because they are important (and there are few of them). + if cr.Hash[i] == v { + releaseNums = append(releaseNums, num) + if k == "go1" { + break loop // Point of no benchmark: test/bench/shootout: update timing.log to Go 1. + } + } + } + nums = append(nums, num) + } + } + todo.CommitNums = orderPrefTodo(nums) + todo.CommitNums = append(todo.CommitNums, releaseNums...) + if _, err = datastore.Put(c, todo.Key(c), todo); err != nil { + return fmt.Errorf("putting PerfTodo: %v", err) + } + return nil + } + return todo, datastore.RunInTransaction(c, tx, nil) +} + +func removeCommitFromPerfTodo(c appengine.Context, builder string, num int) error { + todo := &PerfTodo{Builder: builder} + err := datastore.Get(c, todo.Key(c), todo) + if err != nil && err != datastore.ErrNoSuchEntity { + return fmt.Errorf("fetching PerfTodo: %v", err) + } + if err == datastore.ErrNoSuchEntity { + return nil + } + for i := len(todo.CommitNums) - 1; i >= 0; i-- { + if todo.CommitNums[i] == num { + for ; i < len(todo.CommitNums)-1; i++ { + todo.CommitNums[i] = todo.CommitNums[i+1] + } + todo.CommitNums = todo.CommitNums[:i] + _, err = datastore.Put(c, todo.Key(c), todo) + if err != nil { + return fmt.Errorf("putting PerfTodo: %v", err) + } + break + } + } + return nil +} + // packagesHandler returns a list of the non-Go Packages monitored // by the dashboard. func packagesHandler(r *http.Request) (interface{}, error) { @@ -329,6 +482,202 @@ func resultHandler(r *http.Request) (interface{}, error) { return nil, datastore.RunInTransaction(c, tx, nil) } +// perf-result request payload +type PerfRequest struct { + Builder string + Benchmark string + Hash string + OK bool + Metrics []PerfMetric + Artifacts []PerfArtifact +} + +type PerfMetric struct { + Type string + Val uint64 +} + +type PerfArtifact struct { + Type string + Body string +} + +// perfResultHandler records a becnhmarking result. +func perfResultHandler(r *http.Request) (interface{}, error) { + defer r.Body.Close() + if r.Method != "POST" { + return nil, errBadMethod(r.Method) + } + + req := new(PerfRequest) + if err := json.NewDecoder(r.Body).Decode(req); err != nil { + return nil, fmt.Errorf("decoding Body: %v", err) + } + + c := contextForRequest(r) + defer cache.Tick(c) + + // store the text files if supplied + for i, a := range req.Artifacts { + hash, err := PutLog(c, a.Body) + if err != nil { + return nil, fmt.Errorf("putting Log: %v", err) + } + req.Artifacts[i].Body = hash + } + tx := func(c appengine.Context) error { + return addPerfResult(c, r, req) + } + return nil, datastore.RunInTransaction(c, tx, nil) +} + +// addPerfResult creates PerfResult and updates Commit, PerfTodo, +// PerfMetricRun and PerfConfig. Must be executed within a transaction. +func addPerfResult(c appengine.Context, r *http.Request, req *PerfRequest) error { + // check Package exists + p, err := GetPackage(c, "") + if err != nil { + return fmt.Errorf("GetPackage: %v", err) + } + // add result to Commit + com := &Commit{Hash: req.Hash} + if err := com.AddPerfResult(c, req.Builder, req.Benchmark); err != nil { + return fmt.Errorf("AddPerfResult: %v", err) + } + + // add the result to PerfResult + res := &PerfResult{CommitHash: req.Hash} + if err := datastore.Get(c, res.Key(c), res); err != nil { + return fmt.Errorf("getting PerfResult: %v", err) + } + present := res.AddResult(req) + if _, err := datastore.Put(c, res.Key(c), res); err != nil { + return fmt.Errorf("putting PerfResult: %v", err) + } + + // Meta-done denotes that there are no benchmarks left. + if req.Benchmark == "meta-done" { + // Don't send duplicate emails for the same commit/builder. + // And don't send emails about too old commits. + if !present && com.Num >= p.NextNum-commitsPerPage { + if err := checkPerfChanges(c, r, com, req.Builder, res); err != nil { + return err + } + } + if err := removeCommitFromPerfTodo(c, req.Builder, com.Num); err != nil { + return nil + } + return nil + } + + // update PerfConfig + newBenchmark, err := UpdatePerfConfig(c, r, req) + if err != nil { + return fmt.Errorf("updating PerfConfig: %v", err) + } + if newBenchmark { + // If this is a new benchmark on the builder, delete PerfTodo. + // It will be recreated later with all commits again. + todo := &PerfTodo{Builder: req.Builder} + err = datastore.Delete(c, todo.Key(c)) + if err != nil && err != datastore.ErrNoSuchEntity { + return fmt.Errorf("deleting PerfTodo: %v", err) + } + } + + // add perf metrics + for _, metric := range req.Metrics { + m, err := GetPerfMetricRun(c, req.Builder, req.Benchmark, metric.Type, com.Num) + if err != nil { + return fmt.Errorf("GetPerfMetrics: %v", err) + } + if err = m.AddMetric(c, com.Num, metric.Val); err != nil { + return fmt.Errorf("AddMetric: %v", err) + } + } + + return nil +} + +func checkPerfChanges(c appengine.Context, r *http.Request, com *Commit, builder string, res *PerfResult) error { + pc, err := GetPerfConfig(c, r) + if err != nil { + return err + } + + results := res.ParseData()[builder] + rcNewer := MakePerfResultCache(c, com, true) + rcOlder := MakePerfResultCache(c, com, false) + + // Check whether we need to send failure notification email. + if results["meta-done"].OK { + // This one is successful, see if the next is failed. + nextRes, err := rcNewer.Next(com.Num) + if err != nil { + return err + } + if nextRes != nil && isPerfFailed(nextRes, builder) { + sendPerfFailMail(c, builder, nextRes) + } + } else { + // This one is failed, see if the previous is successful. + prevRes, err := rcOlder.Next(com.Num) + if err != nil { + return err + } + if prevRes != nil && !isPerfFailed(prevRes, builder) { + sendPerfFailMail(c, builder, res) + } + } + + // Now see if there are any performance changes. + // Find the previous and the next results for performance comparison. + prevRes, err := rcOlder.NextForComparison(com.Num, builder) + if err != nil { + return err + } + nextRes, err := rcNewer.NextForComparison(com.Num, builder) + if err != nil { + return err + } + if results["meta-done"].OK { + // This one is successful, compare with a previous one. + if prevRes != nil { + if err := comparePerfResults(c, pc, builder, prevRes, res); err != nil { + return err + } + } + // Compare a next one with the current. + if nextRes != nil { + if err := comparePerfResults(c, pc, builder, res, nextRes); err != nil { + return err + } + } + } else { + // This one is failed, compare a previous one with a next one. + if prevRes != nil && nextRes != nil { + if err := comparePerfResults(c, pc, builder, prevRes, nextRes); err != nil { + return err + } + } + } + + return nil +} + +func comparePerfResults(c appengine.Context, pc *PerfConfig, builder string, prevRes, res *PerfResult) error { + changes := significantPerfChanges(pc, builder, prevRes, res) + if len(changes) == 0 { + return nil + } + com := &Commit{Hash: res.CommitHash} + if err := datastore.Get(c, com.Key(c), com); err != nil { + return fmt.Errorf("getting commit %v: %v", com.Hash, err) + } + sendPerfMailLater.Call(c, com, prevRes.CommitHash, builder, changes) // add task to queue + return nil +} + // logHandler displays log text for a given hash. // It handles paths like "/log/hash". func logHandler(w http.ResponseWriter, r *http.Request) { @@ -422,6 +771,7 @@ func init() { http.HandleFunc(d.RelPath+"commit", AuthHandler(commitHandler)) http.HandleFunc(d.RelPath+"packages", AuthHandler(packagesHandler)) http.HandleFunc(d.RelPath+"result", AuthHandler(resultHandler)) + http.HandleFunc(d.RelPath+"perf-result", AuthHandler(perfResultHandler)) http.HandleFunc(d.RelPath+"tag", AuthHandler(tagHandler)) http.HandleFunc(d.RelPath+"todo", AuthHandler(todoHandler)) diff --git a/dashboard/app/build/init.go b/dashboard/app/build/init.go index 8f94714a..81fc9793 100644 --- a/dashboard/app/build/init.go +++ b/dashboard/app/build/init.go @@ -36,5 +36,9 @@ func initHandler(w http.ResponseWriter, r *http.Request) { return } } + + // Create secret key. + secretKey(c) + fmt.Fprint(w, "OK") } diff --git a/dashboard/app/build/notify.go b/dashboard/app/build/notify.go index 44c23606..3f1d44ac 100644 --- a/dashboard/app/build/notify.go +++ b/dashboard/app/build/notify.go @@ -14,6 +14,7 @@ import ( "io/ioutil" "net/http" "regexp" + "sort" "text/template" "appengine" @@ -99,14 +100,14 @@ func notifyOnFailure(c appengine.Context, com *Commit, builder string) error { broken = com } } - var err error - if broken != nil && !broken.FailNotificationSent { - c.Infof("%s is broken commit; notifying", broken.Hash) - notifyLater.Call(c, broken, builder) // add task to queue - broken.FailNotificationSent = true - _, err = datastore.Put(c, broken.Key(c), broken) + if broken == nil { + return nil } - return err + r := broken.Result(builder, "") + if r == nil { + return fmt.Errorf("finding result for %q: %+v", builder, com) + } + return commonNotify(c, broken, builder, r.LogHash) } // firstMatch executes the query q and loads the first entity into v. @@ -123,27 +124,22 @@ var notifyLater = delay.Func("notify", notify) // notify tries to update the CL for the given Commit with a failure message. // If it doesn't succeed, it sends a failure email to golang-dev. -func notify(c appengine.Context, com *Commit, builder string) { - if !updateCL(c, com, builder) { +func notify(c appengine.Context, com *Commit, builder, logHash string) { + if !updateCL(c, com, builder, logHash) { // Send a mail notification if the CL can't be found. - sendFailMail(c, com, builder) + sendFailMail(c, com, builder, logHash) } } // updateCL updates the CL for the given Commit with a failure message // for the given builder. -func updateCL(c appengine.Context, com *Commit, builder string) bool { +func updateCL(c appengine.Context, com *Commit, builder, logHash string) bool { cl, err := lookupCL(c, com) if err != nil { c.Errorf("could not find CL for %v: %v", com.Hash, err) return false } - res := com.Result(builder, "") - if res == nil { - c.Errorf("finding result for %q: %+v", builder, com) - return false - } - url := fmt.Sprintf("%v?cl=%v&brokebuild=%v&log=%v", gobotBase, cl, builder, res.LogHash) + url := fmt.Sprintf("%v?cl=%v&brokebuild=%v&log=%v", gobotBase, cl, builder, logHash) r, err := urlfetch.Client(c).Post(url, "text/plain", nil) if err != nil { c.Errorf("could not update CL %v: %v", cl, err) @@ -192,30 +188,65 @@ func init() { gob.Register(&Commit{}) // for delay } +var ( + sendPerfMailLater = delay.Func("sendPerfMail", sendPerfMailFunc) + sendPerfMailTmpl = template.Must( + template.New("perf_notify.txt"). + Funcs(template.FuncMap(tmplFuncs)). + ParseFiles("build/perf_notify.txt"), + ) +) + +func sendPerfFailMail(c appengine.Context, builder string, res *PerfResult) error { + com := &Commit{Hash: res.CommitHash} + if err := datastore.Get(c, com.Key(c), com); err != nil { + return fmt.Errorf("getting commit %v: %v", com.Hash, err) + } + logHash := "" + parsed := res.ParseData() + for _, data := range parsed[builder] { + if !data.OK { + logHash = data.Artifacts["log"] + break + } + } + if logHash == "" { + return fmt.Errorf("can not find failed result for commit %v on builder %v", com.Hash, builder) + } + return commonNotify(c, com, builder, logHash) +} + +func commonNotify(c appengine.Context, com *Commit, builder, logHash string) error { + if com.FailNotificationSent { + return nil + } + c.Infof("%s is broken commit; notifying", com.Hash) + notifyLater.Call(c, com, builder, logHash) // add task to queue + com.FailNotificationSent = true + _, err := datastore.Put(c, com.Key(c), com) + return err +} + // sendFailMail sends a mail notification that the build failed on the // provided commit and builder. -func sendFailMail(c appengine.Context, com *Commit, builder string) { - // TODO(adg): handle packages - - // get Result - r := com.Result(builder, "") - if r == nil { - c.Errorf("finding result for %q: %+v", builder, com) - return - } - +func sendFailMail(c appengine.Context, com *Commit, builder, logHash string) { // get Log - k := datastore.NewKey(c, "Log", r.LogHash, 0, nil) + k := datastore.NewKey(c, "Log", logHash, 0, nil) l := new(Log) if err := datastore.Get(c, k, l); err != nil { - c.Errorf("finding Log record %v: %v", r.LogHash, err) + c.Errorf("finding Log record %v: %v", logHash, err) + return + } + logText, err := l.Text() + if err != nil { + c.Errorf("unpacking Log record %v: %v", logHash, err) return } // prepare mail message var body bytes.Buffer - err := sendFailMailTmpl.Execute(&body, map[string]interface{}{ - "Builder": builder, "Commit": com, "Result": r, "Log": l, + err = sendFailMailTmpl.Execute(&body, map[string]interface{}{ + "Builder": builder, "Commit": com, "LogHash": logHash, "LogText": logText, "Hostname": domain, }) if err != nil { @@ -236,3 +267,83 @@ func sendFailMail(c appengine.Context, com *Commit, builder string) { c.Errorf("sending mail: %v", err) } } + +type PerfChangeBenchmark struct { + Name string + Metrics []*PerfChangeMetric +} + +type PerfChangeMetric struct { + Name string + Old uint64 + New uint64 + Delta float64 +} + +type PerfChangeBenchmarkSlice []*PerfChangeBenchmark + +func (l PerfChangeBenchmarkSlice) Len() int { return len(l) } +func (l PerfChangeBenchmarkSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l PerfChangeBenchmarkSlice) Less(i, j int) bool { + b1, p1 := splitBench(l[i].Name) + b2, p2 := splitBench(l[j].Name) + if b1 != b2 { + return b1 < b2 + } + return p1 < p2 +} + +type PerfChangeMetricSlice []*PerfChangeMetric + +func (l PerfChangeMetricSlice) Len() int { return len(l) } +func (l PerfChangeMetricSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l PerfChangeMetricSlice) Less(i, j int) bool { return l[i].Name < l[j].Name } + +func sendPerfMailFunc(c appengine.Context, com *Commit, prevCommitHash, builder string, changes []*PerfChange) { + // Sort the changes into the right order. + var benchmarks []*PerfChangeBenchmark + for _, ch := range changes { + // Find the benchmark. + var b *PerfChangeBenchmark + for _, b1 := range benchmarks { + if b1.Name == ch.bench { + b = b1 + break + } + } + if b == nil { + b = &PerfChangeBenchmark{Name: ch.bench} + benchmarks = append(benchmarks, b) + } + b.Metrics = append(b.Metrics, &PerfChangeMetric{Name: ch.metric, Old: ch.old, New: ch.new, Delta: ch.diff}) + } + for _, b := range benchmarks { + sort.Sort(PerfChangeMetricSlice(b.Metrics)) + } + sort.Sort(PerfChangeBenchmarkSlice(benchmarks)) + + url := fmt.Sprintf("http://%v/perfdetail?commit=%v&commit0=%v&kind=builder&builder=%v", domain, com.Hash, prevCommitHash, builder) + + // prepare mail message + var body bytes.Buffer + err := sendPerfMailTmpl.Execute(&body, map[string]interface{}{ + "Builder": builder, "Commit": com, "Hostname": domain, "Url": url, "Benchmarks": benchmarks, + }) + if err != nil { + c.Errorf("rendering perf mail template: %v", err) + return + } + subject := fmt.Sprintf("Perf changes on %s by %s", builder, shortDesc(com.Desc)) + msg := &mail.Message{ + Sender: mailFrom, + To: []string{failMailTo}, + ReplyTo: failMailTo, + Subject: subject, + Body: body.String(), + } + + // send mail + if err := mail.Send(c, msg); err != nil { + c.Errorf("sending mail: %v", err) + } +} diff --git a/dashboard/app/build/notify.txt b/dashboard/app/build/notify.txt index 6c900670..514191f5 100644 --- a/dashboard/app/build/notify.txt +++ b/dashboard/app/build/notify.txt @@ -1,9 +1,9 @@ Change {{shortHash .Commit.Hash}} broke the {{.Builder}} build: -http://{{.Hostname}}/log/{{.Result.LogHash}} +http://{{.Hostname}}/log/{{.LogHash}} {{.Commit.Desc}} http://code.google.com/p/go/source/detail?r={{shortHash .Commit.Hash}} $ tail -200 < log -{{printf "%s" .Log.Text | tail 200}} +{{printf "%s" .LogText | tail 200}} diff --git a/dashboard/app/build/perf.go b/dashboard/app/build/perf.go new file mode 100644 index 00000000..fab944ba --- /dev/null +++ b/dashboard/app/build/perf.go @@ -0,0 +1,310 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package build + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "appengine" + "appengine/datastore" +) + +var knownTags = map[string]string{ + "go1": "0051c7442fed9c888de6617fa9239a913904d96e", + "go1.1": "d29da2ced72ba2cf48ed6a8f1ec4abc01e4c5bf1", + "go1.2": "b1edf8faa5d6cbc50c6515785df9df9c19296564", +} + +var lastRelease = "go1.2" + +func splitBench(benchProcs string) (string, int) { + ss := strings.Split(benchProcs, "-") + procs, _ := strconv.Atoi(ss[1]) + return ss[0], procs +} + +func dashPerfCommits(c appengine.Context, page int) ([]*Commit, error) { + q := datastore.NewQuery("Commit"). + Ancestor((&Package{}).Key(c)). + Order("-Num"). + Filter("NeedsBenchmarking =", true). + Limit(commitsPerPage). + Offset(page * commitsPerPage) + var commits []*Commit + _, err := q.GetAll(c, &commits) + if err == nil && len(commits) == 0 { + err = fmt.Errorf("no commits") + } + return commits, err +} + +func perfChangeStyle(pc *PerfConfig, v float64, builder, benchmark, metric string) string { + noise := pc.NoiseLevel(builder, benchmark, metric) + if isNoise(v, noise) { + return "noise" + } + if v > 0 { + return "bad" + } + return "good" +} + +func isNoise(diff, noise float64) bool { + rnoise := -100 * noise / (noise + 100) + return diff < noise && diff > rnoise +} + +func perfDiff(old, new uint64) float64 { + return 100*float64(new)/float64(old) - 100 +} + +func isPerfFailed(res *PerfResult, builder string) bool { + data := res.ParseData()[builder] + return data != nil && data["meta-done"] != nil && !data["meta-done"].OK +} + +// PerfResultCache caches a set of PerfResults so that it's easy to access them +// without lots of duplicate accesses to datastore. +// It allows to iterate over newer or older results for some base commit. +type PerfResultCache struct { + c appengine.Context + newer bool + iter *datastore.Iterator + results map[int]*PerfResult +} + +func MakePerfResultCache(c appengine.Context, com *Commit, newer bool) *PerfResultCache { + p := &Package{} + q := datastore.NewQuery("PerfResult").Ancestor(p.Key(c)).Limit(100) + if newer { + q = q.Filter("CommitNum >=", com.Num).Order("CommitNum") + } else { + q = q.Filter("CommitNum <=", com.Num).Order("-CommitNum") + } + rc := &PerfResultCache{c: c, newer: newer, iter: q.Run(c), results: make(map[int]*PerfResult)} + return rc +} + +func (rc *PerfResultCache) Get(commitNum int) *PerfResult { + rc.Next(commitNum) // fetch the commit, if necessary + return rc.results[commitNum] +} + +// Next returns the next PerfResult for the commit commitNum. +// It does not care whether the result has any data, failed or whatever. +func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) { + // See if we have next result in the cache. + next := -1 + for ci := range rc.results { + if rc.newer { + if ci > commitNum && (next == -1 || ci < next) { + next = ci + } + } else { + if ci < commitNum && (next == -1 || ci > next) { + next = ci + } + } + } + //rc.c.Errorf("PerfResultCache.Next: num=%v next=%v", commitNum, next) + if next != -1 { + return rc.results[next], nil + } + // Fetch next result from datastore. + res := new(PerfResult) + _, err := rc.iter.Next(res) + //rc.c.Errorf("PerfResultCache.Next: fetched %v %+v", err, res) + if err == datastore.Done { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("fetching perf results: %v", err) + } + if (rc.newer && res.CommitNum < commitNum) || (!rc.newer && res.CommitNum > commitNum) { + rc.c.Errorf("PerfResultCache.Next: bad commit num") + } + rc.results[res.CommitNum] = res + return res, nil +} + +// NextForComparison returns PerfResult which we need to use for performance comprison. +// It skips failed results, but does not skip results with no data. +func (rc *PerfResultCache) NextForComparison(commitNum int, builder string) (*PerfResult, error) { + for { + res, err := rc.Next(commitNum) + if err != nil { + return nil, err + } + if res == nil { + return nil, nil + } + if res.CommitNum == commitNum { + continue + } + parsed := res.ParseData() + if builder != "" { + // Comparing for a particular builder. + // This is used in perf_changes and in email notifications. + b := parsed[builder] + if b == nil || b["meta-done"] == nil { + // No results yet, must not do the comparison. + return nil, nil + } + if b["meta-done"].OK { + // Have complete results, compare. + return res, nil + } + } else { + // Comparing for all builders, find a result with at least + // one successful meta-done. + // This is used in perf_detail. + for _, benchs := range parsed { + if data := benchs["meta-done"]; data != nil && data.OK { + return res, nil + } + } + } + // Failed, try next result. + commitNum = res.CommitNum + } +} + +type PerfChange struct { + builder string + bench string + metric string + old uint64 + new uint64 + diff float64 +} + +func significantPerfChanges(pc *PerfConfig, builder string, prevRes, res *PerfResult) (changes []*PerfChange) { + // First, collect all significant changes. + for builder1, benchmarks1 := range res.ParseData() { + if builder != "" && builder != builder1 { + // This is not the builder you're looking for, Luke. + continue + } + benchmarks0 := prevRes.ParseData()[builder1] + if benchmarks0 == nil { + continue + } + for benchmark, data1 := range benchmarks1 { + data0 := benchmarks0[benchmark] + if data0 == nil { + continue + } + for metric, val := range data1.Metrics { + val0 := data0.Metrics[metric] + if val0 == 0 { + continue + } + diff := perfDiff(val0, val) + noise := pc.NoiseLevel(builder, benchmark, metric) + if isNoise(diff, noise) { + continue + } + ch := &PerfChange{builder: builder, bench: benchmark, metric: metric, old: val0, new: val, diff: diff} + changes = append(changes, ch) + } + } + } + // Then, strip non-repeatable changes (flakes). + // The hypothesis is that a real change must show up with at least + // 2 different values of GOMAXPROCS. + cnt := make(map[string]int) + for _, ch := range changes { + b, _ := splitBench(ch.bench) + name := b + "|" + ch.metric + inc := 1 + if ch.diff < 0 { + inc = -1 + } + cnt[name] = cnt[name] + inc + } + for i := 0; i < len(changes); i++ { + ch := changes[i] + b, _ := splitBench(ch.bench) + name := b + "|" + ch.metric + if n := cnt[name]; n <= -2 || n >= 2 { + continue + } + last := len(changes) - 1 + changes[i] = changes[last] + changes = changes[:last] + i-- + } + return changes +} + +// orderPrefTodo reorders commit nums for benchmarking todo. +// The resulting order is somewhat tricky. We want 2 things: +// 1. benchmark sequentially backwards (this provides information about most +// recent changes, and allows to estimate noise levels) +// 2. benchmark old commits in "scatter" order (this allows to quickly gather +// brief information about thousands of old commits) +// So this function interleaves the two orders. +func orderPrefTodo(nums []int) []int { + sort.Ints(nums) + n := len(nums) + pow2 := uint32(0) // next power-of-two that is >= n + npow2 := 0 + for npow2 <= n { + pow2++ + npow2 = 1 << pow2 + } + res := make([]int, n) + resPos := n - 1 // result array is filled backwards + present := make([]bool, n) // denotes values that already present in result array + for i0, i1 := n-1, 0; i0 >= 0 || i1 < npow2; { + // i0 represents "benchmark sequentially backwards" sequence + // find the next commit that is not yet present and add it + for cnt := 0; cnt < 2; cnt++ { + for ; i0 >= 0; i0-- { + if !present[i0] { + present[i0] = true + res[resPos] = nums[i0] + resPos-- + i0-- + break + } + } + } + // i1 represents "scatter order" sequence + // find the next commit that is not yet present and add it + for ; i1 < npow2; i1++ { + // do the "recursive split-ordering" trick + idx := 0 // bitwise reverse of i1 + for j := uint32(0); j <= pow2; j++ { + if (i1 & (1 << j)) != 0 { + idx = idx | (1 << (pow2 - j - 1)) + } + } + if idx < n && !present[idx] { + present[idx] = true + res[resPos] = nums[idx] + resPos-- + i1++ + break + } + } + } + // The above can't possibly be correct. Do dump check. + res2 := make([]int, n) + copy(res2, res) + sort.Ints(res2) + for i := range res2 { + if res2[i] != nums[i] { + panic(fmt.Sprintf("diff at %v: expect %v, want %v\nwas: %v\n become: %v", + i, nums[i], res2[i], nums, res2)) + } + } + return res +} diff --git a/dashboard/app/build/test.go b/dashboard/app/build/test.go index 3c9576e7..2adcac48 100644 --- a/dashboard/app/build/test.go +++ b/dashboard/app/build/test.go @@ -32,7 +32,12 @@ func init() { var testEntityKinds = []string{ "Package", "Commit", + "CommitRun", "Result", + "PerfResult", + "PerfMetricRun", + "PerfConfig", + "PerfTodo", "Log", } @@ -47,15 +52,16 @@ var testPackages = []*Package{ var tCommitTime = time.Now().Add(-time.Hour * 24 * 7) -func tCommit(hash, parentHash, path string) *Commit { +func tCommit(hash, parentHash, path string, bench bool) *Commit { tCommitTime.Add(time.Hour) // each commit should have a different time return &Commit{ - PackagePath: path, - Hash: hash, - ParentHash: parentHash, - Time: tCommitTime, - User: "adg", - Desc: "change description " + hash, + PackagePath: path, + Hash: hash, + ParentHash: parentHash, + Time: tCommitTime, + User: "adg", + Desc: "change description " + hash, + NeedsBenchmarking: bench, } } @@ -69,9 +75,9 @@ var testRequests = []struct { {"/packages?kind=subrepo", nil, nil, []*Package{testPackage}}, // Go repo - {"/commit", nil, tCommit("0001", "0000", ""), nil}, - {"/commit", nil, tCommit("0002", "0001", ""), nil}, - {"/commit", nil, tCommit("0003", "0002", ""), nil}, + {"/commit", nil, tCommit("0001", "0000", "", true), nil}, + {"/commit", nil, tCommit("0002", "0001", "", false), nil}, + {"/commit", nil, tCommit("0003", "0002", "", true), nil}, {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, {"/result", nil, &Result{Builder: "linux-386", Hash: "0001", OK: true}, nil}, @@ -95,8 +101,8 @@ var testRequests = []struct { {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}}, // branches - {"/commit", nil, tCommit("0004", "0003", ""), nil}, - {"/commit", nil, tCommit("0005", "0002", ""), nil}, + {"/commit", nil, tCommit("0004", "0003", "", false), nil}, + {"/commit", nil, tCommit("0005", "0002", "", false), nil}, {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}}, {"/result", nil, &Result{Builder: "linux-386", Hash: "0005", OK: true}, nil}, {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}}, @@ -112,9 +118,9 @@ var testRequests = []struct { {"/result", nil, &Result{Builder: "linux-386", Hash: "0003", OK: false, Log: "test"}, nil}, // non-Go repos - {"/commit", nil, tCommit("1001", "1000", testPkg), nil}, - {"/commit", nil, tCommit("1002", "1001", testPkg), nil}, - {"/commit", nil, tCommit("1003", "1002", testPkg), nil}, + {"/commit", nil, tCommit("1001", "1000", testPkg, false), nil}, + {"/commit", nil, tCommit("1002", "1001", testPkg, false), nil}, + {"/commit", nil, tCommit("1003", "1002", testPkg, false), nil}, {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1003"}}}, {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0001", OK: true}, nil}, {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1002"}}}, @@ -128,6 +134,84 @@ var testRequests = []struct { {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}}, {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0005", OK: false, Log: "boo"}, nil}, {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, nil}, + + // benchmarks + // build-go-commit must have precedence over benchmark-go-commit + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}}, + // drain build-go-commit todo + {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0005", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}}, + {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0004", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}}, + {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0002", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0001"}}}, + {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0001", OK: true}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0005", OK: false}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0005", OK: false}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0005", OK: false}, nil}, + // now we must get benchmark todo + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{}}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0003", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http"}}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "json", Hash: "0003", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http", "json"}}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0003", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001", PerfResults: []string{}}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0001", OK: true}, nil}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0001", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil}, + // create new commit, it must appear in todo + {"/commit", nil, tCommit("0006", "0005", "", true), nil}, + // drain build-go-commit todo + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0006"}}}, + {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0006", OK: true}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0006", OK: false}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0006", OK: false}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0006", OK: false}, nil}, + // now we must get benchmark todo + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006", PerfResults: []string{}}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0006", OK: true}, nil}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0006", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil}, + // create new benchmark, all commits must re-appear in todo + {"/commit", nil, tCommit("0007", "0006", "", true), nil}, + // drain build-go-commit todo + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0007"}}}, + {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0007", OK: true}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0007", OK: false}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0007", OK: false}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0007", OK: false}, nil}, + // now we must get benchmark todo + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007", PerfResults: []string{}}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "bson", Hash: "0007", OK: true}, nil}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0007", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007", PerfResults: []string{"bson"}}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0007", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006", PerfResults: []string{"http"}}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0006", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http", "json"}}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0003", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001", PerfResults: []string{"http"}}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0001", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil}, + // attach second builder + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0007"}}}, + // drain build-go-commit todo + {"/result", nil, &Result{Builder: "linux-386", Hash: "0007", OK: true}, nil}, + {"/result", nil, &Result{Builder: "linux-386", Hash: "0006", OK: true}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0007", OK: false}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1002", GoHash: "0007", OK: false}, nil}, + {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0007", OK: false}, nil}, + // now we must get benchmark todo + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007"}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0007", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006"}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0006", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003"}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0003", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001"}}}, + {"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0001", OK: true}, nil}, + {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, nil}, } func testHandler(w http.ResponseWriter, r *http.Request) { @@ -159,7 +243,7 @@ func testHandler(w http.ResponseWriter, r *http.Request) { *r = origReq }() for i, t := range testRequests { - c.Infof("running test %d %s", i, t.path) + c.Infof("running test %d %s vals='%q' req='%q' res='%q'", i, t.path, t.vals, t.req, t.res) errorf := func(format string, args ...interface{}) { fmt.Fprintf(w, "%d %s: ", i, t.path) fmt.Fprintf(w, format, args...) @@ -194,11 +278,12 @@ func testHandler(w http.ResponseWriter, r *http.Request) { errorf(rec.Body.String()) return } + c.Infof("response='%v'", rec.Body.String()) resp := new(dashResponse) // If we're expecting a *Todo value, // prime the Response field with a Todo and a Commit inside it. - if _, ok := t.res.(*Todo); ok { + if t.path == "/todo" { resp.Response = &Todo{Data: &Commit{}} } @@ -241,14 +326,28 @@ func testHandler(w http.ResponseWriter, r *http.Request) { errorf("Response.Data not *Commit: %T", g.Data) return } - if eh := e.Data.(*Commit).Hash; eh != gd.Hash { - errorf("hashes don't match: got %q, want %q", gd.Hash, eh) + if g.Kind != e.Kind { + errorf("kind don't match: got %q, want %q", g.Kind, e.Kind) return } + ed := e.Data.(*Commit) + if ed.Hash != gd.Hash { + errorf("hashes don't match: got %q, want %q", gd.Hash, ed.Hash) + return + } + if len(gd.PerfResults) != len(ed.PerfResults) { + errorf("result data len don't match: got %v, want %v", len(gd.PerfResults), len(ed.PerfResults)) + return + } + for i := range gd.PerfResults { + if gd.PerfResults[i] != ed.PerfResults[i] { + errorf("result data %v don't match: got %v, want %v", i, gd.PerfResults[i], ed.PerfResults[i]) + return + } + } } if t.res == nil && resp.Response != nil { - errorf("response mismatch: got %q expected ", - resp.Response) + errorf("response mismatch: got %q expected ", resp.Response) return } } diff --git a/dashboard/app/cron.yaml b/dashboard/app/cron.yaml new file mode 100644 index 00000000..43cabf78 --- /dev/null +++ b/dashboard/app/cron.yaml @@ -0,0 +1,5 @@ +cron: +- description: updates noise level for benchmarking results + url: /perflearn + schedule: every 24 hours + diff --git a/dashboard/app/index.yaml b/dashboard/app/index.yaml index 8a21baca..76dd09ec 100644 --- a/dashboard/app/index.yaml +++ b/dashboard/app/index.yaml @@ -11,3 +11,29 @@ indexes: properties: - name: Time direction: desc + +- kind: Commit + ancestor: yes + properties: + - name: NeedsBenchmarking + - name: Num + direction: desc + +- kind: CommitRun + ancestor: yes + properties: + - name: StartCommitNum + direction: desc + +- kind: PerfResult + ancestor: yes + properties: + - name: CommitNum + direction: desc + +- kind: PerfResult + ancestor: yes + properties: + - name: CommitNum + direction: asc +