From f579fb3656a7193bb17a8f842f098733df467b6a Mon Sep 17 00:00:00 2001
From: Dmitriy Vyukov <dvyukov@google.com>
Date: Tue, 19 Aug 2014 16:22:27 +0400
Subject: [PATCH] dashboard/app: eliminate more flakes Currently we still see
 some flakes on perf dashboard (e.g. sys-stack is quite frequent). I am
 planning to run real perf builder with 5 different values of GOMAXPROCS, so
 we can require 3 builders to agree on a change instead of 2. This will
 provide 20x improvement in flake detection. At the same time lower noise bar
 from 1.2 to 1.1, as I see some real perf changes gets ignored as noise.

All these magic numbers affect only representation of data, but not the data stored in DB.
So we can continue freely tuning them later. There are no significant risks here.

LGTM=adg
R=adg
CC=golang-codereviews
https://golang.org/cl/127520044
---
 dashboard/app/build/perf.go       | 17 +++++++++--------
 dashboard/app/build/perf_learn.go |  2 +-
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/dashboard/app/build/perf.go b/dashboard/app/build/perf.go
index bce67156..9ec717cd 100644
--- a/dashboard/app/build/perf.go
+++ b/dashboard/app/build/perf.go
@@ -114,14 +114,12 @@ func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) {
 			}
 		}
 	}
-	//rc.c.Errorf("PerfResultCache.Next: num=%v next=%v", commitNum, next)
 	if next != -1 {
 		return rc.results[next], nil
 	}
 	// Fetch next result from datastore.
 	res := new(PerfResult)
 	_, err := rc.iter.Next(res)
-	//rc.c.Errorf("PerfResultCache.Next: fetched %v %+v", err, res)
 	if err == datastore.Done {
 		return nil, nil
 	}
@@ -218,25 +216,28 @@ func significantPerfChanges(pc *PerfConfig, builder string, prevRes, res *PerfRe
 		}
 	}
 	// Then, strip non-repeatable changes (flakes).
-	// The hypothesis is that a real change must show up with at least
-	// 2 different values of GOMAXPROCS.
+	// The hypothesis is that a real change must show up with the majority of GOMAXPROCS values.
+	majority := len(pc.ProcList(builder))/2 + 1
 	cnt := make(map[string]int)
 	for _, ch := range changes {
 		b, _ := splitBench(ch.bench)
 		name := b + "|" + ch.metric
-		inc := 1
 		if ch.diff < 0 {
-			inc = -1
+			name += "--"
 		}
-		cnt[name] = cnt[name] + inc
+		cnt[name] = cnt[name] + 1
 	}
 	for i := 0; i < len(changes); i++ {
 		ch := changes[i]
 		b, _ := splitBench(ch.bench)
 		name := b + "|" + ch.metric
-		if n := cnt[name]; n <= -2 || n >= 2 {
+		if cnt[name] >= majority {
 			continue
 		}
+		if cnt[name+"--"] >= majority {
+			continue
+		}
+		// Remove flake.
 		last := len(changes) - 1
 		changes[i] = changes[last]
 		changes = changes[:last]
diff --git a/dashboard/app/build/perf_learn.go b/dashboard/app/build/perf_learn.go
index bd308ea8..32b8fdf1 100644
--- a/dashboard/app/build/perf_learn.go
+++ b/dashboard/app/build/perf_learn.go
@@ -23,7 +23,7 @@ func init() {
 
 const (
 	learnPercentile       = 0.95
-	learnSignalMultiplier = 1.2
+	learnSignalMultiplier = 1.1
 	learnMinSignal        = 0.5
 )