go.tools/cmd/benchcmp: sort by original benchmark order by default

benchcmp now preserves benchmark order. This restores the original
misc/benchcmp behavior. This also makes the output of benchcmp stable,
and groups together multiple -cpu results.

Magnitude-based sorting is still available via the -mag flag.
It is useful for surfacing items of note (particularly changes
in allocs) when making compiler changes and running broad
benchmarks.

Fixes golang/go#7259.

LGTM=dave
R=dave, mtj
CC=bradfitz, dvyukov, golang-codereviews
https://golang.org/cl/60840045

cmd/benchcmp/benchcmp.go

@@ -16,6 +16,7 @@ import (
 
 var (
 	changedOnly = flag.Bool("changed", false, "show only benchmarks that have changed")
+	magSort     = flag.Bool("mag", false, "sort benchmarks by magnitude of change")
 )
 
 const usageFooter = `
@@ -59,7 +60,11 @@ func main() {
 
 	var header bool // Has the header has been displayed yet for a given block?
 
+	if *magSort {
 		sort.Sort(ByDeltaNsOp(cmps))
+	} else {
+		sort.Sort(ByParseOrder(cmps))
+	}
 	for _, cmp := range cmps {
 		if !cmp.Measured(NsOp) {
 			continue
@@ -74,7 +79,9 @@ func main() {
 	}
 
 	header = false
+	if *magSort {
 		sort.Sort(ByDeltaMbS(cmps))
+	}
 	for _, cmp := range cmps {
 		if !cmp.Measured(MbS) {
 			continue
@@ -89,7 +96,9 @@ func main() {
 	}
 
 	header = false
+	if *magSort {
 		sort.Sort(ByDeltaAllocsOp(cmps))
+	}
 	for _, cmp := range cmps {
 		if !cmp.Measured(AllocsOp) {
 			continue
@@ -104,7 +113,9 @@ func main() {
 	}
 
 	header = false
+	if *magSort {
 		sort.Sort(ByDeltaBOp(cmps))
+	}
 	for _, cmp := range cmps {
 		if !cmp.Measured(BOp) {
 			continue

cmd/benchcmp/compare.go

@@ -96,6 +96,14 @@ func (d Delta) String() string {
 	return fmt.Sprintf("Δ(%f, %f)", d.Before, d.After)
 }
 
+// ByParseOrder sorts BenchCmps to match the order in
+// which the Before benchmarks were presented to Parse.
+type ByParseOrder []BenchCmp
+
+func (x ByParseOrder) Len() int           { return len(x) }
+func (x ByParseOrder) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
+func (x ByParseOrder) Less(i, j int) bool { return x[i].Before.ord < x[j].Before.ord }
+
 // lessByDelta provides lexicographic ordering:
 //   * largest delta by magnitude
 //   * alphabetic by name

cmd/benchcmp/compare_test.go

@@ -107,18 +107,25 @@ func TestCorrelate(t *testing.T) {
 }
 
 func TestBenchCmpSorting(t *testing.T) {
-	// Test just one sort order; they are symmetric.
 	c := []BenchCmp{
-		{&Bench{Name: "BenchmarkMuchFaster", NsOp: 10}, &Bench{Name: "BenchmarkMuchFaster", NsOp: 1}},
+		{&Bench{Name: "BenchmarkMuchFaster", NsOp: 10, ord: 3}, &Bench{Name: "BenchmarkMuchFaster", NsOp: 1}},
-		{&Bench{Name: "BenchmarkSameB", NsOp: 5}, &Bench{Name: "BenchmarkSameB", NsOp: 5}},
+		{&Bench{Name: "BenchmarkSameB", NsOp: 5, ord: 1}, &Bench{Name: "BenchmarkSameB", NsOp: 5}},
-		{&Bench{Name: "BenchmarkSameA", NsOp: 5}, &Bench{Name: "BenchmarkSameA", NsOp: 5}},
+		{&Bench{Name: "BenchmarkSameA", NsOp: 5, ord: 2}, &Bench{Name: "BenchmarkSameA", NsOp: 5}},
-		{&Bench{Name: "BenchmarkSlower", NsOp: 10}, &Bench{Name: "BenchmarkSlower", NsOp: 11}},
+		{&Bench{Name: "BenchmarkSlower", NsOp: 10, ord: 0}, &Bench{Name: "BenchmarkSlower", NsOp: 11}},
 	}
 
+	// Test just one magnitude-based sort order; they are symmetric.
 	sort.Sort(ByDeltaNsOp(c))
 	want := []string{"BenchmarkMuchFaster", "BenchmarkSlower", "BenchmarkSameA", "BenchmarkSameB"}
 	have := []string{c[0].Name(), c[1].Name(), c[2].Name(), c[3].Name()}
 	if !reflect.DeepEqual(want, have) {
 		t.Errorf("ByDeltaNsOp incorrect sorting: want %v have %v", want, have)
 	}
+
+	sort.Sort(ByParseOrder(c))
+	want = []string{"BenchmarkSlower", "BenchmarkSameB", "BenchmarkSameA", "BenchmarkMuchFaster"}
+	have = []string{c[0].Name(), c[1].Name(), c[2].Name(), c[3].Name()}
+	if !reflect.DeepEqual(want, have) {
+		t.Errorf("ByParseOrder incorrect sorting: want %v have %v", want, have)
+	}
 }

cmd/benchcmp/parse.go

@@ -31,6 +31,7 @@ type Bench struct {
 	BOp      uint64  // bytes allocated per iteration
 	AllocsOp uint64  // allocs per iteration
 	Measured int     // which measurements were recorded
+	ord      int     // ordinal position within a benchmark run, used for sorting
 }
 
 // ParseLine extracts a Bench from a single line of testing.B output.
@@ -109,9 +110,12 @@ type BenchSet map[string][]*Bench
 func ParseBenchSet(r io.Reader) (BenchSet, error) {
 	bb := make(BenchSet)
 	scan := bufio.NewScanner(r)
+	ord := 0
 	for scan.Scan() {
 		if b, err := ParseLine(scan.Text()); err == nil {
+			b.ord = ord
 			bb[b.Name] = append(bb[b.Name], b)
+			ord++
 		}
 	}
 

cmd/benchcmp/parse_test.go

@@ -117,6 +117,7 @@ func TestParseBenchSet(t *testing.T) {
 				Name: "BenchmarkReadRequestApachebench",
 				N:    1000000, NsOp: 2960, MbS: 27.70, BOp: 839, AllocsOp: 9,
 				Measured: NsOp | MbS | BOp | AllocsOp,
+				ord:      2,
 			},
 		},
 		"BenchmarkClientServerParallel64": []*Bench{
@@ -124,6 +125,7 @@ func TestParseBenchSet(t *testing.T) {
 				Name: "BenchmarkClientServerParallel64",
 				N:    50000, NsOp: 59192, BOp: 7028, AllocsOp: 60,
 				Measured: NsOp | BOp | AllocsOp,
+				ord:      3,
 			},
 		},
 		"BenchmarkEncrypt": []*Bench{
@@ -131,11 +133,13 @@ func TestParseBenchSet(t *testing.T) {
 				Name: "BenchmarkEncrypt",
 				N:    100000000, NsOp: 19.6,
 				Measured: NsOp,
+				ord:      0,
 			},
 			{
 				Name: "BenchmarkEncrypt",
 				N:    5000000, NsOp: 517,
 				Measured: NsOp,
+				ord:      1,
 			},
 		},
 	}