From ddb83b48153625bab7fe7a5aa331a9a1820bcd3d Mon Sep 17 00:00:00 2001
From: Dmitriy Vyukov <dvyukov@google.com>
Date: Fri, 17 Oct 2014 11:38:11 +0400
Subject: [PATCH] dashboard: faster perf data fetching Use single query to
 fetch commit runs and metric runs (instead of individual selects). Hopefully
 this enables some kind of prefetching. But more importantly it allows to work
 around "gaps" in commit nums, as we only fetch data that is actually in the
 database and don't try to query all commit runs in the "gap".

LGTM=adg
R=adg
CC=golang-codereviews, rsc
https://golang.org/cl/159910045
---
 dashboard/app/build/build.go      | 91 +++++++++++++++++++++++--------
 dashboard/app/build/perf_graph.go |  2 +-
 dashboard/app/build/perf_learn.go |  2 +-
 dashboard/app/index.yaml          | 15 +++++
 4 files changed, 86 insertions(+), 24 deletions(-)

diff --git a/dashboard/app/build/build.go b/dashboard/app/build/build.go
index 0401cd2c..8a135cf9 100644
--- a/dashboard/app/build/build.go
+++ b/dashboard/app/build/build.go
@@ -315,28 +315,50 @@ func GetCommits(c appengine.Context, startCommitNum, n int) ([]*Commit, error) {
 	if startCommitNum < 0 || n <= 0 {
 		return nil, fmt.Errorf("GetCommits: invalid args (%v, %v)", startCommitNum, n)
 	}
-	var res []*Commit
-	for n > 0 {
-		cr, err := GetCommitRun(c, startCommitNum)
+
+	p := &Package{}
+	t := datastore.NewQuery("CommitRun").
+		Ancestor(p.Key(c)).
+		Filter("StartCommitNum >=", startCommitNum/PerfRunLength*PerfRunLength).
+		Order("StartCommitNum").
+		Limit(100).
+		Run(c)
+
+	res := make([]*Commit, n)
+	for {
+		cr := new(CommitRun)
+		_, err := t.Next(cr)
+		if err == datastore.Done {
+			break
+		}
 		if err != nil {
 			return nil, err
 		}
-		idx := startCommitNum - cr.StartCommitNum
-		cnt := PerfRunLength - idx
-		if cnt > n {
-			cnt = n
+		if cr.StartCommitNum >= startCommitNum+n {
+			break
 		}
-		for i := idx; i < idx+cnt; i++ {
+		// Calculate start index for copying.
+		i := 0
+		if cr.StartCommitNum < startCommitNum {
+			i = startCommitNum - cr.StartCommitNum
+		}
+		// Calculate end index for copying.
+		e := PerfRunLength
+		if cr.StartCommitNum+e > startCommitNum+n {
+			e = startCommitNum + n - cr.StartCommitNum
+		}
+		for ; i < e; i++ {
 			com := new(Commit)
 			com.Hash = cr.Hash[i]
 			com.User = cr.User[i]
 			com.Desc = cr.Desc[i]
 			com.Time = cr.Time[i]
 			com.NeedsBenchmarking = cr.NeedsBenchmarking[i]
-			res = append(res, com)
+			res[cr.StartCommitNum-startCommitNum+i] = com
+		}
+		if e != PerfRunLength {
+			break
 		}
-		startCommitNum += cnt
-		n -= cnt
 	}
 	return res, nil
 }
@@ -535,22 +557,47 @@ func GetPerfMetricsForCommits(c appengine.Context, builder, benchmark, metric st
 	if startCommitNum < 0 || n <= 0 {
 		return nil, fmt.Errorf("GetPerfMetricsForCommits: invalid args (%v, %v)", startCommitNum, n)
 	}
-	var res []uint64
-	for n > 0 {
-		metrics, err := GetPerfMetricRun(c, builder, benchmark, metric, startCommitNum)
+
+	p := &Package{}
+	t := datastore.NewQuery("PerfMetricRun").
+		Ancestor(p.Key(c)).
+		Filter("Builder =", builder).
+		Filter("Benchmark =", benchmark).
+		Filter("Metric =", metric).
+		Filter("StartCommitNum >=", startCommitNum/PerfRunLength*PerfRunLength).
+		Order("StartCommitNum").
+		Limit(100).
+		Run(c)
+
+	res := make([]uint64, n)
+	for {
+		metrics := new(PerfMetricRun)
+		_, err := t.Next(metrics)
+		if err == datastore.Done {
+			break
+		}
 		if err != nil {
 			return nil, err
 		}
-		idx := startCommitNum - metrics.StartCommitNum
-		cnt := PerfRunLength - idx
-		if cnt > n {
-			cnt = n
+		if metrics.StartCommitNum >= startCommitNum+n {
+			break
 		}
-		for _, v := range metrics.Vals[idx : idx+cnt] {
-			res = append(res, uint64(v))
+		// Calculate start index for copying.
+		i := 0
+		if metrics.StartCommitNum < startCommitNum {
+			i = startCommitNum - metrics.StartCommitNum
+		}
+		// Calculate end index for copying.
+		e := PerfRunLength
+		if metrics.StartCommitNum+e > startCommitNum+n {
+			e = startCommitNum + n - metrics.StartCommitNum
+		}
+		for ; i < e; i++ {
+			res[metrics.StartCommitNum-startCommitNum+i] = uint64(metrics.Vals[i])
+		}
+		if e != PerfRunLength {
+			break
 		}
-		startCommitNum += cnt
-		n -= cnt
 	}
 	return res, nil
 }
diff --git a/dashboard/app/build/perf_graph.go b/dashboard/app/build/perf_graph.go
index 647d5877..7a660e4b 100644
--- a/dashboard/app/build/perf_graph.go
+++ b/dashboard/app/build/perf_graph.go
@@ -231,7 +231,7 @@ func perfGraphHandler(w http.ResponseWriter, r *http.Request) {
 		idx := 0
 		for i := range vals[0] {
 			com := commits2[i]
-			if !com.NeedsBenchmarking {
+			if com == nil || !com.NeedsBenchmarking {
 				continue
 			}
 			c := perfGraphCommit{Id: idx, Name: fmt.Sprintf("%v (%v)", com.Desc, com.Time.Format("Jan 2, 2006 1:04"))}
diff --git a/dashboard/app/build/perf_learn.go b/dashboard/app/build/perf_learn.go
index 32b8fdf1..683ba60b 100644
--- a/dashboard/app/build/perf_learn.go
+++ b/dashboard/app/build/perf_learn.go
@@ -67,7 +67,7 @@ func perfLearnHandler(w http.ResponseWriter, r *http.Request) {
 					last := uint64(0)
 					for i, v := range values {
 						if v == 0 {
-							if commits[i].NeedsBenchmarking {
+							if com := commits[i]; com == nil || com.NeedsBenchmarking {
 								last = 0
 							}
 							continue
diff --git a/dashboard/app/index.yaml b/dashboard/app/index.yaml
index 76dd09ec..670a667f 100644
--- a/dashboard/app/index.yaml
+++ b/dashboard/app/index.yaml
@@ -37,3 +37,18 @@ indexes:
   - name: CommitNum
     direction: asc
 
+- kind: CommitRun
+  ancestor: yes
+  properties:
+  - name: StartCommitNum
+    direction: asc
+
+- kind: PerfMetricRun
+  ancestor: yes
+  properties:
+  - name: Builder
+  - name: Benchmark
+  - name: Metric
+  - name: StartCommitNum
+    direction: asc
+