tip: fix, update tip.golang.org

Primarily for golang/go#29251 but also update the base Docker layer
from Go 1.9 to Go 1.11, update other deps, and fix some Kubernetes
config cleanups that happened prior without testing apparently.

Fixes golang/go#29251

Change-Id: I0aafccdfedfc0d9ebb75d0c1a3b0819245ea5f19
Reviewed-on: https://go-review.googlesource.com/c/154181
Reviewed-by: Bryan C. Mills <bcmills@google.com>

cmd/tip/Dockerfile

@@ -1,19 +1,23 @@
-FROM golang:1.9
+FROM golang:1.11
 
 RUN apt-get update && apt-get install --no-install-recommends -y -q build-essential git
 
+# For implicit GOCACHE (Issues 29243 and 29251), set HOME:
+RUN mkdir -p /home/gopher
+ENV HOME /home/gopher
+
 # golang puts its go install here (weird but true)
 ENV GOROOT_BOOTSTRAP /usr/local/go
 
 # BEGIN deps (run `make update-deps` to update)
 
-# Repo cloud.google.com/go at 1d0c2da (2018-01-30)
+# Repo cloud.google.com/go at b5eca92 (2018-10-23)
-ENV REV=1d0c2da40456a9b47f5376165f275424acc15c09
+ENV REV=b5eca92245a08e245bc29c4880c9779ea4aeaa9a
-RUN go get -d cloud.google.com/go/compute/metadata `#and 6 other pkgs` &&\
+RUN go get -d cloud.google.com/go/compute/metadata `#and 7 other pkgs` &&\
     (cd /go/src/cloud.google.com/go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
-# Repo github.com/golang/protobuf at 9255415 (2018-01-25)
+# Repo github.com/golang/protobuf at b4deda0 (2018-04-30)
-ENV REV=925541529c1fa6821df4e44ce2723319eb2be768
+ENV REV=b4deda0973fb4c70b50d226b1af49f3da59f5265
 RUN go get -d github.com/golang/protobuf/proto `#and 6 other pkgs` &&\
     (cd /go/src/github.com/golang/protobuf && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
@@ -22,44 +26,54 @@ ENV REV=317e0006254c44a0ac427cc52a0e083ff0b9622f
 RUN go get -d github.com/googleapis/gax-go &&\
     (cd /go/src/github.com/googleapis/gax-go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
-# Repo golang.org/x/build at e879390 (2018-02-01)
+# Repo go.opencensus.io at ebd8d31 (2018-05-16)
-ENV REV=e8793909ba350594eea4c7c6bdb0f0d9a0d0f77a
+ENV REV=ebd8d31470fedf6c27d0e3056653ddff642509b8
+RUN go get -d go.opencensus.io/exporter/stackdriver/propagation `#and 12 other pkgs` &&\
+    (cd /go/src/go.opencensus.io && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
+
+# Repo golang.org/x/build at 7b78c20 (2018-12-13)
+ENV REV=7b78c2042368d5c56ee9dbd92ab5fa988c763944
 RUN go get -d golang.org/x/build/autocertcache &&\
     (cd /go/src/golang.org/x/build && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
-# Repo golang.org/x/crypto at 1875d0a (2018-01-27)
+# Repo golang.org/x/crypto at e4dc69e (2018-11-06)
-ENV REV=1875d0a70c90e57f11972aefd42276df65e895b9
+ENV REV=e4dc69e5b2fd71dcaf8bd5d054eb936deb78d1fa
 RUN go get -d golang.org/x/crypto/acme `#and 2 other pkgs` &&\
     (cd /go/src/golang.org/x/crypto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
-# Repo golang.org/x/net at 6d90978 (2018-02-01)
+# Repo golang.org/x/net at 891ebc4 (2018-12-13)
-ENV REV=6d90978dc4889d44e8cfbd04c05d17b5417823c7
+ENV REV=891ebc4b82d6e74f468c533b06f983c7be918a96
 RUN go get -d golang.org/x/net/context `#and 8 other pkgs` &&\
     (cd /go/src/golang.org/x/net && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
-# Repo golang.org/x/oauth2 at 30785a2 (2018-01-04)
+# Repo golang.org/x/oauth2 at f42d051 (2018-11-06)
-ENV REV=30785a2c434e431ef7c507b54617d6a951d5f2b4
+ENV REV=f42d05182288abf10faef86d16c0d07b8d40ea2d
 RUN go get -d golang.org/x/oauth2 `#and 5 other pkgs` &&\
     (cd /go/src/golang.org/x/oauth2 && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
-# Repo golang.org/x/text at e19ae14 (2017-12-27)
+# Repo golang.org/x/sys at 4d1cda0 (2018-12-13)
-ENV REV=e19ae1496984b1c655b8044a65c0300a3c878dd3
+ENV REV=4d1cda033e0619309c606fc686de3adcf599539e
+RUN go get -d golang.org/x/sys/unix &&\
+    (cd /go/src/golang.org/x/sys && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
+
+# Repo golang.org/x/text at 6f44c5a (2018-10-30)
+ENV REV=6f44c5a2ea40ee3593d98cdcc905cc1fdaa660e2
 RUN go get -d golang.org/x/text/secure/bidirule `#and 4 other pkgs` &&\
     (cd /go/src/golang.org/x/text && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
-# Repo google.golang.org/api at 7d0e2d3 (2018-01-30)
+# Repo google.golang.org/api at 20530fd (2018-05-06)
-ENV REV=7d0e2d350555821bef5a5b8aecf0d12cc1def633
+ENV REV=20530fd5d65ad2caee87891f9896d7547cb400c9
 RUN go get -d google.golang.org/api/gensupport `#and 9 other pkgs` &&\
     (cd /go/src/google.golang.org/api && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
-# Repo google.golang.org/genproto at 4eb30f4 (2018-01-25)
+# Repo google.golang.org/genproto at 86e600f (2018-04-27)
-ENV REV=4eb30f4778eed4c258ba66527a0d4f9ec8a36c45
+ENV REV=86e600f69ee4704c6efbf6a2a40a5c10700e76c2
-RUN go get -d google.golang.org/genproto/googleapis/api/annotations `#and 3 other pkgs` &&\
+RUN go get -d google.golang.org/genproto/googleapis/api/annotations `#and 4 other pkgs` &&\
     (cd /go/src/google.golang.org/genproto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
-# Repo google.golang.org/grpc at 0bd008f (2018-01-25)
+# Repo google.golang.org/grpc at 07ef407 (2018-08-06)
-ENV REV=0bd008f5fadb62d228f12b18d016709e8139a7af
+ENV REV=07ef407d991f1004e6c3367c8f452ed9a02f17ff
-RUN go get -d google.golang.org/grpc `#and 23 other pkgs` &&\
+RUN go get -d google.golang.org/grpc `#and 26 other pkgs` &&\
     (cd /go/src/google.golang.org/grpc && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
 
 # Optimization to speed up iterative development, not necessary for correctness:
@@ -67,6 +81,7 @@ RUN go install cloud.google.com/go/compute/metadata \
 	cloud.google.com/go/iam \
 	cloud.google.com/go/internal \
 	cloud.google.com/go/internal/optional \
+	cloud.google.com/go/internal/trace \
 	cloud.google.com/go/internal/version \
 	cloud.google.com/go/storage \
 	github.com/golang/protobuf/proto \
@@ -76,22 +91,35 @@ RUN go install cloud.google.com/go/compute/metadata \
 	github.com/golang/protobuf/ptypes/duration \
 	github.com/golang/protobuf/ptypes/timestamp \
 	github.com/googleapis/gax-go \
+	go.opencensus.io/exporter/stackdriver/propagation \
+	go.opencensus.io/internal \
+	go.opencensus.io/internal/tagencoding \
+	go.opencensus.io/plugin/ochttp \
+	go.opencensus.io/plugin/ochttp/propagation/b3 \
+	go.opencensus.io/stats \
+	go.opencensus.io/stats/internal \
+	go.opencensus.io/stats/view \
+	go.opencensus.io/tag \
+	go.opencensus.io/trace \
+	go.opencensus.io/trace/internal \
+	go.opencensus.io/trace/propagation \
 	golang.org/x/build/autocertcache \
 	golang.org/x/crypto/acme \
 	golang.org/x/crypto/acme/autocert \
 	golang.org/x/net/context \
 	golang.org/x/net/context/ctxhttp \
+	golang.org/x/net/http/httpguts \
 	golang.org/x/net/http2 \
 	golang.org/x/net/http2/hpack \
 	golang.org/x/net/idna \
 	golang.org/x/net/internal/timeseries \
-	golang.org/x/net/lex/httplex \
 	golang.org/x/net/trace \
 	golang.org/x/oauth2 \
 	golang.org/x/oauth2/google \
 	golang.org/x/oauth2/internal \
 	golang.org/x/oauth2/jws \
 	golang.org/x/oauth2/jwt \
+	golang.org/x/sys/unix \
 	golang.org/x/text/secure/bidirule \
 	golang.org/x/text/transform \
 	golang.org/x/text/unicode/bidi \
@@ -107,6 +135,7 @@ RUN go install cloud.google.com/go/compute/metadata \
 	google.golang.org/api/transport/http \
 	google.golang.org/genproto/googleapis/api/annotations \
 	google.golang.org/genproto/googleapis/iam/v1 \
+	google.golang.org/genproto/googleapis/rpc/code \
 	google.golang.org/genproto/googleapis/rpc/status \
 	google.golang.org/grpc \
 	google.golang.org/grpc/balancer \
@@ -117,9 +146,13 @@ RUN go install cloud.google.com/go/compute/metadata \
 	google.golang.org/grpc/credentials \
 	google.golang.org/grpc/encoding \
 	google.golang.org/grpc/encoding/proto \
-	google.golang.org/grpc/grpclb/grpc_lb_v1/messages \
 	google.golang.org/grpc/grpclog \
 	google.golang.org/grpc/internal \
+	google.golang.org/grpc/internal/backoff \
+	google.golang.org/grpc/internal/channelz \
+	google.golang.org/grpc/internal/envconfig \
+	google.golang.org/grpc/internal/grpcrand \
+	google.golang.org/grpc/internal/transport \
 	google.golang.org/grpc/keepalive \
 	google.golang.org/grpc/metadata \
 	google.golang.org/grpc/naming \
@@ -129,8 +162,7 @@ RUN go install cloud.google.com/go/compute/metadata \
 	google.golang.org/grpc/resolver/passthrough \
 	google.golang.org/grpc/stats \
 	google.golang.org/grpc/status \
-	google.golang.org/grpc/tap \
+	google.golang.org/grpc/tap
-	google.golang.org/grpc/transport
 # END deps.
 
 # golang sets GOPATH=/go

cmd/tip/Makefile

@@ -33,8 +33,8 @@ push-staging: docker-image
 
 deploy-prod: push-prod
 	go install golang.org/x/build/cmd/xb
-	xb --prod kubectl set image deployment/tip-deployment tip=$(IMAGE_PROD):$(VERSION)
+	xb --prod kubectl set image deployment/tipgodoc-deployment tipgodoc=$(IMAGE_PROD):$(VERSION)
 deploy-staging: push-staging
 	go install golang.org/x/build/cmd/xb
-	xb --staging kubectl set image deployment/tip-deployment tip=$(IMAGE_STAGING):$(VERSION)
+	xb --staging kubectl set image deployment/tipgodoc-deployment tipgodoc=$(IMAGE_STAGING):$(VERSION)
 

cmd/tip/README

@@ -23,10 +23,6 @@ New Kubernetes instructions, for tip.golang.org:
 
 Kubernetes instructions:
 
- * build & push images (see Makefile for helpers)
+ * make deploy-prod
- * create/update resources:
-   - kubectl create -f tip-rc.yaml
-   - kubectl create -f tip-service.yaml
 
-TODO(bradfitz): flesh out these instructions as I gain experience
+Also: move talks.golang.org to GKE too?
-with updating this over time. Also: move talks.golang.org to GKE too?

cmd/tip/godoc.go

@@ -8,53 +8,73 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 )
 
-type godocBuilder struct {
+type godocBuilder struct{}
+
+func prefix8(s string) string {
+	if len(s) < 8 {
+		return s
+	}
+	return s[:8]
 }
 
 func (b godocBuilder) Signature(heads map[string]string) string {
-	return fmt.Sprintf("go=%v/tools=%v", heads["go"], heads["tools"])
+	return fmt.Sprintf("go=%v/tools=%v", prefix8(heads["go"]), prefix8(heads["tools"]))
 }
 
-func (b godocBuilder) Init(dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
+func (b godocBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
+
 	goDir := filepath.Join(dir, "go")
 	toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools")
+	logger.Printf("checking out go repo ...")
 	if err := checkout(repoURL+"go", heads["go"], goDir); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("checkout of go: %v", err)
 	}
+	logger.Printf("checking out tools repo ...")
 	if err := checkout(repoURL+"tools", heads["tools"], toolsDir); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("checkout of tools: %v", err)
 	}
 
+	var logWriter io.Writer = toLoggerWriter{logger}
+
 	make := exec.Command(filepath.Join(goDir, "src/make.bash"))
 	make.Dir = filepath.Join(goDir, "src")
-	if err := runErr(make); err != nil {
+	make.Stdout = logWriter
-		return nil, err
+	make.Stderr = logWriter
+	logger.Printf("running make.bash in %s ...", make.Dir)
+	if err := make.Run(); err != nil {
+		return nil, fmt.Errorf("running make.bash: %v", err)
 	}
+
+	logger.Printf("installing godoc ...")
 	goBin := filepath.Join(goDir, "bin/go")
 	goPath := filepath.Join(dir, "gopath")
 	install := exec.Command(goBin, "install", "golang.org/x/tools/cmd/godoc")
-	install.Env = []string{
+	install.Stdout = logWriter
-		"GOROOT=" + goDir,
+	install.Stderr = logWriter
-		"GOPATH=" + goPath,
+	install.Env = append(os.Environ(),
-		"GOROOT_BOOTSTRAP=" + os.Getenv("GOROOT_BOOTSTRAP"),
+		"GOROOT="+goDir,
-	}
+		"GOPATH="+goPath,
-	if err := runErr(install); err != nil {
+		"GOROOT_BOOTSTRAP="+os.Getenv("GOROOT_BOOTSTRAP"),
-		return nil, err
+	)
+	if err := install.Run(); err != nil {
+		return nil, fmt.Errorf("go install golang.org/x/tools/cmd/godoc: %v", err)
 	}
 
+	logger.Printf("starting godoc ...")
 	godocBin := filepath.Join(goPath, "bin/godoc")
 	godoc := exec.Command(godocBin, "-http="+hostport, "-index", "-index_interval=-1s", "-play")
-	godoc.Env = []string{"GOROOT=" + goDir}
+	godoc.Env = append(os.Environ(), "GOROOT="+goDir)
-	// TODO(adg): log this somewhere useful
+	godoc.Stdout = logWriter
-	godoc.Stdout = os.Stdout
+	godoc.Stderr = logWriter
-	godoc.Stderr = os.Stderr
 	if err := godoc.Start(); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("starting godoc: %v", err)
 	}
 	return godoc, nil
 }

cmd/tip/godoc.yaml

@@ -1,17 +0,0 @@
-module: tip
-runtime: custom
-vm: true
-
-manual_scaling:
-  instances: 4
-
-env_variables:
-  TIP_BUILDER: 'godoc'
-
-health_check:
-  enable_health_check: True
-  check_interval_sec: 5
-  timeout_sec: 4
-  unhealthy_threshold: 2
-  healthy_threshold: 2
-  restart_threshold: 240

cmd/tip/talks.go

@@ -8,14 +8,14 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 )
 
-type talksBuilder struct {
+type talksBuilder struct{}
-}
 
 func (b talksBuilder) Signature(heads map[string]string) string {
 	return heads["talks"]
@@ -23,7 +23,8 @@ func (b talksBuilder) Signature(heads map[string]string) string {
 
 const talksToolsRev = "8cab8a1319f0be9798e7fe78b15da75e5f94b2e9"
 
-func (b talksBuilder) Init(dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
+func (b talksBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
+	// TODO: use logger
 	toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools")
 	if err := checkout(repoURL+"tools", talksToolsRev, toolsDir); err != nil {
 		return nil, err
@@ -41,7 +42,7 @@ func (b talksBuilder) Init(dir, hostport string, heads map[string]string) (*exec
 	goPath := filepath.Join(dir, "gopath")
 	presentPath := "golang.org/x/tools/cmd/present"
 	install := exec.Command(goBin, "install", "-tags=appenginevm", presentPath)
-	install.Env = []string{"GOROOT=" + goDir, "GOPATH=" + goPath}
+	install.Env = append(os.Environ(), "GOROOT="+goDir, "GOPATH="+goPath)
 	if err := runErr(install); err != nil {
 		return nil, err
 	}

cmd/tip/tip-prod.yaml

@@ -31,8 +31,8 @@ spec:
         - containerPort: 443
         resources:
           requests:
-            cpu: "1"
-            memory: "2Gi"
-          limits:
             cpu: "2"
             memory: "4Gi"
+          limits:
+            cpu: "2"
+            memory: "8Gi"

cmd/tip/tip.go

@@ -100,7 +100,7 @@ func main() {
 type Proxy struct {
 	builder Builder
 
-	mu       sync.Mutex // protects the followin'
+	mu       sync.Mutex // protects following fields
 	proxy    http.Handler
 	cur      string    // signature of gorepo+toolsrepo
 	cmd      *exec.Cmd // live godoc instance, or nil for none
@@ -111,7 +111,7 @@ type Proxy struct {
 
 type Builder interface {
 	Signature(heads map[string]string) string
-	Init(dir, hostport string, heads map[string]string) (*exec.Cmd, error)
+	Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error)
 	HealthCheck(hostport string) error
 }
 
@@ -177,7 +177,9 @@ func (p *Proxy) serveHealthCheck(w http.ResponseWriter, r *http.Request) {
 
 // run runs in its own goroutine.
 func (p *Proxy) run() {
+	p.mu.Lock()
 	p.side = "a"
+	p.mu.Unlock()
 	for {
 		p.poll()
 		time.Sleep(30 * time.Second)
@@ -225,14 +227,20 @@ func (p *Proxy) poll() {
 	if newSide == "b" {
 		hostport = "localhost:8082"
 	}
-	cmd, err := p.builder.Init(dir, hostport, heads)
+	logger := log.New(os.Stderr, sig+": ", log.LstdFlags)
+
+	cmd, err := p.builder.Init(logger, dir, hostport, heads)
 	if err != nil {
+		logger.Printf("Init failed: %v", err)
 		err = fmt.Errorf("builder.Init: %v", err)
 	} else {
 		go func() {
 			// TODO(adg,bradfitz): be smarter about dead processes
 			if err := cmd.Wait(); err != nil {
-				log.Printf("process in %v exited: %v", dir, err)
+				logger.Printf("process in %v exited: %v (%T)", dir, err, err)
+				if ee, ok := err.(*exec.ExitError); ok {
+					logger.Printf("ProcessState.Sys() = %v", ee.ProcessState.Sys())
+				}
 			}
 		}()
 		err = waitReady(p.builder, hostport)
@@ -302,7 +310,7 @@ func checkout(repo, hash, path string) error {
 		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
 			return fmt.Errorf("mkdir: %v", err)
 		}
-		if err := runErr(exec.Command("git", "clone", repo, path)); err != nil {
+		if err := runErr(exec.Command("git", "clone", "--depth", "1", repo, path)); err != nil {
 			return fmt.Errorf("clone: %v", err)
 		}
 	} else if err != nil {
@@ -424,3 +432,10 @@ func (h httpsOnlyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	h.h.ServeHTTP(w, r)
 }
+
+type toLoggerWriter struct{ logger *log.Logger }
+
+func (w toLoggerWriter) Write(p []byte) (int, error) {
+	w.logger.Printf("%s", p)
+	return len(p), nil
+}