tip: fix, update tip.golang.org

Primarily for golang/go#29251 but also update the base Docker layer
from Go 1.9 to Go 1.11, update other deps, and fix some Kubernetes
config cleanups that happened prior without testing apparently.

Fixes golang/go#29251

Change-Id: I0aafccdfedfc0d9ebb75d0c1a3b0819245ea5f19
Reviewed-on: https://go-review.googlesource.com/c/154181
Reviewed-by: Bryan C. Mills <bcmills@google.com>
This commit is contained in:
Brad Fitzpatrick 2018-12-14 14:45:46 +00:00
parent fe2443f7b9
commit 3c39ce7b61
8 changed files with 131 additions and 84 deletions

View File

@ -1,19 +1,23 @@
FROM golang:1.9
FROM golang:1.11
RUN apt-get update && apt-get install --no-install-recommends -y -q build-essential git
# For implicit GOCACHE (Issues 29243 and 29251), set HOME:
RUN mkdir -p /home/gopher
ENV HOME /home/gopher
# golang puts its go install here (weird but true)
ENV GOROOT_BOOTSTRAP /usr/local/go
# BEGIN deps (run `make update-deps` to update)
# Repo cloud.google.com/go at 1d0c2da (2018-01-30)
ENV REV=1d0c2da40456a9b47f5376165f275424acc15c09
RUN go get -d cloud.google.com/go/compute/metadata `#and 6 other pkgs` &&\
# Repo cloud.google.com/go at b5eca92 (2018-10-23)
ENV REV=b5eca92245a08e245bc29c4880c9779ea4aeaa9a
RUN go get -d cloud.google.com/go/compute/metadata `#and 7 other pkgs` &&\
(cd /go/src/cloud.google.com/go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo github.com/golang/protobuf at 9255415 (2018-01-25)
ENV REV=925541529c1fa6821df4e44ce2723319eb2be768
# Repo github.com/golang/protobuf at b4deda0 (2018-04-30)
ENV REV=b4deda0973fb4c70b50d226b1af49f3da59f5265
RUN go get -d github.com/golang/protobuf/proto `#and 6 other pkgs` &&\
(cd /go/src/github.com/golang/protobuf && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
@ -22,44 +26,54 @@ ENV REV=317e0006254c44a0ac427cc52a0e083ff0b9622f
RUN go get -d github.com/googleapis/gax-go &&\
(cd /go/src/github.com/googleapis/gax-go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/build at e879390 (2018-02-01)
ENV REV=e8793909ba350594eea4c7c6bdb0f0d9a0d0f77a
# Repo go.opencensus.io at ebd8d31 (2018-05-16)
ENV REV=ebd8d31470fedf6c27d0e3056653ddff642509b8
RUN go get -d go.opencensus.io/exporter/stackdriver/propagation `#and 12 other pkgs` &&\
(cd /go/src/go.opencensus.io && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/build at 7b78c20 (2018-12-13)
ENV REV=7b78c2042368d5c56ee9dbd92ab5fa988c763944
RUN go get -d golang.org/x/build/autocertcache &&\
(cd /go/src/golang.org/x/build && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/crypto at 1875d0a (2018-01-27)
ENV REV=1875d0a70c90e57f11972aefd42276df65e895b9
# Repo golang.org/x/crypto at e4dc69e (2018-11-06)
ENV REV=e4dc69e5b2fd71dcaf8bd5d054eb936deb78d1fa
RUN go get -d golang.org/x/crypto/acme `#and 2 other pkgs` &&\
(cd /go/src/golang.org/x/crypto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/net at 6d90978 (2018-02-01)
ENV REV=6d90978dc4889d44e8cfbd04c05d17b5417823c7
# Repo golang.org/x/net at 891ebc4 (2018-12-13)
ENV REV=891ebc4b82d6e74f468c533b06f983c7be918a96
RUN go get -d golang.org/x/net/context `#and 8 other pkgs` &&\
(cd /go/src/golang.org/x/net && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/oauth2 at 30785a2 (2018-01-04)
ENV REV=30785a2c434e431ef7c507b54617d6a951d5f2b4
# Repo golang.org/x/oauth2 at f42d051 (2018-11-06)
ENV REV=f42d05182288abf10faef86d16c0d07b8d40ea2d
RUN go get -d golang.org/x/oauth2 `#and 5 other pkgs` &&\
(cd /go/src/golang.org/x/oauth2 && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/text at e19ae14 (2017-12-27)
ENV REV=e19ae1496984b1c655b8044a65c0300a3c878dd3
# Repo golang.org/x/sys at 4d1cda0 (2018-12-13)
ENV REV=4d1cda033e0619309c606fc686de3adcf599539e
RUN go get -d golang.org/x/sys/unix &&\
(cd /go/src/golang.org/x/sys && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/text at 6f44c5a (2018-10-30)
ENV REV=6f44c5a2ea40ee3593d98cdcc905cc1fdaa660e2
RUN go get -d golang.org/x/text/secure/bidirule `#and 4 other pkgs` &&\
(cd /go/src/golang.org/x/text && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo google.golang.org/api at 7d0e2d3 (2018-01-30)
ENV REV=7d0e2d350555821bef5a5b8aecf0d12cc1def633
# Repo google.golang.org/api at 20530fd (2018-05-06)
ENV REV=20530fd5d65ad2caee87891f9896d7547cb400c9
RUN go get -d google.golang.org/api/gensupport `#and 9 other pkgs` &&\
(cd /go/src/google.golang.org/api && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo google.golang.org/genproto at 4eb30f4 (2018-01-25)
ENV REV=4eb30f4778eed4c258ba66527a0d4f9ec8a36c45
RUN go get -d google.golang.org/genproto/googleapis/api/annotations `#and 3 other pkgs` &&\
# Repo google.golang.org/genproto at 86e600f (2018-04-27)
ENV REV=86e600f69ee4704c6efbf6a2a40a5c10700e76c2
RUN go get -d google.golang.org/genproto/googleapis/api/annotations `#and 4 other pkgs` &&\
(cd /go/src/google.golang.org/genproto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo google.golang.org/grpc at 0bd008f (2018-01-25)
ENV REV=0bd008f5fadb62d228f12b18d016709e8139a7af
RUN go get -d google.golang.org/grpc `#and 23 other pkgs` &&\
# Repo google.golang.org/grpc at 07ef407 (2018-08-06)
ENV REV=07ef407d991f1004e6c3367c8f452ed9a02f17ff
RUN go get -d google.golang.org/grpc `#and 26 other pkgs` &&\
(cd /go/src/google.golang.org/grpc && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Optimization to speed up iterative development, not necessary for correctness:
@ -67,6 +81,7 @@ RUN go install cloud.google.com/go/compute/metadata \
cloud.google.com/go/iam \
cloud.google.com/go/internal \
cloud.google.com/go/internal/optional \
cloud.google.com/go/internal/trace \
cloud.google.com/go/internal/version \
cloud.google.com/go/storage \
github.com/golang/protobuf/proto \
@ -76,22 +91,35 @@ RUN go install cloud.google.com/go/compute/metadata \
github.com/golang/protobuf/ptypes/duration \
github.com/golang/protobuf/ptypes/timestamp \
github.com/googleapis/gax-go \
go.opencensus.io/exporter/stackdriver/propagation \
go.opencensus.io/internal \
go.opencensus.io/internal/tagencoding \
go.opencensus.io/plugin/ochttp \
go.opencensus.io/plugin/ochttp/propagation/b3 \
go.opencensus.io/stats \
go.opencensus.io/stats/internal \
go.opencensus.io/stats/view \
go.opencensus.io/tag \
go.opencensus.io/trace \
go.opencensus.io/trace/internal \
go.opencensus.io/trace/propagation \
golang.org/x/build/autocertcache \
golang.org/x/crypto/acme \
golang.org/x/crypto/acme/autocert \
golang.org/x/net/context \
golang.org/x/net/context/ctxhttp \
golang.org/x/net/http/httpguts \
golang.org/x/net/http2 \
golang.org/x/net/http2/hpack \
golang.org/x/net/idna \
golang.org/x/net/internal/timeseries \
golang.org/x/net/lex/httplex \
golang.org/x/net/trace \
golang.org/x/oauth2 \
golang.org/x/oauth2/google \
golang.org/x/oauth2/internal \
golang.org/x/oauth2/jws \
golang.org/x/oauth2/jwt \
golang.org/x/sys/unix \
golang.org/x/text/secure/bidirule \
golang.org/x/text/transform \
golang.org/x/text/unicode/bidi \
@ -107,6 +135,7 @@ RUN go install cloud.google.com/go/compute/metadata \
google.golang.org/api/transport/http \
google.golang.org/genproto/googleapis/api/annotations \
google.golang.org/genproto/googleapis/iam/v1 \
google.golang.org/genproto/googleapis/rpc/code \
google.golang.org/genproto/googleapis/rpc/status \
google.golang.org/grpc \
google.golang.org/grpc/balancer \
@ -117,9 +146,13 @@ RUN go install cloud.google.com/go/compute/metadata \
google.golang.org/grpc/credentials \
google.golang.org/grpc/encoding \
google.golang.org/grpc/encoding/proto \
google.golang.org/grpc/grpclb/grpc_lb_v1/messages \
google.golang.org/grpc/grpclog \
google.golang.org/grpc/internal \
google.golang.org/grpc/internal/backoff \
google.golang.org/grpc/internal/channelz \
google.golang.org/grpc/internal/envconfig \
google.golang.org/grpc/internal/grpcrand \
google.golang.org/grpc/internal/transport \
google.golang.org/grpc/keepalive \
google.golang.org/grpc/metadata \
google.golang.org/grpc/naming \
@ -129,8 +162,7 @@ RUN go install cloud.google.com/go/compute/metadata \
google.golang.org/grpc/resolver/passthrough \
google.golang.org/grpc/stats \
google.golang.org/grpc/status \
google.golang.org/grpc/tap \
google.golang.org/grpc/transport
google.golang.org/grpc/tap
# END deps.
# golang sets GOPATH=/go

View File

@ -33,8 +33,8 @@ push-staging: docker-image
deploy-prod: push-prod
go install golang.org/x/build/cmd/xb
xb --prod kubectl set image deployment/tip-deployment tip=$(IMAGE_PROD):$(VERSION)
xb --prod kubectl set image deployment/tipgodoc-deployment tipgodoc=$(IMAGE_PROD):$(VERSION)
deploy-staging: push-staging
go install golang.org/x/build/cmd/xb
xb --staging kubectl set image deployment/tip-deployment tip=$(IMAGE_STAGING):$(VERSION)
xb --staging kubectl set image deployment/tipgodoc-deployment tipgodoc=$(IMAGE_STAGING):$(VERSION)

View File

@ -23,10 +23,6 @@ New Kubernetes instructions, for tip.golang.org:
Kubernetes instructions:
* build & push images (see Makefile for helpers)
* create/update resources:
- kubectl create -f tip-rc.yaml
- kubectl create -f tip-service.yaml
* make deploy-prod
TODO(bradfitz): flesh out these instructions as I gain experience
with updating this over time. Also: move talks.golang.org to GKE too?
Also: move talks.golang.org to GKE too?

View File

@ -8,53 +8,73 @@ import (
"bytes"
"errors"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
)
type godocBuilder struct {
type godocBuilder struct{}
func prefix8(s string) string {
if len(s) < 8 {
return s
}
return s[:8]
}
func (b godocBuilder) Signature(heads map[string]string) string {
return fmt.Sprintf("go=%v/tools=%v", heads["go"], heads["tools"])
return fmt.Sprintf("go=%v/tools=%v", prefix8(heads["go"]), prefix8(heads["tools"]))
}
func (b godocBuilder) Init(dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
func (b godocBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
goDir := filepath.Join(dir, "go")
toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools")
logger.Printf("checking out go repo ...")
if err := checkout(repoURL+"go", heads["go"], goDir); err != nil {
return nil, err
return nil, fmt.Errorf("checkout of go: %v", err)
}
logger.Printf("checking out tools repo ...")
if err := checkout(repoURL+"tools", heads["tools"], toolsDir); err != nil {
return nil, err
return nil, fmt.Errorf("checkout of tools: %v", err)
}
var logWriter io.Writer = toLoggerWriter{logger}
make := exec.Command(filepath.Join(goDir, "src/make.bash"))
make.Dir = filepath.Join(goDir, "src")
if err := runErr(make); err != nil {
return nil, err
make.Stdout = logWriter
make.Stderr = logWriter
logger.Printf("running make.bash in %s ...", make.Dir)
if err := make.Run(); err != nil {
return nil, fmt.Errorf("running make.bash: %v", err)
}
logger.Printf("installing godoc ...")
goBin := filepath.Join(goDir, "bin/go")
goPath := filepath.Join(dir, "gopath")
install := exec.Command(goBin, "install", "golang.org/x/tools/cmd/godoc")
install.Env = []string{
install.Stdout = logWriter
install.Stderr = logWriter
install.Env = append(os.Environ(),
"GOROOT="+goDir,
"GOPATH="+goPath,
"GOROOT_BOOTSTRAP="+os.Getenv("GOROOT_BOOTSTRAP"),
}
if err := runErr(install); err != nil {
return nil, err
)
if err := install.Run(); err != nil {
return nil, fmt.Errorf("go install golang.org/x/tools/cmd/godoc: %v", err)
}
logger.Printf("starting godoc ...")
godocBin := filepath.Join(goPath, "bin/godoc")
godoc := exec.Command(godocBin, "-http="+hostport, "-index", "-index_interval=-1s", "-play")
godoc.Env = []string{"GOROOT=" + goDir}
// TODO(adg): log this somewhere useful
godoc.Stdout = os.Stdout
godoc.Stderr = os.Stderr
godoc.Env = append(os.Environ(), "GOROOT="+goDir)
godoc.Stdout = logWriter
godoc.Stderr = logWriter
if err := godoc.Start(); err != nil {
return nil, err
return nil, fmt.Errorf("starting godoc: %v", err)
}
return godoc, nil
}

View File

@ -1,17 +0,0 @@
module: tip
runtime: custom
vm: true
manual_scaling:
instances: 4
env_variables:
TIP_BUILDER: 'godoc'
health_check:
enable_health_check: True
check_interval_sec: 5
timeout_sec: 4
unhealthy_threshold: 2
healthy_threshold: 2
restart_threshold: 240

View File

@ -8,14 +8,14 @@ import (
"bytes"
"errors"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
)
type talksBuilder struct {
}
type talksBuilder struct{}
func (b talksBuilder) Signature(heads map[string]string) string {
return heads["talks"]
@ -23,7 +23,8 @@ func (b talksBuilder) Signature(heads map[string]string) string {
const talksToolsRev = "8cab8a1319f0be9798e7fe78b15da75e5f94b2e9"
func (b talksBuilder) Init(dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
func (b talksBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
// TODO: use logger
toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools")
if err := checkout(repoURL+"tools", talksToolsRev, toolsDir); err != nil {
return nil, err
@ -41,7 +42,7 @@ func (b talksBuilder) Init(dir, hostport string, heads map[string]string) (*exec
goPath := filepath.Join(dir, "gopath")
presentPath := "golang.org/x/tools/cmd/present"
install := exec.Command(goBin, "install", "-tags=appenginevm", presentPath)
install.Env = []string{"GOROOT=" + goDir, "GOPATH=" + goPath}
install.Env = append(os.Environ(), "GOROOT="+goDir, "GOPATH="+goPath)
if err := runErr(install); err != nil {
return nil, err
}

View File

@ -31,8 +31,8 @@ spec:
- containerPort: 443
resources:
requests:
cpu: "1"
memory: "2Gi"
limits:
cpu: "2"
memory: "4Gi"
limits:
cpu: "2"
memory: "8Gi"

View File

@ -100,7 +100,7 @@ func main() {
type Proxy struct {
builder Builder
mu sync.Mutex // protects the followin'
mu sync.Mutex // protects following fields
proxy http.Handler
cur string // signature of gorepo+toolsrepo
cmd *exec.Cmd // live godoc instance, or nil for none
@ -111,7 +111,7 @@ type Proxy struct {
type Builder interface {
Signature(heads map[string]string) string
Init(dir, hostport string, heads map[string]string) (*exec.Cmd, error)
Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error)
HealthCheck(hostport string) error
}
@ -177,7 +177,9 @@ func (p *Proxy) serveHealthCheck(w http.ResponseWriter, r *http.Request) {
// run runs in its own goroutine.
func (p *Proxy) run() {
p.mu.Lock()
p.side = "a"
p.mu.Unlock()
for {
p.poll()
time.Sleep(30 * time.Second)
@ -225,14 +227,20 @@ func (p *Proxy) poll() {
if newSide == "b" {
hostport = "localhost:8082"
}
cmd, err := p.builder.Init(dir, hostport, heads)
logger := log.New(os.Stderr, sig+": ", log.LstdFlags)
cmd, err := p.builder.Init(logger, dir, hostport, heads)
if err != nil {
logger.Printf("Init failed: %v", err)
err = fmt.Errorf("builder.Init: %v", err)
} else {
go func() {
// TODO(adg,bradfitz): be smarter about dead processes
if err := cmd.Wait(); err != nil {
log.Printf("process in %v exited: %v", dir, err)
logger.Printf("process in %v exited: %v (%T)", dir, err, err)
if ee, ok := err.(*exec.ExitError); ok {
logger.Printf("ProcessState.Sys() = %v", ee.ProcessState.Sys())
}
}
}()
err = waitReady(p.builder, hostport)
@ -302,7 +310,7 @@ func checkout(repo, hash, path string) error {
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return fmt.Errorf("mkdir: %v", err)
}
if err := runErr(exec.Command("git", "clone", repo, path)); err != nil {
if err := runErr(exec.Command("git", "clone", "--depth", "1", repo, path)); err != nil {
return fmt.Errorf("clone: %v", err)
}
} else if err != nil {
@ -424,3 +432,10 @@ func (h httpsOnlyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
h.h.ServeHTTP(w, r)
}
type toLoggerWriter struct{ logger *log.Logger }
func (w toLoggerWriter) Write(p []byte) (int, error) {
w.logger.Printf("%s", p)
return len(p), nil
}