[release-branch.go1.11] go/packages: make tests pass with custom GOCACHE

This commit was merged earlier with some failing trybots, so it was
reverted. This is a re-submission.

Before this change, a test would fail:

	$ GOCACHE=$HOME/go/cache go test
	--- FAIL: TestLoadImportsGraph (1.05s)
	    packages_test.go:225: subdir/d.test.Srcs = [cf570d60b25cde4f49bbe5f69d3ed407f2d7f1fbc500b8807da726fb19b8f588-d], want [0.go]
	FAIL

This is because it assumed that the user hadn't set their own GOCACHE,
and thus that all source files in the cache would be under the default
"go-build" cache directory.

We could fix this via os.Getenv("GOCACHE"), but a simpler mechanism is
to see if the source file has an extension. Source files don't have an
extension in GOCACHE, so that's much simpler to detect.

After this change:

	$ GOCACHE=$HOME/go/cache go test
	PASS

On release-branch.go1.11, golist_fallback.go did not yet have the code
added that would need to be fixed, so nothing is being backported to it
in this change.

While at it, gofmt.

Updates golang/go#29445
Fixes golang/go#29944

Change-Id: I21fc59f13f00bea1f9a8a80e0438825f1a36ac3e
Reviewed-on: https://go-review.googlesource.com/c/156977
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Michael Matloob <matloob@golang.org>
Reviewed-on: https://go-review.googlesource.com/c/163780
Run-TryBot: Dmitri Shuralyov <dmitshur@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

cmd/callgraph/main_test.go

@@ -18,6 +18,7 @@ import (
 )
 
 func TestCallgraph(t *testing.T) {
+	t.Skip("golang.org/issue/29201")
 	gopath, err := filepath.Abs("testdata")
 	if err != nil {
 		t.Fatal(err)

cmd/godoc/.gitignore

@@ -0,0 +1,3 @@
+index.split.*
+godoc.index
+godoc.zip

cmd/godoc/Dockerfile.prod

@@ -0,0 +1,67 @@
+# Builder
+#########
+
+FROM golang:1.11 AS build
+
+RUN apt-get update && apt-get install -y \
+      zip # required for generate-index.bash
+
+# Check out the desired version of Go, both to build the godoc binary and serve
+# as the goroot for content serving.
+ARG GO_REF
+RUN test -n "$GO_REF" # GO_REF is required.
+RUN git clone --single-branch --depth=1 -b $GO_REF https://go.googlesource.com/go /goroot
+RUN cd /goroot/src && ./make.bash
+
+ENV GOROOT /goroot
+ENV PATH=/goroot/bin:$PATH
+
+RUN go version
+
+RUN go get -v -d \
+      golang.org/x/net/context \
+      google.golang.org/appengine \
+      cloud.google.com/go/datastore \
+      golang.org/x/build \
+      github.com/gomodule/redigo/redis
+
+COPY . /go/src/golang.org/x/tools
+
+WORKDIR /go/src/golang.org/x/tools/cmd/godoc
+RUN GODOC_DOCSET=/goroot ./generate-index.bash
+
+RUN go build -o /godoc -tags=golangorg golang.org/x/tools/cmd/godoc
+
+# Clean up goroot for the final image.
+RUN cd /goroot && git clean -xdf
+
+# Add build metadata.
+RUN cd /goroot && echo "go repo HEAD: $(git rev-parse HEAD)" >> /goroot/buildinfo
+RUN echo "requested go ref: ${GO_REF}" >> /goroot/buildinfo
+ARG TOOLS_HEAD
+RUN echo "x/tools HEAD: ${TOOLS_HEAD}" >> /goroot/buildinfo
+ARG TOOLS_CLEAN
+RUN echo "x/tools clean: ${TOOLS_CLEAN}" >> /goroot/buildinfo
+ARG DOCKER_TAG
+RUN echo "image: ${DOCKER_TAG}" >> /goroot/buildinfo
+ARG BUILD_ENV
+RUN echo "build env: ${BUILD_ENV}" >> /goroot/buildinfo
+
+RUN rm -rf /goroot/.git
+
+# Final image
+#############
+
+FROM gcr.io/distroless/base
+
+WORKDIR /app
+COPY --from=build /godoc /app/
+COPY --from=build /go/src/golang.org/x/tools/cmd/godoc/hg-git-mapping.bin /app/
+
+COPY --from=build /goroot /goroot
+ENV GOROOT /goroot
+
+COPY --from=build /go/src/golang.org/x/tools/cmd/godoc/index.split.* /app/
+ENV GODOC_INDEX_GLOB index.split.*
+
+CMD ["/app/godoc"]

cmd/godoc/Makefile

@@ -0,0 +1,80 @@
+# Copyright 2018 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+.PHONY: usage
+
+GO_REF ?= release-branch.go1.11
+TOOLS_HEAD := $(shell git rev-parse HEAD)
+TOOLS_CLEAN := $(shell (git status --porcelain | grep -q .) && echo dirty || echo clean)
+ifeq ($(TOOLS_CLEAN),clean)
+	DOCKER_VERSION ?= $(TOOLS_HEAD)
+else
+	DOCKER_VERSION ?= $(TOOLS_HEAD)-dirty
+endif
+GCP_PROJECT := golang-org
+DOCKER_TAG := gcr.io/$(GCP_PROJECT)/godoc:$(DOCKER_VERSION)
+
+usage:
+	@echo "See Makefile and README.godoc-app"
+	@exit 1
+
+cloud-build: Dockerfile.prod
+	gcloud builds submit \
+		--project=$(GCP_PROJECT) \
+		--config=cloudbuild.yaml \
+		--substitutions=_GO_REF=$(GO_REF),_TOOLS_HEAD=$(TOOLS_HEAD),_TOOLS_CLEAN=$(TOOLS_CLEAN),_DOCKER_TAG=$(DOCKER_TAG) \
+		../.. # source code
+
+docker-build: Dockerfile.prod
+	# NOTE(cbro): move up in directory to include entire tools repo.
+	# NOTE(cbro): any changes made to this command must also be made in cloudbuild.yaml.
+	cd ../..; docker build \
+		-f=cmd/godoc/Dockerfile.prod \
+		--build-arg=GO_REF=$(GO_REF) \
+		--build-arg=TOOLS_HEAD=$(TOOLS_HEAD) \
+		--build-arg=TOOLS_CLEAN=$(TOOLS_CLEAN) \
+		--build-arg=DOCKER_TAG=$(DOCKER_TAG) \
+		--build-arg=BUILD_ENV=local \
+		--tag=$(DOCKER_TAG) \
+		.
+
+docker-push: docker-build
+	docker push $(DOCKER_TAG)
+
+deploy:
+	gcloud -q app deploy app.prod.yaml \
+		--project=$(GCP_PROJECT) \
+		--no-promote \
+		--image-url=$(DOCKER_TAG)
+
+get-latest-url:
+	@gcloud app versions list \
+		--service=default \
+		--project=$(GCP_PROJECT) \
+		--sort-by=~version.createTime \
+		--format='value(version.versionUrl)' \
+		--limit 1 | cut -f1 # NOTE(cbro): gcloud prints out createTime as the second field.
+
+get-latest-id:
+	@gcloud app versions list \
+		--service=default \
+		--project=$(GCP_PROJECT) \
+		--sort-by=~version.createTime \
+		--format='value(version.id)' \
+		--limit 1 | cut -f1 # NOTE(cbro): gcloud prints out createTime as the second field.
+
+regtest:
+	go test -v \
+		-regtest.host=$(shell make get-latest-url) \
+		-run=Live
+
+publish: regtest
+	gcloud -q app services set-traffic default \
+		--splits=$(shell make get-latest-id)=1 \
+		--project=$(GCP_PROJECT)
+
+	@echo !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+	@echo Stop and/or delete old versions:
+	@echo "https://console.cloud.google.com/appengine/versions?project=$(GCP_PROJECT)&serviceId=default&versionssize=50"
+	@echo !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

cmd/godoc/README.godoc-app

@@ -1,56 +1,94 @@
-godoc on appengine
-------------------
+godoc on Google App Engine
+==========================
 
 Prerequisites
 -------------
 
-* Go appengine SDK
-  https://developers.google.com/appengine/downloads#Google_App_Engine_SDK_for_Go
+* Google Cloud SDK
+  https://cloud.google.com/sdk/
 
-* Go sources at tip under $GOROOT
+* Redis
 
-* Godoc sources at tip inside $GOPATH
+* Go sources under $GOROOT
+
+* Godoc sources inside $GOPATH
   (go get -d golang.org/x/tools/cmd/godoc)
 
 
-Directory structure
--------------------
+Running locally, in production mode
+-----------------------------------
 
-* Let $APPDIR be the directory containing the app engine files.
-  (e.g., $APPDIR=$HOME/godoc-app)
+Build the app:
 
-* $APPDIR contains the following entries (this may change depending on
-  app-engine release and version of godoc):
+  go build -tags golangorg
 
-	app.yaml
-	golang.org/x/tools/cmd/godoc
-	godoc.zip
-	index.split.*
+Run the app:
 
-* The app.yaml file is set up per app engine documentation.
-  For instance:
+  ./godoc
 
-	application: godoc-app
-	version: 1
-	runtime: go
-	api_version: go1
+godoc should come up at http://localhost:8080
 
-	handlers:
-	- url: /.*
-	  script: _go_app
+Use the PORT environment variable to change the port:
+
+    PORT=8081 ./godoc
+
+Running locally, in production mode, using Docker
+-------------------------------------------------
+
+Build the app's Docker container:
+
+  make docker-build
+
+Make sure redis is running on port 6379:
+
+  $ echo PING | nc localhost 6379
+  +PONG
+  ^C
+
+Run the datastore emulator:
+
+  gcloud beta emulators datastore start --project golang-org
+
+In another terminal window, run the container:
+
+  $(gcloud beta emulators datastore env-init)
+
+  docker run --rm \
+    --net host \
+    --env GODOC_REDIS_ADDR=localhost:6379 \
+    --env DATASTORE_EMULATOR_HOST=$DATASTORE_EMULATOR_HOST \
+    --env DATASTORE_PROJECT_ID=$DATASTORE_PROJECT_ID \
+    gcr.io/golang-org/godoc
+
+godoc should come up at http://localhost:8080
 
 
-Configuring and running godoc
------------------------------
+Deploying to golang.org
+-----------------------
 
-To configure godoc, run
+Make sure you're signed in to gcloud:
 
-	bash setup-godoc-app.bash
+  gcloud auth login
 
-to prepare an $APPDIR as described above. See the script for details on usage.
+Build the image, push it to gcr.io, and deploy to Flex:
 
-To run godoc locally, using the App Engine development server, run
+  make cloud-build deploy
 
-	<path to go_appengine>/dev_appserver.py $APPDIR
+Point the load balancer to the newly deployed version:
+(This also runs regression tests)
 
-godoc should come up at http://localhost:8080 .
+  make publish
+
+Stop and/or delete down any very old versions. (Stopped versions can be re-started.)
+Keep at least one older verson to roll back to, just in case.
+You can also migrate traffic to the new version via this UI.
+
+  https://console.cloud.google.com/appengine/versions?project=golang-org&serviceId=default&versionssize=50
+
+
+Troubleshooting
+---------------
+
+Ensure the Cloud SDK is on your PATH and you have the app-engine-go component
+installed (gcloud components install app-engine-go) and your components are
+up-to-date (gcloud components update)

cmd/godoc/app.dev.yaml

@@ -0,0 +1,13 @@
+runtime: go
+api_version: go1
+instance_class: F4_1G
+
+handlers:
+- url: /s
+  script: _go_app
+  login: admin
+- url: /dl/init
+  script: _go_app
+  login: admin
+- url: /.*
+  script: _go_app

cmd/godoc/app.prod.yaml

@@ -0,0 +1,16 @@
+runtime: custom
+env: flex
+
+env_variables:
+  GODOC_PROD: true
+  GODOC_ENFORCE_HOSTS: true
+  GODOC_REDIS_ADDR: 10.0.0.4:6379 # instance "gophercache"
+  GODOC_ANALYTICS: UA-11222381-2
+  DATASTORE_PROJECT_ID: golang-org
+
+network:
+  name: golang
+
+resources:
+  cpu: 4
+  memory_gb: 7.50

cmd/godoc/appinit.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build appengine
+// +build golangorg
 
 package main
 
@@ -11,25 +11,46 @@ package main
 
 import (
 	"archive/zip"
+	"context"
+	"io"
 	"log"
 	"net/http"
+	"os"
 	"path"
 	"regexp"
+	"runtime"
+	"strings"
 
 	"golang.org/x/tools/godoc"
 	"golang.org/x/tools/godoc/dl"
 	"golang.org/x/tools/godoc/proxy"
+	"golang.org/x/tools/godoc/redirect"
 	"golang.org/x/tools/godoc/short"
 	"golang.org/x/tools/godoc/static"
 	"golang.org/x/tools/godoc/vfs"
+	"golang.org/x/tools/godoc/vfs/gatefs"
 	"golang.org/x/tools/godoc/vfs/mapfs"
 	"golang.org/x/tools/godoc/vfs/zipfs"
 
-	"google.golang.org/appengine"
+	"cloud.google.com/go/datastore"
+	"golang.org/x/tools/internal/memcache"
 )
 
-func init() {
-	enforceHosts = !appengine.IsDevAppServer()
+func main() {
+	log.SetFlags(log.Lshortfile | log.LstdFlags)
+
+	var (
+		// .zip filename
+		zipFilename = os.Getenv("GODOC_ZIP")
+
+		// goroot directory in .zip file
+		zipGoroot = os.Getenv("GODOC_ZIP_PREFIX")
+
+		// glob pattern describing search index files
+		// (if empty, the index is built at run-time)
+		indexFilenames = os.Getenv("GODOC_INDEX_GLOB")
+	)
+
 	playEnabled = true
 
 	log.Println("initializing godoc ...")
@@ -37,16 +58,20 @@ func init() {
 	log.Printf(".zip GOROOT = %s", zipGoroot)
 	log.Printf("index files = %s", indexFilenames)
 
-	goroot := path.Join("/", zipGoroot) // fsHttp paths are relative to '/'
-
-	// read .zip file and set up file systems
-	const zipfile = zipFilename
-	rc, err := zip.OpenReader(zipfile)
-	if err != nil {
-		log.Fatalf("%s: %s\n", zipfile, err)
+	if zipFilename != "" {
+		goroot := path.Join("/", zipGoroot) // fsHttp paths are relative to '/'
+		// read .zip file and set up file systems
+		rc, err := zip.OpenReader(zipFilename)
+		if err != nil {
+			log.Fatalf("%s: %s\n", zipFilename, err)
+		}
+		// rc is never closed (app running forever)
+		fs.Bind("/", zipfs.New(rc, zipFilename), goroot, vfs.BindReplace)
+	} else {
+		rootfs := gatefs.New(vfs.OS(runtime.GOROOT()), make(chan bool, 20))
+		fs.Bind("/", rootfs, "/", vfs.BindReplace)
 	}
-	// rc is never closed (app running forever)
-	fs.Bind("/", zipfs.New(rc, zipFilename), goroot, vfs.BindReplace)
+
 	fs.Bind("/lib/godoc", mapfs.New(static.Files), "/", vfs.BindReplace)
 
 	corpus := godoc.NewCorpus(fs)
@@ -58,6 +83,7 @@ func init() {
 		log.Fatal(err)
 	}
 	corpus.IndexDirectory = indexDirectoryDefault
+	corpus.InitVersionInfo()
 	go corpus.RunIndexer()
 
 	pres = godoc.NewPresentation(corpus)
@@ -66,17 +92,61 @@ func init() {
 	pres.ShowExamples = true
 	pres.DeclLinks = true
 	pres.NotesRx = regexp.MustCompile("BUG")
+	pres.GoogleAnalytics = os.Getenv("GODOC_ANALYTICS")
 
 	readTemplates(pres, true)
 
+	datastoreClient, memcacheClient := getClients()
+
+	// NOTE(cbro): registerHandlers registers itself against DefaultServeMux.
+	// The mux returned has host enforcement, so it's important to register
+	// against this mux and not DefaultServeMux.
 	mux := registerHandlers(pres)
-	dl.RegisterHandlers(mux)
-	short.RegisterHandlers(mux)
+	dl.RegisterHandlers(mux, datastoreClient, memcacheClient)
+	short.RegisterHandlers(mux, datastoreClient, memcacheClient)
 
 	// Register /compile and /share handlers against the default serve mux
 	// so that other app modules can make plain HTTP requests to those
 	// hosts. (For reasons, HTTPS communication between modules is broken.)
 	proxy.RegisterHandlers(http.DefaultServeMux)
 
+	http.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "ok")
+	})
+
+	http.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "User-agent: *\nDisallow: /search\n")
+	})
+
+	if err := redirect.LoadChangeMap("hg-git-mapping.bin"); err != nil {
+		log.Fatalf("LoadChangeMap: %v", err)
+	}
+
 	log.Println("godoc initialization complete")
+
+	// TODO(cbro): add instrumentation via opencensus.
+	port := "8080"
+	if p := os.Getenv("PORT"); p != "" { // PORT is set by GAE flex.
+		port = p
+	}
+	log.Fatal(http.ListenAndServe(":"+port, nil))
+}
+
+func getClients() (*datastore.Client, *memcache.Client) {
+	ctx := context.Background()
+
+	datastoreClient, err := datastore.NewClient(ctx, "")
+	if err != nil {
+		if strings.Contains(err.Error(), "missing project") {
+			log.Fatalf("Missing datastore project. Set the DATASTORE_PROJECT_ID env variable. Use `gcloud beta emulators datastore` to start a local datastore.")
+		}
+		log.Fatalf("datastore.NewClient: %v.", err)
+	}
+
+	redisAddr := os.Getenv("GODOC_REDIS_ADDR")
+	if redisAddr == "" {
+		log.Fatalf("Missing redis server for godoc in production mode. set GODOC_REDIS_ADDR environment variable.")
+	}
+	memcacheClient := memcache.New(redisAddr)
+	return datastoreClient, memcacheClient
 }

cmd/godoc/blog.go

@@ -21,7 +21,7 @@ import (
 
 const (
 	blogRepo = "golang.org/x/blog"
-	blogURL  = "http://blog.golang.org/"
+	blogURL  = "https://blog.golang.org/"
 	blogPath = "/blog/"
 )
 
@@ -42,10 +42,11 @@ func init() {
 }
 
 func blogInit(host string) {
-	// Binary distributions will include the blog content in "/blog".
+	// Binary distributions included the blog content in "/blog".
+	// We stopped including this in Go 1.11.
 	root := filepath.Join(runtime.GOROOT(), "blog")
 
-	// Prefer content from go.blog repository if present.
+	// Prefer content from the golang.org/x/blog repository if present.
 	if pkg, err := build.Import(blogRepo, "", build.FindOnly); err == nil {
 		root = pkg.Dir
 	}

cmd/godoc/cloudbuild.yaml

@@ -0,0 +1,25 @@
+# Copyright 2018 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# NOTE(cbro): any changes to the docker command must also be
+# made in docker-build in the Makefile.
+#
+# Variable substitutions must have a preceding underscore. See:
+# https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values#using_user-defined_substitutions
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args: [
+    'build',
+    '-f=cmd/godoc/Dockerfile.prod',
+    '--build-arg=GO_REF=${_GO_REF}',
+    '--build-arg=TOOLS_HEAD=${_TOOLS_HEAD}',
+    '--build-arg=TOOLS_CLEAN=${_TOOLS_CLEAN}',
+    '--build-arg=DOCKER_TAG=${_DOCKER_TAG}',
+    '--build-arg=BUILD_ENV=cloudbuild',
+    '--tag=${_DOCKER_TAG}',
+    '.',
+  ]
+images: ['${_DOCKER_TAG}']
+options:
+  machineType: 'N1_HIGHCPU_8' # building the godoc index takes a lot of memory.

cmd/godoc/dl.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !appengine
+// +build !golangorg
 
 package main
 

cmd/godoc/generate-index.bash

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+# Copyright 2011 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This script creates a .zip file representing the $GOROOT file system
+# and computes the corresponding search index files.
+#
+# These are used in production (see app.prod.yaml)
+
+set -e -u -x
+
+ZIPFILE=godoc.zip
+INDEXFILE=godoc.index
+SPLITFILES=index.split.
+
+error() {
+	echo "error: $1"
+	exit 2
+}
+
+install() {
+	go install
+}
+
+getArgs() {
+	if [ ! -v GODOC_DOCSET ]; then
+		GODOC_DOCSET="$(go env GOROOT)"
+		echo "GODOC_DOCSET not set explicitly, using GOROOT instead"
+	fi
+
+	# safety checks
+	if [ ! -d "$GODOC_DOCSET" ]; then
+		error "$GODOC_DOCSET is not a directory"
+	fi
+
+	# reporting
+	echo "GODOC_DOCSET = $GODOC_DOCSET"
+}
+
+makeZipfile() {
+	echo "*** make $ZIPFILE"
+	rm -f $ZIPFILE goroot
+	ln -s "$GODOC_DOCSET" goroot
+	zip -q -r $ZIPFILE goroot/* # glob to ignore dotfiles (like .git)
+	rm goroot
+}
+
+makeIndexfile() {
+	echo "*** make $INDEXFILE"
+	godoc=$(go env GOPATH)/bin/godoc
+	# NOTE: run godoc without GOPATH set. Otherwise third-party packages will end up in the index.
+	GOPATH= $godoc -write_index -goroot goroot -index_files=$INDEXFILE -zip=$ZIPFILE
+}
+
+splitIndexfile() {
+	echo "*** split $INDEXFILE"
+	rm -f $SPLITFILES*
+	split -b8m $INDEXFILE $SPLITFILES
+}
+
+cd $(dirname $0)
+
+install
+getArgs "$@"
+makeZipfile
+makeIndexfile
+splitIndexfile
+rm $INDEXFILE
+
+echo "*** setup complete"

cmd/godoc/handlers.go

@@ -21,6 +21,7 @@ import (
 	"text/template"
 
 	"golang.org/x/tools/godoc"
+	"golang.org/x/tools/godoc/env"
 	"golang.org/x/tools/godoc/redirect"
 	"golang.org/x/tools/godoc/vfs"
 )
@@ -30,8 +31,6 @@ var (
 	fs   = vfs.NameSpace{}
 )
 
-var enforceHosts = false // set true in production on app engine
-
 // hostEnforcerHandler redirects requests to "http://foo.golang.org/bar"
 // to "https://golang.org/bar".
 // It permits requests to the host "godoc-test.golang.org" for testing and
@@ -41,11 +40,11 @@ type hostEnforcerHandler struct {
 }
 
 func (h hostEnforcerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if !enforceHosts {
+	if !env.EnforceHosts() {
 		h.h.ServeHTTP(w, r)
 		return
 	}
-	if r.TLS == nil || !h.validHost(r.Host) {
+	if !h.isHTTPS(r) || !h.validHost(r.Host) {
 		r.URL.Scheme = "https"
 		if h.validHost(r.Host) {
 			r.URL.Host = r.Host
@@ -59,9 +58,17 @@ func (h hostEnforcerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.h.ServeHTTP(w, r)
 }
 
+func (h hostEnforcerHandler) isHTTPS(r *http.Request) bool {
+	return r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https"
+}
+
 func (h hostEnforcerHandler) validHost(host string) bool {
 	switch strings.ToLower(host) {
-	case "golang.org", "godoc-test.golang.org", "golang.google.cn":
+	case "golang.org", "golang.google.cn":
+		return true
+	}
+	if strings.HasSuffix(host, "-dot-golang-org.appspot.com") {
+		// staging/test
 		return true
 	}
 	return false

cmd/godoc/main.go

@@ -23,7 +23,7 @@
 //	godoc crypto/block Cipher NewCMAC
 //		- prints doc for Cipher and NewCMAC in package crypto/block
 
-// +build !appengine
+// +build !golangorg
 
 package main
 
@@ -183,6 +183,9 @@ func main() {
 		usage()
 	}
 
+	// Setting the resolved goroot.
+	vfs.GOROOT = *goroot
+
 	var fsGate chan bool
 	fsGate = make(chan bool, 20)
 

cmd/godoc/play.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !appengine
+// +build !golangorg
 
 package main
 

cmd/godoc/regtest_test.go

@@ -0,0 +1,171 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Regression tests to run against a production instance of godoc.
+
+package main_test
+
+import (
+	"bytes"
+	"flag"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+var host = flag.String("regtest.host", "", "host to run regression test against")
+
+func init() {
+	flag.Parse()
+	*host = strings.TrimSuffix(*host, "/")
+}
+
+func TestLiveServer(t *testing.T) {
+	if *host == "" {
+		t.Skip("regtest.host flag missing.")
+	}
+	substringTests := []struct {
+		Message     string
+		Path        string
+		Substring   string
+		Regexp      string
+		NoAnalytics bool // expect the response to not contain GA.
+		PostBody    string
+		StatusCode  int // if 0, expect 2xx status code.
+	}{
+		{
+			Path:      "/doc/faq",
+			Substring: "What is the purpose of the project",
+		},
+		{
+			Path:      "/pkg/",
+			Substring: "Package tar",
+		},
+		{
+			Path:      "/pkg/os/",
+			Substring: "func Open",
+		},
+		{
+			Path:      "/pkg/net/http/",
+			Substring: `title="Added in Go 1.11"`,
+			Message:   "version information not present - failed InitVersionInfo?",
+		},
+		{
+			Path:        "/robots.txt",
+			Substring:   "Disallow: /search",
+			Message:     "robots not present - not deployed from Dockerfile?",
+			NoAnalytics: true,
+		},
+		{
+			Path:        "/change/75944e2e3a63",
+			Substring:   "bdb10cf",
+			Message:     "no change redirect - hg to git mapping not registered?",
+			NoAnalytics: true,
+			StatusCode:  302,
+		},
+		{
+			Path:      "/dl/",
+			Substring: "go1.11.windows-amd64.msi",
+			Message:   "missing data on dl page - misconfiguration of datastore?",
+		},
+		{
+			Path:        "/dl/?mode=json",
+			Substring:   ".windows-amd64.msi",
+			NoAnalytics: true,
+		},
+		{
+			Message:     "broken shortlinks - misconfiguration of datastore or memcache?",
+			Path:        "/s/go2design",
+			Regexp:      "proposal.*Found",
+			NoAnalytics: true,
+			StatusCode:  302,
+		},
+		{
+			Message:   "incorrect search result - broken index?",
+			Path:      "/search?q=IsDir",
+			Substring: "src/os/types.go",
+		},
+		{
+			Path:        "/compile",
+			PostBody:    "body=" + url.QueryEscape("package main; func main() { print(6*7); }"),
+			Regexp:      `^{"compile_errors":"","output":"42"}$`,
+			NoAnalytics: true,
+		},
+		{
+			Path:        "/compile",
+			PostBody:    "body=" + url.QueryEscape("//empty"),
+			Substring:   "expected 'package', found 'EOF'",
+			NoAnalytics: true,
+		},
+		{
+			Path:        "/compile",
+			PostBody:    "version=2&body=package+main%3Bimport+(%22fmt%22%3B%22time%22)%3Bfunc+main()%7Bfmt.Print(%22A%22)%3Btime.Sleep(time.Second)%3Bfmt.Print(%22B%22)%7D",
+			Regexp:      `^{"Errors":"","Events":\[{"Message":"A","Kind":"stdout","Delay":0},{"Message":"B","Kind":"stdout","Delay":1000000000}\]}$`,
+			NoAnalytics: true,
+		},
+		{
+			Path:        "/share",
+			PostBody:    "package main",
+			Substring:   "", // just check it is a 2xx.
+			NoAnalytics: true,
+		},
+	}
+
+	for _, tc := range substringTests {
+		t.Run(tc.Path, func(t *testing.T) {
+			method := "GET"
+			var reqBody io.Reader
+			if tc.PostBody != "" {
+				method = "POST"
+				reqBody = strings.NewReader(tc.PostBody)
+			}
+			req, err := http.NewRequest(method, *host+tc.Path, reqBody)
+			if err != nil {
+				t.Fatalf("NewRequest: %v", err)
+			}
+			if reqBody != nil {
+				req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+			}
+			resp, err := http.DefaultTransport.RoundTrip(req)
+			if err != nil {
+				t.Fatalf("RoundTrip: %v", err)
+			}
+			if tc.StatusCode == 0 {
+				if resp.StatusCode > 299 {
+					t.Errorf("Non-OK status code: %v", resp.StatusCode)
+				}
+			} else if tc.StatusCode != resp.StatusCode {
+				t.Errorf("StatusCode; got %v, want %v", resp.StatusCode, tc.StatusCode)
+			}
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("ReadAll: %v", err)
+			}
+
+			const googleAnalyticsID = "UA-11222381-2" // golang.org analytics ID
+			if !tc.NoAnalytics && !bytes.Contains(body, []byte(googleAnalyticsID)) {
+				t.Errorf("want response to contain analytics tracking ID")
+			}
+
+			if tc.Substring != "" {
+				tc.Regexp = regexp.QuoteMeta(tc.Substring)
+			}
+			re := regexp.MustCompile(tc.Regexp)
+
+			if !re.Match(body) {
+				t.Log("------ actual output -------")
+				t.Log(string(body))
+				t.Log("----------------------------")
+				if tc.Message != "" {
+					t.Log(tc.Message)
+				}
+				t.Fatalf("wanted response to match %s", tc.Regexp)
+			}
+		})
+	}
+}

cmd/godoc/remotesearch.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !appengine
+// +build !golangorg
 
 package main
 

cmd/godoc/setup-godoc-app.bash

@@ -1,134 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2011 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-# This script creates a complete godoc app in $APPDIR.
-# It copies the cmd/godoc and src/go/... sources from GOROOT,
-# synthesizes an app.yaml file, and creates the .zip, index, and
-# configuration files.
-#
-# If an argument is provided it is assumed to be the app-engine godoc directory.
-# Without an argument, $APPDIR is used instead. If GOROOT is not set, "go env"
-# is consulted to find the $GOROOT.
-#
-# The script creates a .zip file representing the $GOROOT file system
-# and computes the corresponding search index files. These files are then
-# copied to $APPDIR. A corresponding godoc configuration file is created
-# in $APPDIR/appconfig.go.
-
-ZIPFILE=godoc.zip
-INDEXFILE=godoc.index
-SPLITFILES=index.split.
-GODOC=golang.org/x/tools/cmd/godoc
-CONFIGFILE=$GODOC/appconfig.go
-
-error() {
-	echo "error: $1"
-	exit 2
-}
-
-getArgs() {
-	if [ -z $APPENGINE_SDK ]; then
-		error "APPENGINE_SDK environment variable not set"
-	fi
-	if [ ! -x $APPENGINE_SDK/goapp ]; then
-		error "couldn't find goapp command in $APPENGINE_SDK"
-	fi
-	if [ -z $GOROOT ]; then
-		GOROOT=$(go env GOROOT)
-		echo "GOROOT not set explicitly, using go env value instead"
-	fi
-	if [ -z $APPDIR ]; then
-		if [ $# == 0 ]; then
-			error "APPDIR not set, and no argument provided"
-		fi
-		APPDIR=$1
-		echo "APPDIR not set, using argument instead"
-	fi
-
-	# safety checks
-	if [ ! -d $GOROOT ]; then
-		error "$GOROOT is not a directory"
-	fi
-	if [ -e $APPDIR ]; then
-		error "$APPDIR exists; check and remove it before trying again"
-	fi
-
-	# reporting
-	echo "GOROOT = $GOROOT"
-	echo "APPDIR = $APPDIR"
-}
-
-fetchGodoc() {
-	echo "*** Fetching godoc (if not already in GOPATH)"
-	unset GOBIN
-	go=$APPENGINE_SDK/goapp
-	$go get -d -tags appengine $GODOC
-	mkdir -p $APPDIR/$GODOC
-	cp $(find $($go list -f '{{.Dir}}' $GODOC) -mindepth 1 -maxdepth 1 -type f) $APPDIR/$GODOC/
-}
-
-makeAppYaml() {
-	echo "*** make $APPDIR/app.yaml"
-	cat > $APPDIR/app.yaml <<EOF
-application: godoc
-version: 1
-runtime: go
-api_version: go1
-
-handlers:
-- url: /.*
-  script: _go_app
-EOF
-}
-
-makeZipfile() {
-	echo "*** make $APPDIR/$ZIPFILE"
-	zip -q -r $APPDIR/$ZIPFILE $GOROOT/*
-}
-
-makeIndexfile() {
-	echo "*** make $APPDIR/$INDEXFILE"
-	GOPATH= godoc -write_index -index_files=$APPDIR/$INDEXFILE -zip=$APPDIR/$ZIPFILE
-}
-
-splitIndexfile() {
-	echo "*** split $APPDIR/$INDEXFILE"
-	split -b8m $APPDIR/$INDEXFILE $APPDIR/$SPLITFILES
-}
-
-makeConfigfile() {
-	echo "*** make $APPDIR/$CONFIGFILE"
-	cat > $APPDIR/$CONFIGFILE <<EOF
-package main
-
-// GENERATED FILE - DO NOT MODIFY BY HAND.
-// (generated by golang.org/x/tools/cmd/godoc/setup-godoc-app.bash)
-
-const (
-	// .zip filename
-	zipFilename = "$ZIPFILE"
-
-	// goroot directory in .zip file
-	zipGoroot = "$GOROOT"
-
-	// glob pattern describing search index files
-	// (if empty, the index is built at run-time)
-	indexFilenames = "$SPLITFILES*"
-)
-EOF
-}
-
-getArgs "$@"
-set -e
-mkdir $APPDIR
-fetchGodoc
-makeAppYaml
-makeZipfile
-makeIndexfile
-splitIndexfile
-makeConfigfile
-
-echo "*** setup complete"

cmd/godoc/x.go

@@ -47,7 +47,9 @@ var xMap = map[string]xRepo{
 	"time":       {"https://go.googlesource.com/time", "git"},
 	"tools":      {"https://go.googlesource.com/tools", "git"},
 	"tour":       {"https://go.googlesource.com/tour", "git"},
-	"vgo":        {"https://go.googlesource.com/vgo", "git"}, // Not included at https://golang.org/pkg/#subrepo.
+	"vgo":        {"https://go.googlesource.com/vgo", "git"},     // Not included at https://golang.org/pkg/#subrepo.
+	"website":    {"https://go.googlesource.com/website", "git"}, // Not included at https://golang.org/pkg/#subrepo.
+	"xerrors":    {"https://go.googlesource.com/xerrors", "git"}, // Not included at https://golang.org/pkg/#subrepo.
 }
 
 func init() {

go/packages/packages_test.go

@@ -1207,19 +1207,19 @@ func TestJSON(t *testing.T) {
 		ID:   "b",
 		Name: "b",
 		Imports: map[string]*packages.Package{
-			"a": &packages.Package{ID: "a"},
+			"a": {ID: "a"},
 		},
 	}, {
 		ID:   "c",
 		Name: "c",
 		Imports: map[string]*packages.Package{
-			"b": &packages.Package{ID: "b"},
+			"b": {ID: "b"},
 		},
 	}, {
 		ID:   "d",
 		Name: "d",
 		Imports: map[string]*packages.Package{
-			"b": &packages.Package{ID: "b"},
+			"b": {ID: "b"},
 		},
 	}} {
 		got := decoded[i]
@@ -1267,12 +1267,13 @@ func srcs(p *packages.Package) []string {
 func cleanPaths(paths []string) []string {
 	result := make([]string, len(paths))
 	for i, src := range paths {
-		// The default location for cache data is a subdirectory named go-build
-		// in the standard user cache directory for the current operating system.
-		if strings.Contains(filepath.ToSlash(src), "/go-build/") {
+		// If the source file doesn't have an extension like .go or .s,
+		// it comes from GOCACHE. The names there aren't predictable.
+		name := filepath.Base(src)
+		if !strings.Contains(name, ".") {
 			result[i] = fmt.Sprintf("%d.go", i) // make cache names predictable
 		} else {
-			result[i] = filepath.Base(src)
+			result[i] = name
 		}
 	}
 	return result
@@ -1321,6 +1322,12 @@ func importGraph(initial []*packages.Package) (string, map[string]*packages.Pack
 						continue
 					}
 				}
+				// math/bits took on a dependency on unsafe in 1.12, which breaks some
+				// tests. As a short term hack, prune that edge.
+				// TODO(matloob): think of a cleaner solution, or remove math/bits from the test.
+				if p.ID == "math/bits" && imp.ID == "unsafe" {
+					continue
+				}
 				edges = append(edges, fmt.Sprintf("%s -> %s", p, imp))
 				visit(imp)
 			}

go/ssa/interp/interp_test.go

@@ -103,8 +103,8 @@ var gorootTestTests = []string{
 	"floatcmp.go",
 	"crlf.go", // doesn't actually assert anything (runoutput)
 	// Slow tests follow.
-	"bom.go", // ~1.7s
-	"gc1.go", // ~1.7s
+	"bom.go",                         // ~1.7s
+	"gc1.go",                         // ~1.7s
 	"cmplxdivide.go cmplxdivide1.go", // ~2.4s
 
 	// Working, but not worth enabling:
@@ -156,6 +156,7 @@ var testdataTests = []string{
 type successPredicate func(exitcode int, output string) error
 
 func run(t *testing.T, dir, input string, success successPredicate) bool {
+	t.Skip("golang.org/issue/27292")
 	if runtime.GOOS == "darwin" {
 		t.Skip("skipping on darwin until golang.org/issue/23166 is fixed")
 	}

godoc/appengine.go

@@ -1,13 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package godoc
-
-import "google.golang.org/appengine"
-
-func init() {
-	onAppengine = !appengine.IsDevAppServer()
-}

godoc/dl/dl.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 // Package dl implements a simple downloads frontend server.
 //
 // It accepts HTTP POST requests to create a new download metadata entity, and
@@ -12,26 +10,13 @@
 package dl
 
 import (
-	"crypto/hmac"
-	"crypto/md5"
-	"encoding/json"
 	"fmt"
-	"html"
 	"html/template"
-	"io"
-	"net/http"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
-
-	"golang.org/x/net/context"
-	"google.golang.org/appengine"
-	"google.golang.org/appengine/datastore"
-	"google.golang.org/appengine/log"
-	"google.golang.org/appengine/memcache"
 )
 
 const (
@@ -40,13 +25,6 @@ const (
 	cacheDuration   = time.Hour
 )
 
-func RegisterHandlers(mux *http.ServeMux) {
-	mux.HandleFunc("/dl", getHandler)
-	mux.HandleFunc("/dl/", getHandler) // also serves listHandler
-	mux.HandleFunc("/dl/upload", uploadHandler)
-	mux.HandleFunc("/dl/init", initHandler)
-}
-
 // File represents a file on the golang.org downloads page.
 // It should be kept in sync with the upload code in x/build/cmd/release.
 type File struct {
@@ -161,12 +139,12 @@ type Feature struct {
 var featuredFiles = []Feature{
 	{
 		Platform:     "Microsoft Windows",
-		Requirements: "Windows XP SP3 or later, Intel 64-bit processor",
+		Requirements: "Windows 7 or later, Intel 64-bit processor",
 		fileRE:       regexp.MustCompile(`\.windows-amd64\.msi$`),
 	},
 	{
 		Platform:     "Apple macOS",
-		Requirements: "macOS 10.8 or later, Intel 64-bit processor",
+		Requirements: "macOS 10.10 or later, Intel 64-bit processor",
 		fileRE:       regexp.MustCompile(`\.darwin-amd64(-osx10\.8)?\.pkg$`),
 	},
 	{
@@ -191,54 +169,6 @@ var (
 	templateFuncs = template.FuncMap{"pretty": pretty}
 )
 
-func listHandler(w http.ResponseWriter, r *http.Request) {
-	if r.Method != "GET" {
-		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
-		return
-	}
-	var (
-		c = appengine.NewContext(r)
-		d listTemplateData
-	)
-	if _, err := memcache.Gob.Get(c, cacheKey, &d); err != nil {
-		if err == memcache.ErrCacheMiss {
-			log.Debugf(c, "cache miss")
-		} else {
-			log.Errorf(c, "cache get error: %v", err)
-		}
-
-		var fs []File
-		_, err := datastore.NewQuery("File").Ancestor(rootKey(c)).GetAll(c, &fs)
-		if err != nil {
-			log.Errorf(c, "error listing: %v", err)
-			return
-		}
-		d.Stable, d.Unstable, d.Archive = filesToReleases(fs)
-		if len(d.Stable) > 0 {
-			d.Featured = filesToFeatured(d.Stable[0].Files)
-		}
-
-		item := &memcache.Item{Key: cacheKey, Object: &d, Expiration: cacheDuration}
-		if err := memcache.Gob.Set(c, item); err != nil {
-			log.Errorf(c, "cache set error: %v", err)
-		}
-	}
-
-	if r.URL.Query().Get("mode") == "json" {
-		w.Header().Set("Content-Type", "application/json")
-		enc := json.NewEncoder(w)
-		enc.SetIndent("", " ")
-		if err := enc.Encode(d.Stable); err != nil {
-			log.Errorf(c, "failed rendering JSON for releases: %v", err)
-		}
-		return
-	}
-
-	if err := listTemplate.ExecuteTemplate(w, "root", d); err != nil {
-		log.Errorf(c, "error executing template: %v", err)
-	}
-}
-
 func filesToFeatured(fs []File) (featured []Feature) {
 	for _, feature := range featuredFiles {
 		for _, file := range fs {
@@ -383,146 +313,19 @@ func parseVersion(v string) (maj, min int, tail string) {
 	return
 }
 
-func uploadHandler(w http.ResponseWriter, r *http.Request) {
-	if r.Method != "POST" {
-		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
-		return
-	}
-	c := appengine.NewContext(r)
-
-	// Authenticate using a user token (same as gomote).
-	user := r.FormValue("user")
-	if !validUser(user) {
-		http.Error(w, "bad user", http.StatusForbidden)
-		return
-	}
-	if r.FormValue("key") != userKey(c, user) {
-		http.Error(w, "bad key", http.StatusForbidden)
-		return
-	}
-
-	var f File
-	defer r.Body.Close()
-	if err := json.NewDecoder(r.Body).Decode(&f); err != nil {
-		log.Errorf(c, "error decoding upload JSON: %v", err)
-		http.Error(w, "Something broke", http.StatusInternalServerError)
-		return
-	}
-	if f.Filename == "" {
-		http.Error(w, "Must provide Filename", http.StatusBadRequest)
-		return
-	}
-	if f.Uploaded.IsZero() {
-		f.Uploaded = time.Now()
-	}
-	k := datastore.NewKey(c, "File", f.Filename, 0, rootKey(c))
-	if _, err := datastore.Put(c, k, &f); err != nil {
-		log.Errorf(c, "putting File entity: %v", err)
-		http.Error(w, "could not put File entity", http.StatusInternalServerError)
-		return
-	}
-	if err := memcache.Delete(c, cacheKey); err != nil {
-		log.Errorf(c, "cache delete error: %v", err)
-	}
-	io.WriteString(w, "OK")
-}
-
-func getHandler(w http.ResponseWriter, r *http.Request) {
-	// For go get golang.org/dl/go1.x.y, we need to serve the
-	// same meta tags at /dl for cmd/go to validate against /dl/go1.x.y:
-	if r.URL.Path == "/dl" && (r.Method == "GET" || r.Method == "HEAD") && r.FormValue("go-get") == "1" {
-		w.Header().Set("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, `<!DOCTYPE html><html><head>
-<meta name="go-import" content="golang.org/dl git https://go.googlesource.com/dl">
-</head></html>`)
-		return
-	}
-	if r.URL.Path == "/dl" {
-		http.Redirect(w, r, "/dl/", http.StatusFound)
-		return
-	}
-
-	name := strings.TrimPrefix(r.URL.Path, "/dl/")
-	if name == "" {
-		listHandler(w, r)
-		return
-	}
-	if fileRe.MatchString(name) {
-		http.Redirect(w, r, downloadBaseURL+name, http.StatusFound)
-		return
-	}
-	if goGetRe.MatchString(name) {
-		var isGoGet bool
-		if r.Method == "GET" || r.Method == "HEAD" {
-			w.Header().Set("Content-Type", "text/html; charset=utf-8")
-			isGoGet = r.FormValue("go-get") == "1"
-		}
-		if !isGoGet {
-			w.Header().Set("Location", "https://golang.org/dl/#"+name)
-			w.WriteHeader(http.StatusFound)
-		}
-		fmt.Fprintf(w, `<!DOCTYPE html>
-<html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
-<meta name="go-import" content="golang.org/dl git https://go.googlesource.com/dl">
-<meta http-equiv="refresh" content="0; url=https://golang.org/dl/#%s">
-</head>
-<body>
-Nothing to see here; <a href="https://golang.org/dl/#%s">move along</a>.
-</body>
-</html>
-`, html.EscapeString(name), html.EscapeString(name))
-		return
-	}
-	http.NotFound(w, r)
-}
-
 func validUser(user string) bool {
 	switch user {
-	case "adg", "bradfitz", "cbro", "andybons", "valsorda":
+	case "adg", "bradfitz", "cbro", "andybons", "valsorda", "dmitshur", "katiehockman":
 		return true
 	}
 	return false
 }
 
-func userKey(c context.Context, user string) string {
-	h := hmac.New(md5.New, []byte(secret(c)))
-	h.Write([]byte("user-" + user))
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
 var (
 	fileRe  = regexp.MustCompile(`^go[0-9a-z.]+\.[0-9a-z.-]+\.(tar\.gz|pkg|msi|zip)$`)
 	goGetRe = regexp.MustCompile(`^go[0-9a-z.]+\.[0-9a-z.-]+$`)
 )
 
-func initHandler(w http.ResponseWriter, r *http.Request) {
-	var fileRoot struct {
-		Root string
-	}
-	c := appengine.NewContext(r)
-	k := rootKey(c)
-	err := datastore.RunInTransaction(c, func(c context.Context) error {
-		err := datastore.Get(c, k, &fileRoot)
-		if err != nil && err != datastore.ErrNoSuchEntity {
-			return err
-		}
-		_, err = datastore.Put(c, k, &fileRoot)
-		return err
-	}, nil)
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-		return
-	}
-	io.WriteString(w, "OK")
-}
-
-// rootKey is the ancestor of all File entities.
-func rootKey(c context.Context) *datastore.Key {
-	return datastore.NewKey(c, "FileRoot", "root", 0, nil)
-}
-
 // pretty returns a human-readable version of the given OS, Arch, or Kind.
 func pretty(s string) string {
 	t, ok := prettyStrings[s]
@@ -547,55 +350,3 @@ var prettyStrings = map[string]string{
 	"installer": "Installer",
 	"source":    "Source",
 }
-
-// Code below copied from x/build/app/key
-
-var theKey struct {
-	sync.RWMutex
-	builderKey
-}
-
-type builderKey struct {
-	Secret string
-}
-
-func (k *builderKey) Key(c context.Context) *datastore.Key {
-	return datastore.NewKey(c, "BuilderKey", "root", 0, nil)
-}
-
-func secret(c context.Context) string {
-	// check with rlock
-	theKey.RLock()
-	k := theKey.Secret
-	theKey.RUnlock()
-	if k != "" {
-		return k
-	}
-
-	// prepare to fill; check with lock and keep lock
-	theKey.Lock()
-	defer theKey.Unlock()
-	if theKey.Secret != "" {
-		return theKey.Secret
-	}
-
-	// fill
-	if err := datastore.Get(c, theKey.Key(c), &theKey.builderKey); err != nil {
-		if err == datastore.ErrNoSuchEntity {
-			// If the key is not stored in datastore, write it.
-			// This only happens at the beginning of a new deployment.
-			// The code is left here for SDK use and in case a fresh
-			// deployment is ever needed.  "gophers rule" is not the
-			// real key.
-			if !appengine.IsDevAppServer() {
-				panic("lost key from datastore")
-			}
-			theKey.Secret = "gophers rule"
-			datastore.Put(c, theKey.Key(c), &theKey.builderKey)
-			return theKey.Secret
-		}
-		panic("cannot load builder key: " + err.Error())
-	}
-
-	return theKey.Secret
-}

godoc/dl/dl_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 package dl
 
 import (

godoc/dl/server.go

@@ -0,0 +1,266 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+// +build golangorg
+
+package dl
+
+import (
+	"crypto/hmac"
+	"crypto/md5"
+	"encoding/json"
+	"fmt"
+	"html"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"cloud.google.com/go/datastore"
+	"golang.org/x/net/context"
+	"golang.org/x/tools/godoc/env"
+	"golang.org/x/tools/internal/memcache"
+)
+
+type server struct {
+	datastore *datastore.Client
+	memcache  *memcache.CodecClient
+}
+
+func RegisterHandlers(mux *http.ServeMux, dc *datastore.Client, mc *memcache.Client) {
+	s := server{dc, mc.WithCodec(memcache.Gob)}
+	mux.HandleFunc("/dl", s.getHandler)
+	mux.HandleFunc("/dl/", s.getHandler) // also serves listHandler
+	mux.HandleFunc("/dl/upload", s.uploadHandler)
+
+	// NOTE(cbro): this only needs to be run once per project,
+	// and should be behind an admin login.
+	// TODO(cbro): move into a locally-run program? or remove?
+	// mux.HandleFunc("/dl/init", initHandler)
+}
+
+// rootKey is the ancestor of all File entities.
+var rootKey = datastore.NameKey("FileRoot", "root", nil)
+
+func (h server) listHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	ctx := r.Context()
+	var d listTemplateData
+
+	if err := h.memcache.Get(ctx, cacheKey, &d); err != nil {
+		if err != memcache.ErrCacheMiss {
+			log.Printf("ERROR cache get error: %v", err)
+			// NOTE(cbro): continue to hit datastore if the memcache is down.
+		}
+
+		var fs []File
+		q := datastore.NewQuery("File").Ancestor(rootKey)
+		if _, err := h.datastore.GetAll(ctx, q, &fs); err != nil {
+			log.Printf("ERROR error listing: %v", err)
+			http.Error(w, "Could not get download page. Try again in a few minutes.", 500)
+			return
+		}
+		d.Stable, d.Unstable, d.Archive = filesToReleases(fs)
+		if len(d.Stable) > 0 {
+			d.Featured = filesToFeatured(d.Stable[0].Files)
+		}
+
+		item := &memcache.Item{Key: cacheKey, Object: &d, Expiration: cacheDuration}
+		if err := h.memcache.Set(ctx, item); err != nil {
+			log.Printf("ERROR cache set error: %v", err)
+		}
+	}
+
+	if r.URL.Query().Get("mode") == "json" {
+		w.Header().Set("Content-Type", "application/json")
+		enc := json.NewEncoder(w)
+		enc.SetIndent("", " ")
+		if err := enc.Encode(d.Stable); err != nil {
+			log.Printf("ERROR rendering JSON for releases: %v", err)
+		}
+		return
+	}
+
+	if err := listTemplate.ExecuteTemplate(w, "root", d); err != nil {
+		log.Printf("ERROR executing template: %v", err)
+	}
+}
+
+func (h server) uploadHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	ctx := r.Context()
+
+	// Authenticate using a user token (same as gomote).
+	user := r.FormValue("user")
+	if !validUser(user) {
+		http.Error(w, "bad user", http.StatusForbidden)
+		return
+	}
+	if r.FormValue("key") != h.userKey(ctx, user) {
+		http.Error(w, "bad key", http.StatusForbidden)
+		return
+	}
+
+	var f File
+	defer r.Body.Close()
+	if err := json.NewDecoder(r.Body).Decode(&f); err != nil {
+		log.Printf("ERROR decoding upload JSON: %v", err)
+		http.Error(w, "Something broke", http.StatusInternalServerError)
+		return
+	}
+	if f.Filename == "" {
+		http.Error(w, "Must provide Filename", http.StatusBadRequest)
+		return
+	}
+	if f.Uploaded.IsZero() {
+		f.Uploaded = time.Now()
+	}
+	k := datastore.NameKey("File", f.Filename, rootKey)
+	if _, err := h.datastore.Put(ctx, k, &f); err != nil {
+		log.Printf("ERROR File entity: %v", err)
+		http.Error(w, "could not put File entity", http.StatusInternalServerError)
+		return
+	}
+	if err := h.memcache.Delete(ctx, cacheKey); err != nil {
+		log.Printf("ERROR delete error: %v", err)
+	}
+	io.WriteString(w, "OK")
+}
+
+func (h server) getHandler(w http.ResponseWriter, r *http.Request) {
+	isGoGet := (r.Method == "GET" || r.Method == "HEAD") && r.FormValue("go-get") == "1"
+	// For go get, we need to serve the same meta tags at /dl for cmd/go to
+	// validate against the import path.
+	if r.URL.Path == "/dl" && isGoGet {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		fmt.Fprintf(w, `<!DOCTYPE html><html><head>
+<meta name="go-import" content="golang.org/dl git https://go.googlesource.com/dl">
+</head></html>`)
+		return
+	}
+	if r.URL.Path == "/dl" {
+		http.Redirect(w, r, "/dl/", http.StatusFound)
+		return
+	}
+
+	name := strings.TrimPrefix(r.URL.Path, "/dl/")
+	var redirectURL string
+	switch {
+	case name == "":
+		h.listHandler(w, r)
+		return
+	case fileRe.MatchString(name):
+		http.Redirect(w, r, downloadBaseURL+name, http.StatusFound)
+		return
+	case name == "gotip":
+		redirectURL = "https://godoc.org/golang.org/dl/gotip"
+	case goGetRe.MatchString(name):
+		redirectURL = "https://golang.org/dl/#" + name
+	default:
+		http.NotFound(w, r)
+		return
+	}
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	if !isGoGet {
+		w.Header().Set("Location", redirectURL)
+	}
+	fmt.Fprintf(w, `<!DOCTYPE html>
+<html>
+<head>
+<meta name="go-import" content="golang.org/dl git https://go.googlesource.com/dl">
+<meta http-equiv="refresh" content="0; url=%s">
+</head>
+<body>
+Nothing to see here; <a href="%s">move along</a>.
+</body>
+</html>
+`, html.EscapeString(redirectURL), html.EscapeString(redirectURL))
+}
+
+func (h server) initHandler(w http.ResponseWriter, r *http.Request) {
+	var fileRoot struct {
+		Root string
+	}
+	ctx := r.Context()
+	k := rootKey
+	_, err := h.datastore.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
+		err := tx.Get(k, &fileRoot)
+		if err != nil && err != datastore.ErrNoSuchEntity {
+			return err
+		}
+		_, err = tx.Put(k, &fileRoot)
+		return err
+	}, nil)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	io.WriteString(w, "OK")
+}
+
+func (h server) userKey(c context.Context, user string) string {
+	hash := hmac.New(md5.New, []byte(h.secret(c)))
+	hash.Write([]byte("user-" + user))
+	return fmt.Sprintf("%x", hash.Sum(nil))
+}
+
+// Code below copied from x/build/app/key
+
+var theKey struct {
+	sync.RWMutex
+	builderKey
+}
+
+type builderKey struct {
+	Secret string
+}
+
+func (k *builderKey) Key() *datastore.Key {
+	return datastore.NameKey("BuilderKey", "root", nil)
+}
+
+func (h server) secret(ctx context.Context) string {
+	// check with rlock
+	theKey.RLock()
+	k := theKey.Secret
+	theKey.RUnlock()
+	if k != "" {
+		return k
+	}
+
+	// prepare to fill; check with lock and keep lock
+	theKey.Lock()
+	defer theKey.Unlock()
+	if theKey.Secret != "" {
+		return theKey.Secret
+	}
+
+	// fill
+	if err := h.datastore.Get(ctx, theKey.Key(), &theKey.builderKey); err != nil {
+		if err == datastore.ErrNoSuchEntity {
+			// If the key is not stored in datastore, write it.
+			// This only happens at the beginning of a new deployment.
+			// The code is left here for SDK use and in case a fresh
+			// deployment is ever needed.  "gophers rule" is not the
+			// real key.
+			if env.IsProd() {
+				panic("lost key from datastore")
+			}
+			theKey.Secret = "gophers rule"
+			h.datastore.Put(ctx, theKey.Key(), &theKey.builderKey)
+			return theKey.Secret
+		}
+		panic("cannot load builder key: " + err.Error())
+	}
+
+	return theKey.Secret
+}

godoc/dl/tmpl.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 package dl
 
 // TODO(adg): refactor this to use the tools/godoc/static template.

godoc/env/env.go

@@ -0,0 +1,41 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package env provides environment information for the godoc server running on
+// golang.org.
+package env
+
+import (
+	"log"
+	"os"
+	"strconv"
+)
+
+var (
+	isProd       = boolEnv("GODOC_PROD")
+	enforceHosts = boolEnv("GODOC_ENFORCE_HOSTS")
+)
+
+// IsProd reports whether the server is running in its production configuration
+// on golang.org.
+func IsProd() bool {
+	return isProd
+}
+
+// EnforceHosts reports whether host filtering should be enforced.
+func EnforceHosts() bool {
+	return enforceHosts
+}
+
+func boolEnv(key string) bool {
+	v := os.Getenv(key)
+	if v == "" {
+		return false
+	}
+	b, err := strconv.ParseBool(v)
+	if err != nil {
+		log.Fatalf("environment variable %s (%q) must be a boolean", key, v)
+	}
+	return b
+}

godoc/page.go

@@ -10,6 +10,8 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+
+	"golang.org/x/tools/godoc/env"
 )
 
 // Page describes the contents of the top-level godoc webpage.
@@ -22,10 +24,11 @@ type Page struct {
 	Body     []byte
 	GoogleCN bool // page is being served from golang.google.cn
 
-	// filled in by servePage
-	SearchBox  bool
-	Playground bool
-	Version    string
+	// filled in by ServePage
+	SearchBox       bool
+	Playground      bool
+	Version         string
+	GoogleAnalytics string
 }
 
 func (p *Presentation) ServePage(w http.ResponseWriter, page Page) {
@@ -35,6 +38,7 @@ func (p *Presentation) ServePage(w http.ResponseWriter, page Page) {
 	page.SearchBox = p.Corpus.IndexEnabled
 	page.Playground = p.ShowPlayground
 	page.Version = runtime.Version()
+	page.GoogleAnalytics = p.GoogleAnalytics
 	applyTemplateToResponseWriter(w, p.GodocHTML, page)
 }
 
@@ -49,20 +53,19 @@ func (p *Presentation) ServeError(w http.ResponseWriter, r *http.Request, relpat
 		}
 	}
 	p.ServePage(w, Page{
-		Title:    "File " + relpath,
-		Subtitle: relpath,
-		Body:     applyTemplate(p.ErrorHTML, "errorHTML", err),
-		GoogleCN: googleCN(r),
+		Title:           "File " + relpath,
+		Subtitle:        relpath,
+		Body:            applyTemplate(p.ErrorHTML, "errorHTML", err),
+		GoogleCN:        googleCN(r),
+		GoogleAnalytics: p.GoogleAnalytics,
 	})
 }
 
-var onAppengine = false // overridden in appengine.go when on app engine
-
 func googleCN(r *http.Request) bool {
 	if r.FormValue("googlecn") != "" {
 		return true
 	}
-	if !onAppengine {
+	if !env.IsProd() {
 		return false
 	}
 	if strings.HasSuffix(r.Host, ".cn") {

godoc/pres.go

@@ -92,6 +92,10 @@ type Presentation struct {
 	// body for displaying search results.
 	SearchResults []SearchResultFunc
 
+	// GoogleAnalytics optionally adds Google Analytics via the provided
+	// tracking ID to each page.
+	GoogleAnalytics string
+
 	initFuncMapOnce sync.Once
 	funcMap         template.FuncMap
 	templateFuncs   template.FuncMap

godoc/proxy/proxy.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 // Package proxy proxies requests to the playground's compile and share handlers.
 // It is designed to run only on the instance of godoc that serves golang.org.
 package proxy
@@ -12,20 +10,19 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"log"
 	"net/http"
-	"net/http/httputil"
-	"net/url"
 	"strings"
 	"time"
 
 	"golang.org/x/net/context"
-
-	"google.golang.org/appengine"
-	"google.golang.org/appengine/log"
-	"google.golang.org/appengine/urlfetch"
+	"golang.org/x/tools/godoc/env"
 )
 
+const playgroundURL = "https://play.golang.org"
+
 type Request struct {
 	Body string
 }
@@ -41,8 +38,6 @@ type Event struct {
 	Delay   time.Duration // time to wait before printing Message
 }
 
-const playgroundURL = "https://play.golang.org"
-
 const expires = 7 * 24 * time.Hour // 1 week
 var cacheControlHeader = fmt.Sprintf("public, max-age=%d", int(expires.Seconds()))
 
@@ -57,21 +52,17 @@ func compile(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	ctx := appengine.NewContext(r)
+	ctx := r.Context()
 
 	body := r.FormValue("body")
 	res := &Response{}
 	req := &Request{Body: body}
 	if err := makeCompileRequest(ctx, req, res); err != nil {
-		log.Errorf(ctx, "compile error: %v", err)
+		log.Printf("ERROR compile error: %v", err)
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
 		return
 	}
 
-	expiresTime := time.Now().Add(expires).UTC()
-	w.Header().Set("Expires", expiresTime.Format(time.RFC1123))
-	w.Header().Set("Cache-Control", cacheControlHeader)
-
 	var out interface{}
 	switch r.FormValue("version") {
 	case "2":
@@ -82,9 +73,17 @@ func compile(w http.ResponseWriter, r *http.Request) {
 			Output        string `json:"output"`
 		}{res.Errors, flatten(res.Events)}
 	}
-	if err := json.NewEncoder(w).Encode(out); err != nil {
-		log.Errorf(ctx, "encoding response: %v", err)
+	b, err := json.Marshal(out)
+	if err != nil {
+		log.Printf("ERROR encoding response: %v", err)
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
 	}
+
+	expiresTime := time.Now().Add(expires).UTC()
+	w.Header().Set("Expires", expiresTime.Format(time.RFC1123))
+	w.Header().Set("Cache-Control", cacheControlHeader)
+	w.Write(b)
 }
 
 // makePlaygroundRequest sends the given Request to the playground compile
@@ -94,17 +93,22 @@ func makeCompileRequest(ctx context.Context, req *Request, res *Response) error
 	if err != nil {
 		return fmt.Errorf("marshalling request: %v", err)
 	}
-	r, err := urlfetch.Client(ctx).Post(playgroundURL+"/compile", "application/json", bytes.NewReader(reqJ))
+	hReq, _ := http.NewRequest("POST", playgroundURL+"/compile", bytes.NewReader(reqJ))
+	hReq.Header.Set("Content-Type", "application/json")
+	hReq = hReq.WithContext(ctx)
+
+	r, err := http.DefaultClient.Do(hReq)
 	if err != nil {
 		return fmt.Errorf("making request: %v", err)
 	}
 	defer r.Body.Close()
+
 	if r.StatusCode != http.StatusOK {
 		b, _ := ioutil.ReadAll(r.Body)
 		return fmt.Errorf("bad status: %v body:\n%s", r.Status, b)
 	}
-	err = json.NewDecoder(r.Body).Decode(res)
-	if err != nil {
+
+	if err := json.NewDecoder(r.Body).Decode(res); err != nil {
 		return fmt.Errorf("unmarshalling response: %v", err)
 	}
 	return nil
@@ -124,17 +128,35 @@ func share(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
 		return
 	}
-	target, _ := url.Parse(playgroundURL)
-	p := httputil.NewSingleHostReverseProxy(target)
-	p.Transport = &urlfetch.Transport{Context: appengine.NewContext(r)}
-	p.ServeHTTP(w, r)
+
+	// HACK(cbro): use a simple proxy rather than httputil.ReverseProxy because of Issue #28168.
+	// TODO: investigate using ReverseProxy with a Director, unsetting whatever's necessary to make that work.
+	req, _ := http.NewRequest("POST", playgroundURL+"/share", r.Body)
+	req.Header.Set("Content-Type", r.Header.Get("Content-Type"))
+	req = req.WithContext(r.Context())
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Printf("ERROR share error: %v", err)
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	copyHeader := func(k string) {
+		if v := resp.Header.Get(k); v != "" {
+			w.Header().Set(k, v)
+		}
+	}
+	copyHeader("Content-Type")
+	copyHeader("Content-Length")
+	defer resp.Body.Close()
+	w.WriteHeader(resp.StatusCode)
+	io.Copy(w, resp.Body)
 }
 
 func googleCN(r *http.Request) bool {
 	if r.FormValue("googlecn") != "" {
 		return true
 	}
-	if appengine.IsDevAppServer() {
+	if !env.IsProd() {
 		return false
 	}
 	if strings.HasSuffix(r.Host, ".cn") {

godoc/redirect/redirect.go

@@ -8,12 +8,18 @@
 package redirect // import "golang.org/x/tools/godoc/redirect"
 
 import (
+	"context"
 	"fmt"
+	"html/template"
 	"net/http"
 	"os"
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/net/context/ctxhttp"
 )
 
 // Register registers HTTP handlers that redirect old godoc paths to their new
@@ -115,7 +121,7 @@ var redirects = map[string]string{
 	"/tour":  "http://tour.golang.org",
 	"/wiki":  "https://github.com/golang/go/wiki",
 
-	"/doc/articles/c_go_cgo.html": "/blog/c-go-cgo",
+	"/doc/articles/c_go_cgo.html":                    "/blog/c-go-cgo",
 	"/doc/articles/concurrency_patterns.html":        "/blog/go-concurrency-patterns-timing-out-and",
 	"/doc/articles/defer_panic_recover.html":         "/blog/defer-panic-and-recover",
 	"/doc/articles/error_handling.html":              "/blog/error-handling-and-go",
@@ -191,9 +197,19 @@ func clHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	target := ""
-	// the first CL in rietveld is about 152046, so only treat the id as
-	// a rietveld CL if it is larger than 150000.
-	if n, err := strconv.Atoi(id); err == nil && n > 150000 {
+
+	if n, err := strconv.Atoi(id); err == nil && isRietveldCL(n) {
+		// Issue 28836: if this Rietveld CL happens to
+		// also be a Gerrit CL, render a disambiguation HTML
+		// page with two links instead. We need to make a
+		// Gerrit API call to figure that out, but we cache
+		// known Gerrit CLs so it's done at most once per CL.
+		if ok, err := isGerritCL(r.Context(), n); err == nil && ok {
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			clDisambiguationHTML.Execute(w, n)
+			return
+		}
+
 		target = "https://codereview.appspot.com/" + id
 	} else {
 		target = "https://go-review.googlesource.com/" + id
@@ -201,6 +217,64 @@ func clHandler(w http.ResponseWriter, r *http.Request) {
 	http.Redirect(w, r, target, http.StatusFound)
 }
 
+var clDisambiguationHTML = template.Must(template.New("").Parse(`<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<title>Go CL {{.}} Disambiguation</title>
+		<meta name="viewport" content="width=device-width">
+	</head>
+	<body>
+		CL number {{.}} exists in both Gerrit (the current code review system)
+		and Rietveld (the previous code review system). Please make a choice:
+
+		<ul>
+			<li><a href="https://go-review.googlesource.com/{{.}}">Gerrit CL {{.}}</a></li>
+			<li><a href="https://codereview.appspot.com/{{.}}">Rietveld CL {{.}}</a></li>
+		</ul>
+	</body>
+</html>`))
+
+// isGerritCL reports whether a Gerrit CL with the specified numeric change ID (e.g., "4247")
+// is known to exist by querying the Gerrit API at https://go-review.googlesource.com.
+// isGerritCL uses gerritCLCache as a cache of Gerrit CL IDs that exist.
+func isGerritCL(ctx context.Context, id int) (bool, error) {
+	// Check cache first.
+	gerritCLCache.Lock()
+	ok := gerritCLCache.exist[id]
+	gerritCLCache.Unlock()
+	if ok {
+		return true, nil
+	}
+
+	// Query the Gerrit API Get Change endpoint, as documented at
+	// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#get-change.
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	resp, err := ctxhttp.Get(ctx, nil, fmt.Sprintf("https://go-review.googlesource.com/changes/%d", id))
+	if err != nil {
+		return false, err
+	}
+	resp.Body.Close()
+	switch resp.StatusCode {
+	case http.StatusOK:
+		// A Gerrit CL with this ID exists. Add it to cache.
+		gerritCLCache.Lock()
+		gerritCLCache.exist[id] = true
+		gerritCLCache.Unlock()
+		return true, nil
+	case http.StatusNotFound:
+		// A Gerrit CL with this ID doesn't exist. It may get created in the future.
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %v", resp.Status)
+	}
+}
+
+var gerritCLCache = struct {
+	sync.Mutex
+	exist map[int]bool // exist is a set of Gerrit CL IDs that are known to exist.
+}{exist: make(map[int]bool)}
+
 var changeMap *hashMap
 
 // LoadChangeMap loads the specified map of Mercurial to Git revisions,

godoc/redirect/redirect_test.go

@@ -62,6 +62,15 @@ func TestRedirects(t *testing.T) {
 		"/cl/1/":         {302, "https://go-review.googlesource.com/1"},
 		"/cl/267120043":  {302, "https://codereview.appspot.com/267120043"},
 		"/cl/267120043/": {302, "https://codereview.appspot.com/267120043"},
+
+		// Verify that we're using the Rietveld CL table:
+		"/cl/152046": {302, "https://codereview.appspot.com/152046"},
+		"/cl/152047": {302, "https://go-review.googlesource.com/152047"},
+		"/cl/152048": {302, "https://codereview.appspot.com/152048"},
+
+		// And verify we're using the the "bigEnoughAssumeRietveld" value:
+		"/cl/299999": {302, "https://go-review.googlesource.com/299999"},
+		"/cl/300000": {302, "https://codereview.appspot.com/300000"},
 	}
 
 	mux := http.NewServeMux()

godoc/short/short.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-// +build appengine
+// +build golangorg
 
 // Package short implements a simple URL shortener, serving an administrative
 // interface at /s and shortened urls from /s/key.
@@ -15,16 +15,15 @@ import (
 	"errors"
 	"fmt"
 	"html/template"
+	"io"
+	"log"
 	"net/http"
 	"net/url"
 	"regexp"
 
+	"cloud.google.com/go/datastore"
 	"golang.org/x/net/context"
-
-	"google.golang.org/appengine"
-	"google.golang.org/appengine/datastore"
-	"google.golang.org/appengine/log"
-	"google.golang.org/appengine/memcache"
+	"golang.org/x/tools/internal/memcache"
 	"google.golang.org/appengine/user"
 )
 
@@ -41,17 +40,32 @@ type Link struct {
 
 var validKey = regexp.MustCompile(`^[a-zA-Z0-9-_.]+$`)
 
-func RegisterHandlers(mux *http.ServeMux) {
-	mux.HandleFunc(prefix, adminHandler)
-	mux.HandleFunc(prefix+"/", linkHandler)
+type server struct {
+	datastore *datastore.Client
+	memcache  *memcache.CodecClient
+}
+
+func RegisterHandlers(mux *http.ServeMux, dc *datastore.Client, mc *memcache.Client) {
+	s := server{dc, mc.WithCodec(memcache.JSON)}
+	mux.HandleFunc(prefix+"/", s.linkHandler)
+
+	// TODO(cbro): move storage of the links to a text file in Gerrit.
+	// Disable the admin handler until that happens, since GAE Flex doesn't support
+	// the "google.golang.org/appengine/user" package.
+	// See golang.org/issue/29988 and golang.org/issue/27205#issuecomment-418673218.
+	// mux.HandleFunc(prefix, adminHandler)
+	mux.HandleFunc(prefix, func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+		io.WriteString(w, "Link creation temporarily unavailable. See golang.org/issue/29988.")
+	})
 }
 
 // linkHandler services requests to short URLs.
 //   http://golang.org/s/key
 // It consults memcache and datastore for the Link for key.
 // It then sends a redirects or an error message.
-func linkHandler(w http.ResponseWriter, r *http.Request) {
-	c := appengine.NewContext(r)
+func (h server) linkHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
 
 	key := r.URL.Path[len(prefix)+1:]
 	if !validKey.MatchString(key) {
@@ -60,16 +74,15 @@ func linkHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	var link Link
-	_, err := memcache.JSON.Get(c, cacheKey(key), &link)
-	if err != nil {
-		k := datastore.NewKey(c, kind, key, 0, nil)
-		err = datastore.Get(c, k, &link)
+	if err := h.memcache.Get(ctx, cacheKey(key), &link); err != nil {
+		k := datastore.NameKey(kind, key, nil)
+		err = h.datastore.Get(ctx, k, &link)
 		switch err {
 		case datastore.ErrNoSuchEntity:
 			http.Error(w, "not found", http.StatusNotFound)
 			return
 		default: // != nil
-			log.Errorf(c, "%q: %v", key, err)
+			log.Printf("ERROR %q: %v", key, err)
 			http.Error(w, "internal server error", http.StatusInternalServerError)
 			return
 		case nil:
@@ -77,8 +90,8 @@ func linkHandler(w http.ResponseWriter, r *http.Request) {
 				Key:    cacheKey(key),
 				Object: &link,
 			}
-			if err := memcache.JSON.Set(c, item); err != nil {
-				log.Warningf(c, "%q: %v", key, err)
+			if err := h.memcache.Set(ctx, item); err != nil {
+				log.Printf("WARNING %q: %v", key, err)
 			}
 		}
 	}
@@ -89,10 +102,10 @@ func linkHandler(w http.ResponseWriter, r *http.Request) {
 var adminTemplate = template.Must(template.New("admin").Parse(templateHTML))
 
 // adminHandler serves an administrative interface.
-func adminHandler(w http.ResponseWriter, r *http.Request) {
-	c := appengine.NewContext(r)
+func (h server) adminHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
 
-	if !user.IsAdmin(c) {
+	if !user.IsAdmin(ctx) {
 		http.Error(w, "forbidden", http.StatusForbidden)
 		return
 	}
@@ -104,24 +117,24 @@ func adminHandler(w http.ResponseWriter, r *http.Request) {
 		switch r.FormValue("do") {
 		case "Add":
 			newLink = &Link{key, r.FormValue("target")}
-			doErr = putLink(c, newLink)
+			doErr = h.putLink(ctx, newLink)
 		case "Delete":
-			k := datastore.NewKey(c, kind, key, 0, nil)
-			doErr = datastore.Delete(c, k)
+			k := datastore.NameKey(kind, key, nil)
+			doErr = h.datastore.Delete(ctx, k)
 		default:
 			http.Error(w, "unknown action", http.StatusBadRequest)
 		}
-		err := memcache.Delete(c, cacheKey(key))
+		err := h.memcache.Delete(ctx, cacheKey(key))
 		if err != nil && err != memcache.ErrCacheMiss {
-			log.Warningf(c, "%q: %v", key, err)
+			log.Printf("WARNING %q: %v", key, err)
 		}
 	}
 
 	var links []*Link
-	_, err := datastore.NewQuery(kind).Order("Key").GetAll(c, &links)
-	if err != nil {
+	q := datastore.NewQuery(kind).Order("Key")
+	if _, err := h.datastore.GetAll(ctx, q, &links); err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
-		log.Errorf(c, "%v", err)
+		log.Printf("ERROR %v", err)
 		return
 	}
 
@@ -150,20 +163,20 @@ func adminHandler(w http.ResponseWriter, r *http.Request) {
 		Error   error
 	}{baseURL, prefix, links, newLink, doErr}
 	if err := adminTemplate.Execute(w, &data); err != nil {
-		log.Criticalf(c, "adminTemplate: %v", err)
+		log.Printf("ERROR adminTemplate: %v", err)
 	}
 }
 
 // putLink validates the provided link and puts it into the datastore.
-func putLink(c context.Context, link *Link) error {
+func (h server) putLink(ctx context.Context, link *Link) error {
 	if !validKey.MatchString(link.Key) {
 		return errors.New("invalid key; must match " + validKey.String())
 	}
 	if _, err := url.Parse(link.Target); err != nil {
 		return fmt.Errorf("bad target: %v", err)
 	}
-	k := datastore.NewKey(c, kind, link.Key, 0, nil)
-	_, err := datastore.Put(c, k, link)
+	k := datastore.NameKey(kind, link.Key, nil)
+	_, err := h.datastore.Put(ctx, k, link)
 	return err
 }
 

godoc/short/tmpl.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 package short
 
 const templateHTML = `

godoc/static/godoc.html

@@ -15,6 +15,19 @@
 {{end}}
 <link rel="stylesheet" href="/lib/godoc/jquery.treeview.css">
 <script>window.initFuncs = [];</script>
+{{with .GoogleAnalytics}}
+<script type="text/javascript">
+var _gaq = _gaq || [];
+_gaq.push(["_setAccount", "{{.}}"]);
+window.trackPageview = function() {
+  _gaq.push(["_trackPageview", location.pathname+location.hash]);
+};
+window.trackPageview();
+window.trackEvent = function(category, action, opt_label, opt_value, opt_noninteraction) {
+  _gaq.push(["_trackEvent", category, action, opt_label, opt_value, opt_noninteraction]);
+};
+</script>
+{{end}}
 <script src="/lib/godoc/jquery.js" defer></script>
 <script src="/lib/godoc/jquery.treeview.js" defer></script>
 <script src="/lib/godoc/jquery.treeview.edit.js" defer></script>
@@ -112,6 +125,15 @@ and code is licensed under a <a href="/LICENSE">BSD license</a>.<br>
 
 </div><!-- .container -->
 </div><!-- #page -->
+{{if .GoogleAnalytics}}
+<script type="text/javascript">
+(function() {
+  var ga = document.createElement("script"); ga.type = "text/javascript"; ga.async = true;
+  ga.src = ("https:" == document.location.protocol ? "https://ssl" : "http://www") + ".google-analytics.com/ga.js";
+  var s = document.getElementsByTagName("script")[0]; s.parentNode.insertBefore(ga, s);
+})();
+</script>
+{{end}}
 </body>
 </html>
 

godoc/vfs/os.go

@@ -14,6 +14,14 @@ import (
 	"runtime"
 )
 
+// GOROOT returns the GOROOT path under which the godoc binary is running.
+// It is needed to check whether a filesystem root is under GOROOT or not.
+// This is set from cmd/godoc/main.go
+
+// We expose a new variable because otherwise we need to copy the findGOROOT logic again
+// from cmd/godoc which is already copied twice from the standard library.
+var GOROOT = runtime.GOROOT()
+
 // OS returns an implementation of FileSystem reading from the
 // tree rooted at root.  Recording a root is convenient everywhere
 // but necessary on Windows, because the slash-separated path
@@ -22,7 +30,7 @@ import (
 func OS(root string) FileSystem {
 	var t RootType
 	switch {
-	case root == runtime.GOROOT():
+	case root == GOROOT:
 		t = RootTypeGoRoot
 	case isGoPath(root):
 		t = RootTypeGoPath

godoc/vfs/zipfs/zipfs.go

@@ -25,7 +25,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"runtime"
 	"sort"
 	"strings"
 	"time"
@@ -87,7 +86,7 @@ func (fs *zipFS) String() string {
 func (fs *zipFS) RootType(abspath string) vfs.RootType {
 	var t vfs.RootType
 	switch {
-	case abspath == runtime.GOROOT():
+	case abspath == vfs.GOROOT:
 		t = vfs.RootTypeGoRoot
 	case isGoPath(abspath):
 		t = vfs.RootTypeGoPath

internal/memcache/memcache.go

@@ -0,0 +1,159 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build golangorg
+
+// Package memcache provides a minimally compatible interface for
+// google.golang.org/appengine/memcache
+// and stores the data in Redis (e.g., via Cloud Memorystore).
+package memcache
+
+import (
+	"bytes"
+	"context"
+	"encoding/gob"
+	"encoding/json"
+	"errors"
+	"time"
+
+	"github.com/gomodule/redigo/redis"
+)
+
+var ErrCacheMiss = errors.New("memcache: cache miss")
+
+func New(addr string) *Client {
+	const maxConns = 20
+
+	pool := redis.NewPool(func() (redis.Conn, error) {
+		return redis.Dial("tcp", addr)
+	}, maxConns)
+
+	return &Client{
+		pool: pool,
+	}
+}
+
+type Client struct {
+	pool *redis.Pool
+}
+
+type CodecClient struct {
+	client *Client
+	codec  Codec
+}
+
+type Item struct {
+	Key        string
+	Value      []byte
+	Object     interface{}   // Used with Codec.
+	Expiration time.Duration // Read-only.
+}
+
+func (c *Client) WithCodec(codec Codec) *CodecClient {
+	return &CodecClient{
+		c, codec,
+	}
+}
+
+func (c *Client) Delete(ctx context.Context, key string) error {
+	conn, err := c.pool.GetContext(ctx)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	_, err = conn.Do("DEL", key)
+	return err
+}
+
+func (c *CodecClient) Delete(ctx context.Context, key string) error {
+	return c.client.Delete(ctx, key)
+}
+
+func (c *Client) Set(ctx context.Context, item *Item) error {
+	if item.Value == nil {
+		return errors.New("nil item value")
+	}
+	return c.set(ctx, item.Key, item.Value, item.Expiration)
+}
+
+func (c *CodecClient) Set(ctx context.Context, item *Item) error {
+	if item.Object == nil {
+		return errors.New("nil object value")
+	}
+	b, err := c.codec.Marshal(item.Object)
+	if err != nil {
+		return err
+	}
+	return c.client.set(ctx, item.Key, b, item.Expiration)
+}
+
+func (c *Client) set(ctx context.Context, key string, value []byte, expiration time.Duration) error {
+	conn, err := c.pool.GetContext(ctx)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	if expiration == 0 {
+		_, err := conn.Do("SET", key, value)
+		return err
+	}
+
+	// NOTE(cbro): redis does not support expiry in units more granular than a second.
+	exp := int64(expiration.Seconds())
+	if exp == 0 {
+		// Redis doesn't allow a zero expiration, delete the key instead.
+		_, err := conn.Do("DEL", key)
+		return err
+	}
+
+	_, err = conn.Do("SETEX", key, exp, value)
+	return err
+}
+
+// Get gets the item.
+func (c *Client) Get(ctx context.Context, key string) ([]byte, error) {
+	conn, err := c.pool.GetContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	b, err := redis.Bytes(conn.Do("GET", key))
+	if err == redis.ErrNil {
+		err = ErrCacheMiss
+	}
+	return b, err
+}
+
+func (c *CodecClient) Get(ctx context.Context, key string, v interface{}) error {
+	b, err := c.client.Get(ctx, key)
+	if err != nil {
+		return err
+	}
+	return c.codec.Unmarshal(b, v)
+}
+
+var (
+	Gob  = Codec{gobMarshal, gobUnmarshal}
+	JSON = Codec{json.Marshal, json.Unmarshal}
+)
+
+type Codec struct {
+	Marshal   func(interface{}) ([]byte, error)
+	Unmarshal func([]byte, interface{}) error
+}
+
+func gobMarshal(v interface{}) ([]byte, error) {
+	var buf bytes.Buffer
+	if err := gob.NewEncoder(&buf).Encode(v); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func gobUnmarshal(data []byte, v interface{}) error {
+	return gob.NewDecoder(bytes.NewBuffer(data)).Decode(v)
+}

internal/memcache/memcache_test.go

@@ -0,0 +1,85 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build golangorg
+
+package memcache
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+)
+
+func getClient(t *testing.T) *Client {
+	t.Helper()
+
+	addr := os.Getenv("GOLANG_REDIS_ADDR")
+	if addr == "" {
+		t.Skip("skipping because GOLANG_REDIS_ADDR is unset")
+	}
+
+	return New(addr)
+}
+
+func TestCacheMiss(t *testing.T) {
+	c := getClient(t)
+	ctx := context.Background()
+
+	if _, err := c.Get(ctx, "doesnotexist"); err != ErrCacheMiss {
+		t.Errorf("got %v; want ErrCacheMiss", err)
+	}
+}
+
+func TestExpiry(t *testing.T) {
+	c := getClient(t).WithCodec(Gob)
+	ctx := context.Background()
+
+	key := "testexpiry"
+
+	firstTime := time.Now()
+	err := c.Set(ctx, &Item{
+		Key:        key,
+		Object:     firstTime,
+		Expiration: 3500 * time.Millisecond, // NOTE: check that non-rounded expiries work.
+	})
+	if err != nil {
+		t.Fatalf("Set: %v", err)
+	}
+
+	var newTime time.Time
+	if err := c.Get(ctx, key, &newTime); err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	if !firstTime.Equal(newTime) {
+		t.Errorf("Get: got value %v, want %v", newTime, firstTime)
+	}
+
+	time.Sleep(4 * time.Second)
+
+	if err := c.Get(ctx, key, &newTime); err != ErrCacheMiss {
+		t.Errorf("Get: got %v, want ErrCacheMiss", err)
+	}
+}
+
+func TestShortExpiry(t *testing.T) {
+	c := getClient(t).WithCodec(Gob)
+	ctx := context.Background()
+
+	key := "testshortexpiry"
+
+	err := c.Set(ctx, &Item{
+		Key:        key,
+		Value:      []byte("ok"),
+		Expiration: time.Millisecond,
+	})
+	if err != nil {
+		t.Fatalf("Set: %v", err)
+	}
+
+	if err := c.Get(ctx, key, nil); err != ErrCacheMiss {
+		t.Errorf("GetBytes: got %v, want ErrCacheMiss", err)
+	}
+}