[release-branch.go1.11] godoc: migrate to App Engine flexible

See bug for more details on exactly what was migrated.

Notably:
* No more Google-internal deployment scripts; see README.godoc-app and
  the Makefile for details.
* Build tag "golangorg" is used for the godoc configuration used for
  golang.org.
* Use of App Engine libraries replaced with GCP client libraries.
* Redis is used to replace App Engine memcache.
* Google analytics is controlled by an environment variable.
* Regression tests have been migrated from Google-internal.
* hg -> git hash map is moved from Google-internal.

Updates golang/go#28893
Updates golang/go#27205

Change-Id: Ia0a983f239c50eda8be2363494c8b784f60c2c6d
Reviewed-on: https://go-review.googlesource.com/133355
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Reviewed-on: https://go-review.googlesource.com/c/150599

cmd/godoc/Dockerfile.prod

@@ -0,0 +1,42 @@
+# Builder
+#########
+
+FROM golang:1.11 AS build
+
+RUN apt-get update && apt-get install -y \
+      zip # required for generate-index.bash
+
+ENV GODOC_REF release-branch.go1.11
+
+RUN go get -v -d \
+      golang.org/x/net/context \
+      google.golang.org/appengine \
+      cloud.google.com/go/datastore \
+      golang.org/x/build \
+      github.com/gomodule/redigo/redis
+
+COPY . /go/src/golang.org/x/tools
+
+WORKDIR /go/src/golang.org/x/tools/cmd/godoc
+RUN git clone --single-branch --depth=1 -b $GODOC_REF https://go.googlesource.com/go /docset
+RUN GODOC_DOCSET=/docset ./generate-index.bash
+
+RUN go build -o /godoc -tags=golangorg golang.org/x/tools/cmd/godoc
+
+
+# Final image
+#############
+
+FROM gcr.io/distroless/base
+
+WORKDIR /app
+COPY --from=build /godoc /app/
+COPY --from=build /go/src/golang.org/x/tools/cmd/godoc/hg-git-mapping.bin /app/
+
+COPY --from=build /docset /goroot
+ENV GOROOT /goroot
+
+COPY --from=build /go/src/golang.org/x/tools/cmd/godoc/index.split.* /app/
+ENV GODOC_INDEX_GLOB index.split.*
+
+CMD ["/app/godoc"]

cmd/godoc/Makefile

@@ -0,0 +1,24 @@
+# Copyright 2018 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+.PHONY: usage
+
+usage:
+	echo "See Makefile"
+	exit 1
+
+docker-prod: Dockerfile.prod
+	cd ../..; docker build -f cmd/godoc/Dockerfile.prod --tag=gcr.io/golang-org/godoc:$(VERSION) .
+
+push-prod: docker-prod
+	docker push gcr.io/golang-org/godoc:$(VERSION)
+
+deploy-prod: push-prod
+	gcloud -q app deploy app.prod.yaml --project golang-org --no-promote --image-url gcr.io/golang-org/godoc:$(VERSION)
+
+get-latest-url:
+	@gcloud app versions list -s default --project golang-org --sort-by=~version.createTime --format='value(version.versionUrl)' --limit 1 | cut -f1
+
+regtest:
+	./regtest.bash $(shell make get-latest-url)

cmd/godoc/README.godoc-app

@@ -7,31 +7,78 @@ Prerequisites
 * Google Cloud SDK
   https://cloud.google.com/sdk/
 
+* Redis
+
 * Go sources under $GOROOT
 
 * Godoc sources inside $GOPATH
   (go get -d golang.org/x/tools/cmd/godoc)
 
 
-Running in dev_appserver.py
+Running locally, in production mode
----------------------------
+-----------------------------------
 
-Use dev_appserver.py to run the server in development mode:
+Build the app:
 
-	dev_appserver.py app.dev.yaml
+  go build -tags golangorg
 
-To run the server with generated zip file and search index:
+Run the app:
 
-	./generate-index.bash
+  ./godoc
-	dev_appserver.py app.prod.yaml
 
 godoc should come up at http://localhost:8080
-Use the --host and --port flags to listen on a different address.
 
-To clean up the index files, use git:
+Use the PORT environment variable to change the port:
 
-	git clean -xn # n is dry run, replace with f
+    PORT=8081 ./godoc
 
+Running locally, in production mode, using Docker
+-------------------------------------------------
+
+Build the app's Docker container:
+
+  VERSION=$(git rev-parse HEAD) make docker-prod
+
+Make sure redis is running on port 6379:
+
+  $ echo PING | nc localhost 6379
+  +PONG
+  ^C
+
+Run the datastore emulator:
+
+  gcloud beta emulators datastore start --project golang-org
+
+In another terminal window, run the container:
+
+  $(gcloud beta emulators datastore env-init)
+
+  docker run --rm \
+    --net host \
+    --env GODOC_REDIS_ADDR=localhost:6379 \
+    --env DATASTORE_EMULATOR_HOST=$DATASTORE_EMULATOR_HOST \
+    --env DATASTORE_PROJECT_ID=$DATASTORE_PROJECT_ID \
+    gcr.io/golang-org/godoc
+
+godoc should come up at http://localhost:8080
+
+
+Deploying to golang.org
+-----------------------
+
+Build the image, push it to gcr.io, and deploy to Flex:
+
+  VERSION=$(git rev-parse HEAD) make deploy-prod
+
+Run regression tests:
+
+  make regtest
+
+Go to the console to migrate traffic to the newly deployed version:
+
+  https://console.cloud.google.com/appengine/versions?project=golang-org&serviceId=default&versionssize=50
+
+Shut down any very old versions (keep at least one to roll back to, just in case).
 
 Troubleshooting
 ---------------

cmd/godoc/app.prod.yaml

@@ -1,18 +1,16 @@
-runtime: go
+runtime: custom
-api_version: go1
+env: flex
-instance_class: F4_1G
-
-handlers:
-- url: /s
-  script: _go_app
-  login: admin
-- url: /dl/init
-  script: _go_app
-  login: admin
-- url: /.*
-  script: _go_app
 
 env_variables:
-  GODOC_ZIP: godoc.zip
+  GODOC_PROD: true
-  GODOC_ZIP_PREFIX: goroot
+  #  GODOC_ENFORCE_HOSTS: true # TODO(cbro): modify host filter to allow version-specific URLs (see issue 27205).
-  GODOC_INDEX_GLOB: 'index.split.*'
+  GODOC_REDIS_ADDR: 10.0.0.4:6379 # instance "gophercache"
+  GODOC_ANALYTICS: UA-11222381-2
+  DATASTORE_PROJECT_ID: golang-org
+
+network:
+  name: golang
+
+resources:
+  cpu: 4
+  memory_gb: 7.50

cmd/godoc/appinit.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build appengine
+// +build golangorg
 
 package main
 
@@ -11,16 +11,20 @@ package main
 
 import (
 	"archive/zip"
+	"context"
+	"io"
 	"log"
 	"net/http"
 	"os"
 	"path"
 	"regexp"
 	"runtime"
+	"strings"
 
 	"golang.org/x/tools/godoc"
 	"golang.org/x/tools/godoc/dl"
 	"golang.org/x/tools/godoc/proxy"
+	"golang.org/x/tools/godoc/redirect"
 	"golang.org/x/tools/godoc/short"
 	"golang.org/x/tools/godoc/static"
 	"golang.org/x/tools/godoc/vfs"
@@ -28,10 +32,13 @@ import (
 	"golang.org/x/tools/godoc/vfs/mapfs"
 	"golang.org/x/tools/godoc/vfs/zipfs"
 
-	"google.golang.org/appengine"
+	"cloud.google.com/go/datastore"
+	"golang.org/x/tools/internal/memcache"
 )
 
-func init() {
+func main() {
+	log.SetFlags(log.Lshortfile | log.LstdFlags)
+
 	var (
 		// .zip filename
 		zipFilename = os.Getenv("GODOC_ZIP")
@@ -44,7 +51,6 @@ func init() {
 		indexFilenames = os.Getenv("GODOC_INDEX_GLOB")
 	)
 
-	enforceHosts = !appengine.IsDevAppServer()
 	playEnabled = true
 
 	log.Println("initializing godoc ...")
@@ -85,17 +91,61 @@ func init() {
 	pres.ShowExamples = true
 	pres.DeclLinks = true
 	pres.NotesRx = regexp.MustCompile("BUG")
+	pres.GoogleAnalytics = os.Getenv("GODOC_ANALYTICS")
 
 	readTemplates(pres, true)
 
+	datastoreClient, memcacheClient := getClients()
+
+	// NOTE(cbro): registerHandlers registers itself against DefaultServeMux.
+	// The mux returned has host enforcement, so it's important to register
+	// against this mux and not DefaultServeMux.
 	mux := registerHandlers(pres)
-	dl.RegisterHandlers(mux)
+	dl.RegisterHandlers(mux, datastoreClient, memcacheClient)
-	short.RegisterHandlers(mux)
+	short.RegisterHandlers(mux, datastoreClient, memcacheClient)
 
 	// Register /compile and /share handlers against the default serve mux
 	// so that other app modules can make plain HTTP requests to those
 	// hosts. (For reasons, HTTPS communication between modules is broken.)
 	proxy.RegisterHandlers(http.DefaultServeMux)
 
+	http.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "ok")
+	})
+
+	http.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "User-agent: *\nDisallow: /search\n")
+	})
+
+	if err := redirect.LoadChangeMap("hg-git-mapping.bin"); err != nil {
+		log.Fatalf("LoadChangeMap: %v", err)
+	}
+
 	log.Println("godoc initialization complete")
+
+	// TODO(cbro): add instrumentation via opencensus.
+	port := "8080"
+	if p := os.Getenv("PORT"); p != "" { // PORT is set by GAE flex.
+		port = p
+	}
+	log.Fatal(http.ListenAndServe(":"+port, nil))
+}
+
+func getClients() (*datastore.Client, *memcache.Client) {
+	ctx := context.Background()
+
+	datastoreClient, err := datastore.NewClient(ctx, "")
+	if err != nil {
+		if strings.Contains(err.Error(), "missing project") {
+			log.Fatalf("Missing datastore project. Set the DATASTORE_PROJECT_ID env variable. Use `gcloud beta emulators datastore` to start a local datastore.")
+		}
+		log.Fatalf("datastore.NewClient: %v.", err)
+	}
+
+	redisAddr := os.Getenv("GODOC_REDIS_ADDR")
+	if redisAddr == "" {
+		log.Fatalf("Missing redis server for godoc in production mode. set GODOC_REDIS_ADDR environment variable.")
+	}
+	memcacheClient := memcache.New(redisAddr)
+	return datastoreClient, memcacheClient
 }

cmd/godoc/dl.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !appengine
+// +build !golangorg
 
 package main
 

cmd/godoc/generate-index.bash

@@ -25,24 +25,24 @@ install() {
 }
 
 getArgs() {
-	if [ ! -v GOROOT ]; then
+	if [ ! -v GODOC_DOCSET ]; then
-		GOROOT="$(go env GOROOT)"
+		GODOC_DOCSET="$(go env GOROOT)"
-		echo "GOROOT not set explicitly, using go env value instead"
+		echo "GODOC_DOCSET not set explicitly, using GOROOT instead"
 	fi
 
 	# safety checks
-	if [ ! -d "$GOROOT" ]; then
+	if [ ! -d "$GODOC_DOCSET" ]; then
-		error "$GOROOT is not a directory"
+		error "$GODOC_DOCSET is not a directory"
 	fi
 
 	# reporting
-	echo "GOROOT = $GOROOT"
+	echo "GODOC_DOCSET = $GODOC_DOCSET"
 }
 
 makeZipfile() {
 	echo "*** make $ZIPFILE"
 	rm -f $ZIPFILE goroot
-	ln -s "$GOROOT" goroot
+	ln -s "$GODOC_DOCSET" goroot
 	zip -q -r $ZIPFILE goroot/* # glob to ignore dotfiles (like .git)
 	rm goroot
 }

cmd/godoc/handlers.go

@@ -21,6 +21,7 @@ import (
 	"text/template"
 
 	"golang.org/x/tools/godoc"
+	"golang.org/x/tools/godoc/env"
 	"golang.org/x/tools/godoc/redirect"
 	"golang.org/x/tools/godoc/vfs"
 )
@@ -30,8 +31,6 @@ var (
 	fs   = vfs.NameSpace{}
 )
 
-var enforceHosts = false // set true in production on app engine
-
 // hostEnforcerHandler redirects requests to "http://foo.golang.org/bar"
 // to "https://golang.org/bar".
 // It permits requests to the host "godoc-test.golang.org" for testing and
@@ -41,7 +40,7 @@ type hostEnforcerHandler struct {
 }
 
 func (h hostEnforcerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if !enforceHosts {
+	if !env.EnforceHosts() {
 		h.h.ServeHTTP(w, r)
 		return
 	}

cmd/godoc/main.go

@@ -23,7 +23,7 @@
 //	godoc crypto/block Cipher NewCMAC
 //		- prints doc for Cipher and NewCMAC in package crypto/block
 
-// +build !appengine
+// +build !golangorg
 
 package main
 

cmd/godoc/play.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !appengine
+// +build !golangorg
 
 package main
 

cmd/godoc/regtest.bash

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+
+# Copyright 2018 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# Regression tests for golang.org.
+# Usage: ./regtest.bash https://golang.org/
+
+#TODO: turn this into a Go program. maybe behind a build tag and "go run regtest.go <url>"
+
+set -e
+
+addr="$(echo $1 | sed -e 's/\/$//')"
+if [ -z "$addr" ]; then
+	echo "usage: $0 <addr>" 1>&2
+	echo "example: $0 https://20180928t023837-dot-golang-org.appspot.com/" 1>&2
+	exit 1
+fi
+
+set -u
+
+# fetch url, check the response with a regexp.
+fetch() {
+	curl -s "${addr}$1" | grep "$2" > /dev/null
+}
+fatal() {
+	log "$1"
+	exit 1
+}
+log() {
+	echo "$1" 1>&2
+}
+logn() {
+	echo -n "$1" 1>&2
+}
+
+log "Checking FAQ..."
+fetch /doc/faq 'What is the purpose of the project' || {
+	fatal "FAQ did not match."
+}
+
+log "Checking package listing..."
+fetch /pkg/ 'Package tar' || {
+	fatal "package listing page did not match."
+}
+
+log "Checking os package..."
+fetch /pkg/os/ 'func Open' || {
+	fatal "os package page did not match."
+}
+
+log "Checking robots.txt..."
+fetch /robots.txt 'Disallow: /search' || {
+	fatal "robots.txt did not match."
+}
+
+log "Checking /change/ redirect..."
+fetch /change/75944e2e3a63 'bdb10cf' || {
+	fatal "/change/ direct did not match."
+}
+
+log "Checking /dl/ page has data..."
+fetch /dl/ 'go1.11.windows-amd64.msi' || {
+	fatal "/dl/ did not match."
+}
+
+log "Checking /dl/?mode=json page has data..."
+fetch /dl/?mode=json 'go1.11.windows-amd64.msi' || {
+	fatal "/dl/?mode=json did not match."
+}
+
+log "Checking shortlinks (/s/go2design)..."
+fetch /s/go2design 'proposal.*Found' || {
+	fatal "/s/go2design did not match."
+}
+
+log "Checking analytics on pages..."
+ga_id="UA-11222381-2"
+fetch / $ga_id || fatal "/ missing GA."
+fetch /dl/ $ga_id || fatal "/dl/ missing GA."
+fetch /project/ $ga_id || fatal "/project missing GA."
+fetch /pkg/context/ $ga_id || fatal "/pkg/context missing GA."
+
+log "Checking search..."
+fetch /search?q=IsDir 'src/os/types.go' || {
+	fatal "search result did not match."
+}
+
+log "Checking compile service..."
+compile="curl -s ${addr}/compile"
+
+p="package main; func main() { print(6*7); }"
+$compile --data-urlencode "body=$p" | tee /tmp/compile.out | grep '^{"compile_errors":"","output":"42"}$' > /dev/null || {
+	cat /tmp/compile.out
+	fatal "compile service output did not match."
+}
+
+$compile --data-urlencode "body=//empty" | tee /tmp/compile.out | grep "expected 'package', found 'EOF'" > /dev/null || {
+	cat /tmp/compile.out
+	fatal "compile service error output did not match."
+}
+
+# Check API version 2
+d="version=2&body=package+main%3Bimport+(%22fmt%22%3B%22time%22)%3Bfunc+main()%7Bfmt.Print(%22A%22)%3Btime.Sleep(time.Second)%3Bfmt.Print(%22B%22)%7D"
+$compile --data "$d" | grep '^{"Errors":"","Events":\[{"Message":"A","Kind":"stdout","Delay":0},{"Message":"B","Kind":"stdout","Delay":1000000000}\]}$' > /dev/null || {
+	fatal "compile service v2 output did not match."
+}
+
+log "All OK"

cmd/godoc/remotesearch.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !appengine
+// +build !golangorg
 
 package main
 

godoc/appengine.go

@@ -1,13 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package godoc
-
-import "google.golang.org/appengine"
-
-func init() {
-	onAppengine = !appengine.IsDevAppServer()
-}

godoc/dl/dl.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 // Package dl implements a simple downloads frontend server.
 //
 // It accepts HTTP POST requests to create a new download metadata entity, and
@@ -19,6 +17,7 @@ import (
 	"html"
 	"html/template"
 	"io"
+	"log"
 	"net/http"
 	"regexp"
 	"sort"
@@ -27,11 +26,10 @@ import (
 	"sync"
 	"time"
 
+	"cloud.google.com/go/datastore"
 	"golang.org/x/net/context"
-	"google.golang.org/appengine"
+	"golang.org/x/tools/godoc/env"
-	"google.golang.org/appengine/datastore"
+	"golang.org/x/tools/internal/memcache"
-	"google.golang.org/appengine/log"
-	"google.golang.org/appengine/memcache"
 )
 
 const (
@@ -40,11 +38,21 @@ const (
 	cacheDuration   = time.Hour
 )
 
-func RegisterHandlers(mux *http.ServeMux) {
+type server struct {
-	mux.HandleFunc("/dl", getHandler)
+	datastore *datastore.Client
-	mux.HandleFunc("/dl/", getHandler) // also serves listHandler
+	memcache  *memcache.CodecClient
-	mux.HandleFunc("/dl/upload", uploadHandler)
+}
-	mux.HandleFunc("/dl/init", initHandler)
+
+func RegisterHandlers(mux *http.ServeMux, dc *datastore.Client, mc *memcache.Client) {
+	s := server{dc, mc.WithCodec(memcache.Gob)}
+	mux.HandleFunc("/dl", s.getHandler)
+	mux.HandleFunc("/dl/", s.getHandler) // also serves listHandler
+	mux.HandleFunc("/dl/upload", s.uploadHandler)
+
+	// NOTE(cbro): this only needs to be run once per project,
+	// and should be behind an admin login.
+	// TODO(cbro): move into a locally-run program? or remove?
+	// mux.HandleFunc("/dl/init", initHandler)
 }
 
 // File represents a file on the golang.org downloads page.
@@ -191,26 +199,25 @@ var (
 	templateFuncs = template.FuncMap{"pretty": pretty}
 )
 
-func listHandler(w http.ResponseWriter, r *http.Request) {
+func (h server) listHandler(w http.ResponseWriter, r *http.Request) {
 	if r.Method != "GET" {
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
 	}
-	var (
+	ctx := r.Context()
-		c = appengine.NewContext(r)
+	var d listTemplateData
-		d listTemplateData
+
-	)
+	if err := h.memcache.Get(ctx, cacheKey, &d); err != nil {
-	if _, err := memcache.Gob.Get(c, cacheKey, &d); err != nil {
+		if err != memcache.ErrCacheMiss {
-		if err == memcache.ErrCacheMiss {
+			log.Printf("ERROR cache get error: %v", err)
-			log.Debugf(c, "cache miss")
+			// NOTE(cbro): continue to hit datastore if the memcache is down.
-		} else {
-			log.Errorf(c, "cache get error: %v", err)
 		}
 
 		var fs []File
-		_, err := datastore.NewQuery("File").Ancestor(rootKey(c)).GetAll(c, &fs)
+		q := datastore.NewQuery("File").Ancestor(rootKey)
-		if err != nil {
+		if _, err := h.datastore.GetAll(ctx, q, &fs); err != nil {
-			log.Errorf(c, "error listing: %v", err)
+			log.Printf("ERROR error listing: %v", err)
+			http.Error(w, "Could not get download page. Try again in a few minutes.", 500)
 			return
 		}
 		d.Stable, d.Unstable, d.Archive = filesToReleases(fs)
@@ -219,8 +226,8 @@ func listHandler(w http.ResponseWriter, r *http.Request) {
 		}
 
 		item := &memcache.Item{Key: cacheKey, Object: &d, Expiration: cacheDuration}
-		if err := memcache.Gob.Set(c, item); err != nil {
+		if err := h.memcache.Set(ctx, item); err != nil {
-			log.Errorf(c, "cache set error: %v", err)
+			log.Printf("ERROR cache set error: %v", err)
 		}
 	}
 
@@ -229,13 +236,13 @@ func listHandler(w http.ResponseWriter, r *http.Request) {
 		enc := json.NewEncoder(w)
 		enc.SetIndent("", " ")
 		if err := enc.Encode(d.Stable); err != nil {
-			log.Errorf(c, "failed rendering JSON for releases: %v", err)
+			log.Printf("ERROR rendering JSON for releases: %v", err)
 		}
 		return
 	}
 
 	if err := listTemplate.ExecuteTemplate(w, "root", d); err != nil {
-		log.Errorf(c, "error executing template: %v", err)
+		log.Printf("ERROR executing template: %v", err)
 	}
 }
 
@@ -383,12 +390,12 @@ func parseVersion(v string) (maj, min int, tail string) {
 	return
 }
 
-func uploadHandler(w http.ResponseWriter, r *http.Request) {
+func (h server) uploadHandler(w http.ResponseWriter, r *http.Request) {
 	if r.Method != "POST" {
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
 	}
-	c := appengine.NewContext(r)
+	ctx := r.Context()
 
 	// Authenticate using a user token (same as gomote).
 	user := r.FormValue("user")
@@ -396,7 +403,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "bad user", http.StatusForbidden)
 		return
 	}
-	if r.FormValue("key") != userKey(c, user) {
+	if r.FormValue("key") != h.userKey(ctx, user) {
 		http.Error(w, "bad key", http.StatusForbidden)
 		return
 	}
@@ -404,7 +411,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 	var f File
 	defer r.Body.Close()
 	if err := json.NewDecoder(r.Body).Decode(&f); err != nil {
-		log.Errorf(c, "error decoding upload JSON: %v", err)
+		log.Printf("ERROR decoding upload JSON: %v", err)
 		http.Error(w, "Something broke", http.StatusInternalServerError)
 		return
 	}
@@ -415,19 +422,19 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 	if f.Uploaded.IsZero() {
 		f.Uploaded = time.Now()
 	}
-	k := datastore.NewKey(c, "File", f.Filename, 0, rootKey(c))
+	k := datastore.NameKey("File", f.Filename, rootKey)
-	if _, err := datastore.Put(c, k, &f); err != nil {
+	if _, err := h.datastore.Put(ctx, k, &f); err != nil {
-		log.Errorf(c, "putting File entity: %v", err)
+		log.Printf("ERROR File entity: %v", err)
 		http.Error(w, "could not put File entity", http.StatusInternalServerError)
 		return
 	}
-	if err := memcache.Delete(c, cacheKey); err != nil {
+	if err := h.memcache.Delete(ctx, cacheKey); err != nil {
-		log.Errorf(c, "cache delete error: %v", err)
+		log.Printf("ERROR delete error: %v", err)
 	}
 	io.WriteString(w, "OK")
 }
 
-func getHandler(w http.ResponseWriter, r *http.Request) {
+func (h server) getHandler(w http.ResponseWriter, r *http.Request) {
 	// For go get golang.org/dl/go1.x.y, we need to serve the
 	// same meta tags at /dl for cmd/go to validate against /dl/go1.x.y:
 	if r.URL.Path == "/dl" && (r.Method == "GET" || r.Method == "HEAD") && r.FormValue("go-get") == "1" {
@@ -444,7 +451,7 @@ func getHandler(w http.ResponseWriter, r *http.Request) {
 
 	name := strings.TrimPrefix(r.URL.Path, "/dl/")
 	if name == "" {
-		listHandler(w, r)
+		h.listHandler(w, r)
 		return
 	}
 	if fileRe.MatchString(name) {
@@ -486,10 +493,10 @@ func validUser(user string) bool {
 	return false
 }
 
-func userKey(c context.Context, user string) string {
+func (h server) userKey(c context.Context, user string) string {
-	h := hmac.New(md5.New, []byte(secret(c)))
+	hash := hmac.New(md5.New, []byte(h.secret(c)))
-	h.Write([]byte("user-" + user))
+	hash.Write([]byte("user-" + user))
-	return fmt.Sprintf("%x", h.Sum(nil))
+	return fmt.Sprintf("%x", hash.Sum(nil))
 }
 
 var (
@@ -497,18 +504,18 @@ var (
 	goGetRe = regexp.MustCompile(`^go[0-9a-z.]+\.[0-9a-z.-]+$`)
 )
 
-func initHandler(w http.ResponseWriter, r *http.Request) {
+func (h server) initHandler(w http.ResponseWriter, r *http.Request) {
 	var fileRoot struct {
 		Root string
 	}
-	c := appengine.NewContext(r)
+	ctx := r.Context()
-	k := rootKey(c)
+	k := rootKey
-	err := datastore.RunInTransaction(c, func(c context.Context) error {
+	_, err := h.datastore.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
-		err := datastore.Get(c, k, &fileRoot)
+		err := tx.Get(k, &fileRoot)
 		if err != nil && err != datastore.ErrNoSuchEntity {
 			return err
 		}
-		_, err = datastore.Put(c, k, &fileRoot)
+		_, err = tx.Put(k, &fileRoot)
 		return err
 	}, nil)
 	if err != nil {
@@ -519,9 +526,7 @@ func initHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 // rootKey is the ancestor of all File entities.
-func rootKey(c context.Context) *datastore.Key {
+var rootKey = datastore.NameKey("FileRoot", "root", nil)
-	return datastore.NewKey(c, "FileRoot", "root", 0, nil)
-}
 
 // pretty returns a human-readable version of the given OS, Arch, or Kind.
 func pretty(s string) string {
@@ -559,11 +564,11 @@ type builderKey struct {
 	Secret string
 }
 
-func (k *builderKey) Key(c context.Context) *datastore.Key {
+func (k *builderKey) Key() *datastore.Key {
-	return datastore.NewKey(c, "BuilderKey", "root", 0, nil)
+	return datastore.NameKey("BuilderKey", "root", nil)
 }
 
-func secret(c context.Context) string {
+func (h server) secret(ctx context.Context) string {
 	// check with rlock
 	theKey.RLock()
 	k := theKey.Secret
@@ -580,18 +585,18 @@ func secret(c context.Context) string {
 	}
 
 	// fill
-	if err := datastore.Get(c, theKey.Key(c), &theKey.builderKey); err != nil {
+	if err := h.datastore.Get(ctx, theKey.Key(), &theKey.builderKey); err != nil {
 		if err == datastore.ErrNoSuchEntity {
 			// If the key is not stored in datastore, write it.
 			// This only happens at the beginning of a new deployment.
 			// The code is left here for SDK use and in case a fresh
 			// deployment is ever needed.  "gophers rule" is not the
 			// real key.
-			if !appengine.IsDevAppServer() {
+			if env.IsProd() {
 				panic("lost key from datastore")
 			}
 			theKey.Secret = "gophers rule"
-			datastore.Put(c, theKey.Key(c), &theKey.builderKey)
+			h.datastore.Put(ctx, theKey.Key(), &theKey.builderKey)
 			return theKey.Secret
 		}
 		panic("cannot load builder key: " + err.Error())

godoc/dl/dl_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 package dl
 
 import (

godoc/dl/tmpl.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 package dl
 
 // TODO(adg): refactor this to use the tools/godoc/static template.

godoc/env/env.go

@@ -0,0 +1,41 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package env provides environment information for the godoc server running on
+// golang.org.
+package env
+
+import (
+	"log"
+	"os"
+	"strconv"
+)
+
+var (
+	isProd       = boolEnv("GODOC_PROD")
+	enforceHosts = boolEnv("GODOC_ENFORCE_HOSTS")
+)
+
+// IsProd reports whether the server is running in its production configuration
+// on golang.org.
+func IsProd() bool {
+	return isProd
+}
+
+// EnforceHosts reports whether host filtering should be enforced.
+func EnforceHosts() bool {
+	return enforceHosts
+}
+
+func boolEnv(key string) bool {
+	v := os.Getenv(key)
+	if v == "" {
+		return false
+	}
+	b, err := strconv.ParseBool(v)
+	if err != nil {
+		log.Fatalf("environment variable %s (%q) must be a boolean", key, v)
+	}
+	return b
+}

godoc/page.go

@@ -10,6 +10,8 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+
+	"golang.org/x/tools/godoc/env"
 )
 
 // Page describes the contents of the top-level godoc webpage.
@@ -22,10 +24,11 @@ type Page struct {
 	Body     []byte
 	GoogleCN bool // page is being served from golang.google.cn
 
-	// filled in by servePage
+	// filled in by ServePage
 	SearchBox       bool
 	Playground      bool
 	Version         string
+	GoogleAnalytics string
 }
 
 func (p *Presentation) ServePage(w http.ResponseWriter, page Page) {
@@ -35,6 +38,7 @@ func (p *Presentation) ServePage(w http.ResponseWriter, page Page) {
 	page.SearchBox = p.Corpus.IndexEnabled
 	page.Playground = p.ShowPlayground
 	page.Version = runtime.Version()
+	page.GoogleAnalytics = p.GoogleAnalytics
 	applyTemplateToResponseWriter(w, p.GodocHTML, page)
 }
 
@@ -53,16 +57,15 @@ func (p *Presentation) ServeError(w http.ResponseWriter, r *http.Request, relpat
 		Subtitle:        relpath,
 		Body:            applyTemplate(p.ErrorHTML, "errorHTML", err),
 		GoogleCN:        googleCN(r),
+		GoogleAnalytics: p.GoogleAnalytics,
 	})
 }
 
-var onAppengine = false // overridden in appengine.go when on app engine
-
 func googleCN(r *http.Request) bool {
 	if r.FormValue("googlecn") != "" {
 		return true
 	}
-	if !onAppengine {
+	if !env.IsProd() {
 		return false
 	}
 	if strings.HasSuffix(r.Host, ".cn") {

godoc/pres.go

@@ -92,6 +92,10 @@ type Presentation struct {
 	// body for displaying search results.
 	SearchResults []SearchResultFunc
 
+	// GoogleAnalytics optionally adds Google Analytics via the provided
+	// tracking ID to each page.
+	GoogleAnalytics string
+
 	initFuncMapOnce sync.Once
 	funcMap         template.FuncMap
 	templateFuncs   template.FuncMap

godoc/proxy/proxy.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 // Package proxy proxies requests to the playground's compile and share handlers.
 // It is designed to run only on the instance of godoc that serves golang.org.
 package proxy
@@ -13,6 +11,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -20,12 +19,18 @@ import (
 	"time"
 
 	"golang.org/x/net/context"
-
+	"golang.org/x/tools/godoc/env"
-	"google.golang.org/appengine"
-	"google.golang.org/appengine/log"
-	"google.golang.org/appengine/urlfetch"
 )
 
+const playgroundURL = "https://play.golang.org"
+
+var proxy *httputil.ReverseProxy
+
+func init() {
+	target, _ := url.Parse(playgroundURL)
+	proxy = httputil.NewSingleHostReverseProxy(target)
+}
+
 type Request struct {
 	Body string
 }
@@ -41,8 +46,6 @@ type Event struct {
 	Delay   time.Duration // time to wait before printing Message
 }
 
-const playgroundURL = "https://play.golang.org"
-
 const expires = 7 * 24 * time.Hour // 1 week
 var cacheControlHeader = fmt.Sprintf("public, max-age=%d", int(expires.Seconds()))
 
@@ -57,21 +60,17 @@ func compile(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	ctx := appengine.NewContext(r)
+	ctx := r.Context()
 
 	body := r.FormValue("body")
 	res := &Response{}
 	req := &Request{Body: body}
 	if err := makeCompileRequest(ctx, req, res); err != nil {
-		log.Errorf(ctx, "compile error: %v", err)
+		log.Printf("ERROR compile error: %v", err)
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
 		return
 	}
 
-	expiresTime := time.Now().Add(expires).UTC()
-	w.Header().Set("Expires", expiresTime.Format(time.RFC1123))
-	w.Header().Set("Cache-Control", cacheControlHeader)
-
 	var out interface{}
 	switch r.FormValue("version") {
 	case "2":
@@ -82,9 +81,17 @@ func compile(w http.ResponseWriter, r *http.Request) {
 			Output        string `json:"output"`
 		}{res.Errors, flatten(res.Events)}
 	}
-	if err := json.NewEncoder(w).Encode(out); err != nil {
+	b, err := json.Marshal(out)
-		log.Errorf(ctx, "encoding response: %v", err)
+	if err != nil {
+		log.Printf("ERROR encoding response: %v", err)
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
 	}
+
+	expiresTime := time.Now().Add(expires).UTC()
+	w.Header().Set("Expires", expiresTime.Format(time.RFC1123))
+	w.Header().Set("Cache-Control", cacheControlHeader)
+	w.Write(b)
 }
 
 // makePlaygroundRequest sends the given Request to the playground compile
@@ -94,17 +101,22 @@ func makeCompileRequest(ctx context.Context, req *Request, res *Response) error
 	if err != nil {
 		return fmt.Errorf("marshalling request: %v", err)
 	}
-	r, err := urlfetch.Client(ctx).Post(playgroundURL+"/compile", "application/json", bytes.NewReader(reqJ))
+	hReq, _ := http.NewRequest("POST", playgroundURL+"/compile", bytes.NewReader(reqJ))
+	hReq.Header.Set("Content-Type", "application/json")
+	hReq = hReq.WithContext(ctx)
+
+	r, err := http.DefaultClient.Do(hReq)
 	if err != nil {
 		return fmt.Errorf("making request: %v", err)
 	}
 	defer r.Body.Close()
+
 	if r.StatusCode != http.StatusOK {
 		b, _ := ioutil.ReadAll(r.Body)
 		return fmt.Errorf("bad status: %v body:\n%s", r.Status, b)
 	}
-	err = json.NewDecoder(r.Body).Decode(res)
+
-	if err != nil {
+	if err := json.NewDecoder(r.Body).Decode(res); err != nil {
 		return fmt.Errorf("unmarshalling response: %v", err)
 	}
 	return nil
@@ -124,17 +136,14 @@ func share(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
 		return
 	}
-	target, _ := url.Parse(playgroundURL)
+	proxy.ServeHTTP(w, r)
-	p := httputil.NewSingleHostReverseProxy(target)
-	p.Transport = &urlfetch.Transport{Context: appengine.NewContext(r)}
-	p.ServeHTTP(w, r)
 }
 
 func googleCN(r *http.Request) bool {
 	if r.FormValue("googlecn") != "" {
 		return true
 	}
-	if appengine.IsDevAppServer() {
+	if !env.IsProd() {
 		return false
 	}
 	if strings.HasSuffix(r.Host, ".cn") {

godoc/short/short.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 // Package short implements a simple URL shortener, serving an administrative
 // interface at /s and shortened urls from /s/key.
 // It is designed to run only on the instance of godoc that serves golang.org.
@@ -15,16 +13,15 @@ import (
 	"errors"
 	"fmt"
 	"html/template"
+	"io"
+	"log"
 	"net/http"
 	"net/url"
 	"regexp"
 
+	"cloud.google.com/go/datastore"
 	"golang.org/x/net/context"
-
+	"golang.org/x/tools/internal/memcache"
-	"google.golang.org/appengine"
-	"google.golang.org/appengine/datastore"
-	"google.golang.org/appengine/log"
-	"google.golang.org/appengine/memcache"
 	"google.golang.org/appengine/user"
 )
 
@@ -41,17 +38,32 @@ type Link struct {
 
 var validKey = regexp.MustCompile(`^[a-zA-Z0-9-_.]+$`)
 
-func RegisterHandlers(mux *http.ServeMux) {
+type server struct {
-	mux.HandleFunc(prefix, adminHandler)
+	datastore *datastore.Client
-	mux.HandleFunc(prefix+"/", linkHandler)
+	memcache  *memcache.CodecClient
+}
+
+func RegisterHandlers(mux *http.ServeMux, dc *datastore.Client, mc *memcache.Client) {
+	s := server{dc, mc.WithCodec(memcache.JSON)}
+	mux.HandleFunc(prefix+"/", s.linkHandler)
+
+	// TODO(cbro): move storage of the links to a text file in Gerrit.
+	// Disable the admin handler until that happens, since GAE Flex doesn't support
+	// the "google.golang.org/appengine/user" package.
+	// See golang.org/issue/27205#issuecomment-418673218
+	// mux.HandleFunc(prefix, adminHandler)
+	mux.HandleFunc(prefix, func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+		io.WriteString(w, "Link creation temporarily unavailable. See golang.org/issue/27205.")
+	})
 }
 
 // linkHandler services requests to short URLs.
 //   http://golang.org/s/key
 // It consults memcache and datastore for the Link for key.
 // It then sends a redirects or an error message.
-func linkHandler(w http.ResponseWriter, r *http.Request) {
+func (h server) linkHandler(w http.ResponseWriter, r *http.Request) {
-	c := appengine.NewContext(r)
+	ctx := r.Context()
 
 	key := r.URL.Path[len(prefix)+1:]
 	if !validKey.MatchString(key) {
@@ -60,16 +72,15 @@ func linkHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	var link Link
-	_, err := memcache.JSON.Get(c, cacheKey(key), &link)
+	if err := h.memcache.Get(ctx, cacheKey(key), &link); err != nil {
-	if err != nil {
+		k := datastore.NameKey(kind, key, nil)
-		k := datastore.NewKey(c, kind, key, 0, nil)
+		err = h.datastore.Get(ctx, k, &link)
-		err = datastore.Get(c, k, &link)
 		switch err {
 		case datastore.ErrNoSuchEntity:
 			http.Error(w, "not found", http.StatusNotFound)
 			return
 		default: // != nil
-			log.Errorf(c, "%q: %v", key, err)
+			log.Printf("ERROR %q: %v", key, err)
 			http.Error(w, "internal server error", http.StatusInternalServerError)
 			return
 		case nil:
@@ -77,8 +88,8 @@ func linkHandler(w http.ResponseWriter, r *http.Request) {
 				Key:    cacheKey(key),
 				Object: &link,
 			}
-			if err := memcache.JSON.Set(c, item); err != nil {
+			if err := h.memcache.Set(ctx, item); err != nil {
-				log.Warningf(c, "%q: %v", key, err)
+				log.Printf("WARNING %q: %v", key, err)
 			}
 		}
 	}
@@ -89,10 +100,10 @@ func linkHandler(w http.ResponseWriter, r *http.Request) {
 var adminTemplate = template.Must(template.New("admin").Parse(templateHTML))
 
 // adminHandler serves an administrative interface.
-func adminHandler(w http.ResponseWriter, r *http.Request) {
+func (h server) adminHandler(w http.ResponseWriter, r *http.Request) {
-	c := appengine.NewContext(r)
+	ctx := r.Context()
 
-	if !user.IsAdmin(c) {
+	if !user.IsAdmin(ctx) {
 		http.Error(w, "forbidden", http.StatusForbidden)
 		return
 	}
@@ -104,24 +115,24 @@ func adminHandler(w http.ResponseWriter, r *http.Request) {
 		switch r.FormValue("do") {
 		case "Add":
 			newLink = &Link{key, r.FormValue("target")}
-			doErr = putLink(c, newLink)
+			doErr = h.putLink(ctx, newLink)
 		case "Delete":
-			k := datastore.NewKey(c, kind, key, 0, nil)
+			k := datastore.NameKey(kind, key, nil)
-			doErr = datastore.Delete(c, k)
+			doErr = h.datastore.Delete(ctx, k)
 		default:
 			http.Error(w, "unknown action", http.StatusBadRequest)
 		}
-		err := memcache.Delete(c, cacheKey(key))
+		err := h.memcache.Delete(ctx, cacheKey(key))
 		if err != nil && err != memcache.ErrCacheMiss {
-			log.Warningf(c, "%q: %v", key, err)
+			log.Printf("WARNING %q: %v", key, err)
 		}
 	}
 
 	var links []*Link
-	_, err := datastore.NewQuery(kind).Order("Key").GetAll(c, &links)
+	q := datastore.NewQuery(kind).Order("Key")
-	if err != nil {
+	if _, err := h.datastore.GetAll(ctx, q, &links); err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
-		log.Errorf(c, "%v", err)
+		log.Printf("ERROR %v", err)
 		return
 	}
 
@@ -150,20 +161,20 @@ func adminHandler(w http.ResponseWriter, r *http.Request) {
 		Error   error
 	}{baseURL, prefix, links, newLink, doErr}
 	if err := adminTemplate.Execute(w, &data); err != nil {
-		log.Criticalf(c, "adminTemplate: %v", err)
+		log.Printf("ERROR adminTemplate: %v", err)
 	}
 }
 
 // putLink validates the provided link and puts it into the datastore.
-func putLink(c context.Context, link *Link) error {
+func (h server) putLink(ctx context.Context, link *Link) error {
 	if !validKey.MatchString(link.Key) {
 		return errors.New("invalid key; must match " + validKey.String())
 	}
 	if _, err := url.Parse(link.Target); err != nil {
 		return fmt.Errorf("bad target: %v", err)
 	}
-	k := datastore.NewKey(c, kind, link.Key, 0, nil)
+	k := datastore.NameKey(kind, link.Key, nil)
-	_, err := datastore.Put(c, k, link)
+	_, err := h.datastore.Put(ctx, k, link)
 	return err
 }
 

godoc/short/tmpl.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-// +build appengine
-
 package short
 
 const templateHTML = `

godoc/static/godoc.html

@@ -15,6 +15,19 @@
 {{end}}
 <link rel="stylesheet" href="/lib/godoc/jquery.treeview.css">
 <script>window.initFuncs = [];</script>
+{{with .GoogleAnalytics}}
+<script type="text/javascript">
+var _gaq = _gaq || [];
+_gaq.push(["_setAccount", "{{.}}"]);
+window.trackPageview = function() {
+  _gaq.push(["_trackPageview", location.pathname+location.hash]);
+};
+window.trackPageview();
+window.trackEvent = function(category, action, opt_label, opt_value, opt_noninteraction) {
+  _gaq.push(["_trackEvent", category, action, opt_label, opt_value, opt_noninteraction]);
+};
+</script>
+{{end}}
 <script src="/lib/godoc/jquery.js" defer></script>
 <script src="/lib/godoc/jquery.treeview.js" defer></script>
 <script src="/lib/godoc/jquery.treeview.edit.js" defer></script>
@@ -112,6 +125,15 @@ and code is licensed under a <a href="/LICENSE">BSD license</a>.<br>
 
 </div><!-- .container -->
 </div><!-- #page -->
+{{if .GoogleAnalytics}}
+<script type="text/javascript">
+(function() {
+  var ga = document.createElement("script"); ga.type = "text/javascript"; ga.async = true;
+  ga.src = ("https:" == document.location.protocol ? "https://ssl" : "http://www") + ".google-analytics.com/ga.js";
+  var s = document.getElementsByTagName("script")[0]; s.parentNode.insertBefore(ga, s);
+})();
+</script>
+{{end}}
 </body>
 </html>
 

internal/memcache/memcache.go

@@ -0,0 +1,157 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package memcache provides a minimally compatible interface for
+// google.golang.org/appengine/memcache
+// and stores the data in Redis (e.g., via Cloud Memorystore).
+package memcache
+
+import (
+	"bytes"
+	"context"
+	"encoding/gob"
+	"encoding/json"
+	"errors"
+	"time"
+
+	"github.com/gomodule/redigo/redis"
+)
+
+var ErrCacheMiss = errors.New("memcache: cache miss")
+
+func New(addr string) *Client {
+	const maxConns = 20
+
+	pool := redis.NewPool(func() (redis.Conn, error) {
+		return redis.Dial("tcp", addr)
+	}, maxConns)
+
+	return &Client{
+		pool: pool,
+	}
+}
+
+type Client struct {
+	pool *redis.Pool
+}
+
+type CodecClient struct {
+	client *Client
+	codec  Codec
+}
+
+type Item struct {
+	Key        string
+	Value      []byte
+	Object     interface{}   // Used with Codec.
+	Expiration time.Duration // Read-only.
+}
+
+func (c *Client) WithCodec(codec Codec) *CodecClient {
+	return &CodecClient{
+		c, codec,
+	}
+}
+
+func (c *Client) Delete(ctx context.Context, key string) error {
+	conn, err := c.pool.GetContext(ctx)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	_, err = conn.Do("DEL", key)
+	return err
+}
+
+func (c *CodecClient) Delete(ctx context.Context, key string) error {
+	return c.client.Delete(ctx, key)
+}
+
+func (c *Client) Set(ctx context.Context, item *Item) error {
+	if item.Value == nil {
+		return errors.New("nil item value")
+	}
+	return c.set(ctx, item.Key, item.Value, item.Expiration)
+}
+
+func (c *CodecClient) Set(ctx context.Context, item *Item) error {
+	if item.Object == nil {
+		return errors.New("nil object value")
+	}
+	b, err := c.codec.Marshal(item.Object)
+	if err != nil {
+		return err
+	}
+	return c.client.set(ctx, item.Key, b, item.Expiration)
+}
+
+func (c *Client) set(ctx context.Context, key string, value []byte, expiration time.Duration) error {
+	conn, err := c.pool.GetContext(ctx)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	if expiration == 0 {
+		_, err := conn.Do("SET", key, value)
+		return err
+	}
+
+	// NOTE(cbro): redis does not support expiry in units more granular than a second.
+	exp := int64(expiration.Seconds())
+	if exp == 0 {
+		// Redis doesn't allow a zero expiration, delete the key instead.
+		_, err := conn.Do("DEL", key)
+		return err
+	}
+
+	_, err = conn.Do("SETEX", key, exp, value)
+	return err
+}
+
+// Get gets the item.
+func (c *Client) Get(ctx context.Context, key string) ([]byte, error) {
+	conn, err := c.pool.GetContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	b, err := redis.Bytes(conn.Do("GET", key))
+	if err == redis.ErrNil {
+		err = ErrCacheMiss
+	}
+	return b, err
+}
+
+func (c *CodecClient) Get(ctx context.Context, key string, v interface{}) error {
+	b, err := c.client.Get(ctx, key)
+	if err != nil {
+		return err
+	}
+	return c.codec.Unmarshal(b, v)
+}
+
+var (
+	Gob  = Codec{gobMarshal, gobUnmarshal}
+	JSON = Codec{json.Marshal, json.Unmarshal}
+)
+
+type Codec struct {
+	Marshal   func(interface{}) ([]byte, error)
+	Unmarshal func([]byte, interface{}) error
+}
+
+func gobMarshal(v interface{}) ([]byte, error) {
+	var buf bytes.Buffer
+	if err := gob.NewEncoder(&buf).Encode(v); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func gobUnmarshal(data []byte, v interface{}) error {
+	return gob.NewDecoder(bytes.NewBuffer(data)).Decode(v)
+}

internal/memcache/memcache_test.go

@@ -0,0 +1,83 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package memcache
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+)
+
+func getClient(t *testing.T) *Client {
+	t.Helper()
+
+	addr := os.Getenv("GOLANG_REDIS_ADDR")
+	if addr == "" {
+		t.Skip("skipping because GOLANG_REDIS_ADDR is unset")
+	}
+
+	return New(addr)
+}
+
+func TestCacheMiss(t *testing.T) {
+	c := getClient(t)
+	ctx := context.Background()
+
+	if _, err := c.Get(ctx, "doesnotexist"); err != ErrCacheMiss {
+		t.Errorf("got %v; want ErrCacheMiss", err)
+	}
+}
+
+func TestExpiry(t *testing.T) {
+	c := getClient(t).WithCodec(Gob)
+	ctx := context.Background()
+
+	key := "testexpiry"
+
+	firstTime := time.Now()
+	err := c.Set(ctx, &Item{
+		Key:        key,
+		Object:     firstTime,
+		Expiration: 3500 * time.Millisecond, // NOTE: check that non-rounded expiries work.
+	})
+	if err != nil {
+		t.Fatalf("Set: %v", err)
+	}
+
+	var newTime time.Time
+	if err := c.Get(ctx, key, &newTime); err != nil {
+		t.Fatalf("Get: %v", err)
+	}
+	if !firstTime.Equal(newTime) {
+		t.Errorf("Get: got value %v, want %v", newTime, firstTime)
+	}
+
+	time.Sleep(4 * time.Second)
+
+	if err := c.Get(ctx, key, &newTime); err != ErrCacheMiss {
+		t.Errorf("Get: got %v, want ErrCacheMiss", err)
+	}
+}
+
+func TestShortExpiry(t *testing.T) {
+	c := getClient(t).WithCodec(Gob)
+	ctx := context.Background()
+
+	key := "testshortexpiry"
+
+	err := c.Set(ctx, &Item{
+		Key:        key,
+		Value:      []byte("ok"),
+		Expiration: time.Millisecond,
+	})
+	if err != nil {
+		t.Fatalf("Set: %v", err)
+	}
+
+	if err := c.Get(ctx, key, nil); err != ErrCacheMiss {
+		t.Errorf("GetBytes: got %v, want ErrCacheMiss", err)
+	}
+}