[release-branch.go1.11] cmd/godoc: simplify dev and prod environment for App Engine

Remove all of the code generation and the concept of "APPDIR" - just
generate godoc.zip and index files in the app directory.

Simplify generation of the zip - use a symlink so that every file in
godoc.zip is under the "goroot" directory, regardless of the
environment. Previously, the prefix would be dependent on the location
of the user's GOROOT.

Running the setup script is now optional - it's now possible to run
dev_appserver.py on a regular checkout of cmd/godoc without godoc.zip
and search index files. Use environment variables to switch whether the
zip file is used vs reading GOROOT from the filesystem.

Updates golang/go#28893

Change-Id: I1ce95c891717fe2da975f979778fd775b23f18c8
Reviewed-on: https://go-review.googlesource.com/46725
Reviewed-by: Andrew Bonventre <andybons@golang.org>
(cherry picked from commit e9ca907325)
Reviewed-on: https://go-review.googlesource.com/c/150597
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

cmd/godoc/.gitignore

@@ -0,0 +1,3 @@
+index.split.*
+godoc.index
+godoc.zip

cmd/godoc/README.godoc-app

@@ -1,56 +1,41 @@
-godoc on appengine
+godoc on Google App Engine
-------------------
+==========================
 
 Prerequisites
 -------------
 
-* Go appengine SDK
+* Google Cloud SDK
-  https://developers.google.com/appengine/downloads#Google_App_Engine_SDK_for_Go
+  https://cloud.google.com/sdk/
 
-* Go sources at tip under $GOROOT
+* Go sources under $GOROOT
 
-* Godoc sources at tip inside $GOPATH
+* Godoc sources inside $GOPATH
   (go get -d golang.org/x/tools/cmd/godoc)
 
 
-Directory structure
+Running in dev_appserver.py
--------------------
+---------------------------
 
-* Let $APPDIR be the directory containing the app engine files.
+Use dev_appserver.py to run the server in development mode:
-  (e.g., $APPDIR=$HOME/godoc-app)
 
-* $APPDIR contains the following entries (this may change depending on
+	dev_appserver.py app.dev.yaml
-  app-engine release and version of godoc):
 
-	app.yaml
+To run the server with generated zip file and search index:
-	golang.org/x/tools/cmd/godoc
-	godoc.zip
-	index.split.*
 
-* The app.yaml file is set up per app engine documentation.
+	./generate-index.bash
-  For instance:
+	dev_appserver.py app.prod.yaml
 
-	application: godoc-app
+godoc should come up at http://localhost:8080
-	version: 1
+Use the --host and --port flags to listen on a different address.
-	runtime: go
-	api_version: go1
 
-	handlers:
+To clean up the index files, use git:
-	- url: /.*
+
-	  script: _go_app
+	git clean -xn # n is dry run, replace with f
 
 
-Configuring and running godoc
+Troubleshooting
------------------------------
+---------------
 
-To configure godoc, run
+Ensure the Cloud SDK is on your PATH and you have the app-engine-go component
-
+installed (gcloud components install app-engine-go) and your components are
-	bash setup-godoc-app.bash
+up-to-date (gcloud components update)
-
-to prepare an $APPDIR as described above. See the script for details on usage.
-
-To run godoc locally, using the App Engine development server, run
-
-	<path to go_appengine>/dev_appserver.py $APPDIR
-
-godoc should come up at http://localhost:8080 .

cmd/godoc/app.dev.yaml

@@ -0,0 +1,13 @@
+runtime: go
+api_version: go1
+instance_class: F4_1G
+
+handlers:
+- url: /s
+  script: _go_app
+  login: admin
+- url: /dl/init
+  script: _go_app
+  login: admin
+- url: /.*
+  script: _go_app

cmd/godoc/app.prod.yaml

@@ -0,0 +1,18 @@
+runtime: go
+api_version: go1
+instance_class: F4_1G
+
+handlers:
+- url: /s
+  script: _go_app
+  login: admin
+- url: /dl/init
+  script: _go_app
+  login: admin
+- url: /.*
+  script: _go_app
+
+env_variables:
+  GODOC_ZIP: godoc.zip
+  GODOC_ZIP_PREFIX: goroot
+  GODOC_INDEX_GLOB: 'index.split.*'

cmd/godoc/appinit.go

@@ -13,8 +13,10 @@ import (
 	"archive/zip"
 	"log"
 	"net/http"
+	"os"
 	"path"
 	"regexp"
+	"runtime"
 
 	"golang.org/x/tools/godoc"
 	"golang.org/x/tools/godoc/dl"
@@ -22,6 +24,7 @@ import (
 	"golang.org/x/tools/godoc/short"
 	"golang.org/x/tools/godoc/static"
 	"golang.org/x/tools/godoc/vfs"
+	"golang.org/x/tools/godoc/vfs/gatefs"
 	"golang.org/x/tools/godoc/vfs/mapfs"
 	"golang.org/x/tools/godoc/vfs/zipfs"
 
@@ -29,6 +32,18 @@ import (
 )
 
 func init() {
+	var (
+		// .zip filename
+		zipFilename = os.Getenv("GODOC_ZIP")
+
+		// goroot directory in .zip file
+		zipGoroot = os.Getenv("GODOC_ZIP_PREFIX")
+
+		// glob pattern describing search index files
+		// (if empty, the index is built at run-time)
+		indexFilenames = os.Getenv("GODOC_INDEX_GLOB")
+	)
+
 	enforceHosts = !appengine.IsDevAppServer()
 	playEnabled = true
 
@@ -37,16 +52,20 @@ func init() {
 	log.Printf(".zip GOROOT = %s", zipGoroot)
 	log.Printf("index files = %s", indexFilenames)
 
-	goroot := path.Join("/", zipGoroot) // fsHttp paths are relative to '/'
+	if zipFilename != "" {
-
+		goroot := path.Join("/", zipGoroot) // fsHttp paths are relative to '/'
-	// read .zip file and set up file systems
+		// read .zip file and set up file systems
-	const zipfile = zipFilename
+		rc, err := zip.OpenReader(zipFilename)
-	rc, err := zip.OpenReader(zipfile)
+		if err != nil {
-	if err != nil {
+			log.Fatalf("%s: %s\n", zipFilename, err)
-		log.Fatalf("%s: %s\n", zipfile, err)
+		}
+		// rc is never closed (app running forever)
+		fs.Bind("/", zipfs.New(rc, zipFilename), goroot, vfs.BindReplace)
+	} else {
+		rootfs := gatefs.New(vfs.OS(runtime.GOROOT()), make(chan bool, 20))
+		fs.Bind("/", rootfs, "/", vfs.BindReplace)
 	}
-	// rc is never closed (app running forever)
+
-	fs.Bind("/", zipfs.New(rc, zipFilename), goroot, vfs.BindReplace)
 	fs.Bind("/lib/godoc", mapfs.New(static.Files), "/", vfs.BindReplace)
 
 	corpus := godoc.NewCorpus(fs)

cmd/godoc/generate-index.bash

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+# Copyright 2011 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This script creates a .zip file representing the $GOROOT file system
+# and computes the corresponding search index files.
+#
+# These are used in production (see app.prod.yaml)
+
+set -e -u -x
+
+ZIPFILE=godoc.zip
+INDEXFILE=godoc.index
+SPLITFILES=index.split.
+
+error() {
+	echo "error: $1"
+	exit 2
+}
+
+install() {
+	go install
+}
+
+getArgs() {
+	if [ ! -v GOROOT ]; then
+		GOROOT="$(go env GOROOT)"
+		echo "GOROOT not set explicitly, using go env value instead"
+	fi
+
+	# safety checks
+	if [ ! -d "$GOROOT" ]; then
+		error "$GOROOT is not a directory"
+	fi
+
+	# reporting
+	echo "GOROOT = $GOROOT"
+}
+
+makeZipfile() {
+	echo "*** make $ZIPFILE"
+	rm -f $ZIPFILE goroot
+	ln -s "$GOROOT" goroot
+	zip -q -r $ZIPFILE goroot/* # glob to ignore dotfiles (like .git)
+	rm goroot
+}
+
+makeIndexfile() {
+	echo "*** make $INDEXFILE"
+	godoc=$(go env GOPATH)/bin/godoc
+	# NOTE: run godoc without GOPATH set. Otherwise third-party packages will end up in the index.
+	GOPATH= $godoc -write_index -goroot goroot -index_files=$INDEXFILE -zip=$ZIPFILE
+}
+
+splitIndexfile() {
+	echo "*** split $INDEXFILE"
+	rm -f $SPLITFILES*
+	split -b8m $INDEXFILE $SPLITFILES
+}
+
+cd $(dirname $0)
+
+install
+getArgs "$@"
+makeZipfile
+makeIndexfile
+splitIndexfile
+rm $INDEXFILE
+
+echo "*** setup complete"

cmd/godoc/setup-godoc-app.bash

@@ -1,134 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2011 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-# This script creates a complete godoc app in $APPDIR.
-# It copies the cmd/godoc and src/go/... sources from GOROOT,
-# synthesizes an app.yaml file, and creates the .zip, index, and
-# configuration files.
-#
-# If an argument is provided it is assumed to be the app-engine godoc directory.
-# Without an argument, $APPDIR is used instead. If GOROOT is not set, "go env"
-# is consulted to find the $GOROOT.
-#
-# The script creates a .zip file representing the $GOROOT file system
-# and computes the corresponding search index files. These files are then
-# copied to $APPDIR. A corresponding godoc configuration file is created
-# in $APPDIR/appconfig.go.
-
-ZIPFILE=godoc.zip
-INDEXFILE=godoc.index
-SPLITFILES=index.split.
-GODOC=golang.org/x/tools/cmd/godoc
-CONFIGFILE=$GODOC/appconfig.go
-
-error() {
-	echo "error: $1"
-	exit 2
-}
-
-getArgs() {
-	if [ -z $APPENGINE_SDK ]; then
-		error "APPENGINE_SDK environment variable not set"
-	fi
-	if [ ! -x $APPENGINE_SDK/goapp ]; then
-		error "couldn't find goapp command in $APPENGINE_SDK"
-	fi
-	if [ -z $GOROOT ]; then
-		GOROOT=$(go env GOROOT)
-		echo "GOROOT not set explicitly, using go env value instead"
-	fi
-	if [ -z $APPDIR ]; then
-		if [ $# == 0 ]; then
-			error "APPDIR not set, and no argument provided"
-		fi
-		APPDIR=$1
-		echo "APPDIR not set, using argument instead"
-	fi
-
-	# safety checks
-	if [ ! -d $GOROOT ]; then
-		error "$GOROOT is not a directory"
-	fi
-	if [ -e $APPDIR ]; then
-		error "$APPDIR exists; check and remove it before trying again"
-	fi
-
-	# reporting
-	echo "GOROOT = $GOROOT"
-	echo "APPDIR = $APPDIR"
-}
-
-fetchGodoc() {
-	echo "*** Fetching godoc (if not already in GOPATH)"
-	unset GOBIN
-	go=$APPENGINE_SDK/goapp
-	$go get -d -tags appengine $GODOC
-	mkdir -p $APPDIR/$GODOC
-	cp $(find $($go list -f '{{.Dir}}' $GODOC) -mindepth 1 -maxdepth 1 -type f) $APPDIR/$GODOC/
-}
-
-makeAppYaml() {
-	echo "*** make $APPDIR/app.yaml"
-	cat > $APPDIR/app.yaml <<EOF
-application: godoc
-version: 1
-runtime: go
-api_version: go1
-
-handlers:
-- url: /.*
-  script: _go_app
-EOF
-}
-
-makeZipfile() {
-	echo "*** make $APPDIR/$ZIPFILE"
-	zip -q -r $APPDIR/$ZIPFILE $GOROOT/*
-}
-
-makeIndexfile() {
-	echo "*** make $APPDIR/$INDEXFILE"
-	GOPATH= godoc -write_index -index_files=$APPDIR/$INDEXFILE -zip=$APPDIR/$ZIPFILE
-}
-
-splitIndexfile() {
-	echo "*** split $APPDIR/$INDEXFILE"
-	split -b8m $APPDIR/$INDEXFILE $APPDIR/$SPLITFILES
-}
-
-makeConfigfile() {
-	echo "*** make $APPDIR/$CONFIGFILE"
-	cat > $APPDIR/$CONFIGFILE <<EOF
-package main
-
-// GENERATED FILE - DO NOT MODIFY BY HAND.
-// (generated by golang.org/x/tools/cmd/godoc/setup-godoc-app.bash)
-
-const (
-	// .zip filename
-	zipFilename = "$ZIPFILE"
-
-	// goroot directory in .zip file
-	zipGoroot = "$GOROOT"
-
-	// glob pattern describing search index files
-	// (if empty, the index is built at run-time)
-	indexFilenames = "$SPLITFILES*"
-)
-EOF
-}
-
-getArgs "$@"
-set -e
-mkdir $APPDIR
-fetchGodoc
-makeAppYaml
-makeZipfile
-makeIndexfile
-splitIndexfile
-makeConfigfile
-
-echo "*** setup complete"