From f160a88a353b733224bd2db316a764cc8da2f544 Mon Sep 17 00:00:00 2001 From: Chris Broadfoot Date: Tue, 2 Oct 2018 18:41:55 -0700 Subject: [PATCH] [release-branch.go1.11] cmd/godoc: move regression tests to a go test Run them separately from the other tests in godoc_test by requiring a regtest.host flag and by filtering on the test name. Updates golang/go#28893 Updates golang/go#27205 Change-Id: I166d2278a3f6954307f7c935567a81e73f78e7bb Reviewed-on: https://go-review.googlesource.com/c/139238 Reviewed-by: Brad Fitzpatrick Reviewed-on: https://go-review.googlesource.com/c/150680 --- cmd/godoc/Makefile | 8 +- cmd/godoc/regtest.bash | 110 ---------------------------- cmd/godoc/regtest_test.go | 150 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 113 deletions(-) delete mode 100755 cmd/godoc/regtest.bash create mode 100644 cmd/godoc/regtest_test.go diff --git a/cmd/godoc/Makefile b/cmd/godoc/Makefile index aae6b763..cc75464c 100644 --- a/cmd/godoc/Makefile +++ b/cmd/godoc/Makefile @@ -16,8 +16,8 @@ GCP_PROJECT := golang-org DOCKER_TAG := gcr.io/$(GCP_PROJECT)/godoc:$(DOCKER_VERSION) usage: - echo "See Makefile and README.godoc-app" - exit 1 + @echo "See Makefile and README.godoc-app" + @exit 1 docker: Dockerfile.prod # NOTE(cbro): move up in directory to include entire tools repo. @@ -48,4 +48,6 @@ get-latest-url: --limit 1 | cut -f1 # NOTE(cbro): gcloud prints out createTime as the second field. regtest: - ./regtest.bash $(shell make get-latest-url) + go test -v \ + -regtest.host=$(shell make get-latest-url) \ + -run=Live diff --git a/cmd/godoc/regtest.bash b/cmd/godoc/regtest.bash deleted file mode 100755 index 9b596fae..00000000 --- a/cmd/godoc/regtest.bash +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# Regression tests for golang.org. -# Usage: ./regtest.bash https://golang.org/ - -#TODO: turn this into a Go program. maybe behind a build tag and "go run regtest.go " - -set -e - -addr="$(echo $1 | sed -e 's/\/$//')" -if [ -z "$addr" ]; then - echo "usage: $0 " 1>&2 - echo "example: $0 https://20180928t023837-dot-golang-org.appspot.com/" 1>&2 - exit 1 -fi - -set -u - -# fetch url, check the response with a regexp. -fetch() { - curl -s "${addr}$1" | grep "$2" > /dev/null -} -fatal() { - log "$1" - exit 1 -} -log() { - echo "$1" 1>&2 -} -logn() { - echo -n "$1" 1>&2 -} - -log "Checking FAQ..." -fetch /doc/faq 'What is the purpose of the project' || { - fatal "FAQ did not match." -} - -log "Checking package listing..." -fetch /pkg/ 'Package tar' || { - fatal "package listing page did not match." -} - -log "Checking os package..." -fetch /pkg/os/ 'func Open' || { - fatal "os package page did not match." -} - -log "Checking robots.txt..." -fetch /robots.txt 'Disallow: /search' || { - fatal "robots.txt did not match." -} - -log "Checking /change/ redirect..." -fetch /change/75944e2e3a63 'bdb10cf' || { - fatal "/change/ direct did not match." -} - -log "Checking /dl/ page has data..." -fetch /dl/ 'go1.11.windows-amd64.msi' || { - fatal "/dl/ did not match." -} - -log "Checking /dl/?mode=json page has data..." -fetch /dl/?mode=json 'go1.11.windows-amd64.msi' || { - fatal "/dl/?mode=json did not match." -} - -log "Checking shortlinks (/s/go2design)..." -fetch /s/go2design 'proposal.*Found' || { - fatal "/s/go2design did not match." -} - -log "Checking analytics on pages..." -ga_id="UA-11222381-2" -fetch / $ga_id || fatal "/ missing GA." -fetch /dl/ $ga_id || fatal "/dl/ missing GA." -fetch /project/ $ga_id || fatal "/project missing GA." -fetch /pkg/context/ $ga_id || fatal "/pkg/context missing GA." - -log "Checking search..." -fetch /search?q=IsDir 'src/os/types.go' || { - fatal "search result did not match." -} - -log "Checking compile service..." -compile="curl -s ${addr}/compile" - -p="package main; func main() { print(6*7); }" -$compile --data-urlencode "body=$p" | tee /tmp/compile.out | grep '^{"compile_errors":"","output":"42"}$' > /dev/null || { - cat /tmp/compile.out - fatal "compile service output did not match." -} - -$compile --data-urlencode "body=//empty" | tee /tmp/compile.out | grep "expected 'package', found 'EOF'" > /dev/null || { - cat /tmp/compile.out - fatal "compile service error output did not match." -} - -# Check API version 2 -d="version=2&body=package+main%3Bimport+(%22fmt%22%3B%22time%22)%3Bfunc+main()%7Bfmt.Print(%22A%22)%3Btime.Sleep(time.Second)%3Bfmt.Print(%22B%22)%7D" -$compile --data "$d" | grep '^{"Errors":"","Events":\[{"Message":"A","Kind":"stdout","Delay":0},{"Message":"B","Kind":"stdout","Delay":1000000000}\]}$' > /dev/null || { - fatal "compile service v2 output did not match." -} - -log "All OK" diff --git a/cmd/godoc/regtest_test.go b/cmd/godoc/regtest_test.go new file mode 100644 index 00000000..dcfa9154 --- /dev/null +++ b/cmd/godoc/regtest_test.go @@ -0,0 +1,150 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Regression tests to run against a production instance of godoc. + +package main_test + +import ( + "bytes" + "flag" + "io" + "io/ioutil" + "net/http" + "net/url" + "regexp" + "strings" + "testing" +) + +var host = flag.String("regtest.host", "", "host to run regression test against") + +func init() { + flag.Parse() + *host = strings.TrimSuffix(*host, "/") +} + +func TestLiveServer(t *testing.T) { + if *host == "" { + t.Skip("regtest.host flag missing.") + } + substringTests := []struct { + Message string + Path string + Substring string + Regexp string + NoAnalytics bool // expect the response to not contain GA. + PostBody string + }{ + { + Path: "/doc/faq", + Substring: "What is the purpose of the project", + }, + { + Path: "/pkg/", + Substring: "Package tar", + }, + { + Path: "/pkg/os/", + Substring: "func Open", + }, + { + Path: "/robots.txt", + Substring: "Disallow: /search", + Message: "robots not present - not deployed from Dockerfile?", + NoAnalytics: true, + }, + { + Path: "/change/75944e2e3a63", + Substring: "bdb10cf", + Message: "no change redirect - hg to git mapping not registered?", + NoAnalytics: true, + }, + { + Path: "/dl/", + Substring: "go1.11.windows-amd64.msi", + Message: "missing data on dl page - misconfiguration of datastore?", + }, + { + Path: "/dl/?mode=json", + Substring: ".windows-amd64.msi", + NoAnalytics: true, + }, + { + Message: "broken shortlinks - misconfiguration of datastore or memcache?", + Path: "/s/go2design", + Regexp: "proposal.*Found", + NoAnalytics: true, + }, + { + Message: "incorrect search result - broken index?", + Path: "/search?q=IsDir", + Substring: "src/os/types.go", + }, + { + Path: "/compile", + PostBody: "body=" + url.QueryEscape("package main; func main() { print(6*7); }"), + Regexp: `^{"compile_errors":"","output":"42"}$`, + NoAnalytics: true, + }, + { + Path: "/compile", + PostBody: "body=" + url.QueryEscape("//empty"), + Substring: "expected 'package', found 'EOF'", + NoAnalytics: true, + }, + { + Path: "/compile", + PostBody: "version=2&body=package+main%3Bimport+(%22fmt%22%3B%22time%22)%3Bfunc+main()%7Bfmt.Print(%22A%22)%3Btime.Sleep(time.Second)%3Bfmt.Print(%22B%22)%7D", + Regexp: `^{"Errors":"","Events":\[{"Message":"A","Kind":"stdout","Delay":0},{"Message":"B","Kind":"stdout","Delay":1000000000}\]}$`, + NoAnalytics: true, + }, + } + + for _, tc := range substringTests { + t.Run(tc.Path, func(t *testing.T) { + method := "GET" + var reqBody io.Reader + if tc.PostBody != "" { + method = "POST" + reqBody = strings.NewReader(tc.PostBody) + } + req, err := http.NewRequest(method, *host+tc.Path, reqBody) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + if reqBody != nil { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + resp, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + t.Fatalf("RoundTrip: %v", err) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + + const googleAnalyticsID = "UA-11222381-2" // golang.org analytics ID + if !tc.NoAnalytics && !bytes.Contains(body, []byte(googleAnalyticsID)) { + t.Errorf("want response to contain analytics tracking ID") + } + + if tc.Substring != "" { + tc.Regexp = regexp.QuoteMeta(tc.Substring) + } + re := regexp.MustCompile(tc.Regexp) + + if !re.Match(body) { + t.Log("------ actual output -------") + t.Log(string(body)) + t.Log("----------------------------") + if tc.Message != "" { + t.Log(tc.Message) + } + t.Fatalf("wanted response to match %s", tc.Regexp) + } + }) + } +}