cmd/tip: move side health check to main health check. Use 503.

500 is considered to be healthy. Use 503 instead, so the instance is
considered unhealthy while the side is coming up.

Add some extra log statements that help debug instances from
application logs.

Fixes golang/go#13682.

Change-Id: I713c8c2fa75de4e275f632b999edc98cedd257bd
Reviewed-on: https://go-review.googlesource.com/18547
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Reviewed-by: Andrew Gerrand <adg@golang.org>

cmd/tip/godoc.yaml

@@ -1,15 +1,18 @@
 module: tip
 runtime: custom
-api_version: go1
 vm: true
 
 automatic_scaling:
   min_num_instances: 1
   max_num_instances: 2
 
-handlers:
-- url: /.*
-  script: _go_app
-
 env_variables:
   TIP_BUILDER: 'godoc'
+
+health_check:
+  enable_health_check: True
+  check_interval_sec: 5
+  timeout_sec: 4
+  unhealthy_threshold: 2
+  healthy_threshold: 2
+  restart_threshold: 240

cmd/tip/talks.yaml

@@ -1,15 +1,18 @@
 module: talks
 runtime: custom
-api_version: go1
 vm: true
 
 automatic_scaling:
   min_num_instances: 1
   max_num_instances: 5
 
-handlers:
-- url: /.*
-  script: _go_app
-
 env_variables:
   TIP_BUILDER: 'talks'
+
+health_check:
+  enable_health_check: True
+  check_interval_sec: 5
+  timeout_sec: 4
+  unhealthy_threshold: 2
+  healthy_threshold: 2
+  restart_threshold: 240

cmd/tip/tip.go

@@ -47,6 +47,8 @@ func main() {
 	http.Handle("/", p)
 	http.HandleFunc("/_ah/health", p.serveHealthCheck)
 
+	log.Print("Starting up")
+
 	if err := http.ListenAndServe(":8080", nil); err != nil {
 		p.stop()
 		log.Fatal(err)
@@ -90,14 +92,6 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, s, http.StatusInternalServerError)
 		return
 	}
-	if r.URL.Path == "/_ah/health" {
-		if err := p.builder.HealthCheck(p.hostport); err != nil {
-			http.Error(w, "Health check failde: "+err.Error(), http.StatusInternalServerError)
-			return
-		}
-		fmt.Fprintln(w, "OK")
-		return
-	}
 	proxy.ServeHTTP(w, r)
 }
 
@@ -110,8 +104,17 @@ func (p *Proxy) serveStatus(w http.ResponseWriter, r *http.Request) {
 func (p *Proxy) serveHealthCheck(w http.ResponseWriter, r *http.Request) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
+	// NOTE: Status 502, 503, 504 are the only status codes that signify an unhealthy app.
+	// So long as this handler returns one of those codes, this instance will not be sent any requests.
 	if p.proxy == nil {
-		http.Error(w, "not ready", 500)
+		log.Printf("Health check: not ready")
+		http.Error(w, "Not ready", http.StatusServiceUnavailable)
+		return
+	}
+
+	if err := p.builder.HealthCheck(p.hostport); err != nil {
+		log.Printf("Health check failed: %v", err)
+		http.Error(w, "Health check failed", http.StatusServiceUnavailable)
 		return
 	}
 	io.WriteString(w, "ok")