Zero down-time deploy and server checks

This change makes Dokku start up the new container, run a set of checks
against it, and only switch traffic over to the new containers if all
checks complete successfully. No requests are dropped during the switch
over.

To specify checks, add a CHECKS file to the root of your project
directory. This is a text file with one line per check. Empty lines and
lines starting with # are ignored.

A check is a relative URL and may be followed by expected content from
the page, for example:

  /about     Our Amazing Team

Even if you don’t use any checks, this change will prevent downtime
during switching from old to new container.

See: https://labnotes.org/zero-downtime-deploy-with-dokku/

dokku

@@ -57,21 +57,40 @@ case "$1" in
     APP="$2"; IMAGE="dokku/$APP"
     pluginhook pre-deploy $APP
 
-    # kill the app when running
     if [[ -f "$DOKKU_ROOT/$APP/CONTAINER" ]]; then
       oldid=$(< "$DOKKU_ROOT/$APP/CONTAINER")
-      docker inspect $oldid &> /dev/null && docker kill $oldid > /dev/null
     fi
 
     # start the app
     DOCKER_ARGS=$(: | pluginhook docker-args $APP)
     id=$(docker run -d -p 5000 -e PORT=5000 $DOCKER_ARGS $IMAGE /bin/bash -c "/start web")
-    echo $id > "$DOKKU_ROOT/$APP/CONTAINER"
     port=$(docker port $id 5000 | sed 's/0.0.0.0://')
+
+    # if we can't post-deploy successfully, kill new container
+    function kill_new {
+      docker inspect $id &> /dev/null && docker kill $id > /dev/null
+      trap - INT TERM EXIT
+      kill -9 $$
+    }
+
+    # run checks first, then post-deploy hooks, which switches Nginx traffic
+    trap kill_new INT TERM EXIT
+    echo "-----> Running pre-flight checks"
+    pluginhook check-deploy $id $APP $port
+    echo "-----> Running post-deploy"
+    pluginhook post-deploy  $APP $port
+    trap -        INT TERM EXIT
+
+    # now using the new container
+    echo $id > "$DOKKU_ROOT/$APP/CONTAINER"
     echo $port > "$DOKKU_ROOT/$APP/PORT"
     echo "http://$(< "$DOKKU_ROOT/HOSTNAME"):$port" > "$DOKKU_ROOT/$APP/URL"
 
-    pluginhook post-deploy $APP $port
+    # kill the old container
+    if [[ -n "$oldid" ]]; then
+      docker inspect $oldid &> /dev/null && docker kill $oldid > /dev/null
+    fi
+
     ;;
 
   cleanup)

plugins/checks/check-deploy

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Hook to check server against list of checks specified in CHECKS file.  Each
+# check is a relative path and, optionally, expected content.
+#
+# For example:
+#   /                       My Amazing App
+#   /stylesheets/index.css  .body
+#   /scripts/index.js       $(function()
+#   /images/logo.png
+#
+# Waits 5 seconds, giving server time to start, before running the checks.  For
+# shorter/longer wait, change the DOKKU_CHECKS_WAIT environment variable (value
+# in seconds).
+
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+CONTAINERID="$1"; APP="$2"; PORT="$3" ; HOSTNAME="${4:-localhost}"
+
+# source in app env to get DOKKU_CHECKS_WAIT and any other necessary vars
+[[ -f "$DOKKU_ROOT/$APP/ENV" ]] && source $DOKKU_ROOT/$APP/ENV
+# echo "DOKKU_CHECKS_WAIT is $DOKKU_CHECKS_WAIT"
+FILENAME="$DOKKU_ROOT/$APP/CHECKS"
+WAIT="${DOKKU_CHECKS_WAIT:-5}"
+
+# try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist
+# docker cp exits with status 1 when run as non-root user when it tries to chown the file
+# after successfully copying the file. Thus, we suppress stderr.
+# ref: https://github.com/dotcloud/docker/issues/3986
+if [[ ! -f "$FILENAME" ]] ; then
+    echo "       check-deploy: $FILENAME not found. attempting to retrieve it from container ..."
+    TMPDIR=$(mktemp -d /tmp/CHECKS.XXXXX)
+    docker cp $CONTAINERID:/app/CHECKS $TMPDIR 2> /dev/null || true
+    if [[ ! -s "${TMPDIR}/CHECKS" ]] ; then
+        echo "       CHECKS file not found in container. skipping checks."
+        rm -rf $TMPDIR
+        exit 0
+    else
+        echo "       CHECKS file found in container"
+        FILENAME=${TMPDIR}/CHECKS
+
+        function cleanup() {
+          echo "       removing CHECKS file copied from container"
+          rm -rf $TMPDIR
+        }
+        trap cleanup EXIT
+    fi
+fi
+
+echo "Waiting $WAIT seconds ..."
+sleep $WAIT
+
+# -q           do not use .curlrc (must come first)
+# --compressed Test compression handled correctly
+# --fail       Fail on server errors (4xx, 5xx)
+# --location   Follow redirects
+CURL_OPTIONS="-q --compressed --fail --location --max-time 30"
+
+cat "$FILENAME" | while read PATHNAME EXPECTED ; do
+  # Ignore empty lines and lines starting with #
+  [[ -z "$PATHNAME" || "$PATHNAME" =~ ^\# ]] && continue
+
+  URL="http://$HOSTNAME:$PORT$PATHNAME"
+
+  echo "checking with: curl $CURL_OPTIONS $URL"
+  HTML=$(curl $CURL_OPTIONS $URL)
+  if [[ -n "$EXPECTED" && ! "$HTML" =~ "$EXPECTED" ]] ; then
+    echo -e "\033[31m\033[1m$URL: expected to but did not find: \"$EXPECTED\"\033[0m"
+    exit 1
+  else
+    echo -e "\033[32m\033[1m$URL => \"$EXPECTED\"\033[0m"
+  fi
+done
+
+echo -e "\033[32m\033[1mAll checks successful!\033[0m"

plugins/nginx-vhosts/post-deploy

@@ -84,6 +84,8 @@ EOF
 
   echo "http://$hostname" > "$DOKKU_ROOT/$APP/URL"
   fi
-  pluginhook nginx-pre-reload $APP
+  echo "-----> Running nginx-pre-reload"
+  pluginhook nginx-pre-reload $APP $PORT
+  echo "       Reloading nginx"
   sudo /etc/init.d/nginx reload > /dev/null
 fi