Merge pull request #562 from assaf/zero-downtime

Zero down-time deploy and server checks

dokku

@@ -57,21 +57,40 @@ case "$1" in
     APP="$2"; IMAGE="dokku/$APP"
     pluginhook pre-deploy $APP
 
-    # kill the app when running
     if [[ -f "$DOKKU_ROOT/$APP/CONTAINER" ]]; then
       oldid=$(< "$DOKKU_ROOT/$APP/CONTAINER")
-      docker inspect $oldid &> /dev/null && docker kill $oldid > /dev/null
     fi
 
     # start the app
     DOCKER_ARGS=$(: | pluginhook docker-args $APP)
     id=$(docker run -d -p 5000 -e PORT=5000 $DOCKER_ARGS $IMAGE /bin/bash -c "/start web")
-    echo $id > "$DOKKU_ROOT/$APP/CONTAINER"
     port=$(docker port $id 5000 | sed 's/0.0.0.0://')
+
+    # if we can't post-deploy successfully, kill new container
+    function kill_new {
+      docker inspect $id &> /dev/null && docker kill $id > /dev/null
+      trap - INT TERM EXIT
+      kill -9 $$
+    }
+
+    # run checks first, then post-deploy hooks, which switches Nginx traffic
+    trap kill_new INT TERM EXIT
+    echo "-----> Running pre-flight checks"
+    pluginhook check-deploy $id $APP $port
+    echo "-----> Running post-deploy"
+    pluginhook post-deploy  $APP $port
+    trap -        INT TERM EXIT
+
+    # now using the new container
+    echo $id > "$DOKKU_ROOT/$APP/CONTAINER"
     echo $port > "$DOKKU_ROOT/$APP/PORT"
     echo "http://$(< "$DOKKU_ROOT/HOSTNAME"):$port" > "$DOKKU_ROOT/$APP/URL"
 
-    pluginhook post-deploy $APP $port
+    # kill the old container
+    if [[ -n "$oldid" ]]; then
+      docker inspect $oldid &> /dev/null && docker kill $oldid > /dev/null
+    fi
+
     ;;
 
   cleanup)

plugins/checks/check-deploy

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Hook to check server against list of checks specified in CHECKS file.  Each
+# check is a relative path and, optionally, expected content.
+#
+# For example:
+#   /                       My Amazing App
+#   /stylesheets/index.css  .body
+#   /scripts/index.js       $(function()
+#   /images/logo.png
+#
+# Waits 5 seconds, giving server time to start, before running the checks.  For
+# shorter/longer wait, change the DOKKU_CHECKS_WAIT environment variable (value
+# in seconds).
+
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+CONTAINERID="$1"; APP="$2"; PORT="$3" ; HOSTNAME="${4:-localhost}"
+
+# source in app env to get DOKKU_CHECKS_WAIT and any other necessary vars
+[[ -f "$DOKKU_ROOT/$APP/ENV" ]] && source $DOKKU_ROOT/$APP/ENV
+# echo "DOKKU_CHECKS_WAIT is $DOKKU_CHECKS_WAIT"
+FILENAME="$DOKKU_ROOT/$APP/CHECKS"
+WAIT="${DOKKU_CHECKS_WAIT:-5}"
+
+# try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist
+# docker cp exits with status 1 when run as non-root user when it tries to chown the file
+# after successfully copying the file. Thus, we suppress stderr.
+# ref: https://github.com/dotcloud/docker/issues/3986
+if [[ ! -f "$FILENAME" ]] ; then
+    echo "       check-deploy: $FILENAME not found. attempting to retrieve it from container ..."
+    TMPDIR=$(mktemp -d /tmp/CHECKS.XXXXX)
+    docker cp $CONTAINERID:/app/CHECKS $TMPDIR 2> /dev/null || true
+    if [[ ! -s "${TMPDIR}/CHECKS" ]] ; then
+        echo "       CHECKS file not found in container. skipping checks."
+        rm -rf $TMPDIR
+        exit 0
+    else
+        echo "       CHECKS file found in container"
+        FILENAME=${TMPDIR}/CHECKS
+
+        function cleanup() {
+          echo "       removing CHECKS file copied from container"
+          rm -rf $TMPDIR
+        }
+        trap cleanup EXIT
+    fi
+fi
+
+echo "Waiting $WAIT seconds ..."
+sleep $WAIT
+
+# -q           do not use .curlrc (must come first)
+# --compressed Test compression handled correctly
+# --fail       Fail on server errors (4xx, 5xx)
+# --location   Follow redirects
+CURL_OPTIONS="-q --compressed --fail --location --max-time 30"
+
+cat "$FILENAME" | while read PATHNAME EXPECTED ; do
+  # Ignore empty lines and lines starting with #
+  [[ -z "$PATHNAME" || "$PATHNAME" =~ ^\# ]] && continue
+
+  URL="http://$HOSTNAME:$PORT$PATHNAME"
+
+  echo "checking with: curl $CURL_OPTIONS $URL"
+  HTML=$(curl $CURL_OPTIONS $URL)
+  if [[ -n "$EXPECTED" && ! "$HTML" =~ "$EXPECTED" ]] ; then
+    echo -e "\033[31m\033[1m$URL: expected to but did not find: \"$EXPECTED\"\033[0m"
+    exit 1
+  else
+    echo -e "\033[32m\033[1m$URL => \"$EXPECTED\"\033[0m"
+  fi
+done
+
+echo -e "\033[32m\033[1mAll checks successful!\033[0m"

plugins/nginx-vhosts/post-deploy

@@ -84,6 +84,8 @@ EOF
 
   echo "http://$hostname" > "$DOKKU_ROOT/$APP/URL"
   fi
-  pluginhook nginx-pre-reload $APP
+  echo "-----> Running nginx-pre-reload"
+  pluginhook nginx-pre-reload $APP $PORT
+  echo "       Reloading nginx"
   sudo /etc/init.d/nginx reload > /dev/null
 fi