From 00ec004871c3c8bc1dfdc496c3aefbf0922edabe Mon Sep 17 00:00:00 2001 From: Assaf Arkin Date: Mon, 28 Apr 2014 21:28:50 -0700 Subject: [PATCH] Zero down-time deploy and server checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change makes Dokku start up the new container, run a set of checks against it, and only switch traffic over to the new containers if all checks complete successfully. No requests are dropped during the switch over. To specify checks, add a CHECKS file to the root of your project directory. This is a text file with one line per check. Empty lines and lines starting with # are ignored. A check is a relative URL and may be followed by expected content from the page, for example: /about Our Amazing Team Even if you don’t use any checks, this change will prevent downtime during switching from old to new container. See: https://labnotes.org/zero-downtime-deploy-with-dokku/ --- dokku | 27 ++++++++++-- plugins/checks/check-deploy | 74 ++++++++++++++++++++++++++++++++ plugins/nginx-vhosts/post-deploy | 4 +- 3 files changed, 100 insertions(+), 5 deletions(-) create mode 100755 plugins/checks/check-deploy diff --git a/dokku b/dokku index e0b61ba63..dd445a9ba 100755 --- a/dokku +++ b/dokku @@ -57,21 +57,40 @@ case "$1" in APP="$2"; IMAGE="dokku/$APP" pluginhook pre-deploy $APP - # kill the app when running if [[ -f "$DOKKU_ROOT/$APP/CONTAINER" ]]; then oldid=$(< "$DOKKU_ROOT/$APP/CONTAINER") - docker inspect $oldid &> /dev/null && docker kill $oldid > /dev/null fi # start the app DOCKER_ARGS=$(: | pluginhook docker-args $APP) id=$(docker run -d -p 5000 -e PORT=5000 $DOCKER_ARGS $IMAGE /bin/bash -c "/start web") - echo $id > "$DOKKU_ROOT/$APP/CONTAINER" port=$(docker port $id 5000 | sed 's/0.0.0.0://') + + # if we can't post-deploy successfully, kill new container + function kill_new { + docker inspect $id &> /dev/null && docker kill $id > /dev/null + trap - INT TERM EXIT + kill -9 $$ + } + + # run checks first, then post-deploy hooks, which switches Nginx traffic + trap kill_new INT TERM EXIT + echo "-----> Running pre-flight checks" + pluginhook check-deploy $id $APP $port + echo "-----> Running post-deploy" + pluginhook post-deploy $APP $port + trap - INT TERM EXIT + + # now using the new container + echo $id > "$DOKKU_ROOT/$APP/CONTAINER" echo $port > "$DOKKU_ROOT/$APP/PORT" echo "http://$(< "$DOKKU_ROOT/HOSTNAME"):$port" > "$DOKKU_ROOT/$APP/URL" - pluginhook post-deploy $APP $port + # kill the old container + if [[ -n "$oldid" ]]; then + docker inspect $oldid &> /dev/null && docker kill $oldid > /dev/null + fi + ;; cleanup) diff --git a/plugins/checks/check-deploy b/plugins/checks/check-deploy new file mode 100755 index 000000000..6db3f6444 --- /dev/null +++ b/plugins/checks/check-deploy @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Hook to check server against list of checks specified in CHECKS file. Each +# check is a relative path and, optionally, expected content. +# +# For example: +# / My Amazing App +# /stylesheets/index.css .body +# /scripts/index.js $(function() +# /images/logo.png +# +# Waits 5 seconds, giving server time to start, before running the checks. For +# shorter/longer wait, change the DOKKU_CHECKS_WAIT environment variable (value +# in seconds). + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +CONTAINERID="$1"; APP="$2"; PORT="$3" ; HOSTNAME="${4:-localhost}" + +# source in app env to get DOKKU_CHECKS_WAIT and any other necessary vars +[[ -f "$DOKKU_ROOT/$APP/ENV" ]] && source $DOKKU_ROOT/$APP/ENV +# echo "DOKKU_CHECKS_WAIT is $DOKKU_CHECKS_WAIT" +FILENAME="$DOKKU_ROOT/$APP/CHECKS" +WAIT="${DOKKU_CHECKS_WAIT:-5}" + +# try to copy CHECKS from container if not in APP dir & quit gracefully if it doesn't exist +# docker cp exits with status 1 when run as non-root user when it tries to chown the file +# after successfully copying the file. Thus, we suppress stderr. +# ref: https://github.com/dotcloud/docker/issues/3986 +if [[ ! -f "$FILENAME" ]] ; then + echo " check-deploy: $FILENAME not found. attempting to retrieve it from container ..." + TMPDIR=$(mktemp -d /tmp/CHECKS.XXXXX) + docker cp $CONTAINERID:/app/CHECKS $TMPDIR 2> /dev/null || true + if [[ ! -s "${TMPDIR}/CHECKS" ]] ; then + echo " CHECKS file not found in container. skipping checks." + rm -rf $TMPDIR + exit 0 + else + echo " CHECKS file found in container" + FILENAME=${TMPDIR}/CHECKS + + function cleanup() { + echo " removing CHECKS file copied from container" + rm -rf $TMPDIR + } + trap cleanup EXIT + fi +fi + +echo "Waiting $WAIT seconds ..." +sleep $WAIT + +# -q do not use .curlrc (must come first) +# --compressed Test compression handled correctly +# --fail Fail on server errors (4xx, 5xx) +# --location Follow redirects +CURL_OPTIONS="-q --compressed --fail --location --max-time 30" + +cat "$FILENAME" | while read PATHNAME EXPECTED ; do + # Ignore empty lines and lines starting with # + [[ -z "$PATHNAME" || "$PATHNAME" =~ ^\# ]] && continue + + URL="http://$HOSTNAME:$PORT$PATHNAME" + + echo "checking with: curl $CURL_OPTIONS $URL" + HTML=$(curl $CURL_OPTIONS $URL) + if [[ -n "$EXPECTED" && ! "$HTML" =~ "$EXPECTED" ]] ; then + echo -e "\033[31m\033[1m$URL: expected to but did not find: \"$EXPECTED\"\033[0m" + exit 1 + else + echo -e "\033[32m\033[1m$URL => \"$EXPECTED\"\033[0m" + fi +done + +echo -e "\033[32m\033[1mAll checks successful!\033[0m" diff --git a/plugins/nginx-vhosts/post-deploy b/plugins/nginx-vhosts/post-deploy index be7fd2007..1e0b1e7fe 100755 --- a/plugins/nginx-vhosts/post-deploy +++ b/plugins/nginx-vhosts/post-deploy @@ -84,6 +84,8 @@ EOF echo "http://$hostname" > "$DOKKU_ROOT/$APP/URL" fi - pluginhook nginx-pre-reload $APP + echo "-----> Running nginx-pre-reload" + pluginhook nginx-pre-reload $APP $PORT + echo " Reloading nginx" sudo /etc/init.d/nginx reload > /dev/null fi