From ab8eb5a727d2c77c017f49c95bbd873203cab06e Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Wed, 16 Sep 2015 23:47:33 -0700
Subject: [PATCH] Handle crashing containers by using restart=on-failure policy

Previously a crashed container would stay down, regardless of exit status. In some cases, it may be useful to restart the container. For example, an application may not be correctly implementing their error handling, or the crash may be caused by a transient error.

By setting the restart policy to `on-failure:N` - where N is a number of max restarts - we can help developers guard against crashing applications. Note that this is not a replacement for proper error handling, nor does this include notifications to a developer when a container is restarted. Those patterns should be implemented application side, or via a feature request to docker.

The value is configurable at the app-level by setting DOKKU_RESTART_LIMIT to a number. By default, containers will be restarted a max of 10 times. If a container crashes during the check-deploy plugin trigger, then the deploy will be marked as a failure.

Closes #216
Closes #398
Closes #1327
---
 dokku                       | 4 ++++
 plugins/checks/check-deploy | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/dokku b/dokku
index d335e8508..e9082ce7b 100755
--- a/dokku
+++ b/dokku
@@ -77,6 +77,9 @@ case "$1" in
     DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE"
     oldids=$(get_app_container_ids $APP)
 
+    DOKKU_RESTART_LIMIT=$(dokku config:get $APP DOKKU_RESTART_LIMIT || echo 10)
+    DOKKU_RESTART_LIMIT=${DOKKU_RESTART_LIMIT:=10}
+
     while read line || [[ -n "$line" ]]; do
       TRIM=${line%#*}
       PROC_TYPE=${TRIM%%=*}
@@ -91,6 +94,7 @@ case "$1" in
 
         # start the app
         DOCKER_ARGS=" -e DYNO='$PROC_TYPE.$CONTAINER_INDEX' "
+        DOCKER_ARGS=" --restart=on-failure:$DOKKU_RESTART_LIMIT"
         DOCKER_ARGS+=$(: | plugn trigger docker-args-deploy $APP $IMAGE_TAG)
         [[ "$DOKKU_TRACE" ]] && DOCKER_ARGS+=" -e TRACE=true "
         BIND_EXTERNAL=$(plugn trigger bind-external-ip $APP)
diff --git a/plugins/checks/check-deploy b/plugins/checks/check-deploy
index ef159a99d..6b3713852 100755
--- a/plugins/checks/check-deploy
+++ b/plugins/checks/check-deploy
@@ -97,6 +97,12 @@ if [[ ! -s "${TMPDIR}/CHECKS" ]] || [[ "$DOKKU_APP_CONTAINER_TYPE" != "web" ]];
   sleep $DOKKU_DEFAULT_CHECKS_WAIT
 
   ! (is_container_running $DOKKU_APP_CONTAINER_ID) && dokku_log_fail "App container failed to start!!"
+  container_restarts=$(docker inspect -f "{{ .RestartCount }}" $DOKKU_APP_CONTAINER_ID)
+  if [[ $container_restarts -ne 0 ]]; then
+    docker stop "$DOKKU_APP_CONTAINER_ID" || true
+    dokku_log_fail "App container failed to start!!"
+  fi
+
   dokku_log_info1 "Default container check successful!" && exit 0
 fi