Merge pull request #3236 from dokku/retire-old-containers

Retire old containers

plugins/00_dokku-standard/install

@@ -9,7 +9,7 @@ if [[ ! -f  "$DOKKU_ROOT/VHOST" ]]; then
   [[ $(dig +short "$(< "$DOKKU_ROOT/HOSTNAME")") ]] && cp "$DOKKU_ROOT/HOSTNAME" "$DOKKU_ROOT/VHOST"
 fi
 
-dokku_path=$(which dokku)
+dokku_path="$(which dokku)"
 
 # temporary hack for https://github.com/dokku/dokku/issues/82
 # redeploys all apps after a reboot

plugins/common/functions

@@ -943,15 +943,30 @@ acquire_app_deploy_lock() {
   declare desc="acquire advisory lock for use in git/tar deploys"
   local APP="$1"; verify_app_name "$APP"
   local LOCK_TYPE="${2:-waiting}"
-  local APP_DEPLOY_LOCK_FD="200"
   local APP_DEPLOY_LOCK_FILE="$DOKKU_ROOT/$APP/.deploy.lock"
   local LOCK_WAITING_MSG="$APP currently has a deploy lock in place. Waiting..."
   local LOCK_FAILED_MSG="$APP currently has a deploy lock in place. Exiting..."
+
+  acquire_advisory_lock "$APP_DEPLOY_LOCK_FILE" "$LOCK_TYPE" "$LOCK_WAITING_MSG" "$LOCK_FAILED_MSG"
+}
+
+release_app_deploy_lock() {
+  declare desc="release advisory lock used in git/tar deploys"
+  local APP="$1"; verify_app_name "$APP"
+  local APP_DEPLOY_LOCK_FILE="$DOKKU_ROOT/$APP/.deploy.lock"
+
+  release_advisory_lock "$APP_DEPLOY_LOCK_FILE"
+}
+
+acquire_advisory_lock() {
+  declare desc="acquire advisory lock"
+  local LOCK_FILE="$1" LOCK_TYPE="$2" LOCK_WAITING_MSG="$3" LOCK_FAILED_MSG="$4"
+  local LOCK_FD="200"
   local SHOW_MSG=true
 
-  eval "exec $APP_DEPLOY_LOCK_FD>$APP_DEPLOY_LOCK_FILE"
+  eval "exec $LOCK_FD>$LOCK_FILE"
   if [[ "$LOCK_TYPE" == "waiting" ]]; then
-    while [[ $(flock -n "$APP_DEPLOY_LOCK_FD" &>/dev/null ; echo $?) -ne 0 ]]; do
+    while [[ $(flock -n "$LOCK_FD" &>/dev/null ; echo $?) -ne 0 ]]; do
       if [[ "$SHOW_MSG" == "true" ]]; then
         echo "$LOCK_WAITING_MSG"
         SHOW_MSG=false
@@ -959,17 +974,16 @@ acquire_app_deploy_lock() {
       sleep 1
     done
   else
-    flock -n "$APP_DEPLOY_LOCK_FD" &>/dev/null || dokku_log_fail "$LOCK_FAILED_MSG"
+    flock -n "$LOCK_FD" &>/dev/null || dokku_log_fail "$LOCK_FAILED_MSG"
   fi
 }
 
-release_app_deploy_lock() {
-  declare desc="release advisory lock used in git/tar deploys"
-  local APP="$1"; verify_app_name "$APP"
-  local APP_DEPLOY_LOCK_FD="200"
-  local APP_DEPLOY_LOCK_FILE="$DOKKU_ROOT/$APP/.deploy.lock"
+release_advisory_lock() {
+  declare desc="release advisory lock"
+  local LOCK_FILE="$1"
+  local LOCK_FD="200"
 
-  flock -u "$APP_DEPLOY_LOCK_FD" && rm -f "$APP_DEPLOY_LOCK_FILE" &> /dev/null
+  flock -u "$LOCK_FD" && rm -f "$LOCK_FILE" &> /dev/null
 }
 
 suppress_output() {

plugins/common/properties.go

@@ -144,13 +144,6 @@ func setPermissions(path string, fileMode os.FileMode) (err error) {
 
 	systemGroup := os.Getenv("DOKKU_SYSTEM_GROUP")
 	systemUser := os.Getenv("DOKKU_SYSTEM_USER")
-	if systemGroup == "" {
-		systemGroup = "dokku"
-	}
-	if systemUser == "" {
-		systemUser = "dokku"
-	}
-
 	group, err := user.LookupGroup(systemGroup)
 	if err != nil {
 		return

plugins/ps/install

@@ -4,11 +4,14 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
 source "$PLUGIN_AVAILABLE_PATH/docker-options/functions"
 source "$PLUGIN_AVAILABLE_PATH/ps/functions"
 
-set_default_restart_policies() {
+trigger-ps-install() {
   declare desc="set the default restart policy for all applications if there is not one already set"
   local APPS="$(dokku_apps)"
   local APP
 
+  mkdir -p "${DOKKU_LIB_ROOT}/data/ps"
+  chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/ps"
+
   for APP in $APPS; do
     local RESTART_POLICIES=$(get_restart_policies "$(get_phase_file_path "deploy")")
     if [[ -z "$RESTART_POLICIES" ]]; then
@@ -18,4 +21,4 @@ set_default_restart_policies() {
   done
 }
 
-set_default_restart_policies "$@"
+trigger-ps-install "$@"

plugins/ps/subcommands/retire

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_AVAILABLE_PATH/config/functions"
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+
+ps_retire_cmd() {
+  declare desc="ensures old containers are retired"
+  local cmd="ps:retire"
+  local LOCK_FILE="${DOKKU_LIB_ROOT}/data/ps/retire"
+  local DOKKU_SCHEDULER=$(config_get "--global" DOKKU_SCHEDULER || echo "docker-local")
+
+  acquire_advisory_lock "$LOCK_FILE" "exclusive" "" "Failed to acquire ps:retire lock"
+  plugn trigger scheduler-retire "$DOKKU_SCHEDULER"
+  release_advisory_lock "$LOCK_FILE"
+}
+
+ps_retire_cmd "$@"

plugins/scheduler-docker-local/install

@@ -1,15 +1,54 @@
 #!/usr/bin/env bash
 set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
 source "$PLUGIN_CORE_AVAILABLE_PATH/common/property-functions"
 
 scheduler-docker-local-install() {
   declare desc="scheduler-docker-local install plugin trigger"
   declare trigger="scheduler-docker-local-install"
+  local DOKKU_PATH
 
   mkdir -p "${DOKKU_LIB_ROOT}/data/scheduler-docker-local"
   chown -R "${DOKKU_SYSTEM_USER}:${DOKKU_SYSTEM_GROUP}" "${DOKKU_LIB_ROOT}/data/scheduler-docker-local"
 
   fn-plugin-property-setup "scheduler-docker-local"
+
+  DOKKU_PATH="$(which dokku)"
+
+if [[ $(systemctl 2> /dev/null) =~ -\.mount ]]; then
+  cat<<EOF > /etc/systemd/system/dokku-retire.service
+[Unit]
+Description=Dokku retire service
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+User=$DOKKU_SYSTEM_USER
+ExecStart=$DOKKU_PATH ps:restore
+
+[Install]
+WantedBy=docker.service
+EOF
+
+  cat<<EOF > /etc/systemd/system/dokku-retire.timer
+[Unit]
+Description=Run dokku-retire.service every 2 minutes
+
+[Timer]
+OnCalendar=*:0/2
+EOF
+  if command -v systemctl &>/dev/null; then
+    systemctl reenable dokku-redeploy
+  fi
+else
+  cat<<EOF > /etc/cron.d/dokku-retire
+PATH=/usr/local/bin:/usr/bin:/bin
+SHELL=/bin/bash
+
+2 * * * * $DOKKU_SYSTEM_USER $DOKKU_PATH ps:retire
+EOF
+fi
 }
 
 scheduler-docker-local-install "$@"

plugins/scheduler-docker-local/internal-functions

@@ -87,3 +87,36 @@ cmd-scheduler-docker-local-help() {
 help_desc
   fi
 }
+
+fn-scheduler-docker-local-retire-container() {
+  declare APP="$1" CID="$2" DEAD_TIME="$3"
+
+  if ! docker inspect "${CID}" > /dev/null; then
+    return
+  fi
+
+  DOKKU_DOCKER_STOP_TIMEOUT="$(config_get "$APP" DOKKU_DOCKER_STOP_TIMEOUT || true)"
+  [[ $DOKKU_DOCKER_STOP_TIMEOUT ]] && DOCKER_STOP_TIME_ARG="--time=${DOKKU_DOCKER_STOP_TIMEOUT}"
+
+  # Attempt to stop, if that fails, then force a kill as docker seems
+  # to not send SIGKILL as the docs would indicate. If that fails, move
+  # on to the next.
+  # shellcheck disable=SC2086
+  docker stop $DOCKER_STOP_TIME_ARG "$oldid" \
+  || docker kill "$oldid" \
+  || dokku_log_warn "Unable to kill container ${CID}"
+
+  if ! docker kill "$CID"; then
+    dokku_log_warn "Unable to kill container ${CID}"
+  fi
+}
+
+fn-scheduler-docker-local-register-retired-container() {
+  declare APP="$1" CID="$2" WAIT="$3"
+  local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
+  local CURRENT_TIME DEAD_TIME
+
+  CURRENT_TIME="$(date +%s)"
+  DEAD_TIME=$((CURRENT_TIME + WAIT))
+  echo "${APP} ${CID} ${DEAD_TIME}" >> "${DEAD_CONTAINER_FILE}"
+}

plugins/scheduler-docker-local/scheduler-deploy

@@ -4,6 +4,7 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
 source "$PLUGIN_AVAILABLE_PATH/checks/functions"
 source "$PLUGIN_AVAILABLE_PATH/config/functions"
 source "$PLUGIN_AVAILABLE_PATH/ps/functions"
+source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions"
 
 scheduler-docker-local-scheduler-deploy() {
   declare desc="deploys an image tag for a given application"
@@ -155,6 +156,7 @@ scheduler-docker-local-scheduler-deploy() {
     local oldid
     for oldid in $oldids; do
       dokku_log_info2 "$oldid"
+      fn-scheduler-docker-local-register-retired-container "$APP" "$oldid" "$WAIT"
     done
     (
       exec >/dev/null 2>/dev/null </dev/null

plugins/scheduler-docker-local/scheduler-retire

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions"
+
+scheduler-docker-local-scheduler-retire() {
+  declare desc="retires all old containers once they have aged out"
+  declare trigger="scheduler-docker-local scheduler-retire"
+
+  local DEAD_CONTAINER_FILE="${DOKKU_LIB_ROOT}/data/scheduler-docker-local/dead-containers"
+  local APP CID CURRENT_TIME DEAD_TIME
+
+  DEAD_CONTAINERS=()
+  while read line; do
+    CURRENT_TIME="$(date +%s)"
+    APP="$(echo "$line" | cut -d ' ' -f1)"
+    CID="$(echo "$line" | cut -d ' ' -f2)"
+    DEAD_TIME="$(echo "$line" | cut -d ' ' -f3)"
+
+    if [[ "$CURRENT_TIME" -le "$DEAD_TIME" ]]; then
+      continue
+    fi
+
+    fn-scheduler-docker-local-retire-container "$APP" "$CID" "$DEAD_TIME"
+    if docker ps -aq -f "id=${CID}" -f "status=running" > /dev/null 2>&1; then
+      dokku_log_warn "Container ${CID} still running"
+      continue
+    fi
+
+    docker rm -f "$CID" > /dev/null 2>&1 || true
+    if docker inspect "${CID}" > /dev/null 2>&1; then
+      dokku_log_warn "Container ${CID} still running"
+      continue
+    fi
+
+    DEAD_CONTAINERS+=("$CID")
+  done < "$DEAD_CONTAINER_FILE"
+
+  for CID in "${DEAD_CONTAINERS[@]}"; do
+    sed -i "/${CID}/d" "$DEAD_CONTAINER_FILE"
+  done
+}
+
+scheduler-docker-local-scheduler-retire "$@"