feat: implement cron and one-off run task TTLs

Closes #7938

docs/development/plugin-triggers.md

@@ -2805,6 +2805,26 @@ DOKKU_SCHEDULER="$1"; APP="$2"; CONTAINER="$3"; TAIL="$4"; PRETTY_PRINT="$5"; NU
 # TODO
 ```
 
+### `scheduler-run-retire`
+
+> [!WARNING]
+> The scheduler plugin trigger apis are under development and may change
+> between minor releases until the 1.0 release.
+
+- Description: Allows you to retire containers started by the `run` command
+- Invoked by: `dokku run:retire`
+- Arguments: `$DOKKU_SCHEDULER $APP`
+- Example:
+
+```shell
+#!/usr/bin/env bash
+
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+DOKKU_SCHEDULER="$1"; APP="$2";
+
+# TODO
+```
+
 ### `scheduler-run-stop`
 
 > [!WARNING]

docs/processes/one-off-tasks.md

@@ -1,8 +1,8 @@
 # One-off Tasks
 
 ```
-run [-e|--env KEY=VALUE] [--no-tty] <app> <cmd>              # Run a command in a new container using the current app image
-run:detached [-e|-env KEY=VALUE] [--force-tty] <app> <cmd>         # Run a command in a new detached container using the current app image
+run [-e|--env KEY=VALUE] [--no-tty] [--ttl-seconds SECONDS] <app> <cmd>              # Run a command in a new container using the current app image
+run:detached [-e|-env KEY=VALUE] [--force-tty] [--ttl-seconds SECONDS] <app> <cmd>         # Run a command in a new detached container using the current app image
 run:list [--format json|stdout] [<app>]                      # List all run containers for an app
 run:logs <app|--container CONTAINER> [-h] [-t] [-n num] [-q] # Display recent log output for run containers
 run:stop <app|--container CONTAINER>                         # Stops all run containers for an app or a specified run container
@@ -29,6 +29,16 @@ dokku run --env "NODE_ENV=development" --env "PATH=/custom/path" node-js-app npm
 
 One off containers are removed at the end of process execution.
 
+In addition, one-off containers also run for a max of 24 hours (86400 seconds) by default, after which they are reaped. This can be changed by setting the `--ttl-seconds` argument:
+
+
+```shell
+# runs for 10 minutes
+dokku run --ttl-seconds 600 node-js-app npm run mytask
+```
+
+One-off containers that exceed their runtime are reaped every 5 minutes, so the ttl-seconds is an approximation and your app may end up running for up to 5 minutes longer than expected.
+
 #### Running Procfile commands
 
 The `run` command can also be used to run a command defined in the app `Procfile`:

docs/processes/scheduled-cron-tasks.md

@@ -43,6 +43,8 @@ A cron task takes the following properties:
 
 Zero or more cron tasks can be specified per app. Cron tasks are validated after the build artifact is created but before the app is deployed, and the cron schedule is updated during the post-deploy phase.
 
+Cron tasks can run for a maximum of 24 hours via the docker-local scheduler, after which they are reaped from the system.
+
 See the [app.json location documentation](/docs/advanced-usage/deployment-tasks.md#changing-the-appjson-location) for more information on where to place your `app.json` file.
 
 #### Task Environment

plugins/ps/subcommands.go

@@ -142,6 +142,19 @@ func CommandRetire(appName string) error {
 		Args:        []string{scheduler, appName},
 		StreamStdio: true,
 	})
+	if err != nil {
+		return fmt.Errorf("Error retiring containers: %w", err)
+	}
+
+	common.LogInfo1("Retiring expired run containers")
+	_, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
+		Trigger:     "scheduler-run-retire",
+		StreamStdio: true,
+	})
+	if err != nil {
+		return fmt.Errorf("Error retiring expired run containers: %w", err)
+	}
+
 	return err
 }
 

plugins/run/help-functions

@@ -27,10 +27,11 @@ help_desc
 fn-help-content() {
   declare desc="return help content"
   cat <<help_content
-    run [-e|--env KEY=VALUE] [--no-tty] <app> <cmd>, Run a command in a new container using the current app image
-    run:detached [-e|-env KEY=VALUE] [--force-tty] <app> <cmd>, Run a command in a new detached container using the current app image
+    run [-e|--env KEY=VALUE] [--no-tty] [--ttl-seconds SECONDS] <app> <cmd>, Run a command in a new container using the current app image
+    run:detached [-e|-env KEY=VALUE] [--force-tty] [--ttl-seconds SECONDS] <app> <cmd>, Run a command in a new detached container using the current app image
     run:list [--format json|stdout] <app>, List all run containers for an app
     run:logs <app|--container CONTAINER> [-h] [-t] [-n num] [-q], Display recent log output for run containers
+    run:retire [<app>], Stop all run containers for an app that have exceeded their active deadline
     run:stop <app|--container CONTAINER>, Stops all run containers for an app or a specified run container
 help_content
 }

plugins/run/internal-functions

@@ -9,7 +9,7 @@ fn-run() {
   shift 1
 
   declare APP=""
-  local CRON_ID CONCURRENCY_POLICY="allow"
+  local CRON_ID CONCURRENCY_POLICY="allow" DOKKU_RUN_TTL_SECONDS="86400"
   declare -a RUN_ENV
   RUN_ENV=()
   while [[ $# -gt 0 ]]; do
@@ -61,6 +61,19 @@ fn-run() {
         RUN_ENV+=("$2")
         shift 2
         ;;
+      --ttl-seconds=*)
+        local arg=$(printf "%s" "$1" | sed -E 's/(^--ttl-seconds=)//g')
+        DOKKU_RUN_TTL_SECONDS="$arg"
+        shift
+        ;;
+      --ttl-seconds)
+        if [[ ! $2 ]]; then
+          dokku_log_warn "expected $1 to have an argument"
+          break
+        fi
+        DOKKU_RUN_TTL_SECONDS="$2"
+        shift 2
+        ;;
       *)
         APP="$1"
         shift
@@ -69,6 +82,10 @@ fn-run() {
     esac
   done
 
+  if [[ -z "$DOKKU_RUN_TTL_SECONDS" ]]; then
+    DOKKU_RUN_TTL_SECONDS="86400"
+  fi
+
   if [[ "$CMD" == "run:detached" ]] && [[ "$DOKKU_FORCE_TTY" != "true" ]]; then
     export DOKKU_DISABLE_TTY=true
   fi
@@ -80,7 +97,7 @@ fn-run() {
   verify_app_name "$APP"
 
   local DOKKU_SCHEDULER=$(get_app_scheduler "$APP")
-  DOKKU_CRON_ID="$CRON_ID" DOKKU_CONCURRENCY_POLICY="$CONCURRENCY_POLICY" plugn trigger scheduler-run "$DOKKU_SCHEDULER" "$APP" "${#RUN_ENV[@]}" "${RUN_ENV[@]}" -- "$@"
+  DOKKU_CRON_ID="$CRON_ID" DOKKU_CONCURRENCY_POLICY="$CONCURRENCY_POLICY" DOKKU_RUN_TTL_SECONDS="$DOKKU_RUN_TTL_SECONDS" plugn trigger scheduler-run "$DOKKU_SCHEDULER" "$APP" "${#RUN_ENV[@]}" "${RUN_ENV[@]}" -- "$@"
 }
 
 cmd-run() {
@@ -124,6 +141,10 @@ cmd-run-list() {
         FORMAT="$2"
         shift 2
         ;;
+      --quiet)
+        export DOKKU_QUIET_OUTPUT=true
+        shift
+        ;;
       *)
         APP="$1"
         shift
@@ -201,8 +222,23 @@ cmd-run-logs() {
   plugn trigger scheduler-run-logs "$DOKKU_SCHEDULER" "$APP" "$CONTAINER_NAME" "$TAIL" "$PRETTY_PRINT" "$NUM"
 }
 
+cmd-run-retire() {
+  declare desc="stop all run containers that have exceeded their active deadline"
+  declare cmd="run:retire"
+  [[ "$1" == "$cmd" ]] && shift 1
+  declare APP="$1"
+
+  if [[ -z "$APP" ]]; then
+    plugn trigger scheduler-run-retire
+  else
+    verify_app_name "$APP"
+    local DOKKU_SCHEDULER="$(get_app_scheduler "$APP")"
+    plugn trigger scheduler-run-retire "$DOKKU_SCHEDULER" "$APP"
+  fi
+}
+
 cmd-run-stop() {
-  declare desc="Stops all run containers for an app or a specified run container"
+  declare desc="stops all run containers for an app or a specified run container"
   declare cmd="run:stop"
   [[ "$1" == "$cmd" ]] && shift 1
   declare APP CONTAINER_NAME

plugins/run/subcommands/retire

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -eo pipefail
+source "$PLUGIN_AVAILABLE_PATH/run/internal-functions"
+[[ $DOKKU_TRACE ]] && set -x
+
+cmd-run-retire "$@"

plugins/scheduler-docker-local/scheduler-run

@@ -119,6 +119,10 @@ trigger-scheduler-docker-local-scheduler-run() {
     DOCKER_ARGS+=" --platform=linux/amd64"
   fi
 
+  if [[ -n "$DOKKU_RUN_TTL_SECONDS" ]]; then
+    DOCKER_ARGS+=" --label=com.dokku.active-deadline-seconds=$DOKKU_RUN_TTL_SECONDS"
+  fi
+
   # shellcheck disable=SC2124
   DOCKER_ARGS+=" --label=com.dokku.container-type=$PROCESS_TYPE"
   DOCKER_ARGS+=" --label=com.dokku.app-name=$APP"

plugins/scheduler-docker-local/scheduler-run-list

@@ -15,7 +15,11 @@ trigger-scheduler-docker-local-scheduler-run-list() {
 
   if [[ "$FORMAT" == "stdout" ]]; then
     dokku_log_info2_quiet "$APP run containers"
-    "$DOCKER_BIN" container ls --all --no-trunc --filter "label=com.dokku.app-name=$APP" --filter "label=com.dokku.container-type=run" --format "table {{.Names}}\t{{.Command}}\t{{.RunningFor}}"
+    if [[ -n "$DOKKU_QUIET_OUTPUT" ]]; then
+      "$DOCKER_BIN" container ls --all --no-trunc --filter "label=com.dokku.app-name=$APP" --filter "label=com.dokku.container-type=run" --format "{{.Names}}\t{{.Command}}\t{{.RunningFor}}"
+    else
+      "$DOCKER_BIN" container ls --all --no-trunc --filter "label=com.dokku.app-name=$APP" --filter "label=com.dokku.container-type=run" --format "table {{.Names}}\t{{.Command}}\t{{.RunningFor}}"
+      fi
   else
     "$DOCKER_BIN" container ls --all --no-trunc --filter "label=com.dokku.app-name=$APP" --filter "label=com.dokku.container-type=run" --format "{{ json . }}" | jq -s -M 'map({name: .Names, state:.State, command:.Command, created_at:.CreatedAt})'
   fi

plugins/scheduler-docker-local/scheduler-run-retire

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+set -eo pipefail
+[[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+source "$PLUGIN_AVAILABLE_PATH/config/functions"
+source "$PLUGIN_AVAILABLE_PATH/scheduler-docker-local/internal-functions"
+
+fn-scheduler-docker-local-run-retire-container() {
+  declare desc="stop a container"
+  declare CONTAINER_ID="$1"
+
+  local DOKKU_DOCKER_STOP_TIMEOUT="$(plugn trigger ps-get-property "$APP" stop-timeout-seconds || true)"
+
+  [[ -n "$DOKKU_DOCKER_STOP_TIMEOUT" ]] && DOCKER_STOP_TIME_ARG="--time=${DOKKU_DOCKER_STOP_TIMEOUT}"
+
+  "$DOCKER_BIN" container update --restart=no "$CONTAINER_ID" &>/dev/null || true
+  "$DOCKER_BIN" container stop $DOCKER_STOP_TIME_ARG "$CONTAINER_ID" &>/dev/null || true
+  "$DOCKER_BIN" container kill "$CONTAINER_ID" &>/dev/null || true
+  "$DOCKER_BIN" container rm "$CONTAINER_ID" &>/dev/null || true
+
+  if "$DOCKER_BIN" container inspect "$CONTAINER_ID" &>/dev/null; then
+    dokku_log_warn "Unable to retire container ${CONTAINER_ID}"
+    return 1
+  fi
+}
+
+fn-scheduler-docker-local-run-retire() {
+  declare desc="stop all run containers that have exceeded their active deadline"
+  declare APP="$1"
+  local containers=""
+
+  # find all run containers for the specified app
+  container_type_filter="label=com.dokku.container-type=run"
+  if [[ -n "$APP" ]]; then
+    app_filter="label=com.dokku.app-name=$APP"
+    containers="$("$DOCKER_BIN" container ls --filter "$container_type_filter" --filter "$app_filter" --format '{{.ID}} {{.CreatedAt}}')"
+  else
+    containers="$("$DOCKER_BIN" container ls --filter "$container_type_filter" --format '{{.ID}} {{.CreatedAt}}')"
+  fi
+
+  if [[ -z "$containers" ]]; then
+    return
+  fi
+
+  # iterate over all containers, ignoring the timezone in the last two columns
+  echo "$containers" | awk '{NF-=2} 1' | while read -r container_id container_date container_time; do
+    # get the cutoff time in seconds from the container's `com.dokku.active-deadline-seconds` label
+    active_deadline_seconds="$("$DOCKER_BIN" container inspect "$container_id" --format '{{ index .Config.Labels "com.dokku.active-deadline-seconds" }}')"
+    if [[ -z "$active_deadline_seconds" ]]; then
+      continue
+    fi
+
+    # convert the active deadline seconds to a unix timestamp
+    cutoff_time="$(date --date="$active_deadline_seconds seconds ago" "+%s")"
+
+    # convert the container start time to unix timestamp
+    start_time="$(date --date="$container_date $container_time" "+%s")"
+
+    # if the container start time is before the cutoff time, stop the container
+    if [[ "$start_time" -lt "$cutoff_time" ]]; then
+      dokku_log_verbose_quiet "Retiring container ${container_id}"
+      fn-scheduler-docker-local-run-retire-container "$container_id"
+    fi
+  done
+}
+
+trigger-scheduler-docker-local-scheduler-run-retire() {
+  declare desc="retires all run containers for an app that have exceeded their active deadline"
+  declare trigger="scheduler-run-retire"
+  declare DOKKU_SCHEDULER="$1" APP="$2"
+
+  if [[ -z "$DOKKU_SCHEDULER" ]]; then
+    dokku_log_info1_quiet "Retiring all run containers"
+    fn-scheduler-docker-local-run-retire
+    return "$?"
+  fi
+
+  if [[ "$DOKKU_SCHEDULER" != "docker-local" ]]; then
+    return
+  fi
+
+  dokku_log_info1_quiet "Retiring run containers for app ${APP}"
+  fn-scheduler-docker-local-run-retire "$APP"
+}
+
+trigger-scheduler-docker-local-scheduler-run-retire "$@"

plugins/scheduler-k3s/templates/chart/cron-job.yaml

@@ -50,6 +50,7 @@ spec:
       backoffLimit: 0
       podReplacementPolicy: Failed
       ttlSecondsAfterFinished: 60
+      activeDeadlineSeconds: 86400
       template:
         metadata:
           annotations:

tests/unit/run_1.bats

@@ -98,3 +98,45 @@ teardown() {
   echo "status: $status"
   assert_success
 }
+
+@test "(run) run:retire" {
+  run deploy_app
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku run:detached --ttl-seconds=1 $TEST_APP sleep 300"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  container_id="$output"
+
+  # check the labels on the container to ensure the active deadline seconds is set
+  run /bin/bash -c "docker container inspect $container_id --format '{{ index .Config.Labels \"com.dokku.active-deadline-seconds\" }}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "1"
+
+  run /bin/bash -c "dokku run:list --quiet $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "$container_id"
+
+  run /bin/bash -c "sleep 2"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku run:retire"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku run:list --quiet $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_not_contains "$container_id"
+}