test: add scheduler-k3s run parity tests across builders

Extends bats coverage for the scheduler-k3s scheduler so that
herokuish and dockerfile builders match the cnb test surface for
`dokku run`, `dokku run:detached`, `dokku cron:run`, deployment
manifests, cronjob manifests, and Procfile-key resolution. Adds
the corresponding `app-cron-procfile.json` fixture for the python
app and `app-cron.json` / `app-cron-procfile.json` fixtures for
the dockerfile-procfile app.

tests/apps/dockerfile-procfile/app-cron-procfile.json

@@ -0,0 +1,21 @@
+{
+  "cron": [
+    {
+      "command": "release",
+      "schedule": "5 5 5 5 5"
+    }
+  ],
+  "healthchecks": {
+    "web": [
+      {
+        "attempts": 2,
+        "content": "nodejs/express",
+        "name": "check-1",
+        "path": "/",
+        "timeout": 5,
+        "type": "startup",
+        "wait": 2
+      }
+    ]
+  }
+}

tests/apps/dockerfile-procfile/app-cron.json

@@ -0,0 +1,27 @@
+{
+  "cron": [
+    {
+      "command": "echo hello-from-cron",
+      "schedule": "5 5 5 5 5"
+    }
+  ],
+  "healthchecks": {
+    "web": [
+      {
+        "attempts": 2,
+        "content": "nodejs/express",
+        "name": "check-1",
+        "path": "/",
+        "timeout": 5,
+        "type": "startup",
+        "wait": 2
+      }
+    ]
+  },
+  "scripts": {
+    "dokku": {
+      "postdeploy": "touch /app/postdeploy.test",
+      "predeploy": "touch /app/predeploy.test"
+    }
+  }
+}

tests/apps/python/app-cron-procfile.json

@@ -0,0 +1,21 @@
+{
+  "cron": [
+    {
+      "command": "task",
+      "schedule": "5 5 5 5 5"
+    }
+  ],
+  "healthchecks": {
+    "web": [
+      {
+        "attempts": 2,
+        "content": "python/http.server",
+        "name": "check-1",
+        "path": "/",
+        "timeout": 5,
+        "type": "startup",
+        "wait": 2
+      }
+    ]
+  }
+}

tests/unit/scheduler-k3s-cnb.bats

@@ -199,6 +199,61 @@ teardown() {
   assert_output_contains "['task.py', 'test']"
 }
 
+@test "(scheduler-k3s) cnb dokku run:detached returns pod name with launcher entrypoint" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku builder:set $TEST_APP selected pack"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt_cnb
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains 'from cnb stack'
+
+  run /bin/bash -c "dokku --quiet run:detached $TEST_APP sleep 300"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+  pod_name="$output"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.labels.app\.kubernetes\.io/part-of}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$TEST_APP"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.annotations.dokku\.com/builder-type}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "pack"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].command[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "launcher"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "sleep"
+}
+
 cron_run_wrapper() {
   local APP="$1"
   local APP_REPO_DIR="$2"

tests/unit/scheduler-k3s-dockerfile.bats

@@ -0,0 +1,240 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+}
+
+@test "(scheduler-k3s) dockerfile deployment sets command for non-web process" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:scale $TEST_APP web=1 worker=1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].command}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "node"
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[1]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "worker.js"
+}
+
+@test "(scheduler-k3s) dockerfile cronjob manifest sets command" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_wrapper
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].command}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "echo"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "hello-from-cron"
+}
+
+@test "(scheduler-k3s) dockerfile dokku run executes command" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP echo hello-from-run"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "hello-from-run"
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP env"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "SECRET_KEY=fjdkslafjdk"
+}
+
+@test "(scheduler-k3s) dockerfile dokku run resolves Procfile key" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app dockerfile-procfile dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_procfile_wrapper
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP release"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "SECRET_KEY: fjdkslafjdk"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "SECRET_KEY: fjdkslafjdk"
+}
+
+@test "(scheduler-k3s) dockerfile dokku run:detached returns pod name" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app dockerfile dokku@$DOKKU_DOMAIN:$TEST_APP
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run:detached $TEST_APP sleep 300"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+  pod_name="$output"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.labels.app\.kubernetes\.io/part-of}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$TEST_APP"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.annotations.dokku\.com/builder-type}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "dockerfile"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].command}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+}
+
+cron_run_wrapper() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  mv -f "$APP_REPO_DIR/app-cron.json" "$APP_REPO_DIR/app.json"
+}
+
+cron_run_procfile_wrapper() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  mv -f "$APP_REPO_DIR/app-cron-procfile.json" "$APP_REPO_DIR/app.json"
+}

tests/unit/scheduler-k3s-herokuish.bats

@@ -0,0 +1,247 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+}
+
+@test "(scheduler-k3s) herokuish deployment sets start command for non-web process" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:scale $TEST_APP web=1 worker=1"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].command}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "/start"
+
+  run /bin/bash -c "kubectl get deployment $TEST_APP-worker -o=jsonpath='{.spec.template.spec.containers[0].args[1]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "worker"
+}
+
+@test "(scheduler-k3s) herokuish cronjob manifest sets start command" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_wrapper
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].command}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output ""
+
+  run /bin/bash -c "kubectl get cronjob -o=jsonpath='{.items[0].spec.jobTemplate.spec.template.spec.containers[0].args[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "python3"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'some', 'cron', 'task']"
+}
+
+@test "(scheduler-k3s) herokuish dokku run uses /exec entrypoint" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP python3 task.py test"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'test']"
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP env"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "SECRET_KEY=fjdkslafjdk"
+}
+
+@test "(scheduler-k3s) herokuish dokku run resolves Procfile key" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP SECRET_KEY=fjdkslafjdk"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP cron_run_procfile_wrapper
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run $TEST_APP task"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'test']"
+
+  cron_id="$(dokku cron:list $TEST_APP --format json | jq -r '.[0].id')"
+  run /bin/bash -c "echo $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+
+  run /bin/bash -c "dokku --quiet cron:run $TEST_APP $cron_id"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "['task.py', 'test']"
+}
+
+@test "(scheduler-k3s) herokuish dokku run:detached returns pod name" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP add_requirements_txt
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku --quiet run:detached $TEST_APP sleep 300"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_exists
+  pod_name="$output"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.labels.app\.kubernetes\.io/part-of}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "$TEST_APP"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.metadata.annotations.dokku\.com/builder-type}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "herokuish"
+
+  run /bin/bash -c "kubectl get pod $pod_name -o=jsonpath='{.spec.containers[0].command[0]}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "/exec"
+}
+
+cron_run_wrapper() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  add_requirements_txt "$APP" "$APP_REPO_DIR"
+  mv -f "$APP_REPO_DIR/app-cron.json" "$APP_REPO_DIR/app.json"
+}
+
+cron_run_procfile_wrapper() {
+  local APP="$1"
+  local APP_REPO_DIR="$2"
+  [[ -z "$APP" ]] && local APP="$TEST_APP"
+  APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+
+  add_requirements_txt "$APP" "$APP_REPO_DIR"
+  mv -f "$APP_REPO_DIR/app-cron-procfile.json" "$APP_REPO_DIR/app.json"
+}