fix: do not require a local image for k3s deploys

Kubernetes pulls the app image itself, so a k3s host is free to reap its local copy while the workload keeps running, which the `registry` plugin already does on its own. Deploys, restarts, `dokku run`, and in-cluster cron no longer assert that the image is present locally, falling back to the metadata recorded in the app's current Helm release. A `ps:restart` naming a single process type now rolls only that process type's pods rather than silently redeploying every one. Apps with an `app.json` postdeploy task still require the image locally, as that task runs on the Dokku host.

docs/deployment/schedulers/k3s.md

@@ -268,6 +268,20 @@ The global default value may be set by passing an empty value for the option.
 dokku scheduler-k3s:set --global deploy-timeout
 ```
 
+### Restarting apps
+
+A `ps:restart` re-renders the app's Helm chart from its current configuration and upgrades the release, which is how configuration changes are picked up. Pods cycle because each Deployment's pod template carries an `app.kubernetes.io/version` annotation that changes on restart.
+
+A single process type may be targeted, in which case only that process type's pods are replaced. The rest of the release is still upgraded so configuration converges everywhere, but the untargeted Deployments keep their existing annotation and are left running:
+
+```shell
+dokku ps:restart node-js-app web
+```
+
+The app image does not need to be present on the Dokku host. Kubernetes pulls it from the registry, so a host that has reaped its local copy - as the `registry` plugin does on its own once an app has been deployed a number of times - can still restart, scale, and run one-off commands against the app. The builder type and working directory needed to render the chart are read back from the app's current Helm release when the image is unavailable locally.
+
+There is one exception. An `app.json` with a `scripts.dokku.postdeploy` task runs that task in a container on the Dokku host rather than in the cluster, and so does require the image locally. Apps without a postdeploy task are unaffected.
+
 ### Displaying the scheduler report
 
 Configured properties can be inspected with the `scheduler-k3s:report` command. Without arguments, it iterates every app. Passing an app name scopes the report to that app, while `--global` reports the scheduler-wide properties on their own:
@@ -1079,6 +1093,8 @@ This plugin implements various functionality through `plugn` triggers to integra
     - Properties set by the `nginx` plugin will be respected, either by turning them into annotations or creating a custom server/location snippet that the `ingress-nginx` project can use. A `ps:restart` after changing any nginx properties is required in order to have them apply.
     - The `nginx:access-logs` and `nginx:error-logs` commands will fetch logs from one running `ingress-nginx` pod.
     - The `nginx:show-config` command will retrieve any `server` blocks associated with a domain attached to the app from one running `ingress-nginx` pod.
+- `ps:restart`
+    - Supports targeting a single process type, see [Restarting apps](#restarting-apps)
 - `ps:stop`
 - `run`
     - The `scheduler-post-run` trigger is not always triggered

docs/development/plugin-triggers.md

@@ -2562,16 +2562,16 @@ source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
 > The scheduler plugin trigger apis are under development and may change
 > between minor releases until the 1.0 release.
 
-- Description: Allows you to run scheduler commands when an app is deployed
+- Description: Allows you to run scheduler commands when an app is deployed. `$PROCESS_TYPE` is empty for a normal deploy and set when a single process type is targeted, as by `dokku ps:restart <app> <process-type>`, in which case only that process type should be redeployed.
 - Invoked by: `dokku deploy`
-- Arguments: `$DOKKU_SCHEDULER $APP $IMAGE_TAG`
+- Arguments: `$DOKKU_SCHEDULER $APP $IMAGE_TAG $PROCESS_TYPE`
 - Example:
 
 ```shell
 #!/usr/bin/env bash
 
 set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
-DOKKU_SCHEDULER="$1"; APP="$2"; IMAGE_TAG="$3";
+DOKKU_SCHEDULER="$1"; APP="$2"; IMAGE_TAG="$3"; PROCESS_TYPE="$4";
 
 # TODO
 ```

plugins/app-json/triggers.go

@@ -186,7 +186,20 @@ func TriggerPostDelete(appName string) error {
 }
 
 // TriggerPostDeploy is a trigger to execute the postdeploy deployment task
+//
+// The task is looked up before the image name is resolved because resolving it
+// asserts the image exists on the local docker daemon. That assertion does not
+// hold for schedulers running workloads off this host, which are free to reap
+// the local copy while the app keeps running in a cluster. An app with no
+// postdeploy task needs no image at all, so it should not be made to fail here.
 func TriggerPostDeploy(appName string, imageTag string) error {
+	command, err := getPhaseScript(appName, "postdeploy")
+	if err == nil && command == "" {
+		common.LogInfo1("Checking for postdeploy task")
+		common.LogVerbose("No postdeploy task found, skipping")
+		return nil
+	}
+
 	image, err := common.GetDeployingAppImageName(appName, imageTag, "")
 	if err != nil {
 		return err

plugins/common/common.go

@@ -468,8 +468,27 @@ func GetGlobalScheduler() string {
 	return "docker-local"
 }
 
-// GetDeployingAppImageName returns deploying image identifier for a given app, tag tuple. validate if tag is presented
+// GetDeployingAppImageName returns deploying image identifier for a given app, tag tuple,
+// erroring when the image is not present on the local docker daemon. Callers that hand the
+// image off to a remote runtime should use ResolveDeployingAppImageName instead.
 func GetDeployingAppImageName(appName, imageTag, imageRepo string) (string, error) {
+	imageName, err := ResolveDeployingAppImageName(appName, imageTag, imageRepo)
+	if err != nil {
+		return "", err
+	}
+
+	if !VerifyImage(imageName) {
+		return "", fmt.Errorf("App image (%s) not found", imageName)
+	}
+
+	return imageName, nil
+}
+
+// ResolveDeployingAppImageName returns the deploying image identifier for a given app, tag
+// tuple without asserting that the image exists locally. Schedulers that run workloads off
+// the dokku host - where the image is pulled from a registry by the remote runtime and may
+// have been reaped locally - need the name without the local existence check.
+func ResolveDeployingAppImageName(appName, imageTag, imageRepo string) (string, error) {
 	imageRemoteRepository := ""
 	newImageTag := ""
 	newImageRepo := ""
@@ -526,11 +545,7 @@ func GetDeployingAppImageName(appName, imageTag, imageRepo string) (string, erro
 		imageTag = "latest"
 	}
 
-	imageName := fmt.Sprintf("%s%s:%s", imageRemoteRepository, imageRepo, imageTag)
-	if !VerifyImage(imageName) {
-		return "", fmt.Errorf("App image (%s) not found", imageName)
-	}
-	return imageName, nil
+	return fmt.Sprintf("%s%s:%s", imageRemoteRepository, imageRepo, imageTag), nil
 }
 
 // GetAppImageRepo is the central definition of a dokku image repo pattern

plugins/scheduler-k3s/chart.go

@@ -45,6 +45,25 @@ type BuildOptions struct {
 	// yet (e.g. a never-deployed app being previewed). The deploy path
 	// leaves this false so a missing image still errors.
 	AllowMissingImage bool
+
+	// FallbackImageMetadata supplies the builder type and working directory to
+	// use when the image is absent from the local docker daemon. It is only
+	// consulted after a local inspect fails - it never overrides a present
+	// image - so callers pass the values recorded in the app's current Helm
+	// release and let a deploy proceed on a host that has since reaped its
+	// local copy of the image.
+	FallbackImageMetadata *ImageMetadata
+
+	// RestartProcessType, when set, limits which process type receives a fresh
+	// deployment id. Helm still upgrades the whole release so configuration
+	// converges everywhere, but only this process type's pod template changes,
+	// so only its pods roll. Empty means every process type rolls.
+	RestartProcessType string
+
+	// PriorProcessDeploymentIDs carries the deployment ids currently deployed
+	// per process type, used to hold untargeted processes steady when
+	// RestartProcessType is set.
+	PriorProcessDeploymentIDs map[string]string
 }
 
 // BuildAppChart constructs a Helm chart on disk that reflects dokku's configured
@@ -72,14 +91,26 @@ func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOpti
 		return result, err
 	}
 
+	if opts.RestartProcessType != "" {
+		if _, ok := processes[opts.RestartProcessType]; !ok {
+			return result, fmt.Errorf("Process type %s not found in the scale for app %s", opts.RestartProcessType, appName)
+		}
+	}
+
 	namespace := getComputedNamespace(appName)
 
-	image, err := common.GetDeployingAppImageName(appName, imageTag, "")
+	image, err := common.ResolveDeployingAppImageName(appName, imageTag, "")
+	if err != nil {
+		return result, fmt.Errorf("Error getting deploying app image name: %w", err)
+	}
+
+	imageMetadata, err := resolveImageMetadata(appName, image, opts.FallbackImageMetadata)
 	if err != nil {
 		if !opts.AllowMissingImage {
-			return result, fmt.Errorf("Error getting deploying app image name: %w", err)
+			return result, err
 		}
 		image = fmt.Sprintf("dokku/%s:not-yet-deployed", appName)
+		imageMetadata = ImageMetadata{SourceType: "dockerfile"}
 	}
 
 	deployTimeout := getComputedDeployTimeout(appName)
@@ -97,12 +128,7 @@ func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOpti
 		return result, fmt.Errorf("Error parsing rollback-on-failure value as boolean: %w", err)
 	}
 
-	imageSourceType := "dockerfile"
-	if common.IsImageCnbBased(image) {
-		imageSourceType = "pack"
-	} else if common.IsImageHerokuishBased(image, appName) {
-		imageSourceType = "herokuish"
-	}
+	imageSourceType := imageMetadata.SourceType
 
 	env, err := config.LoadMergedAppEnv(appName)
 	if err != nil {
@@ -189,7 +215,7 @@ func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOpti
 		return cleanup(fmt.Errorf("Error getting app.json for deployment: %w", err))
 	}
 
-	workingDir := common.GetWorkingDir(appName, image)
+	workingDir := imageMetadata.WorkingDir
 
 	cronTasks, err := cron.FetchCronTasks(cron.FetchCronTasksInput{AppName: appName})
 	if err != nil {
@@ -379,6 +405,7 @@ func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOpti
 			Annotations:  annotations,
 			Autoscaling:  autoscaling,
 			Args:         args,
+			DeploymentID: resolveProcessDeploymentID(processType, deploymentId, opts.RestartProcessType, opts.PriorProcessDeploymentIDs),
 			Healthchecks: processHealthchecks,
 			Labels:       labels,
 			ProcessType:  ProcessType_Worker,

plugins/scheduler-k3s/preview.go

@@ -69,6 +69,7 @@ func CommandPreview(appName string, diffContext int, showSecrets bool, showSecre
 	buildOpts := BuildOptions{AllowMissingImage: true}
 	if currentRelease != nil {
 		buildOpts.OverrideDeploymentID = deploymentIDFromRelease(currentRelease)
+		buildOpts.FallbackImageMetadata = imageMetadataFromValues(releaseValues(helmAgent, appName))
 	}
 
 	chartResult, err := BuildAppChart(ctx, appName, "", buildOpts)

plugins/scheduler-k3s/release_values.go

@@ -0,0 +1,146 @@
+package scheduler_k3s
+
+import (
+	"fmt"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// ImageMetadata holds the image-derived values a chart or job needs that would
+// otherwise require the image to be present on the local docker daemon.
+type ImageMetadata struct {
+	SourceType string
+	WorkingDir string
+}
+
+// resolveImageMetadata determines the builder type and working directory for an
+// image, preferring a local inspect and falling back to values carried by the
+// app's current Helm release.
+//
+// k3s workloads run in the cluster and kubelet pulls the image itself, so the
+// dokku host is free to reap its local copy while the app keeps running - the
+// registry plugin's own imageCleanup does exactly that. Without a fallback,
+// every later deploy of such an app would fail. A fallback alone is not enough
+// either: common.IsImageCnbBased, common.IsImageHerokuishBased and
+// common.GetWorkingDir all report false/"" when the inspect errors, so a
+// missing image would otherwise be silently misread as a dockerfile app with no
+// working directory, dropping the herokuish /start wrapper from the start
+// command.
+func resolveImageMetadata(appName string, image string, fallback *ImageMetadata) (ImageMetadata, error) {
+	if common.VerifyImage(image) {
+		metadata := ImageMetadata{SourceType: "dockerfile"}
+		if common.IsImageCnbBased(image) {
+			metadata.SourceType = "pack"
+		} else if common.IsImageHerokuishBased(image, appName) {
+			metadata.SourceType = "herokuish"
+		}
+
+		metadata.WorkingDir = common.GetWorkingDir(appName, image)
+		return metadata, nil
+	}
+
+	if fallback != nil && fallback.SourceType != "" {
+		return *fallback, nil
+	}
+
+	return ImageMetadata{}, fmt.Errorf("App image (%s) not found locally and no deployed release to read image metadata from", image)
+}
+
+// releaseValues reads an app's current Helm release values, which every deploy
+// writes and which therefore make the cluster the authoritative record of how
+// the running workload was built. Returns nil when there is no readable
+// release, leaving callers to treat the values as unavailable rather than
+// failing - a first deploy has no release yet. Existence is checked up front so
+// a never-deployed app does not surface a not-found through the agent's logger.
+func releaseValues(helmAgent *HelmAgent, releaseName string) map[string]interface{} {
+	exists, err := helmAgent.ChartExists(releaseName)
+	if err != nil || !exists {
+		return nil
+	}
+
+	values, err := helmAgent.GetValues(releaseName)
+	if err != nil {
+		return nil
+	}
+
+	return values
+}
+
+// imageMetadataFromValues extracts image metadata from a set of Helm release values.
+func imageMetadataFromValues(values map[string]interface{}) *ImageMetadata {
+	globalValues, ok := values["global"].(map[string]interface{})
+	if !ok {
+		return nil
+	}
+
+	imageValues, ok := globalValues["image"].(map[string]interface{})
+	if !ok {
+		return nil
+	}
+
+	sourceType, ok := imageValues["type"].(string)
+	if !ok || sourceType == "" {
+		return nil
+	}
+
+	workingDir, _ := imageValues["working_dir"].(string)
+	return &ImageMetadata{
+		SourceType: sourceType,
+		WorkingDir: workingDir,
+	}
+}
+
+// processDeploymentIDsFromValues extracts the per-process deployment ids
+// recorded in an app's current Helm release, keyed by process type.
+//
+// A targeted restart must leave the untargeted processes' pod templates byte
+// identical, otherwise Kubernetes rolls them too. Their ids therefore have to
+// come from what is already deployed rather than being regenerated. Releases
+// written before per-process ids existed only carry a global id, so that is
+// used as the per-process default.
+func processDeploymentIDsFromValues(values map[string]interface{}) map[string]string {
+	deploymentIDs := map[string]string{}
+
+	globalDeploymentID := ""
+	if globalValues, ok := values["global"].(map[string]interface{}); ok {
+		globalDeploymentID, _ = globalValues["deployment_id"].(string)
+	}
+
+	processes, ok := values["processes"].(map[string]interface{})
+	if !ok {
+		return deploymentIDs
+	}
+
+	for processType, rawValues := range processes {
+		processValues, ok := rawValues.(map[string]interface{})
+		if !ok {
+			continue
+		}
+
+		deploymentID, _ := processValues["deployment_id"].(string)
+		if deploymentID == "" {
+			deploymentID = globalDeploymentID
+		}
+		if deploymentID != "" {
+			deploymentIDs[processType] = deploymentID
+		}
+	}
+
+	return deploymentIDs
+}
+
+// resolveProcessDeploymentID picks the deployment id for a single process type.
+// Every process type gets the fresh id unless a different one was targeted for
+// restart and a prior id is known for this one, in which case holding that id
+// steady leaves the process's pod template untouched so Kubernetes does not roll
+// it. A targeted restart against an app with no prior release rolls everything,
+// which is correct - there is nothing running to preserve.
+func resolveProcessDeploymentID(processType string, deploymentID int64, restartProcessType string, priorDeploymentIDs map[string]string) string {
+	if restartProcessType != "" && restartProcessType != processType {
+		if priorDeploymentID := priorDeploymentIDs[processType]; priorDeploymentID != "" {
+			return priorDeploymentID
+		}
+	}
+
+	return fmt.Sprint(deploymentID)
+}

plugins/scheduler-k3s/release_values_test.go

@@ -0,0 +1,186 @@
+package scheduler_k3s
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestImageMetadataFromValues(t *testing.T) {
+	t.Run("reads the builder type and working directory", func(t *testing.T) {
+		metadata := imageMetadataFromValues(map[string]interface{}{
+			"global": map[string]interface{}{
+				"image": map[string]interface{}{
+					"name":        "registry.example.com/dokku/myapp:5",
+					"type":        "herokuish",
+					"working_dir": "/app",
+				},
+			},
+		})
+
+		if metadata == nil {
+			t.Fatal("imageMetadataFromValues() = nil, want metadata")
+		}
+		if metadata.SourceType != "herokuish" {
+			t.Errorf("SourceType = %q, want herokuish", metadata.SourceType)
+		}
+		if metadata.WorkingDir != "/app" {
+			t.Errorf("WorkingDir = %q, want /app", metadata.WorkingDir)
+		}
+	})
+
+	t.Run("allows an empty working directory", func(t *testing.T) {
+		metadata := imageMetadataFromValues(map[string]interface{}{
+			"global": map[string]interface{}{
+				"image": map[string]interface{}{"type": "dockerfile"},
+			},
+		})
+
+		if metadata == nil {
+			t.Fatal("imageMetadataFromValues() = nil, want metadata")
+		}
+		if metadata.WorkingDir != "" {
+			t.Errorf("WorkingDir = %q, want empty", metadata.WorkingDir)
+		}
+	})
+
+	t.Run("nil when the release carries no usable type", func(t *testing.T) {
+		cases := map[string]map[string]interface{}{
+			"no global key": {},
+			"no image key":  {"global": map[string]interface{}{}},
+			"no type key": {
+				"global": map[string]interface{}{"image": map[string]interface{}{"name": "myapp:latest"}},
+			},
+			"empty type": {
+				"global": map[string]interface{}{"image": map[string]interface{}{"type": ""}},
+			},
+			"non-string type": {
+				"global": map[string]interface{}{"image": map[string]interface{}{"type": 3}},
+			},
+		}
+
+		for name, values := range cases {
+			t.Run(name, func(t *testing.T) {
+				if metadata := imageMetadataFromValues(values); metadata != nil {
+					t.Errorf("imageMetadataFromValues() = %+v, want nil", metadata)
+				}
+			})
+		}
+	})
+}
+
+func TestResolveImageMetadata(t *testing.T) {
+	// These cases all rely on the image being absent from the local docker
+	// daemon, which is what a reaped k3s image looks like.
+	const missingImage = "registry.example.invalid/dokku/does-not-exist:0"
+
+	t.Run("uses the fallback when the image is not present locally", func(t *testing.T) {
+		metadata, err := resolveImageMetadata("myapp", missingImage, &ImageMetadata{
+			SourceType: "herokuish",
+			WorkingDir: "/app",
+		})
+		if err != nil {
+			t.Fatalf("resolveImageMetadata() error = %v, want nil", err)
+		}
+		if metadata.SourceType != "herokuish" {
+			t.Errorf("SourceType = %q, want herokuish", metadata.SourceType)
+		}
+		if metadata.WorkingDir != "/app" {
+			t.Errorf("WorkingDir = %q, want /app", metadata.WorkingDir)
+		}
+	})
+
+	t.Run("errors when there is no image and no fallback", func(t *testing.T) {
+		_, err := resolveImageMetadata("myapp", missingImage, nil)
+		if err == nil {
+			t.Fatal("resolveImageMetadata() error = nil, want an error")
+		}
+		if !strings.Contains(err.Error(), missingImage) {
+			t.Errorf("error %q does not name the image", err.Error())
+		}
+	})
+
+	t.Run("ignores a fallback carrying no builder type", func(t *testing.T) {
+		if _, err := resolveImageMetadata("myapp", missingImage, &ImageMetadata{WorkingDir: "/app"}); err == nil {
+			t.Fatal("resolveImageMetadata() error = nil, want an error")
+		}
+	})
+}
+
+func TestProcessDeploymentIDsFromValues(t *testing.T) {
+	t.Run("reads per-process ids", func(t *testing.T) {
+		deploymentIDs := processDeploymentIDsFromValues(map[string]interface{}{
+			"global": map[string]interface{}{"deployment_id": "100"},
+			"processes": map[string]interface{}{
+				"web":    map[string]interface{}{"deployment_id": "200"},
+				"worker": map[string]interface{}{"deployment_id": "300"},
+			},
+		})
+
+		if deploymentIDs["web"] != "200" {
+			t.Errorf("web = %q, want 200", deploymentIDs["web"])
+		}
+		if deploymentIDs["worker"] != "300" {
+			t.Errorf("worker = %q, want 300", deploymentIDs["worker"])
+		}
+	})
+
+	t.Run("falls back to the global id for releases predating per-process ids", func(t *testing.T) {
+		deploymentIDs := processDeploymentIDsFromValues(map[string]interface{}{
+			"global": map[string]interface{}{"deployment_id": "100"},
+			"processes": map[string]interface{}{
+				"web":    map[string]interface{}{"replicas": 1},
+				"worker": map[string]interface{}{"deployment_id": "300"},
+			},
+		})
+
+		if deploymentIDs["web"] != "100" {
+			t.Errorf("web = %q, want the global 100", deploymentIDs["web"])
+		}
+		if deploymentIDs["worker"] != "300" {
+			t.Errorf("worker = %q, want 300", deploymentIDs["worker"])
+		}
+	})
+
+	t.Run("empty when there is nothing to read", func(t *testing.T) {
+		cases := map[string]map[string]interface{}{
+			"no values":       {},
+			"no processes":    {"global": map[string]interface{}{"deployment_id": "100"}},
+			"no ids anywhere": {"processes": map[string]interface{}{"web": map[string]interface{}{"replicas": 1}}},
+		}
+
+		for name, values := range cases {
+			t.Run(name, func(t *testing.T) {
+				if deploymentIDs := processDeploymentIDsFromValues(values); len(deploymentIDs) != 0 {
+					t.Errorf("processDeploymentIDsFromValues() = %v, want empty", deploymentIDs)
+				}
+			})
+		}
+	})
+}
+
+func TestResolveProcessDeploymentID(t *testing.T) {
+	priorDeploymentIDs := map[string]string{"web": "100", "worker": "100"}
+
+	t.Run("an untargeted restart rolls every process", func(t *testing.T) {
+		for _, processType := range []string{"web", "worker"} {
+			if got := resolveProcessDeploymentID(processType, 200, "", priorDeploymentIDs); got != "200" {
+				t.Errorf("resolveProcessDeploymentID(%q) = %q, want 200", processType, got)
+			}
+		}
+	})
+
+	t.Run("a targeted restart holds the other processes steady", func(t *testing.T) {
+		if got := resolveProcessDeploymentID("web", 200, "web", priorDeploymentIDs); got != "200" {
+			t.Errorf("targeted process = %q, want 200", got)
+		}
+		if got := resolveProcessDeploymentID("worker", 200, "web", priorDeploymentIDs); got != "100" {
+			t.Errorf("untargeted process = %q, want the prior 100", got)
+		}
+	})
+
+	t.Run("a process with no prior id takes the fresh one", func(t *testing.T) {
+		if got := resolveProcessDeploymentID("worker", 200, "web", map[string]string{}); got != "200" {
+			t.Errorf("resolveProcessDeploymentID() = %q, want 200", got)
+		}
+	})
+}

plugins/scheduler-k3s/src/triggers/triggers.go

@@ -66,7 +66,8 @@ func main() {
 		scheduler := flag.Arg(0)
 		appName := flag.Arg(1)
 		imageTag := flag.Arg(2)
-		err = scheduler_k3s.TriggerSchedulerDeploy(scheduler, appName, imageTag)
+		processType := flag.Arg(3)
+		err = scheduler_k3s.TriggerSchedulerDeploy(scheduler, appName, imageTag, processType)
 	case "scheduler-enter":
 		scheduler := flag.Arg(0)
 		appName := flag.Arg(1)

plugins/scheduler-k3s/template.go

@@ -108,6 +108,7 @@ type ProcessValues struct {
 	Args         []string            `yaml:"args,omitempty"`
 	Autoscaling  ProcessAutoscaling  `yaml:"autoscaling,omitempty"`
 	Cron         ProcessCron         `yaml:"cron,omitempty"`
+	DeploymentID string              `yaml:"deployment_id,omitempty"`
 	Healthchecks ProcessHealthchecks `yaml:"healthchecks,omitempty"`
 	Labels       ProcessLabels       `yaml:"labels,omitempty"`
 	ProcessType  ProcessType         `yaml:"process_type"`

plugins/scheduler-k3s/template_test.go

@@ -56,6 +56,17 @@ func TestToCoreV1PodSecurityContext(t *testing.T) {
 func renderDeploymentTemplate(t *testing.T, globalValues map[string]interface{}) string {
 	t.Helper()
 
+	return renderDeploymentTemplateWithProcesses(t, globalValues, map[string]interface{}{
+		"worker": map[string]interface{}{
+			"args":     []interface{}{"echo", "hello"},
+			"replicas": 1,
+		},
+	})
+}
+
+func renderDeploymentTemplateWithProcesses(t *testing.T, globalValues map[string]interface{}, processes map[string]interface{}) string {
+	t.Helper()
+
 	chartDir := t.TempDir()
 	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), 0o755); err != nil {
 		t.Fatalf("mkdir: %v", err)
@@ -95,13 +106,8 @@ func renderDeploymentTemplate(t *testing.T, globalValues map[string]interface{})
 	}
 
 	values := map[string]interface{}{
-		"global": global,
-		"processes": map[string]interface{}{
-			"worker": map[string]interface{}{
-				"args":     []interface{}{"echo", "hello"},
-				"replicas": 1,
-			},
-		},
+		"global":    global,
+		"processes": processes,
 	}
 
 	renderValues, err := chartutil.ToRenderValues(loaded, values, chartutil.ReleaseOptions{Name: "test", Namespace: "default"}, nil)
@@ -197,3 +203,47 @@ func TestDeploymentSysctlsRendering(t *testing.T) {
 		}
 	})
 }
+
+// TestDeploymentDeploymentIDRendering asserts each Deployment's pod template
+// carries its own process's deployment id. A targeted ps:restart works by
+// giving only the named process a fresh id, so an untargeted process whose id
+// is unchanged must render an unchanged pod template - otherwise Kubernetes
+// rolls its pods too and the targeting is meaningless.
+func TestDeploymentDeploymentIDRendering(t *testing.T) {
+	t.Run("each process renders its own deployment id", func(t *testing.T) {
+		manifest := renderDeploymentTemplateWithProcesses(t, map[string]interface{}{}, map[string]interface{}{
+			"web": map[string]interface{}{
+				"args":          []interface{}{"echo", "web"},
+				"deployment_id": "200",
+				"replicas":      1,
+			},
+			"worker": map[string]interface{}{
+				"args":          []interface{}{"echo", "worker"},
+				"deployment_id": "100",
+				"replicas":      1,
+			},
+		})
+
+		if !strings.Contains(manifest, `app.kubernetes.io/version: "200"`) {
+			t.Errorf("rendered deployment missing the targeted process id:\n%s", manifest)
+		}
+		if !strings.Contains(manifest, `app.kubernetes.io/version: "100"`) {
+			t.Errorf("rendered deployment missing the untargeted process id:\n%s", manifest)
+		}
+	})
+
+	t.Run("falls back to the global id when a process has none", func(t *testing.T) {
+		manifest := renderDeploymentTemplateWithProcesses(t, map[string]interface{}{
+			"deployment_id": "42",
+		}, map[string]interface{}{
+			"worker": map[string]interface{}{
+				"args":     []interface{}{"echo", "worker"},
+				"replicas": 1,
+			},
+		})
+
+		if !strings.Contains(manifest, `app.kubernetes.io/version: "42"`) {
+			t.Errorf("rendered deployment did not fall back to the global id:\n%s", manifest)
+		}
+	})
+}

plugins/scheduler-k3s/templates/chart/deployment.yaml

@@ -15,7 +15,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
+    app.kubernetes.io/version: {{ $config.deployment_id | default $.Values.global.deployment_id | quote }}
     dokku.com/builder-type: {{ $.Values.global.image.type | quote }}
     dokku.com/managed: "true"
     kubectl.kubernetes.io/default-container: {{ printf "%s-%s" $.Values.global.app_name $processName | quote }}
@@ -45,7 +45,7 @@ spec:
   template:
     metadata:
       annotations:
-        app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
+        app.kubernetes.io/version: {{ $config.deployment_id | default $.Values.global.deployment_id | quote }}
         dokku.com/builder-type: {{ $.Values.global.image.type }}
         dokku.com/managed: "true"
         kubectl.kubernetes.io/default-container: {{ $.Values.global.app_name }}-{{ $processName }}

plugins/scheduler-k3s/triggers.go

@@ -263,7 +263,7 @@ func TriggerPostCertsUpdate(appName string) error {
 	}
 
 	common.LogInfo1(fmt.Sprintf("Triggering redeploy for %s to update ingress configuration", appName))
-	return TriggerSchedulerDeploy("k3s", appName, imageTag)
+	return TriggerSchedulerDeploy("k3s", appName, imageTag, "")
 }
 
 // TriggerPostCertsRemove handles post-certs-remove trigger
@@ -299,7 +299,7 @@ func TriggerPostCertsRemove(appName string) error {
 	}
 
 	common.LogInfo1(fmt.Sprintf("Triggering redeploy for %s to update ingress configuration", appName))
-	return TriggerSchedulerDeploy("k3s", appName, imageTag)
+	return TriggerSchedulerDeploy("k3s", appName, imageTag, "")
 }
 
 // TriggerPostAppCloneSetup creates new scheduler-k3s files
@@ -464,7 +464,7 @@ func TriggerSchedulerCronWrite(scheduler string, appName string) error {
 }
 
 // TriggerSchedulerDeploy deploys an image tag for a given application
-func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) error {
+func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string, processType string) error {
 	if scheduler != "k3s" {
 		return nil
 	}
@@ -486,7 +486,17 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error creating kubernetes namespace for deployment: %w", err)
 	}
 
-	chartResult, err := BuildAppChart(ctx, appName, imageTag, BuildOptions{})
+	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("Error creating helm agent: %w", err)
+	}
+
+	priorValues := releaseValues(helmAgent, appName)
+	chartResult, err := BuildAppChart(ctx, appName, imageTag, BuildOptions{
+		FallbackImageMetadata:     imageMetadataFromValues(priorValues),
+		RestartProcessType:        processType,
+		PriorProcessDeploymentIDs: processDeploymentIDsFromValues(priorValues),
+	})
 	if err != nil {
 		return err
 	}
@@ -529,11 +539,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
-	helmAgent, err := NewHelmAgent(chartResult.Namespace, DeployLogPrinter)
-	if err != nil {
-		return fmt.Errorf("Error creating helm agent: %w", err)
-	}
-
 	ingresses, err := clientset.ListIngresses(ctx, ListIngressesInput{
 		Namespace:     chartResult.Namespace,
 		LabelSelector: fmt.Sprintf("app.kubernetes.io/instance=%s-web", appName),
@@ -934,18 +939,26 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 	if err != nil {
 		return fmt.Errorf("Error getting running image tag: %w", err)
 	}
-	image, err := common.GetDeployingAppImageName(appName, imageTag, "")
+	image, err := common.ResolveDeployingAppImageName(appName, imageTag, "")
 	if err != nil {
 		return fmt.Errorf("Error getting deploying app image name: %w", err)
 	}
 
-	imageStage, err := common.DockerInspect(image, "{{ index .Config.Labels \"com.dokku.image-stage\" }}")
-	if err != nil {
-		return fmt.Errorf("Error getting image stage: %w", err)
-	}
-	if imageStage != "release" {
-		common.LogWarn("Invalid image stage detected: expected 'release', got '$IMAGE_STAGE'")
-		return fmt.Errorf("Successfully deploy your app to fix dokku run calls")
+	// The image-stage guard exists to catch a build-stage image being run before
+	// the app was ever deployed. It can only be answered from a local copy of the
+	// image, which a k3s host is free to reap since the cluster pulls its own. In
+	// that case the guard is skipped: resolveImageMetadata below falls back to the
+	// app's Helm release, and the existence of that release is itself proof the
+	// app got past the release stage.
+	if common.VerifyImage(image) {
+		imageStage, err := common.DockerInspect(image, "{{ index .Config.Labels \"com.dokku.image-stage\" }}")
+		if err != nil {
+			return fmt.Errorf("Error getting image stage: %w", err)
+		}
+		if imageStage != "release" {
+			common.LogWarn(fmt.Sprintf("Invalid image stage detected: expected 'release', got '%s'", imageStage))
+			return fmt.Errorf("Successfully deploy your app to fix dokku run calls")
+		}
 	}
 
 	dokkuRmContainer := os.Getenv("DOKKU_RM_CONTAINER")
@@ -1026,11 +1039,18 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		}
 	}
 
-	imageSourceType, err := common.DockerInspect(image, "{{ index .Config.Labels \"com.dokku.builder-type\" }}")
+	helmAgent, err := NewHelmAgent(namespace, DevNullPrinter)
 	if err != nil {
-		return fmt.Errorf("Error getting image builder type: %w", err)
+		return fmt.Errorf("Error creating helm agent: %w", err)
 	}
 
+	values := releaseValues(helmAgent, appName)
+	imageMetadata, err := resolveImageMetadata(appName, image, imageMetadataFromValues(values))
+	if err != nil {
+		return err
+	}
+	imageSourceType := imageMetadata.SourceType
+
 	// todo: do something with docker args
 	command := args
 	commandShell := common.GetDokkuAppShell(appName)
@@ -1059,16 +1079,6 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		entrypoint = "launcher"
 	}
 
-	helmAgent, err := NewHelmAgent(namespace, DevNullPrinter)
-	if err != nil {
-		return fmt.Errorf("Error creating helm agent: %w", err)
-	}
-
-	values, err := helmAgent.GetValues(appName)
-	if err != nil {
-		return fmt.Errorf("Error getting helm values: %w", err)
-	}
-
 	globalValues, ok := values["global"].(map[string]interface{})
 	if !ok {
 		return errors.New("Global helm values not found")
@@ -1123,7 +1133,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		}
 	}
 
-	workingDir := common.GetWorkingDir(appName, image)
+	workingDir := imageMetadata.WorkingDir
 	job, err := templateKubernetesJob(Job{
 		ActiveDeadlineSeconds: activeDeadlineSeconds,
 		Annotations:           annotations,