Merge pull request #8533 from dokku/8531-split-out-imagepullsecrets-and-config-env-into-separate-helm-charts

Split env config and image pull secret into separate helm releases

docs/appendices/0.38.0-migration-guide.md

@@ -21,6 +21,7 @@
 - The `docker-local` scheduler now sends `SIGTERM` to old containers immediately after a successful deploy, rather than waiting `wait-to-retire` seconds before signaling. This matches Heroku's graceful-shutdown contract and lets applications begin draining in-flight work as soon as proxy traffic switches. The `wait-to-retire` grace period and `stop-timeout-seconds` hard-stop continue to apply as before. See the [zero downtime deploys documentation](/docs/deployment/zero-downtime-deploys.md#wait-to-retire) for more details.
 - The `docker-local` scheduler no longer queues an image for retirement when another running container of the same app still uses it. This fixes the case where a `ps:rebuild` against an image-based deploy (`git:from-image`) produced an identical-SHA image and the `dokku-retire` cron timer would log `Image ... has running containers, skipping rm` on every run. Stuck entries from prior versions are pruned automatically on the next `ps:retire` run.
 - All `:report` subcommands now accept the `--global` flag, which scopes the report to globally-configured properties. The flag composes with `--format json`, so a JSON report of global properties can be obtained via, for example, `dokku scheduler:report --global --format json`. Previously, combining `--global` with `--format json` was rejected with an "info flag" error, and `--global` on its own was treated as an unknown flag.
+- The `scheduler-k3s` plugin now manages env config and the dokku-generated image pull Secret as their own helm releases with stable names (`config-{app}` and `pull-secret-{app}`) rather than bundling them into the app helm chart with a per-deploy timestamp suffix (`env-{app}.{ts}` / `ims-{app}.{ts}`). This fixes two bugs: a helm rollback of the app chart no longer deletes Secrets that older ReplicaSets still reference, and the Deployment's `imagePullSecrets` list no longer accumulates references to nonexistent Secrets across deploys. The next deploy of an app switches the Deployment's `envFrom` and `imagePullSecrets` references to the stable names and prunes any leaked entries; existing live Deployments do not need to be patched manually. App rename now also uninstalls the old `tls-{app}`, `config-{app}`, and `pull-secret-{app}` releases under the previous app name; the new name's releases are recreated on the next deploy or certs sync.
 
 ### TLS handshake behavior change
 

plugins/scheduler-k3s/config_secret.go

@@ -0,0 +1,188 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// ConfigSecretValues contains the values for the config secret helm chart
+type ConfigSecretValues struct {
+	Global ConfigSecretGlobalValues `yaml:"global"`
+}
+
+// ConfigSecretGlobalValues contains the global values for the config secret chart
+type ConfigSecretGlobalValues struct {
+	Annotations map[string]string `yaml:"annotations,omitempty"`
+	AppName     string            `yaml:"app_name"`
+	Labels      map[string]string `yaml:"labels,omitempty"`
+	Namespace   string            `yaml:"namespace"`
+	Secrets     map[string]string `yaml:"secrets,omitempty"`
+}
+
+// GetConfigSecretReleaseName returns the helm release name for the config secret
+func GetConfigSecretReleaseName(appName string) string {
+	return fmt.Sprintf("config-%s", appName)
+}
+
+// GetConfigSecretName returns the kubernetes secret name for the config env
+func GetConfigSecretName(appName string) string {
+	return fmt.Sprintf("config-%s", appName)
+}
+
+// CreateOrUpdateConfigSecretInput contains the inputs to CreateOrUpdateConfigSecret
+type CreateOrUpdateConfigSecretInput struct {
+	AppName     string
+	Env         map[string]string
+	Annotations map[string]string
+	Labels      map[string]string
+}
+
+// CreateOrUpdateConfigSecret creates or updates the config env secret helm chart for an app
+func CreateOrUpdateConfigSecret(ctx context.Context, input CreateOrUpdateConfigSecretInput) error {
+	appName := input.AppName
+	if err := isKubernetesAvailable(); err != nil {
+		common.LogDebug("kubernetes not available, skipping config secret creation")
+		return nil
+	}
+
+	scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "")
+	globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local")
+	if scheduler == "" {
+		scheduler = globalScheduler
+	}
+	if scheduler != "k3s" {
+		common.LogDebug("app does not use k3s scheduler, skipping config secret creation")
+		return nil
+	}
+
+	chartDir, err := os.MkdirTemp("", "dokku-config-secret-chart-")
+	if err != nil {
+		return fmt.Errorf("error creating chart directory: %w", err)
+	}
+	defer os.RemoveAll(chartDir)
+
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
+		return fmt.Errorf("error creating chart templates directory: %w", err)
+	}
+
+	namespace := getComputedNamespace(appName)
+	releaseName := GetConfigSecretReleaseName(appName)
+
+	chart := &Chart{
+		ApiVersion: "v2",
+		AppVersion: "1.0.0",
+		Name:       releaseName,
+		Icon:       "https://dokku.com/assets/dokku-logo.svg",
+		Version:    "0.0.1",
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: chart,
+		Path:   filepath.Join(chartDir, "Chart.yaml"),
+	})
+	if err != nil {
+		return fmt.Errorf("error writing chart: %w", err)
+	}
+
+	b, err := templates.ReadFile("templates/config-secret-chart/templates/config-secret.yaml")
+	if err != nil {
+		return fmt.Errorf("error reading config-secret template: %w", err)
+	}
+
+	filename := filepath.Join(chartDir, "templates", "config-secret.yaml")
+	err = os.WriteFile(filename, b, os.FileMode(0644))
+	if err != nil {
+		return fmt.Errorf("error writing config-secret template: %w", err)
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(filename)
+	}
+
+	encodedSecrets := map[string]string{}
+	for key, value := range input.Env {
+		encodedSecrets[key] = base64.StdEncoding.EncodeToString([]byte(value))
+	}
+
+	values := &ConfigSecretValues{
+		Global: ConfigSecretGlobalValues{
+			Annotations: input.Annotations,
+			AppName:     appName,
+			Labels:      input.Labels,
+			Namespace:   namespace,
+			Secrets:     encodedSecrets,
+		},
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: values,
+		Path:   filepath.Join(chartDir, "values.yaml"),
+	})
+	if err != nil {
+		return fmt.Errorf("error writing values: %w", err)
+	}
+
+	if err := createKubernetesNamespace(ctx, namespace); err != nil {
+		return fmt.Errorf("error creating namespace: %w", err)
+	}
+
+	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("error creating helm agent: %w", err)
+	}
+
+	chartPath, err := filepath.Abs(chartDir)
+	if err != nil {
+		return fmt.Errorf("error getting chart path: %w", err)
+	}
+
+	common.LogVerboseQuiet(fmt.Sprintf("Installing config secret for %s", appName))
+	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
+		ChartPath:   chartPath,
+		Namespace:   namespace,
+		ReleaseName: releaseName,
+		Wait:        false,
+	})
+	if err != nil {
+		return fmt.Errorf("error installing config secret chart: %w", err)
+	}
+
+	return nil
+}
+
+// DeleteConfigSecret deletes the config env secret helm chart for an app
+func DeleteConfigSecret(ctx context.Context, appName string) error {
+	if err := isKubernetesAvailable(); err != nil {
+		common.LogDebug("kubernetes not available, skipping config secret deletion")
+		return nil
+	}
+
+	namespace := getComputedNamespace(appName)
+	releaseName := GetConfigSecretReleaseName(appName)
+
+	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("error creating helm agent: %w", err)
+	}
+
+	exists, err := helmAgent.ChartExists(releaseName)
+	if err != nil {
+		return fmt.Errorf("error checking if config secret chart exists: %w", err)
+	}
+
+	if !exists {
+		return nil
+	}
+
+	common.LogVerboseQuiet(fmt.Sprintf("Removing config secret for %s", appName))
+	if err := helmAgent.UninstallChart(releaseName); err != nil {
+		return fmt.Errorf("error uninstalling config secret chart: %w", err)
+	}
+
+	return nil
+}

plugins/scheduler-k3s/config_secret_test.go

@@ -0,0 +1,65 @@
+package scheduler_k3s
+
+import (
+	"encoding/base64"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestGetConfigSecretReleaseName(t *testing.T) {
+	got := GetConfigSecretReleaseName("myapp")
+	want := "config-myapp"
+	if got != want {
+		t.Fatalf("GetConfigSecretReleaseName(\"myapp\") = %q, want %q", got, want)
+	}
+}
+
+func TestGetConfigSecretName(t *testing.T) {
+	got := GetConfigSecretName("myapp")
+	want := "config-myapp"
+	if got != want {
+		t.Fatalf("GetConfigSecretName(\"myapp\") = %q, want %q", got, want)
+	}
+}
+
+func TestConfigSecretValuesEncoding(t *testing.T) {
+	values := &ConfigSecretValues{
+		Global: ConfigSecretGlobalValues{
+			AppName:   "myapp",
+			Namespace: "default",
+			Secrets: map[string]string{
+				"DATABASE_URL": base64.StdEncoding.EncodeToString([]byte("postgres://localhost/db")),
+				"REDIS_URL":    base64.StdEncoding.EncodeToString([]byte("redis://localhost:6379")),
+			},
+		},
+	}
+
+	data, err := yaml.Marshal(values)
+	if err != nil {
+		t.Fatalf("yaml.Marshal returned error: %v", err)
+	}
+
+	var roundTrip ConfigSecretValues
+	if err := yaml.Unmarshal(data, &roundTrip); err != nil {
+		t.Fatalf("yaml.Unmarshal returned error: %v", err)
+	}
+
+	if roundTrip.Global.AppName != "myapp" {
+		t.Errorf("AppName = %q, want \"myapp\"", roundTrip.Global.AppName)
+	}
+	if roundTrip.Global.Namespace != "default" {
+		t.Errorf("Namespace = %q, want \"default\"", roundTrip.Global.Namespace)
+	}
+	if len(roundTrip.Global.Secrets) != 2 {
+		t.Errorf("Secrets length = %d, want 2", len(roundTrip.Global.Secrets))
+	}
+
+	decoded, err := base64.StdEncoding.DecodeString(roundTrip.Global.Secrets["DATABASE_URL"])
+	if err != nil {
+		t.Fatalf("base64 decode failed: %v", err)
+	}
+	if string(decoded) != "postgres://localhost/db" {
+		t.Errorf("DATABASE_URL decoded = %q, want \"postgres://localhost/db\"", string(decoded))
+	}
+}

plugins/scheduler-k3s/functions.go

@@ -808,6 +808,63 @@ func getComputedImagePullSecrets(appName string) string {
 	return imagePullSecrets
 }
 
+// pruneStaleImagePullSecretsFromDeployments rewrites the imagePullSecrets list on existing app
+// Deployments to contain only the names in keepNames. This removes references that helm has
+// lost track of due to strategic-merge accumulation on PodSpec.ImagePullSecrets, which is the
+// root cause of stale-secret pod hard crashes after rollbacks. The patch is a no-op when the
+// live list already matches.
+func pruneStaleImagePullSecretsFromDeployments(ctx context.Context, clientset KubernetesClient, namespace string, appName string, keepNames []string) error {
+	deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{
+		Namespace:     namespace,
+		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
+	})
+	if err != nil {
+		return fmt.Errorf("error listing deployments: %w", err)
+	}
+
+	keepSet := map[string]struct{}{}
+	for _, name := range keepNames {
+		if name != "" {
+			keepSet[name] = struct{}{}
+		}
+	}
+
+	for _, deployment := range deployments {
+		live := deployment.Spec.Template.Spec.ImagePullSecrets
+		if needsImagePullSecretsPrune(live, keepSet) {
+			common.LogVerboseQuiet(fmt.Sprintf("Pruning stale imagePullSecrets entries from deployment %s", deployment.Name))
+			err := clientset.SetDeploymentImagePullSecrets(ctx, SetDeploymentImagePullSecretsInput{
+				Name:             deployment.Name,
+				Namespace:        namespace,
+				ImagePullSecrets: keepNames,
+			})
+			if err != nil {
+				return fmt.Errorf("error pruning deployment %s: %w", deployment.Name, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// needsImagePullSecretsPrune reports whether the live imagePullSecrets list differs from the
+// desired keep-set (either contains entries not in the keep-set or is missing entries in it).
+func needsImagePullSecretsPrune(live []corev1.LocalObjectReference, keepSet map[string]struct{}) bool {
+	if len(live) != len(keepSet) {
+		return true
+	}
+
+	seen := map[string]struct{}{}
+	for _, ref := range live {
+		if _, ok := keepSet[ref.Name]; !ok {
+			return true
+		}
+		seen[ref.Name] = struct{}{}
+	}
+
+	return len(seen) != len(keepSet)
+}
+
 func getGlobalIngressClass() string {
 	return common.PropertyGetDefault("scheduler-k3s", "--global", "ingress-class", DefaultIngressClass)
 }

plugins/scheduler-k3s/functions_test.go

@@ -0,0 +1,93 @@
+package scheduler_k3s
+
+import (
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+func TestNeedsImagePullSecretsPrune(t *testing.T) {
+	cases := []struct {
+		name    string
+		live    []corev1.LocalObjectReference
+		keepSet map[string]struct{}
+		want    bool
+	}{
+		{
+			name:    "empty live, empty keep",
+			live:    []corev1.LocalObjectReference{},
+			keepSet: map[string]struct{}{},
+			want:    false,
+		},
+		{
+			name: "matches keep set exactly",
+			live: []corev1.LocalObjectReference{{Name: "pull-secret-foo"}},
+			keepSet: map[string]struct{}{
+				"pull-secret-foo": {},
+			},
+			want: false,
+		},
+		{
+			name: "leaked entries beyond keep set",
+			live: []corev1.LocalObjectReference{
+				{Name: "ims-foo.111"},
+				{Name: "ims-foo.222"},
+				{Name: "pull-secret-foo"},
+			},
+			keepSet: map[string]struct{}{
+				"pull-secret-foo": {},
+			},
+			want: true,
+		},
+		{
+			name: "live missing the keep entry",
+			live: []corev1.LocalObjectReference{
+				{Name: "ims-foo.111"},
+			},
+			keepSet: map[string]struct{}{
+				"pull-secret-foo": {},
+			},
+			want: true,
+		},
+		{
+			name:    "live populated, keep set empty",
+			live:    []corev1.LocalObjectReference{{Name: "ims-foo.111"}},
+			keepSet: map[string]struct{}{},
+			want:    true,
+		},
+		{
+			name:    "empty live, keep set populated",
+			live:    []corev1.LocalObjectReference{},
+			keepSet: map[string]struct{}{"pull-secret-foo": {}},
+			want:    true,
+		},
+		{
+			name: "user override only, no leaked entries",
+			live: []corev1.LocalObjectReference{{Name: "my-custom-secret"}},
+			keepSet: map[string]struct{}{
+				"my-custom-secret": {},
+			},
+			want: false,
+		},
+		{
+			name: "user override with leaked dokku-managed entries",
+			live: []corev1.LocalObjectReference{
+				{Name: "ims-foo.111"},
+				{Name: "my-custom-secret"},
+			},
+			keepSet: map[string]struct{}{
+				"my-custom-secret": {},
+			},
+			want: true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := needsImagePullSecretsPrune(tc.live, tc.keepSet)
+			if got != tc.want {
+				t.Errorf("needsImagePullSecretsPrune() = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}

plugins/scheduler-k3s/k8s.go

@@ -3,6 +3,7 @@ package scheduler_k3s
 import (
 	"bufio"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -977,6 +978,64 @@ func (k KubernetesClient) ScaleDeployment(ctx context.Context, input ScaleDeploy
 	return nil
 }
 
+// SetDeploymentImagePullSecretsInput contains all the information needed to overwrite a Deployment's imagePullSecrets list
+type SetDeploymentImagePullSecretsInput struct {
+	// Name is the Kubernetes deployment name
+	Name string
+
+	// Namespace is the Kubernetes namespace
+	Namespace string
+
+	// ImagePullSecrets is the desired list of imagePullSecret names. May be empty.
+	ImagePullSecrets []string
+}
+
+// SetDeploymentImagePullSecrets overwrites the deployment pod template's imagePullSecrets list with exactly
+// the names supplied in input.ImagePullSecrets. Strategic-merge patches honor patchMergeKey semantics, which
+// is precisely what causes leaked entries to accumulate in the first place. To force a hard replace we use
+// a JSON merge patch on the imagePullSecrets path; JSON merge patch replaces array fields wholesale.
+func (k KubernetesClient) SetDeploymentImagePullSecrets(ctx context.Context, input SetDeploymentImagePullSecretsInput) error {
+	type patchPodSpec struct {
+		ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets"`
+	}
+	type patchPodTemplate struct {
+		Spec patchPodSpec `json:"spec"`
+	}
+	type patchSpec struct {
+		Template patchPodTemplate `json:"template"`
+	}
+	type patchBody struct {
+		Spec patchSpec `json:"spec"`
+	}
+
+	refs := make([]corev1.LocalObjectReference, 0, len(input.ImagePullSecrets))
+	for _, name := range input.ImagePullSecrets {
+		refs = append(refs, corev1.LocalObjectReference{Name: name})
+	}
+
+	body := patchBody{
+		Spec: patchSpec{
+			Template: patchPodTemplate{
+				Spec: patchPodSpec{
+					ImagePullSecrets: refs,
+				},
+			},
+		},
+	}
+
+	data, err := json.Marshal(body)
+	if err != nil {
+		return fmt.Errorf("error marshalling deployment patch: %w", err)
+	}
+
+	_, err = k.Client.AppsV1().Deployments(input.Namespace).Patch(ctx, input.Name, types.MergePatchType, data, metav1.PatchOptions{})
+	if err != nil {
+		return fmt.Errorf("error patching deployment imagePullSecrets: %w", err)
+	}
+
+	return nil
+}
+
 type StreamLogsInput struct {
 	// ContainerName is the Kubernetes container name
 	ContainerName string

plugins/scheduler-k3s/pull_secret.go

@@ -0,0 +1,183 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// ImagePullSecretValues contains the values for the dokku-managed image pull secret helm chart
+type ImagePullSecretValues struct {
+	Global ImagePullSecretGlobalValues `yaml:"global"`
+}
+
+// ImagePullSecretGlobalValues contains the global values for the image pull secret chart
+type ImagePullSecretGlobalValues struct {
+	Annotations      map[string]string `yaml:"annotations,omitempty"`
+	AppName          string            `yaml:"app_name"`
+	Labels           map[string]string `yaml:"labels,omitempty"`
+	Namespace        string            `yaml:"namespace"`
+	PullSecretBase64 string            `yaml:"pull_secret_base64"`
+}
+
+// GetImagePullSecretReleaseName returns the helm release name for the image pull secret
+func GetImagePullSecretReleaseName(appName string) string {
+	return fmt.Sprintf("pull-secret-%s", appName)
+}
+
+// GetImagePullSecretName returns the kubernetes secret name for the dokku-managed image pull secret
+func GetImagePullSecretName(appName string) string {
+	return fmt.Sprintf("pull-secret-%s", appName)
+}
+
+// CreateOrUpdateImagePullSecretInput contains the inputs to CreateOrUpdateImagePullSecret
+type CreateOrUpdateImagePullSecretInput struct {
+	AppName          string
+	DockerConfigJSON []byte
+	Annotations      map[string]string
+	Labels           map[string]string
+}
+
+// CreateOrUpdateImagePullSecret creates or updates the image pull secret helm chart for an app
+func CreateOrUpdateImagePullSecret(ctx context.Context, input CreateOrUpdateImagePullSecretInput) error {
+	appName := input.AppName
+	if err := isKubernetesAvailable(); err != nil {
+		common.LogDebug("kubernetes not available, skipping image pull secret creation")
+		return nil
+	}
+
+	scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "")
+	globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local")
+	if scheduler == "" {
+		scheduler = globalScheduler
+	}
+	if scheduler != "k3s" {
+		common.LogDebug("app does not use k3s scheduler, skipping image pull secret creation")
+		return nil
+	}
+
+	chartDir, err := os.MkdirTemp("", "dokku-pull-secret-chart-")
+	if err != nil {
+		return fmt.Errorf("error creating chart directory: %w", err)
+	}
+	defer os.RemoveAll(chartDir)
+
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
+		return fmt.Errorf("error creating chart templates directory: %w", err)
+	}
+
+	namespace := getComputedNamespace(appName)
+	releaseName := GetImagePullSecretReleaseName(appName)
+
+	chart := &Chart{
+		ApiVersion: "v2",
+		AppVersion: "1.0.0",
+		Name:       releaseName,
+		Icon:       "https://dokku.com/assets/dokku-logo.svg",
+		Version:    "0.0.1",
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: chart,
+		Path:   filepath.Join(chartDir, "Chart.yaml"),
+	})
+	if err != nil {
+		return fmt.Errorf("error writing chart: %w", err)
+	}
+
+	b, err := templates.ReadFile("templates/pull-secret-chart/templates/pull-secret.yaml")
+	if err != nil {
+		return fmt.Errorf("error reading pull-secret template: %w", err)
+	}
+
+	filename := filepath.Join(chartDir, "templates", "pull-secret.yaml")
+	err = os.WriteFile(filename, b, os.FileMode(0644))
+	if err != nil {
+		return fmt.Errorf("error writing pull-secret template: %w", err)
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(filename)
+	}
+
+	values := &ImagePullSecretValues{
+		Global: ImagePullSecretGlobalValues{
+			Annotations:      input.Annotations,
+			AppName:          appName,
+			Labels:           input.Labels,
+			Namespace:        namespace,
+			PullSecretBase64: base64.StdEncoding.EncodeToString(input.DockerConfigJSON),
+		},
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: values,
+		Path:   filepath.Join(chartDir, "values.yaml"),
+	})
+	if err != nil {
+		return fmt.Errorf("error writing values: %w", err)
+	}
+
+	if err := createKubernetesNamespace(ctx, namespace); err != nil {
+		return fmt.Errorf("error creating namespace: %w", err)
+	}
+
+	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("error creating helm agent: %w", err)
+	}
+
+	chartPath, err := filepath.Abs(chartDir)
+	if err != nil {
+		return fmt.Errorf("error getting chart path: %w", err)
+	}
+
+	common.LogVerboseQuiet(fmt.Sprintf("Installing image pull secret for %s", appName))
+	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
+		ChartPath:   chartPath,
+		Namespace:   namespace,
+		ReleaseName: releaseName,
+		Wait:        false,
+	})
+	if err != nil {
+		return fmt.Errorf("error installing image pull secret chart: %w", err)
+	}
+
+	return nil
+}
+
+// DeleteImagePullSecret deletes the image pull secret helm chart for an app
+func DeleteImagePullSecret(ctx context.Context, appName string) error {
+	if err := isKubernetesAvailable(); err != nil {
+		common.LogDebug("kubernetes not available, skipping image pull secret deletion")
+		return nil
+	}
+
+	namespace := getComputedNamespace(appName)
+	releaseName := GetImagePullSecretReleaseName(appName)
+
+	helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("error creating helm agent: %w", err)
+	}
+
+	exists, err := helmAgent.ChartExists(releaseName)
+	if err != nil {
+		return fmt.Errorf("error checking if image pull secret chart exists: %w", err)
+	}
+
+	if !exists {
+		return nil
+	}
+
+	common.LogVerboseQuiet(fmt.Sprintf("Removing image pull secret for %s", appName))
+	if err := helmAgent.UninstallChart(releaseName); err != nil {
+		return fmt.Errorf("error uninstalling image pull secret chart: %w", err)
+	}
+
+	return nil
+}

plugins/scheduler-k3s/pull_secret_test.go

@@ -0,0 +1,61 @@
+package scheduler_k3s
+
+import (
+	"encoding/base64"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestGetImagePullSecretReleaseName(t *testing.T) {
+	got := GetImagePullSecretReleaseName("myapp")
+	want := "pull-secret-myapp"
+	if got != want {
+		t.Fatalf("GetImagePullSecretReleaseName(\"myapp\") = %q, want %q", got, want)
+	}
+}
+
+func TestGetImagePullSecretName(t *testing.T) {
+	got := GetImagePullSecretName("myapp")
+	want := "pull-secret-myapp"
+	if got != want {
+		t.Fatalf("GetImagePullSecretName(\"myapp\") = %q, want %q", got, want)
+	}
+}
+
+func TestImagePullSecretValuesEncoding(t *testing.T) {
+	dockerConfig := []byte(`{"auths":{"https://index.docker.io/v1/":{"auth":"dXNlcjpwYXNz"}}}`)
+
+	values := &ImagePullSecretValues{
+		Global: ImagePullSecretGlobalValues{
+			AppName:          "myapp",
+			Namespace:        "default",
+			PullSecretBase64: base64.StdEncoding.EncodeToString(dockerConfig),
+		},
+	}
+
+	data, err := yaml.Marshal(values)
+	if err != nil {
+		t.Fatalf("yaml.Marshal returned error: %v", err)
+	}
+
+	var roundTrip ImagePullSecretValues
+	if err := yaml.Unmarshal(data, &roundTrip); err != nil {
+		t.Fatalf("yaml.Unmarshal returned error: %v", err)
+	}
+
+	if roundTrip.Global.AppName != "myapp" {
+		t.Errorf("AppName = %q, want \"myapp\"", roundTrip.Global.AppName)
+	}
+	if roundTrip.Global.Namespace != "default" {
+		t.Errorf("Namespace = %q, want \"default\"", roundTrip.Global.Namespace)
+	}
+
+	decoded, err := base64.StdEncoding.DecodeString(roundTrip.Global.PullSecretBase64)
+	if err != nil {
+		t.Fatalf("base64 decode failed: %v", err)
+	}
+	if string(decoded) != string(dockerConfig) {
+		t.Errorf("PullSecretBase64 decoded = %q, want %q", string(decoded), string(dockerConfig))
+	}
+}

plugins/scheduler-k3s/template.go

@@ -50,14 +50,12 @@ type GlobalValues struct {
 	Keda            GlobalKedaValues   `yaml:"keda"`
 	Namespace       string             `yaml:"namespace"`
 	Network         GlobalNetwork      `yaml:"network"`
-	Secrets         map[string]string  `yaml:"secrets,omitempty"`
 	SecurityContext SecurityContext    `yaml:"security_context,omitempty"`
 }
 
 type GlobalImage struct {
 	ImagePullSecrets string `yaml:"image_pull_secrets"`
 	Name             string `yaml:"name"`
-	PullSecretBase64 string `yaml:"pull_secret_base64"`
 	Type             string `yaml:"type"`
 	WorkingDir       string `yaml:"working_dir"`
 }
@@ -425,7 +423,7 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
 	}
 
 	maps.Copy(labels, input.Labels)
-	secretName := fmt.Sprintf("env-%s.%d", input.AppName, input.DeploymentID)
+	secretName := GetConfigSecretName(input.AppName)
 
 	env := []corev1.EnvVar{}
 	for key, value := range input.Env {

plugins/scheduler-k3s/templates/chart/cron-job.yaml

@@ -77,7 +77,7 @@ spec:
             {{- end }}
             envFrom:
             - secretRef:
-                name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }}
+                name: config-{{ $.Values.global.app_name }}
                 optional: true
             image: {{ $.Values.global.image.name }}
             imagePullPolicy: Always

plugins/scheduler-k3s/templates/chart/deployment.yaml

@@ -74,7 +74,7 @@ spec:
         {{- end }}
         envFrom:
         - secretRef:
-            name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }}
+            name: config-{{ $.Values.global.app_name }}
             optional: true
         image: {{ $.Values.global.image.name }}
         imagePullPolicy: Always

plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml

@@ -1,20 +0,0 @@
-{{- if $.Values.global.image.pull_secret_base64 }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  annotations:
-    app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
-    dokku.com/managed: "true"
-    {{ include "print.annotations" (dict "config" $.Values.global "key" "secret") | indent 4 }}
-  labels:
-    app.kubernetes.io/instance: ims-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }}
-    app.kubernetes.io/name: ims-{{ $.Values.global.app_name }}
-    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
-    {{ include "print.labels" (dict "config" $.Values.global "key" "secret") | indent 4 }}
-  name: ims-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }}
-  namespace: {{ $.Values.global.namespace }}
-data:
-  .dockerconfigjson: "{{ $.Values.global.image.pull_secret_base64 }}"
-type: kubernetes.io/dockerconfigjson
-{{- end }}

plugins/scheduler-k3s/templates/chart/secret.yaml

@@ -1,19 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
-  annotations:
-    app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }}
-    dokku.com/managed: "true"
-    {{ include "print.annotations" (dict "config" $.Values.global "key" "secret") | indent 4 }}
-  labels:
-    app.kubernetes.io/instance: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }}
-    app.kubernetes.io/name: env-{{ $.Values.global.app_name }}
-    app.kubernetes.io/part-of: {{ $.Values.global.app_name }}
-    {{ include "print.labels" (dict "config" $.Values.global "key" "secret") | indent 4 }}
-  name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }}
-  namespace: {{ $.Values.global.namespace }}
-{{- with .Values.global.secrets }}
-data:
-  {{- toYaml . | nindent 2 }}
-{{- end }}

plugins/scheduler-k3s/templates/config-secret-chart/templates/config-secret.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: config-{{ .Values.global.app_name }}
+  namespace: {{ .Values.global.namespace }}
+  labels:
+    app.kubernetes.io/name: config-{{ .Values.global.app_name }}
+    app.kubernetes.io/part-of: {{ .Values.global.app_name }}
+    {{- range $k, $v := .Values.global.labels }}
+    {{ $k }}: {{ $v | quote }}
+    {{- end }}
+  annotations:
+    dokku.com/managed: "true"
+    {{- range $k, $v := .Values.global.annotations }}
+    {{ $k }}: {{ $v | quote }}
+    {{- end }}
+{{- with .Values.global.secrets }}
+data:
+  {{- toYaml . | nindent 2 }}
+{{- end }}

plugins/scheduler-k3s/templates/pull-secret-chart/templates/pull-secret.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/dockerconfigjson
+metadata:
+  name: pull-secret-{{ .Values.global.app_name }}
+  namespace: {{ .Values.global.namespace }}
+  labels:
+    app.kubernetes.io/name: pull-secret-{{ .Values.global.app_name }}
+    app.kubernetes.io/part-of: {{ .Values.global.app_name }}
+    {{- range $k, $v := .Values.global.labels }}
+    {{ $k }}: {{ $v | quote }}
+    {{- end }}
+  annotations:
+    dokku.com/managed: "true"
+    {{- range $k, $v := .Values.global.annotations }}
+    {{ $k }}: {{ $v | quote }}
+    {{- end }}
+data:
+  .dockerconfigjson: {{ .Values.global.pull_secret_base64 | quote }}

plugins/scheduler-k3s/triggers.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -170,6 +169,21 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error {
 		return err
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	if _, err := DeleteTLSSecret(ctx, oldAppName); err != nil {
+		common.LogWarn(fmt.Sprintf("Error deleting TLS secret for old app name %s: %v", oldAppName, err))
+	}
+
+	if err := DeleteConfigSecret(ctx, oldAppName); err != nil {
+		common.LogWarn(fmt.Sprintf("Error deleting config secret for old app name %s: %v", oldAppName, err))
+	}
+
+	if err := DeleteImagePullSecret(ctx, oldAppName); err != nil {
+		common.LogWarn(fmt.Sprintf("Error deleting image pull secret for old app name %s: %v", oldAppName, err))
+	}
+
 	return nil
 }
 
@@ -403,8 +417,9 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 	}
 
 	deploymentId := time.Now().Unix()
-	pullSecretBase64 := base64.StdEncoding.EncodeToString([]byte(""))
 	imagePullSecrets := getComputedImagePullSecrets(appName)
+	dokkuManagedPullSecret := false
+	var dokkuPullSecretBytes []byte
 	if imagePullSecrets == "" {
 		dockerConfigPath := filepath.Join(registry.GetComputedAppRegistryConfigDir(appName), "config.json")
 		if fi, err := os.Stat(dockerConfigPath); err == nil && !fi.IsDir() {
@@ -413,12 +428,13 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 				return fmt.Errorf("Error reading docker config: %w", err)
 			}
 
-			imagePullSecrets = fmt.Sprintf("ims-%s.%d", appName, deploymentId)
+			imagePullSecrets = GetImagePullSecretName(appName)
-			pullSecretBase64 = base64.StdEncoding.EncodeToString(b)
+			dokkuManagedPullSecret = true
+			dokkuPullSecretBytes = b
 		}
 	}
 
-	globalTemplateFiles := []string{"service-account", "secret", "image-pull-secret"}
+	globalTemplateFiles := []string{"service-account"}
 	for _, templateName := range globalTemplateFiles {
 		b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
 		if err != nil {
@@ -541,7 +557,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 			Keda:         kedaValues,
 			Image: GlobalImage{
 				ImagePullSecrets: imagePullSecrets,
-				PullSecretBase64: pullSecretBase64,
 				Name:             image,
 				Type:             imageSourceType,
 				WorkingDir:       workingDir,
@@ -553,7 +568,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 				PrimaryPort:        primaryPort,
 				PrimaryServicePort: primaryServicePort,
 			},
-			Secrets:         map[string]string{},
 			SecurityContext: securityContext,
 		},
 		Processes: map[string]ProcessValues{},
@@ -851,10 +865,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		}
 	}
 
-	for key, value := range env.Map() {
-		values.Global.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value))
-	}
-
 	b, err := templates.ReadFile("templates/chart/_helpers.tpl")
 	if err != nil {
 		return fmt.Errorf("Error reading _helpers template: %w", err)
@@ -925,6 +935,38 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		}
 	}
 
+	if err := CreateOrUpdateConfigSecret(ctx, CreateOrUpdateConfigSecretInput{
+		AppName:     appName,
+		Env:         env.Map(),
+		Annotations: globalAnnotations.SecretAnnotations,
+		Labels:      globalLabels.SecretLabels,
+	}); err != nil {
+		return fmt.Errorf("Error syncing config secret: %w", err)
+	}
+
+	if dokkuManagedPullSecret {
+		if err := CreateOrUpdateImagePullSecret(ctx, CreateOrUpdateImagePullSecretInput{
+			AppName:          appName,
+			DockerConfigJSON: dokkuPullSecretBytes,
+			Annotations:      globalAnnotations.SecretAnnotations,
+			Labels:           globalLabels.SecretLabels,
+		}); err != nil {
+			return fmt.Errorf("Error syncing image pull secret: %w", err)
+		}
+	} else {
+		if err := DeleteImagePullSecret(ctx, appName); err != nil {
+			return fmt.Errorf("Error removing stale image pull secret: %w", err)
+		}
+	}
+
+	keepImagePullSecrets := []string{}
+	if imagePullSecrets != "" {
+		keepImagePullSecrets = []string{imagePullSecrets}
+	}
+	if err := pruneStaleImagePullSecretsFromDeployments(ctx, clientset, namespace, appName, keepImagePullSecrets); err != nil {
+		return fmt.Errorf("Error pruning stale imagePullSecrets entries: %w", err)
+	}
+
 	kustomizeRootPath := ""
 	if hasKustomizeDirectory(appName) {
 		kustomizeRootPath = getProcessSpecificKustomizeRootPath(appName)
@@ -1402,7 +1444,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 
 	imagePullSecrets := getComputedImagePullSecrets(appName)
 	if imagePullSecrets == "" {
-		imagePullSecrets = fmt.Sprintf("ims-%s.%d", appName, deploymentID)
+		imagePullSecrets = GetImagePullSecretName(appName)
 		_, err := clientset.GetSecret(context.Background(), GetSecretInput{
 			Name:      imagePullSecrets,
 			Namespace: namespace,
@@ -1723,6 +1765,14 @@ func TriggerSchedulerPostDelete(scheduler string, appName string) error {
 		common.LogWarn(fmt.Sprintf("Error deleting TLS secret for %s: %v", appName, err))
 	}
 
+	if err := DeleteConfigSecret(ctx, appName); err != nil {
+		common.LogWarn(fmt.Sprintf("Error deleting config secret for %s: %v", appName, err))
+	}
+
+	if err := DeleteImagePullSecret(ctx, appName); err != nil {
+		common.LogWarn(fmt.Sprintf("Error deleting image pull secret for %s: %v", appName, err))
+	}
+
 	return nil
 }
 

tests/unit/scheduler-k3s-secrets-deploy.bats

@@ -0,0 +1,166 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+}
+
+@test "(scheduler-k3s:secrets) helm rollback of app chart leaves config and pull secret releases intact" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP HELLO=world"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku ps:rebuild $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "sleep 30"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "helm rollback $TEST_APP 1 -n default"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get secret config-$TEST_APP -n default"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get secret pull-secret-$TEST_APP -n default"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.containers[0].envFrom[0].secretRef.name}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "config-$TEST_APP"
+}
+
+@test "(scheduler-k3s:secrets) user image-pull-secrets override skips dokku-managed pull secret release" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku scheduler-k3s:set $TEST_APP image-pull-secrets my-custom-pull-secret"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get secret pull-secret-$TEST_APP -n default"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.imagePullSecrets[*].name}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "my-custom-pull-secret"
+}
+
+@test "(scheduler-k3s:secrets) leaked imagePullSecrets entries get pruned on next deploy" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl patch deployment ${TEST_APP}-web -n default --type=json -p='[{\"op\":\"add\",\"path\":\"/spec/template/spec/imagePullSecrets/-\",\"value\":{\"name\":\"ims-${TEST_APP}.111\"}},{\"op\":\"add\",\"path\":\"/spec/template/spec/imagePullSecrets/-\",\"value\":{\"name\":\"ims-${TEST_APP}.222\"}}]'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.imagePullSecrets[*].name}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "ims-${TEST_APP}.111"
+
+  run /bin/bash -c "dokku ps:rebuild $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "sleep 30"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.imagePullSecrets[*].name}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "pull-secret-$TEST_APP"
+}

tests/unit/scheduler-k3s-secrets.bats

@@ -0,0 +1,162 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+TEST_APP="rdmtestapp"
+
+setup() {
+  uninstall_k3s || true
+  global_setup
+  dokku nginx:stop
+  export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+}
+
+teardown() {
+  global_teardown
+  dokku nginx:start
+  uninstall_k3s || true
+}
+
+@test "(scheduler-k3s:secrets) deploy creates stable config and pull secrets" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP HELLO=world"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get secret config-$TEST_APP -n default"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get secret pull-secret-$TEST_APP -n default"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.containers[0].envFrom[0].secretRef.name}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "config-$TEST_APP"
+
+  run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.imagePullSecrets[*].name}'"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "pull-secret-$TEST_APP"
+
+  run /bin/bash -c "kubectl get secret config-$TEST_APP -n default -o jsonpath='{.data.HELLO}' | base64 --decode"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "world"
+}
+
+@test "(scheduler-k3s:secrets) config:set updates secret in place" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set --no-restart $TEST_APP HELLO=world"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku config:set $TEST_APP HELLO=mars"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "sleep 10"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get secret config-$TEST_APP -n default -o jsonpath='{.data.HELLO}' | base64 --decode"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output "mars"
+}
+
+@test "(scheduler-k3s:secrets) apps:destroy cleans up secret releases" {
+  if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
+    skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
+  fi
+
+  INGRESS_CLASS=nginx install_k3s
+
+  run /bin/bash -c "dokku apps:create $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get secret config-$TEST_APP -n default"
+  assert_success
+
+  run /bin/bash -c "kubectl get secret pull-secret-$TEST_APP -n default"
+  assert_success
+
+  run /bin/bash -c "dokku --force apps:destroy $TEST_APP"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "kubectl get secret config-$TEST_APP -n default"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "kubectl get secret pull-secret-$TEST_APP -n default"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+}