diff --git a/docs/appendices/0.38.0-migration-guide.md b/docs/appendices/0.38.0-migration-guide.md index 7a7eb618e..dbf297da8 100644 --- a/docs/appendices/0.38.0-migration-guide.md +++ b/docs/appendices/0.38.0-migration-guide.md @@ -21,6 +21,7 @@ - The `docker-local` scheduler now sends `SIGTERM` to old containers immediately after a successful deploy, rather than waiting `wait-to-retire` seconds before signaling. This matches Heroku's graceful-shutdown contract and lets applications begin draining in-flight work as soon as proxy traffic switches. The `wait-to-retire` grace period and `stop-timeout-seconds` hard-stop continue to apply as before. See the [zero downtime deploys documentation](/docs/deployment/zero-downtime-deploys.md#wait-to-retire) for more details. - The `docker-local` scheduler no longer queues an image for retirement when another running container of the same app still uses it. This fixes the case where a `ps:rebuild` against an image-based deploy (`git:from-image`) produced an identical-SHA image and the `dokku-retire` cron timer would log `Image ... has running containers, skipping rm` on every run. Stuck entries from prior versions are pruned automatically on the next `ps:retire` run. - All `:report` subcommands now accept the `--global` flag, which scopes the report to globally-configured properties. The flag composes with `--format json`, so a JSON report of global properties can be obtained via, for example, `dokku scheduler:report --global --format json`. Previously, combining `--global` with `--format json` was rejected with an "info flag" error, and `--global` on its own was treated as an unknown flag. +- The `scheduler-k3s` plugin now manages env config and the dokku-generated image pull Secret as their own helm releases with stable names (`config-{app}` and `pull-secret-{app}`) rather than bundling them into the app helm chart with a per-deploy timestamp suffix (`env-{app}.{ts}` / `ims-{app}.{ts}`). This fixes two bugs: a helm rollback of the app chart no longer deletes Secrets that older ReplicaSets still reference, and the Deployment's `imagePullSecrets` list no longer accumulates references to nonexistent Secrets across deploys. The next deploy of an app switches the Deployment's `envFrom` and `imagePullSecrets` references to the stable names and prunes any leaked entries; existing live Deployments do not need to be patched manually. App rename now also uninstalls the old `tls-{app}`, `config-{app}`, and `pull-secret-{app}` releases under the previous app name; the new name's releases are recreated on the next deploy or certs sync. ### TLS handshake behavior change diff --git a/plugins/scheduler-k3s/config_secret.go b/plugins/scheduler-k3s/config_secret.go new file mode 100644 index 000000000..c36c6ba31 --- /dev/null +++ b/plugins/scheduler-k3s/config_secret.go @@ -0,0 +1,188 @@ +package scheduler_k3s + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "path/filepath" + + "github.com/dokku/dokku/plugins/common" +) + +// ConfigSecretValues contains the values for the config secret helm chart +type ConfigSecretValues struct { + Global ConfigSecretGlobalValues `yaml:"global"` +} + +// ConfigSecretGlobalValues contains the global values for the config secret chart +type ConfigSecretGlobalValues struct { + Annotations map[string]string `yaml:"annotations,omitempty"` + AppName string `yaml:"app_name"` + Labels map[string]string `yaml:"labels,omitempty"` + Namespace string `yaml:"namespace"` + Secrets map[string]string `yaml:"secrets,omitempty"` +} + +// GetConfigSecretReleaseName returns the helm release name for the config secret +func GetConfigSecretReleaseName(appName string) string { + return fmt.Sprintf("config-%s", appName) +} + +// GetConfigSecretName returns the kubernetes secret name for the config env +func GetConfigSecretName(appName string) string { + return fmt.Sprintf("config-%s", appName) +} + +// CreateOrUpdateConfigSecretInput contains the inputs to CreateOrUpdateConfigSecret +type CreateOrUpdateConfigSecretInput struct { + AppName string + Env map[string]string + Annotations map[string]string + Labels map[string]string +} + +// CreateOrUpdateConfigSecret creates or updates the config env secret helm chart for an app +func CreateOrUpdateConfigSecret(ctx context.Context, input CreateOrUpdateConfigSecretInput) error { + appName := input.AppName + if err := isKubernetesAvailable(); err != nil { + common.LogDebug("kubernetes not available, skipping config secret creation") + return nil + } + + scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "") + globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local") + if scheduler == "" { + scheduler = globalScheduler + } + if scheduler != "k3s" { + common.LogDebug("app does not use k3s scheduler, skipping config secret creation") + return nil + } + + chartDir, err := os.MkdirTemp("", "dokku-config-secret-chart-") + if err != nil { + return fmt.Errorf("error creating chart directory: %w", err) + } + defer os.RemoveAll(chartDir) + + if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil { + return fmt.Errorf("error creating chart templates directory: %w", err) + } + + namespace := getComputedNamespace(appName) + releaseName := GetConfigSecretReleaseName(appName) + + chart := &Chart{ + ApiVersion: "v2", + AppVersion: "1.0.0", + Name: releaseName, + Icon: "https://dokku.com/assets/dokku-logo.svg", + Version: "0.0.1", + } + + err = writeYaml(WriteYamlInput{ + Object: chart, + Path: filepath.Join(chartDir, "Chart.yaml"), + }) + if err != nil { + return fmt.Errorf("error writing chart: %w", err) + } + + b, err := templates.ReadFile("templates/config-secret-chart/templates/config-secret.yaml") + if err != nil { + return fmt.Errorf("error reading config-secret template: %w", err) + } + + filename := filepath.Join(chartDir, "templates", "config-secret.yaml") + err = os.WriteFile(filename, b, os.FileMode(0644)) + if err != nil { + return fmt.Errorf("error writing config-secret template: %w", err) + } + + if os.Getenv("DOKKU_TRACE") == "1" { + common.CatFile(filename) + } + + encodedSecrets := map[string]string{} + for key, value := range input.Env { + encodedSecrets[key] = base64.StdEncoding.EncodeToString([]byte(value)) + } + + values := &ConfigSecretValues{ + Global: ConfigSecretGlobalValues{ + Annotations: input.Annotations, + AppName: appName, + Labels: input.Labels, + Namespace: namespace, + Secrets: encodedSecrets, + }, + } + + err = writeYaml(WriteYamlInput{ + Object: values, + Path: filepath.Join(chartDir, "values.yaml"), + }) + if err != nil { + return fmt.Errorf("error writing values: %w", err) + } + + if err := createKubernetesNamespace(ctx, namespace); err != nil { + return fmt.Errorf("error creating namespace: %w", err) + } + + helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter) + if err != nil { + return fmt.Errorf("error creating helm agent: %w", err) + } + + chartPath, err := filepath.Abs(chartDir) + if err != nil { + return fmt.Errorf("error getting chart path: %w", err) + } + + common.LogVerboseQuiet(fmt.Sprintf("Installing config secret for %s", appName)) + err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{ + ChartPath: chartPath, + Namespace: namespace, + ReleaseName: releaseName, + Wait: false, + }) + if err != nil { + return fmt.Errorf("error installing config secret chart: %w", err) + } + + return nil +} + +// DeleteConfigSecret deletes the config env secret helm chart for an app +func DeleteConfigSecret(ctx context.Context, appName string) error { + if err := isKubernetesAvailable(); err != nil { + common.LogDebug("kubernetes not available, skipping config secret deletion") + return nil + } + + namespace := getComputedNamespace(appName) + releaseName := GetConfigSecretReleaseName(appName) + + helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter) + if err != nil { + return fmt.Errorf("error creating helm agent: %w", err) + } + + exists, err := helmAgent.ChartExists(releaseName) + if err != nil { + return fmt.Errorf("error checking if config secret chart exists: %w", err) + } + + if !exists { + return nil + } + + common.LogVerboseQuiet(fmt.Sprintf("Removing config secret for %s", appName)) + if err := helmAgent.UninstallChart(releaseName); err != nil { + return fmt.Errorf("error uninstalling config secret chart: %w", err) + } + + return nil +} diff --git a/plugins/scheduler-k3s/config_secret_test.go b/plugins/scheduler-k3s/config_secret_test.go new file mode 100644 index 000000000..aedd9727e --- /dev/null +++ b/plugins/scheduler-k3s/config_secret_test.go @@ -0,0 +1,65 @@ +package scheduler_k3s + +import ( + "encoding/base64" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestGetConfigSecretReleaseName(t *testing.T) { + got := GetConfigSecretReleaseName("myapp") + want := "config-myapp" + if got != want { + t.Fatalf("GetConfigSecretReleaseName(\"myapp\") = %q, want %q", got, want) + } +} + +func TestGetConfigSecretName(t *testing.T) { + got := GetConfigSecretName("myapp") + want := "config-myapp" + if got != want { + t.Fatalf("GetConfigSecretName(\"myapp\") = %q, want %q", got, want) + } +} + +func TestConfigSecretValuesEncoding(t *testing.T) { + values := &ConfigSecretValues{ + Global: ConfigSecretGlobalValues{ + AppName: "myapp", + Namespace: "default", + Secrets: map[string]string{ + "DATABASE_URL": base64.StdEncoding.EncodeToString([]byte("postgres://localhost/db")), + "REDIS_URL": base64.StdEncoding.EncodeToString([]byte("redis://localhost:6379")), + }, + }, + } + + data, err := yaml.Marshal(values) + if err != nil { + t.Fatalf("yaml.Marshal returned error: %v", err) + } + + var roundTrip ConfigSecretValues + if err := yaml.Unmarshal(data, &roundTrip); err != nil { + t.Fatalf("yaml.Unmarshal returned error: %v", err) + } + + if roundTrip.Global.AppName != "myapp" { + t.Errorf("AppName = %q, want \"myapp\"", roundTrip.Global.AppName) + } + if roundTrip.Global.Namespace != "default" { + t.Errorf("Namespace = %q, want \"default\"", roundTrip.Global.Namespace) + } + if len(roundTrip.Global.Secrets) != 2 { + t.Errorf("Secrets length = %d, want 2", len(roundTrip.Global.Secrets)) + } + + decoded, err := base64.StdEncoding.DecodeString(roundTrip.Global.Secrets["DATABASE_URL"]) + if err != nil { + t.Fatalf("base64 decode failed: %v", err) + } + if string(decoded) != "postgres://localhost/db" { + t.Errorf("DATABASE_URL decoded = %q, want \"postgres://localhost/db\"", string(decoded)) + } +} diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 09490fb75..faaddffca 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -808,6 +808,63 @@ func getComputedImagePullSecrets(appName string) string { return imagePullSecrets } +// pruneStaleImagePullSecretsFromDeployments rewrites the imagePullSecrets list on existing app +// Deployments to contain only the names in keepNames. This removes references that helm has +// lost track of due to strategic-merge accumulation on PodSpec.ImagePullSecrets, which is the +// root cause of stale-secret pod hard crashes after rollbacks. The patch is a no-op when the +// live list already matches. +func pruneStaleImagePullSecretsFromDeployments(ctx context.Context, clientset KubernetesClient, namespace string, appName string, keepNames []string) error { + deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{ + Namespace: namespace, + LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName), + }) + if err != nil { + return fmt.Errorf("error listing deployments: %w", err) + } + + keepSet := map[string]struct{}{} + for _, name := range keepNames { + if name != "" { + keepSet[name] = struct{}{} + } + } + + for _, deployment := range deployments { + live := deployment.Spec.Template.Spec.ImagePullSecrets + if needsImagePullSecretsPrune(live, keepSet) { + common.LogVerboseQuiet(fmt.Sprintf("Pruning stale imagePullSecrets entries from deployment %s", deployment.Name)) + err := clientset.SetDeploymentImagePullSecrets(ctx, SetDeploymentImagePullSecretsInput{ + Name: deployment.Name, + Namespace: namespace, + ImagePullSecrets: keepNames, + }) + if err != nil { + return fmt.Errorf("error pruning deployment %s: %w", deployment.Name, err) + } + } + } + + return nil +} + +// needsImagePullSecretsPrune reports whether the live imagePullSecrets list differs from the +// desired keep-set (either contains entries not in the keep-set or is missing entries in it). +func needsImagePullSecretsPrune(live []corev1.LocalObjectReference, keepSet map[string]struct{}) bool { + if len(live) != len(keepSet) { + return true + } + + seen := map[string]struct{}{} + for _, ref := range live { + if _, ok := keepSet[ref.Name]; !ok { + return true + } + seen[ref.Name] = struct{}{} + } + + return len(seen) != len(keepSet) +} + func getGlobalIngressClass() string { return common.PropertyGetDefault("scheduler-k3s", "--global", "ingress-class", DefaultIngressClass) } diff --git a/plugins/scheduler-k3s/functions_test.go b/plugins/scheduler-k3s/functions_test.go new file mode 100644 index 000000000..10bf58d6c --- /dev/null +++ b/plugins/scheduler-k3s/functions_test.go @@ -0,0 +1,93 @@ +package scheduler_k3s + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func TestNeedsImagePullSecretsPrune(t *testing.T) { + cases := []struct { + name string + live []corev1.LocalObjectReference + keepSet map[string]struct{} + want bool + }{ + { + name: "empty live, empty keep", + live: []corev1.LocalObjectReference{}, + keepSet: map[string]struct{}{}, + want: false, + }, + { + name: "matches keep set exactly", + live: []corev1.LocalObjectReference{{Name: "pull-secret-foo"}}, + keepSet: map[string]struct{}{ + "pull-secret-foo": {}, + }, + want: false, + }, + { + name: "leaked entries beyond keep set", + live: []corev1.LocalObjectReference{ + {Name: "ims-foo.111"}, + {Name: "ims-foo.222"}, + {Name: "pull-secret-foo"}, + }, + keepSet: map[string]struct{}{ + "pull-secret-foo": {}, + }, + want: true, + }, + { + name: "live missing the keep entry", + live: []corev1.LocalObjectReference{ + {Name: "ims-foo.111"}, + }, + keepSet: map[string]struct{}{ + "pull-secret-foo": {}, + }, + want: true, + }, + { + name: "live populated, keep set empty", + live: []corev1.LocalObjectReference{{Name: "ims-foo.111"}}, + keepSet: map[string]struct{}{}, + want: true, + }, + { + name: "empty live, keep set populated", + live: []corev1.LocalObjectReference{}, + keepSet: map[string]struct{}{"pull-secret-foo": {}}, + want: true, + }, + { + name: "user override only, no leaked entries", + live: []corev1.LocalObjectReference{{Name: "my-custom-secret"}}, + keepSet: map[string]struct{}{ + "my-custom-secret": {}, + }, + want: false, + }, + { + name: "user override with leaked dokku-managed entries", + live: []corev1.LocalObjectReference{ + {Name: "ims-foo.111"}, + {Name: "my-custom-secret"}, + }, + keepSet: map[string]struct{}{ + "my-custom-secret": {}, + }, + want: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := needsImagePullSecretsPrune(tc.live, tc.keepSet) + if got != tc.want { + t.Errorf("needsImagePullSecretsPrune() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go index 2af20e8f3..205cab96c 100644 --- a/plugins/scheduler-k3s/k8s.go +++ b/plugins/scheduler-k3s/k8s.go @@ -3,6 +3,7 @@ package scheduler_k3s import ( "bufio" "context" + "encoding/json" "errors" "fmt" "io" @@ -977,6 +978,64 @@ func (k KubernetesClient) ScaleDeployment(ctx context.Context, input ScaleDeploy return nil } +// SetDeploymentImagePullSecretsInput contains all the information needed to overwrite a Deployment's imagePullSecrets list +type SetDeploymentImagePullSecretsInput struct { + // Name is the Kubernetes deployment name + Name string + + // Namespace is the Kubernetes namespace + Namespace string + + // ImagePullSecrets is the desired list of imagePullSecret names. May be empty. + ImagePullSecrets []string +} + +// SetDeploymentImagePullSecrets overwrites the deployment pod template's imagePullSecrets list with exactly +// the names supplied in input.ImagePullSecrets. Strategic-merge patches honor patchMergeKey semantics, which +// is precisely what causes leaked entries to accumulate in the first place. To force a hard replace we use +// a JSON merge patch on the imagePullSecrets path; JSON merge patch replaces array fields wholesale. +func (k KubernetesClient) SetDeploymentImagePullSecrets(ctx context.Context, input SetDeploymentImagePullSecretsInput) error { + type patchPodSpec struct { + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets"` + } + type patchPodTemplate struct { + Spec patchPodSpec `json:"spec"` + } + type patchSpec struct { + Template patchPodTemplate `json:"template"` + } + type patchBody struct { + Spec patchSpec `json:"spec"` + } + + refs := make([]corev1.LocalObjectReference, 0, len(input.ImagePullSecrets)) + for _, name := range input.ImagePullSecrets { + refs = append(refs, corev1.LocalObjectReference{Name: name}) + } + + body := patchBody{ + Spec: patchSpec{ + Template: patchPodTemplate{ + Spec: patchPodSpec{ + ImagePullSecrets: refs, + }, + }, + }, + } + + data, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("error marshalling deployment patch: %w", err) + } + + _, err = k.Client.AppsV1().Deployments(input.Namespace).Patch(ctx, input.Name, types.MergePatchType, data, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("error patching deployment imagePullSecrets: %w", err) + } + + return nil +} + type StreamLogsInput struct { // ContainerName is the Kubernetes container name ContainerName string diff --git a/plugins/scheduler-k3s/pull_secret.go b/plugins/scheduler-k3s/pull_secret.go new file mode 100644 index 000000000..47ccb02df --- /dev/null +++ b/plugins/scheduler-k3s/pull_secret.go @@ -0,0 +1,183 @@ +package scheduler_k3s + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "path/filepath" + + "github.com/dokku/dokku/plugins/common" +) + +// ImagePullSecretValues contains the values for the dokku-managed image pull secret helm chart +type ImagePullSecretValues struct { + Global ImagePullSecretGlobalValues `yaml:"global"` +} + +// ImagePullSecretGlobalValues contains the global values for the image pull secret chart +type ImagePullSecretGlobalValues struct { + Annotations map[string]string `yaml:"annotations,omitempty"` + AppName string `yaml:"app_name"` + Labels map[string]string `yaml:"labels,omitempty"` + Namespace string `yaml:"namespace"` + PullSecretBase64 string `yaml:"pull_secret_base64"` +} + +// GetImagePullSecretReleaseName returns the helm release name for the image pull secret +func GetImagePullSecretReleaseName(appName string) string { + return fmt.Sprintf("pull-secret-%s", appName) +} + +// GetImagePullSecretName returns the kubernetes secret name for the dokku-managed image pull secret +func GetImagePullSecretName(appName string) string { + return fmt.Sprintf("pull-secret-%s", appName) +} + +// CreateOrUpdateImagePullSecretInput contains the inputs to CreateOrUpdateImagePullSecret +type CreateOrUpdateImagePullSecretInput struct { + AppName string + DockerConfigJSON []byte + Annotations map[string]string + Labels map[string]string +} + +// CreateOrUpdateImagePullSecret creates or updates the image pull secret helm chart for an app +func CreateOrUpdateImagePullSecret(ctx context.Context, input CreateOrUpdateImagePullSecretInput) error { + appName := input.AppName + if err := isKubernetesAvailable(); err != nil { + common.LogDebug("kubernetes not available, skipping image pull secret creation") + return nil + } + + scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "") + globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local") + if scheduler == "" { + scheduler = globalScheduler + } + if scheduler != "k3s" { + common.LogDebug("app does not use k3s scheduler, skipping image pull secret creation") + return nil + } + + chartDir, err := os.MkdirTemp("", "dokku-pull-secret-chart-") + if err != nil { + return fmt.Errorf("error creating chart directory: %w", err) + } + defer os.RemoveAll(chartDir) + + if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil { + return fmt.Errorf("error creating chart templates directory: %w", err) + } + + namespace := getComputedNamespace(appName) + releaseName := GetImagePullSecretReleaseName(appName) + + chart := &Chart{ + ApiVersion: "v2", + AppVersion: "1.0.0", + Name: releaseName, + Icon: "https://dokku.com/assets/dokku-logo.svg", + Version: "0.0.1", + } + + err = writeYaml(WriteYamlInput{ + Object: chart, + Path: filepath.Join(chartDir, "Chart.yaml"), + }) + if err != nil { + return fmt.Errorf("error writing chart: %w", err) + } + + b, err := templates.ReadFile("templates/pull-secret-chart/templates/pull-secret.yaml") + if err != nil { + return fmt.Errorf("error reading pull-secret template: %w", err) + } + + filename := filepath.Join(chartDir, "templates", "pull-secret.yaml") + err = os.WriteFile(filename, b, os.FileMode(0644)) + if err != nil { + return fmt.Errorf("error writing pull-secret template: %w", err) + } + + if os.Getenv("DOKKU_TRACE") == "1" { + common.CatFile(filename) + } + + values := &ImagePullSecretValues{ + Global: ImagePullSecretGlobalValues{ + Annotations: input.Annotations, + AppName: appName, + Labels: input.Labels, + Namespace: namespace, + PullSecretBase64: base64.StdEncoding.EncodeToString(input.DockerConfigJSON), + }, + } + + err = writeYaml(WriteYamlInput{ + Object: values, + Path: filepath.Join(chartDir, "values.yaml"), + }) + if err != nil { + return fmt.Errorf("error writing values: %w", err) + } + + if err := createKubernetesNamespace(ctx, namespace); err != nil { + return fmt.Errorf("error creating namespace: %w", err) + } + + helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter) + if err != nil { + return fmt.Errorf("error creating helm agent: %w", err) + } + + chartPath, err := filepath.Abs(chartDir) + if err != nil { + return fmt.Errorf("error getting chart path: %w", err) + } + + common.LogVerboseQuiet(fmt.Sprintf("Installing image pull secret for %s", appName)) + err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{ + ChartPath: chartPath, + Namespace: namespace, + ReleaseName: releaseName, + Wait: false, + }) + if err != nil { + return fmt.Errorf("error installing image pull secret chart: %w", err) + } + + return nil +} + +// DeleteImagePullSecret deletes the image pull secret helm chart for an app +func DeleteImagePullSecret(ctx context.Context, appName string) error { + if err := isKubernetesAvailable(); err != nil { + common.LogDebug("kubernetes not available, skipping image pull secret deletion") + return nil + } + + namespace := getComputedNamespace(appName) + releaseName := GetImagePullSecretReleaseName(appName) + + helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter) + if err != nil { + return fmt.Errorf("error creating helm agent: %w", err) + } + + exists, err := helmAgent.ChartExists(releaseName) + if err != nil { + return fmt.Errorf("error checking if image pull secret chart exists: %w", err) + } + + if !exists { + return nil + } + + common.LogVerboseQuiet(fmt.Sprintf("Removing image pull secret for %s", appName)) + if err := helmAgent.UninstallChart(releaseName); err != nil { + return fmt.Errorf("error uninstalling image pull secret chart: %w", err) + } + + return nil +} diff --git a/plugins/scheduler-k3s/pull_secret_test.go b/plugins/scheduler-k3s/pull_secret_test.go new file mode 100644 index 000000000..204f72f5a --- /dev/null +++ b/plugins/scheduler-k3s/pull_secret_test.go @@ -0,0 +1,61 @@ +package scheduler_k3s + +import ( + "encoding/base64" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestGetImagePullSecretReleaseName(t *testing.T) { + got := GetImagePullSecretReleaseName("myapp") + want := "pull-secret-myapp" + if got != want { + t.Fatalf("GetImagePullSecretReleaseName(\"myapp\") = %q, want %q", got, want) + } +} + +func TestGetImagePullSecretName(t *testing.T) { + got := GetImagePullSecretName("myapp") + want := "pull-secret-myapp" + if got != want { + t.Fatalf("GetImagePullSecretName(\"myapp\") = %q, want %q", got, want) + } +} + +func TestImagePullSecretValuesEncoding(t *testing.T) { + dockerConfig := []byte(`{"auths":{"https://index.docker.io/v1/":{"auth":"dXNlcjpwYXNz"}}}`) + + values := &ImagePullSecretValues{ + Global: ImagePullSecretGlobalValues{ + AppName: "myapp", + Namespace: "default", + PullSecretBase64: base64.StdEncoding.EncodeToString(dockerConfig), + }, + } + + data, err := yaml.Marshal(values) + if err != nil { + t.Fatalf("yaml.Marshal returned error: %v", err) + } + + var roundTrip ImagePullSecretValues + if err := yaml.Unmarshal(data, &roundTrip); err != nil { + t.Fatalf("yaml.Unmarshal returned error: %v", err) + } + + if roundTrip.Global.AppName != "myapp" { + t.Errorf("AppName = %q, want \"myapp\"", roundTrip.Global.AppName) + } + if roundTrip.Global.Namespace != "default" { + t.Errorf("Namespace = %q, want \"default\"", roundTrip.Global.Namespace) + } + + decoded, err := base64.StdEncoding.DecodeString(roundTrip.Global.PullSecretBase64) + if err != nil { + t.Fatalf("base64 decode failed: %v", err) + } + if string(decoded) != string(dockerConfig) { + t.Errorf("PullSecretBase64 decoded = %q, want %q", string(decoded), string(dockerConfig)) + } +} diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index b850df321..f58fccc4d 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -50,14 +50,12 @@ type GlobalValues struct { Keda GlobalKedaValues `yaml:"keda"` Namespace string `yaml:"namespace"` Network GlobalNetwork `yaml:"network"` - Secrets map[string]string `yaml:"secrets,omitempty"` SecurityContext SecurityContext `yaml:"security_context,omitempty"` } type GlobalImage struct { ImagePullSecrets string `yaml:"image_pull_secrets"` Name string `yaml:"name"` - PullSecretBase64 string `yaml:"pull_secret_base64"` Type string `yaml:"type"` WorkingDir string `yaml:"working_dir"` } @@ -425,7 +423,7 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) { } maps.Copy(labels, input.Labels) - secretName := fmt.Sprintf("env-%s.%d", input.AppName, input.DeploymentID) + secretName := GetConfigSecretName(input.AppName) env := []corev1.EnvVar{} for key, value := range input.Env { diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml index 4771674eb..fec53e1f2 100644 --- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml +++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml @@ -77,7 +77,7 @@ spec: {{- end }} envFrom: - secretRef: - name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }} + name: config-{{ $.Values.global.app_name }} optional: true image: {{ $.Values.global.image.name }} imagePullPolicy: Always diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index da68f0f26..2eb5946c0 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -74,7 +74,7 @@ spec: {{- end }} envFrom: - secretRef: - name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }} + name: config-{{ $.Values.global.app_name }} optional: true image: {{ $.Values.global.image.name }} imagePullPolicy: Always diff --git a/plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml b/plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml deleted file mode 100644 index 75d0f83e1..000000000 --- a/plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if $.Values.global.image.pull_secret_base64 }} ---- -apiVersion: v1 -kind: Secret -metadata: - annotations: - app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} - dokku.com/managed: "true" - {{ include "print.annotations" (dict "config" $.Values.global "key" "secret") | indent 4 }} - labels: - app.kubernetes.io/instance: ims-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }} - app.kubernetes.io/name: ims-{{ $.Values.global.app_name }} - app.kubernetes.io/part-of: {{ $.Values.global.app_name }} - {{ include "print.labels" (dict "config" $.Values.global "key" "secret") | indent 4 }} - name: ims-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }} - namespace: {{ $.Values.global.namespace }} -data: - .dockerconfigjson: "{{ $.Values.global.image.pull_secret_base64 }}" -type: kubernetes.io/dockerconfigjson -{{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/secret.yaml b/plugins/scheduler-k3s/templates/chart/secret.yaml deleted file mode 100644 index 7a349f6fd..000000000 --- a/plugins/scheduler-k3s/templates/chart/secret.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - annotations: - app.kubernetes.io/version: {{ $.Values.global.deployment_id | quote }} - dokku.com/managed: "true" - {{ include "print.annotations" (dict "config" $.Values.global "key" "secret") | indent 4 }} - labels: - app.kubernetes.io/instance: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }} - app.kubernetes.io/name: env-{{ $.Values.global.app_name }} - app.kubernetes.io/part-of: {{ $.Values.global.app_name }} - {{ include "print.labels" (dict "config" $.Values.global "key" "secret") | indent 4 }} - name: env-{{ $.Values.global.app_name }}.{{ $.Values.global.deployment_id }} - namespace: {{ $.Values.global.namespace }} -{{- with .Values.global.secrets }} -data: - {{- toYaml . | nindent 2 }} -{{- end }} diff --git a/plugins/scheduler-k3s/templates/config-secret-chart/templates/config-secret.yaml b/plugins/scheduler-k3s/templates/config-secret-chart/templates/config-secret.yaml new file mode 100644 index 000000000..cbb2ed52f --- /dev/null +++ b/plugins/scheduler-k3s/templates/config-secret-chart/templates/config-secret.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Secret +metadata: + name: config-{{ .Values.global.app_name }} + namespace: {{ .Values.global.namespace }} + labels: + app.kubernetes.io/name: config-{{ .Values.global.app_name }} + app.kubernetes.io/part-of: {{ .Values.global.app_name }} + {{- range $k, $v := .Values.global.labels }} + {{ $k }}: {{ $v | quote }} + {{- end }} + annotations: + dokku.com/managed: "true" + {{- range $k, $v := .Values.global.annotations }} + {{ $k }}: {{ $v | quote }} + {{- end }} +{{- with .Values.global.secrets }} +data: + {{- toYaml . | nindent 2 }} +{{- end }} diff --git a/plugins/scheduler-k3s/templates/pull-secret-chart/templates/pull-secret.yaml b/plugins/scheduler-k3s/templates/pull-secret-chart/templates/pull-secret.yaml new file mode 100644 index 000000000..778c491b3 --- /dev/null +++ b/plugins/scheduler-k3s/templates/pull-secret-chart/templates/pull-secret.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Secret +type: kubernetes.io/dockerconfigjson +metadata: + name: pull-secret-{{ .Values.global.app_name }} + namespace: {{ .Values.global.namespace }} + labels: + app.kubernetes.io/name: pull-secret-{{ .Values.global.app_name }} + app.kubernetes.io/part-of: {{ .Values.global.app_name }} + {{- range $k, $v := .Values.global.labels }} + {{ $k }}: {{ $v | quote }} + {{- end }} + annotations: + dokku.com/managed: "true" + {{- range $k, $v := .Values.global.annotations }} + {{ $k }}: {{ $v | quote }} + {{- end }} +data: + .dockerconfigjson: {{ .Values.global.pull_secret_base64 | quote }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index f95ccc2c9..bc5d406a9 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/rand" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -170,6 +169,21 @@ func TriggerPostAppRenameSetup(oldAppName string, newAppName string) error { return err } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if _, err := DeleteTLSSecret(ctx, oldAppName); err != nil { + common.LogWarn(fmt.Sprintf("Error deleting TLS secret for old app name %s: %v", oldAppName, err)) + } + + if err := DeleteConfigSecret(ctx, oldAppName); err != nil { + common.LogWarn(fmt.Sprintf("Error deleting config secret for old app name %s: %v", oldAppName, err)) + } + + if err := DeleteImagePullSecret(ctx, oldAppName); err != nil { + common.LogWarn(fmt.Sprintf("Error deleting image pull secret for old app name %s: %v", oldAppName, err)) + } + return nil } @@ -403,8 +417,9 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } deploymentId := time.Now().Unix() - pullSecretBase64 := base64.StdEncoding.EncodeToString([]byte("")) imagePullSecrets := getComputedImagePullSecrets(appName) + dokkuManagedPullSecret := false + var dokkuPullSecretBytes []byte if imagePullSecrets == "" { dockerConfigPath := filepath.Join(registry.GetComputedAppRegistryConfigDir(appName), "config.json") if fi, err := os.Stat(dockerConfigPath); err == nil && !fi.IsDir() { @@ -413,12 +428,13 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error reading docker config: %w", err) } - imagePullSecrets = fmt.Sprintf("ims-%s.%d", appName, deploymentId) - pullSecretBase64 = base64.StdEncoding.EncodeToString(b) + imagePullSecrets = GetImagePullSecretName(appName) + dokkuManagedPullSecret = true + dokkuPullSecretBytes = b } } - globalTemplateFiles := []string{"service-account", "secret", "image-pull-secret"} + globalTemplateFiles := []string{"service-account"} for _, templateName := range globalTemplateFiles { b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName)) if err != nil { @@ -541,7 +557,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e Keda: kedaValues, Image: GlobalImage{ ImagePullSecrets: imagePullSecrets, - PullSecretBase64: pullSecretBase64, Name: image, Type: imageSourceType, WorkingDir: workingDir, @@ -553,7 +568,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e PrimaryPort: primaryPort, PrimaryServicePort: primaryServicePort, }, - Secrets: map[string]string{}, SecurityContext: securityContext, }, Processes: map[string]ProcessValues{}, @@ -851,10 +865,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } } - for key, value := range env.Map() { - values.Global.Secrets[key] = base64.StdEncoding.EncodeToString([]byte(value)) - } - b, err := templates.ReadFile("templates/chart/_helpers.tpl") if err != nil { return fmt.Errorf("Error reading _helpers template: %w", err) @@ -925,6 +935,38 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } } + if err := CreateOrUpdateConfigSecret(ctx, CreateOrUpdateConfigSecretInput{ + AppName: appName, + Env: env.Map(), + Annotations: globalAnnotations.SecretAnnotations, + Labels: globalLabels.SecretLabels, + }); err != nil { + return fmt.Errorf("Error syncing config secret: %w", err) + } + + if dokkuManagedPullSecret { + if err := CreateOrUpdateImagePullSecret(ctx, CreateOrUpdateImagePullSecretInput{ + AppName: appName, + DockerConfigJSON: dokkuPullSecretBytes, + Annotations: globalAnnotations.SecretAnnotations, + Labels: globalLabels.SecretLabels, + }); err != nil { + return fmt.Errorf("Error syncing image pull secret: %w", err) + } + } else { + if err := DeleteImagePullSecret(ctx, appName); err != nil { + return fmt.Errorf("Error removing stale image pull secret: %w", err) + } + } + + keepImagePullSecrets := []string{} + if imagePullSecrets != "" { + keepImagePullSecrets = []string{imagePullSecrets} + } + if err := pruneStaleImagePullSecretsFromDeployments(ctx, clientset, namespace, appName, keepImagePullSecrets); err != nil { + return fmt.Errorf("Error pruning stale imagePullSecrets entries: %w", err) + } + kustomizeRootPath := "" if hasKustomizeDirectory(appName) { kustomizeRootPath = getProcessSpecificKustomizeRootPath(appName) @@ -1402,7 +1444,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] imagePullSecrets := getComputedImagePullSecrets(appName) if imagePullSecrets == "" { - imagePullSecrets = fmt.Sprintf("ims-%s.%d", appName, deploymentID) + imagePullSecrets = GetImagePullSecretName(appName) _, err := clientset.GetSecret(context.Background(), GetSecretInput{ Name: imagePullSecrets, Namespace: namespace, @@ -1723,6 +1765,14 @@ func TriggerSchedulerPostDelete(scheduler string, appName string) error { common.LogWarn(fmt.Sprintf("Error deleting TLS secret for %s: %v", appName, err)) } + if err := DeleteConfigSecret(ctx, appName); err != nil { + common.LogWarn(fmt.Sprintf("Error deleting config secret for %s: %v", appName, err)) + } + + if err := DeleteImagePullSecret(ctx, appName); err != nil { + common.LogWarn(fmt.Sprintf("Error deleting image pull secret for %s: %v", appName, err)) + } + return nil } diff --git a/tests/unit/scheduler-k3s-secrets-deploy.bats b/tests/unit/scheduler-k3s-secrets-deploy.bats new file mode 100644 index 000000000..58295bb95 --- /dev/null +++ b/tests/unit/scheduler-k3s-secrets-deploy.bats @@ -0,0 +1,166 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true +} + +@test "(scheduler-k3s:secrets) helm rollback of app chart leaves config and pull secret releases intact" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set --no-restart $TEST_APP HELLO=world" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 30" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "helm rollback $TEST_APP 1 -n default" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret config-$TEST_APP -n default" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret pull-secret-$TEST_APP -n default" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.containers[0].envFrom[0].secretRef.name}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "config-$TEST_APP" +} + +@test "(scheduler-k3s:secrets) user image-pull-secrets override skips dokku-managed pull secret release" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:set $TEST_APP image-pull-secrets my-custom-pull-secret" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret pull-secret-$TEST_APP -n default" + echo "output: $output" + echo "status: $status" + assert_failure + + run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.imagePullSecrets[*].name}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "my-custom-pull-secret" +} + +@test "(scheduler-k3s:secrets) leaked imagePullSecrets entries get pruned on next deploy" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl patch deployment ${TEST_APP}-web -n default --type=json -p='[{\"op\":\"add\",\"path\":\"/spec/template/spec/imagePullSecrets/-\",\"value\":{\"name\":\"ims-${TEST_APP}.111\"}},{\"op\":\"add\",\"path\":\"/spec/template/spec/imagePullSecrets/-\",\"value\":{\"name\":\"ims-${TEST_APP}.222\"}}]'" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.imagePullSecrets[*].name}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "ims-${TEST_APP}.111" + + run /bin/bash -c "dokku ps:rebuild $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 30" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.imagePullSecrets[*].name}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "pull-secret-$TEST_APP" +} diff --git a/tests/unit/scheduler-k3s-secrets.bats b/tests/unit/scheduler-k3s-secrets.bats new file mode 100644 index 000000000..830e6759a --- /dev/null +++ b/tests/unit/scheduler-k3s-secrets.bats @@ -0,0 +1,162 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true +} + +@test "(scheduler-k3s:secrets) deploy creates stable config and pull secrets" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set --no-restart $TEST_APP HELLO=world" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret config-$TEST_APP -n default" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret pull-secret-$TEST_APP -n default" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.containers[0].envFrom[0].secretRef.name}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "config-$TEST_APP" + + run /bin/bash -c "kubectl get deployment ${TEST_APP}-web -n default -o jsonpath='{.spec.template.spec.imagePullSecrets[*].name}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "pull-secret-$TEST_APP" + + run /bin/bash -c "kubectl get secret config-$TEST_APP -n default -o jsonpath='{.data.HELLO}' | base64 --decode" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "world" +} + +@test "(scheduler-k3s:secrets) config:set updates secret in place" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set --no-restart $TEST_APP HELLO=world" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP HELLO=mars" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 10" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret config-$TEST_APP -n default -o jsonpath='{.data.HELLO}' | base64 --decode" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "mars" +} + +@test "(scheduler-k3s:secrets) apps:destroy cleans up secret releases" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret config-$TEST_APP -n default" + assert_success + + run /bin/bash -c "kubectl get secret pull-secret-$TEST_APP -n default" + assert_success + + run /bin/bash -c "dokku --force apps:destroy $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret config-$TEST_APP -n default" + echo "output: $output" + echo "status: $status" + assert_failure + + run /bin/bash -c "kubectl get secret pull-secret-$TEST_APP -n default" + echo "output: $output" + echo "status: $status" + assert_failure +}