From b8e8ea74fffe879c03b1980586a56024fc364300 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 8 Jan 2026 01:05:40 -0500 Subject: [PATCH] feat: use certificates imported by certs plugin when deploying via scheduler-k3s Closes #7257 --- docs/deployment/schedulers/k3s.md | 30 ++ docs/development/plugin-triggers.md | 24 ++ plugins/certs/certs-get | 24 ++ plugins/scheduler-k3s/.gitignore | 1 + plugins/scheduler-k3s/Makefile | 2 +- plugins/scheduler-k3s/functions.go | 61 ++++ .../scheduler-k3s/src/triggers/triggers.go | 6 + plugins/scheduler-k3s/template.go | 5 +- .../templates/chart/certificate.yaml | 2 +- .../templates/chart/ingress-route.yaml | 4 + .../templates/chart/ingress.yaml | 4 + .../templates/tls-secret.yaml | 16 + plugins/scheduler-k3s/tls.go | 332 ++++++++++++++++++ plugins/scheduler-k3s/triggers.go | 143 ++++++-- tests/unit/scheduler-k3s-certs-deploy.bats | 143 ++++++++ tests/unit/scheduler-k3s-certs.bats | 140 ++++++++ 16 files changed, 899 insertions(+), 38 deletions(-) create mode 100755 plugins/certs/certs-get create mode 100644 plugins/scheduler-k3s/templates/tls-secret-chart/templates/tls-secret.yaml create mode 100644 plugins/scheduler-k3s/tls.go create mode 100644 tests/unit/scheduler-k3s-certs-deploy.bats create mode 100755 tests/unit/scheduler-k3s-certs.bats diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 2c98958ec..69de38b3d 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -307,6 +307,36 @@ The server can also be disabled globally, and then conditionally enabled on a pe dokku scheduler-k3s:set --global letsencrypt-server false ``` +#### Using imported SSL certificates + +SSL certificates imported via the `certs` plugin can be used with the k3s scheduler. When a certificate is imported, it is automatically synced to Kubernetes as a TLS secret and will be used for the app's ingress configuration. + +To import a certificate: + +```shell +dokku certs:add node-js-app server.crt server.key +``` + +When a certificate is imported: + +- A Kubernetes TLS secret named `tls-` is created in the app's namespace +- The ingress configuration is updated to use the imported certificate +- Automatic Let's Encrypt certificate generation is disabled for the app + +Imported certificates take precedence over Let's Encrypt certificates. If you have both an imported certificate and Let's Encrypt configured, the imported certificate will be used. + +To remove an imported certificate: + +```shell +dokku certs:remove node-js-app +``` + +When a certificate is removed: + +- The TLS secret is deleted from Kubernetes +- The app is automatically redeployed to update the ingress configuration +- If Let's Encrypt is configured, automatic certificate generation will resume + ### Customizing Annotations and Labels > [!NOTE] diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 77788ad3b..35e809137 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -387,6 +387,30 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x # TODO ``` +### `certs-get` + +- Description: Returns the certificate or key content for an app +- Invoked by: `scheduler-k3s` +- Arguments: `$APP $KEY_TYPE` +- Example: + +```shell +#!/usr/bin/env bash +# Returns the certificate or key content for an app +# KEY_TYPE should be "crt" or "key" + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +APP="$1"; KEY_TYPE="$2" +APP_SSL_PATH="$DOKKU_ROOT/$APP/tls" + +if [[ "$KEY_TYPE" == "crt" ]]; then + cat "$APP_SSL_PATH/server.crt" +elif [[ "$KEY_TYPE" == "key" ]]; then + cat "$APP_SSL_PATH/server.key" +fi +``` + ### `check-deploy` - Description: Allows you to run checks on a deploy before Dokku allows the container to handle requests. diff --git a/plugins/certs/certs-get b/plugins/certs/certs-get new file mode 100755 index 000000000..b4192ae63 --- /dev/null +++ b/plugins/certs/certs-get @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/certs/functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-certs-certs-get() { + declare desc="returns certificate or key content" + declare trigger="certs-get" + declare APP="$1" KEY_TYPE="$2" + + local APP_SSL_PATH="$DOKKU_ROOT/$APP/tls" + + if [[ "$KEY_TYPE" != "key" ]] && [[ "$KEY_TYPE" != "crt" ]]; then + return 1 + fi + + if ! is_ssl_enabled "$APP"; then + return 1 + fi + + cat "$APP_SSL_PATH/server.$KEY_TYPE" +} + +trigger-certs-certs-get "$@" diff --git a/plugins/scheduler-k3s/.gitignore b/plugins/scheduler-k3s/.gitignore index c6bcef78b..120774657 100644 --- a/plugins/scheduler-k3s/.gitignore +++ b/plugins/scheduler-k3s/.gitignore @@ -6,3 +6,4 @@ /install /post-* /report +/core-* diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index 3c46a33ef..fce3bb816 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/annotations:set subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall -TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write +TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index fe718153b..3cbbc3251 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1262,6 +1262,67 @@ func getGlobalGlobalToken() string { return common.PropertyGet("scheduler-k3s", "--global", "token") } +func getDeployedAppImageTag(appName string) (string, error) { + appValues, err := helmValuesForApp(appName) + if err != nil { + return "", err + } + + if appValues.Global.Image.Name == "" { + return "", fmt.Errorf("image name not found in helm release") + } + + return appValues.Global.Image.Name, nil +} + +func helmValuesForApp(appName string) (AppValues, error) { + namespace := getComputedNamespace(appName) + helmAgent, err := NewHelmAgent(namespace, DevNullPrinter) + if err != nil { + return AppValues{}, fmt.Errorf("error creating helm agent: %w", err) + } + + exists, err := helmAgent.ChartExists(appName) + if err != nil { + return AppValues{}, fmt.Errorf("error checking if chart exists: %w", err) + } + if !exists { + return AppValues{}, fmt.Errorf("app %s is not deployed", appName) + } + + values, err := helmAgent.GetValues(appName) + if err != nil { + return AppValues{}, fmt.Errorf("error getting helm values: %w", err) + } + + b, err := yaml.Marshal(values) + if err != nil { + return AppValues{}, fmt.Errorf("error marshaling helm values: %w", err) + } + + var appValues AppValues + if err := yaml.Unmarshal(b, &appValues); err != nil { + return AppValues{}, fmt.Errorf("error unmarshaling helm values: %w", err) + } + + return appValues, nil +} + +func isAppDeployed(appName string) bool { + namespace := getComputedNamespace(appName) + helmAgent, err := NewHelmAgent(namespace, DevNullPrinter) + if err != nil { + return false + } + + exists, err := helmAgent.ChartExists(appName) + if err != nil { + return false + } + + return exists +} + func getProcessHealtchecks(healthchecks []appjson.Healthcheck, primaryPort int32) ProcessHealthchecks { if len(healthchecks) == 0 { return ProcessHealthchecks{} diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go index 15caf5146..d78606e72 100644 --- a/plugins/scheduler-k3s/src/triggers/triggers.go +++ b/plugins/scheduler-k3s/src/triggers/triggers.go @@ -42,6 +42,12 @@ func main() { case "post-create": appName := flag.Arg(0) err = scheduler_k3s.TriggerPostCreate(appName) + case "post-certs-update": + appName := flag.Arg(0) + err = scheduler_k3s.TriggerPostCertsUpdate(appName) + case "post-certs-remove": + appName := flag.Arg(0) + err = scheduler_k3s.TriggerPostCertsRemove(appName) case "post-delete": appName := flag.Arg(0) err = scheduler_k3s.TriggerPostDelete(appName) diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index 4f57700ba..b850df321 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -336,8 +336,9 @@ func (a NameSorter) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a NameSorter) Less(i, j int) bool { return a[i].Name < a[j].Name } type ProcessTls struct { - Enabled bool `yaml:"enabled"` - IssuerName string `yaml:"issuer_name"` + Enabled bool `yaml:"enabled"` + IssuerName string `yaml:"issuer_name"` + UseImportedCert bool `yaml:"use_imported_cert"` } type ClusterIssuer struct { diff --git a/plugins/scheduler-k3s/templates/chart/certificate.yaml b/plugins/scheduler-k3s/templates/chart/certificate.yaml index 077470ce8..0f3fe05a9 100644 --- a/plugins/scheduler-k3s/templates/chart/certificate.yaml +++ b/plugins/scheduler-k3s/templates/chart/certificate.yaml @@ -4,7 +4,7 @@ {{- continue }} {{- end }} -{{- if and $config.web.tls.enabled $config.web.domains }} +{{- if and $config.web.tls.enabled (not $config.web.tls.use_imported_cert) $config.web.domains }} --- apiVersion: cert-manager.io/v1 kind: Certificate diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index fb137204e..24dd8e2ef 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -53,7 +53,11 @@ spec: {{- end }} {{- if $config.web.tls.enabled }} tls: + {{- if $config.web.tls.use_imported_cert }} + secretName: tls-{{ $.Values.global.app_name }} + {{- else }} secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/ingress.yaml b/plugins/scheduler-k3s/templates/chart/ingress.yaml index 1b5cd1081..3644fd2b8 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress.yaml @@ -38,7 +38,11 @@ spec: tls: - hosts: - {{ $domain.name | quote }} + {{- if $config.web.tls.use_imported_cert }} + secretName: tls-{{ $.Values.global.app_name }} + {{- else }} secretName: tls-{{ $.Values.global.app_name }}-{{ $processName }} + {{- end }} {{- end }} rules: - host: {{ $domain.name | quote }} diff --git a/plugins/scheduler-k3s/templates/tls-secret-chart/templates/tls-secret.yaml b/plugins/scheduler-k3s/templates/tls-secret-chart/templates/tls-secret.yaml new file mode 100644 index 000000000..ee25cac09 --- /dev/null +++ b/plugins/scheduler-k3s/templates/tls-secret-chart/templates/tls-secret.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + name: tls-{{ .Values.global.app_name }} + namespace: {{ .Values.global.namespace }} + labels: + app.kubernetes.io/name: tls-{{ .Values.global.app_name }} + app.kubernetes.io/part-of: {{ .Values.global.app_name }} + dokku.com/cert-source: "imported" + dokku.com/cert-checksum: {{ .Values.global.cert_checksum | quote }} + annotations: + dokku.com/managed: "true" +data: + tls.crt: {{ .Values.global.tls_crt }} + tls.key: {{ .Values.global.tls_key }} diff --git a/plugins/scheduler-k3s/tls.go b/plugins/scheduler-k3s/tls.go new file mode 100644 index 000000000..4c8c9f0c4 --- /dev/null +++ b/plugins/scheduler-k3s/tls.go @@ -0,0 +1,332 @@ +package scheduler_k3s + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/dokku/dokku/plugins/common" +) + +// TLSSecretValues contains the values for a TLS secret helm chart +type TLSSecretValues struct { + Global TLSSecretGlobalValues `yaml:"global"` +} + +// TLSSecretGlobalValues contains the global values for TLS secret +type TLSSecretGlobalValues struct { + AppName string `yaml:"app_name"` + Namespace string `yaml:"namespace"` + CertChecksum string `yaml:"cert_checksum"` + TLSCrt string `yaml:"tls_crt"` + TLSKey string `yaml:"tls_key"` +} + +// GetCertContent retrieves certificate content via the certs-get trigger +func GetCertContent(appName string, keyType string) (string, error) { + result, err := common.CallPlugnTrigger(common.PlugnTriggerInput{ + Trigger: "certs-get", + Args: []string{appName, keyType}, + }) + if err != nil { + return "", fmt.Errorf("failed to get cert content: %w", err) + } + if result.ExitCode != 0 { + return "", fmt.Errorf("certs-get trigger returned non-zero exit code") + } + return result.StdoutContents(), nil +} + +// CertsExist checks if certificates exist for an app +func CertsExist(appName string) bool { + result, err := common.CallPlugnTrigger(common.PlugnTriggerInput{ + Trigger: "certs-exists", + Args: []string{appName}, + }) + if err != nil { + return false + } + return strings.TrimSpace(result.StdoutContents()) == "true" +} + +// ComputeCertChecksum computes sha224 of combined cert+key content +// SHA224 produces a 56-character hex string which fits within Kubernetes label limit of 63 chars +func ComputeCertChecksum(certContent, keyContent string) string { + combined := certContent + keyContent + hash := sha256.Sum224([]byte(combined)) + return hex.EncodeToString(hash[:]) +} + +// GetTLSSecretReleaseName returns the helm release name for TLS secret +func GetTLSSecretReleaseName(appName string) string { + return fmt.Sprintf("tls-%s", appName) +} + +// CreateOrUpdateTLSSecret creates or updates a TLS secret helm chart for an app +func CreateOrUpdateTLSSecret(ctx context.Context, appName string) error { + if err := isKubernetesAvailable(); err != nil { + common.LogDebug("kubernetes not available, skipping TLS secret creation") + return nil + } + + scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "") + globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local") + if scheduler == "" { + scheduler = globalScheduler + } + if scheduler != "k3s" { + common.LogDebug("app does not use k3s scheduler, skipping TLS secret creation") + return nil + } + + certContent, err := GetCertContent(appName, "crt") + if err != nil { + return fmt.Errorf("failed to get certificate: %w", err) + } + + keyContent, err := GetCertContent(appName, "key") + if err != nil { + return fmt.Errorf("failed to get key: %w", err) + } + + checksum := ComputeCertChecksum(certContent, keyContent) + + chartDir, err := os.MkdirTemp("", "dokku-tls-chart-") + if err != nil { + return fmt.Errorf("error creating chart directory: %w", err) + } + defer os.RemoveAll(chartDir) + + if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil { + return fmt.Errorf("error creating chart templates directory: %w", err) + } + + namespace := getComputedNamespace(appName) + releaseName := GetTLSSecretReleaseName(appName) + + chart := &Chart{ + ApiVersion: "v2", + AppVersion: "1.0.0", + Name: releaseName, + Icon: "https://dokku.com/assets/dokku-logo.svg", + Version: "0.0.1", + } + + err = writeYaml(WriteYamlInput{ + Object: chart, + Path: filepath.Join(chartDir, "Chart.yaml"), + }) + if err != nil { + return fmt.Errorf("error writing chart: %w", err) + } + + b, err := templates.ReadFile("templates/tls-secret-chart/templates/tls-secret.yaml") + if err != nil { + return fmt.Errorf("error reading tls-secret template: %w", err) + } + + filename := filepath.Join(chartDir, "templates", "tls-secret.yaml") + err = os.WriteFile(filename, b, os.FileMode(0644)) + if err != nil { + return fmt.Errorf("error writing tls-secret template: %w", err) + } + + if os.Getenv("DOKKU_TRACE") == "1" { + common.CatFile(filename) + } + + values := &TLSSecretValues{ + Global: TLSSecretGlobalValues{ + AppName: appName, + Namespace: namespace, + CertChecksum: checksum, + TLSCrt: base64.StdEncoding.EncodeToString([]byte(certContent)), + TLSKey: base64.StdEncoding.EncodeToString([]byte(keyContent)), + }, + } + + err = writeYaml(WriteYamlInput{ + Object: values, + Path: filepath.Join(chartDir, "values.yaml"), + }) + if err != nil { + return fmt.Errorf("error writing values: %w", err) + } + + if err := createKubernetesNamespace(ctx, namespace); err != nil { + return fmt.Errorf("error creating namespace: %w", err) + } + + helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter) + if err != nil { + return fmt.Errorf("error creating helm agent: %w", err) + } + + chartPath, err := filepath.Abs(chartDir) + if err != nil { + return fmt.Errorf("error getting chart path: %w", err) + } + + common.LogInfo1(fmt.Sprintf("Installing TLS certificate for %s", appName)) + err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{ + ChartPath: chartPath, + Namespace: namespace, + ReleaseName: releaseName, + Wait: true, + }) + if err != nil { + return fmt.Errorf("error installing TLS secret chart: %w", err) + } + + if err := common.PropertyWrite("scheduler-k3s", appName, "tls-cert-imported", "true"); err != nil { + return fmt.Errorf("error setting tls-cert-imported property: %w", err) + } + + return nil +} + +// DeleteTLSSecret deletes the TLS secret helm chart for an app +func DeleteTLSSecret(ctx context.Context, appName string) (bool, error) { + if err := isKubernetesAvailable(); err != nil { + common.LogDebug("kubernetes not available, skipping TLS secret deletion") + return false, nil + } + + namespace := getComputedNamespace(appName) + releaseName := GetTLSSecretReleaseName(appName) + + helmAgent, err := NewHelmAgent(namespace, DeployLogPrinter) + if err != nil { + return false, fmt.Errorf("error creating helm agent: %w", err) + } + + exists, err := helmAgent.ChartExists(releaseName) + if err != nil { + return false, fmt.Errorf("error checking if TLS secret chart exists: %w", err) + } + + if !exists { + if err := common.PropertyDelete("scheduler-k3s", appName, "tls-cert-imported"); err != nil { + common.LogDebug(fmt.Sprintf("error clearing tls-cert-imported property: %v", err)) + } + return false, nil + } + + common.LogInfo1(fmt.Sprintf("Removing TLS certificate for %s", appName)) + err = helmAgent.UninstallChart(releaseName) + if err != nil { + return false, fmt.Errorf("error uninstalling TLS secret chart: %w", err) + } + + if err := common.PropertyDelete("scheduler-k3s", appName, "tls-cert-imported"); err != nil { + common.LogDebug(fmt.Sprintf("error clearing tls-cert-imported property: %v", err)) + } + + return true, nil +} + +// HasImportedTLSCert checks if an app has an imported TLS certificate +func HasImportedTLSCert(appName string) bool { + value := common.PropertyGetDefault("scheduler-k3s", appName, "tls-cert-imported", "") + return value == "true" +} + +// TLSSecretExists checks if the TLS secret helm release exists in k8s +func TLSSecretExists(ctx context.Context, appName string) (bool, error) { + namespace := getComputedNamespace(appName) + releaseName := GetTLSSecretReleaseName(appName) + + helmAgent, err := NewHelmAgent(namespace, DevNullPrinter) + if err != nil { + return false, fmt.Errorf("error creating helm agent: %w", err) + } + + return helmAgent.ChartExists(releaseName) +} + +// TLSSecretNeedsUpdate checks if the TLS secret needs updating by comparing checksums +func TLSSecretNeedsUpdate(ctx context.Context, appName string) (bool, error) { + clientset, err := NewKubernetesClient() + if err != nil { + return false, fmt.Errorf("error creating kubernetes client: %w", err) + } + + namespace := getComputedNamespace(appName) + secretName := fmt.Sprintf("tls-%s", appName) + + secret, err := clientset.GetSecret(ctx, GetSecretInput{ + Name: secretName, + Namespace: namespace, + }) + if err != nil { + return true, nil + } + + existingChecksum, ok := secret.Labels["dokku.com/cert-checksum"] + if !ok { + return true, nil + } + + certContent, err := GetCertContent(appName, "crt") + if err != nil { + return false, err + } + + keyContent, err := GetCertContent(appName, "key") + if err != nil { + return false, err + } + + currentChecksum := ComputeCertChecksum(certContent, keyContent) + return existingChecksum != currentChecksum, nil +} + +// syncExistingCertificates syncs certificates for all apps using k3s scheduler +func syncExistingCertificates() error { + if err := isKubernetesAvailable(); err != nil { + common.LogDebug("kubernetes not available, skipping certificate sync") + return nil + } + + ctx := context.Background() + + apps, err := common.DokkuApps() + if err != nil { + return fmt.Errorf("failed to list apps: %w", err) + } + + for _, appName := range apps { + scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "") + globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local") + if scheduler == "" { + scheduler = globalScheduler + } + if scheduler != "k3s" { + continue + } + + if !CertsExist(appName) { + continue + } + + needsUpdate, err := TLSSecretNeedsUpdate(ctx, appName) + if err != nil { + common.LogDebug(fmt.Sprintf("Error checking TLS secret for %s: %v", appName, err)) + continue + } + + if needsUpdate { + common.LogInfo1(fmt.Sprintf("Syncing TLS certificate for %s", appName)) + if err := CreateOrUpdateTLSSecret(ctx, appName); err != nil { + common.LogWarn(fmt.Sprintf("Failed to sync TLS certificate for %s: %v", appName, err)) + } + } + } + + return nil +} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 4ca9e52dc..da6d531a8 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -42,8 +42,6 @@ func TriggerCorePostDeploy(appName string) error { {Path: "kustomization", IsDirectory: true}, }, }) - - return nil } // TriggerCorePostExtract moves a configured kustomize root path to be in the app root dir @@ -76,9 +74,81 @@ func TriggerInstall() error { return err } + if err := syncExistingCertificates(); err != nil { + common.LogWarn(fmt.Sprintf("Warning: failed to sync existing certificates: %v", err)) + } + return nil } +// TriggerPostCertsUpdate handles post-certs-update trigger +func TriggerPostCertsUpdate(appName string) error { + scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "") + globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local") + if scheduler == "" { + scheduler = globalScheduler + } + if scheduler != "k3s" { + return nil + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + common.LogInfo1(fmt.Sprintf("Syncing TLS certificate for %s to kubernetes", appName)) + if err := CreateOrUpdateTLSSecret(ctx, appName); err != nil { + return err + } + + if !isAppDeployed(appName) { + return nil + } + + imageTag, err := getDeployedAppImageTag(appName) + if err != nil { + return nil + } + + common.LogInfo1(fmt.Sprintf("Triggering redeploy for %s to update ingress configuration", appName)) + return TriggerSchedulerDeploy("k3s", appName, imageTag) +} + +// TriggerPostCertsRemove handles post-certs-remove trigger +func TriggerPostCertsRemove(appName string) error { + scheduler := common.PropertyGetDefault("scheduler", appName, "selected", "") + globalScheduler := common.PropertyGetDefault("scheduler", "--global", "selected", "docker-local") + if scheduler == "" { + scheduler = globalScheduler + } + if scheduler != "k3s" { + return nil + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + deleted, err := DeleteTLSSecret(ctx, appName) + if err != nil { + return err + } + + if !deleted { + return nil + } + + if !isAppDeployed(appName) { + return nil + } + + imageTag, err := getDeployedAppImageTag(appName) + if err != nil { + return nil + } + + common.LogInfo1(fmt.Sprintf("Triggering redeploy for %s to update ingress configuration", appName)) + return TriggerSchedulerDeploy("k3s", appName, imageTag) +} + // TriggerPostAppCloneSetup creates new scheduler-k3s files func TriggerPostAppCloneSetup(oldAppName string, newAppName string) error { err := common.PropertyClone("scheduler-k3s", oldAppName, newAppName) @@ -284,24 +354,41 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error loading environment for deployment: %w", err) } - server := getComputedLetsencryptServer(appName) - letsencryptEmailStag := getGlobalLetsencryptEmailStag() - letsencryptEmailProd := getGlobalLetsencryptEmailProd() - tlsEnabled := false + // Check for imported TLS certificate first + importedCertExists := false + if HasImportedTLSCert(appName) { + exists, err := TLSSecretExists(ctx, appName) + if err == nil && exists { + importedCertExists = true + } + } + // Determine TLS configuration + tlsEnabled := false issuerName := "" - switch server { - case "prod", "production": - issuerName = "letsencrypt-prod" - tlsEnabled = letsencryptEmailProd != "" - case "stag", "staging": - issuerName = "letsencrypt-stag" - tlsEnabled = letsencryptEmailStag != "" - case "false": - issuerName = "" - tlsEnabled = false - default: - return fmt.Errorf("Invalid letsencrypt server config: %s", server) + useImportedCert := false + + if importedCertExists { + tlsEnabled = true + useImportedCert = true + } else { + server := getComputedLetsencryptServer(appName) + letsencryptEmailStag := getGlobalLetsencryptEmailStag() + letsencryptEmailProd := getGlobalLetsencryptEmailProd() + + switch server { + case "prod", "production": + issuerName = "letsencrypt-prod" + tlsEnabled = letsencryptEmailProd != "" + case "stag", "staging": + issuerName = "letsencrypt-stag" + tlsEnabled = letsencryptEmailStag != "" + case "false": + issuerName = "" + tlsEnabled = false + default: + return fmt.Errorf("Invalid letsencrypt server config: %s", server) + } } chartDir, err := os.MkdirTemp("", "dokku-chart-") @@ -577,8 +664,9 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e Domains: domainValues, PortMaps: []ProcessPortMap{}, TLS: ProcessTls{ - Enabled: tlsEnabled, - IssuerName: issuerName, + Enabled: tlsEnabled, + IssuerName: issuerName, + UseImportedCert: useImportedCert, }, } @@ -883,20 +971,7 @@ func TriggerSchedulerIsDeployed(scheduler string, appName string) error { return nil } - // check if there are any helm revisions for the specified appName - helmAgent, err := NewHelmAgent(getComputedNamespace(appName), DeployLogPrinter) - if err != nil { - return fmt.Errorf("Error creating helm agent: %w", err) - } - - revisions, err := helmAgent.ListRevisions(ListRevisionsInput{ - ReleaseName: appName, - }) - if err != nil { - return fmt.Errorf("Error listing helm revisions: %w", err) - } - - if len(revisions) > 0 { + if isAppDeployed(appName) { return nil } diff --git a/tests/unit/scheduler-k3s-certs-deploy.bats b/tests/unit/scheduler-k3s-certs-deploy.bats new file mode 100644 index 000000000..e87db7b6d --- /dev/null +++ b/tests/unit/scheduler-k3s-certs-deploy.bats @@ -0,0 +1,143 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup_local_tls() { + TLS=$BATS_TMPDIR/tls + mkdir -p $TLS + tar xf $BATS_TEST_DIRNAME/server_ssl.tar -C $TLS + sudo chown -R dokku:dokku $TLS +} + +teardown_local_tls() { + TLS=$BATS_TMPDIR/tls + rm -R $TLS +} + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" + setup_local_tls +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true + teardown_local_tls +} + +@test "(scheduler-k3s:certs) deploy uses imported certificate when present" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku certs:add $TEST_APP $BATS_TMPDIR/tls/server.crt $BATS_TMPDIR/tls/server.key" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 30" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get ingress ${TEST_APP}-web-${TEST_APP}-dokku-me -n default -o jsonpath='{.spec.tls[0].secretName}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "tls-$TEST_APP" +} + +@test "(scheduler-k3s:certs) deploy falls back to letsencrypt when no imported cert" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku scheduler-k3s:set --global letsencrypt-email-stag test@dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler-k3s:set --global letsencrypt-server staging" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP $TEST_APP.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 30" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get ingress ${TEST_APP}-web-${TEST_APP}-dokku-me -n default -o jsonpath='{.spec.tls[0].secretName}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "tls-${TEST_APP}-web" +} + +@test "(scheduler-k3s:certs) no TLS secret created for non-k3s scheduler app" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler:set $TEST_APP selected docker-local" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku certs:add $TEST_APP $BATS_TMPDIR/tls/server.crt $BATS_TMPDIR/tls/server.key" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret tls-$TEST_APP -n default 2>&1" + echo "output: $output" + echo "status: $status" + assert_failure +} diff --git a/tests/unit/scheduler-k3s-certs.bats b/tests/unit/scheduler-k3s-certs.bats new file mode 100755 index 000000000..28115e167 --- /dev/null +++ b/tests/unit/scheduler-k3s-certs.bats @@ -0,0 +1,140 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup_local_tls() { + TLS=$BATS_TMPDIR/tls + mkdir -p $TLS + tar xf $BATS_TEST_DIRNAME/server_ssl.tar -C $TLS + sudo chown -R dokku:dokku $TLS +} + +teardown_local_tls() { + TLS=$BATS_TMPDIR/tls + rm -R $TLS +} + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" + setup_local_tls +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true + teardown_local_tls +} + +@test "(scheduler-k3s:certs) certificate import creates k8s TLS secret" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler:set $TEST_APP selected k3s" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku certs:add $TEST_APP $BATS_TMPDIR/tls/server.crt $BATS_TMPDIR/tls/server.key" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Installing TLS certificate for $TEST_APP" + + run /bin/bash -c "kubectl get secret tls-$TEST_APP -n default" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret tls-$TEST_APP -n default -o jsonpath='{.metadata.labels.dokku\\.com/cert-source}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "imported" +} + +@test "(scheduler-k3s:certs) certificate update updates k8s TLS secret" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler:set $TEST_APP selected k3s" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku certs:add $TEST_APP $BATS_TMPDIR/tls/server.crt $BATS_TMPDIR/tls/server.key" + echo "output: $output" + echo "status: $status" + assert_success + + CHECKSUM1=$(kubectl get secret tls-$TEST_APP -n default -o jsonpath='{.metadata.labels.dokku\.com/cert-checksum}') + + run /bin/bash -c "dokku certs:add $TEST_APP $BATS_TMPDIR/tls/server.crt $BATS_TMPDIR/tls/server.key" + echo "output: $output" + echo "status: $status" + assert_success + + CHECKSUM2=$(kubectl get secret tls-$TEST_APP -n default -o jsonpath='{.metadata.labels.dokku\.com/cert-checksum}') + + [ "$CHECKSUM1" = "$CHECKSUM2" ] +} + +@test "(scheduler-k3s:certs) certificate removal deletes k8s TLS secret" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku scheduler:set $TEST_APP selected k3s" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku certs:add $TEST_APP $BATS_TMPDIR/tls/server.crt $BATS_TMPDIR/tls/server.key" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get secret tls-$TEST_APP -n default" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku certs:remove $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Removing TLS certificate for $TEST_APP" + + run /bin/bash -c "kubectl get secret tls-$TEST_APP -n default" + echo "output: $output" + echo "status: $status" + assert_failure +}