Files
dokku/plugins/scheduler-k3s/chart.go
Jose Diaz-Gonzalez 96d726c43c fix: retire cron containers past their active deadline
Cron containers were never reaped once they exceeded their active deadline, so a hung cron task ran indefinitely instead of being retired after 24 hours as documented. `cron:run` now also accepts a `--ttl-seconds` argument, matching the one `dokku run` already takes.
2026-08-12 13:11:34 -04:00

655 lines
20 KiB
Go

package scheduler_k3s
import (
"context"
"crypto/rand"
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
appjson "github.com/dokku/dokku/plugins/app-json"
"github.com/dokku/dokku/plugins/common"
"github.com/dokku/dokku/plugins/config"
"github.com/dokku/dokku/plugins/cron"
"github.com/dokku/dokku/plugins/registry"
"github.com/kballard/go-shellquote"
)
// BuildResult is the materialized output of BuildAppChart. It points at a
// temporary chart directory on disk; the caller is responsible for removing
// it once Helm is done with it.
type BuildResult struct {
ChartDir string
ChartPath string
ReleaseName string
Namespace string
Timeout time.Duration
RollbackOnFailure bool
KustomizeRootPath string
}
// BuildOptions configures non-default behavior for BuildAppChart.
type BuildOptions struct {
// OverrideDeploymentID, when non-zero, replaces the timestamp-derived
// deployment id. CommandPreview uses this to reuse the currently-deployed
// release's id so app.kubernetes.io/version annotation churn does not
// appear in the diff.
OverrideDeploymentID int64
// AllowMissingImage, when true, lets BuildAppChart substitute a
// placeholder image string when no real image is available for the app
// yet (e.g. a never-deployed app being previewed). The deploy path
// leaves this false so a missing image still errors.
AllowMissingImage bool
// FallbackImageMetadata supplies the builder type and working directory to
// use when the image is absent from the local docker daemon. It is only
// consulted after a local inspect fails - it never overrides a present
// image - so callers pass the values recorded in the app's current Helm
// release and let a deploy proceed on a host that has since reaped its
// local copy of the image.
FallbackImageMetadata *ImageMetadata
// RestartProcessType, when set, limits which process type receives a fresh
// deployment id. Helm still upgrades the whole release so configuration
// converges everywhere, but only this process type's pod template changes,
// so only its pods roll. Empty means every process type rolls.
RestartProcessType string
// PriorProcessDeploymentIDs carries the deployment ids currently deployed
// per process type, used to hold untargeted processes steady when
// RestartProcessType is set.
PriorProcessDeploymentIDs map[string]string
}
// BuildAppChart constructs a Helm chart on disk that reflects dokku's configured
// state for the given app and image tag. It reads from the Kubernetes cluster
// where required (for KEDA values and existing cron-job suffixes) but performs
// no writes to the cluster - no namespace, secrets, ingresses, or helm releases
// are created or modified.
//
// The returned BuildResult holds the absolute chart path and the parameters a
// subsequent Helm Install/Upgrade (dry-run or real) needs. The caller must
// os.RemoveAll(result.ChartDir) when finished.
func BuildAppChart(ctx context.Context, appName, imageTag string, opts BuildOptions) (BuildResult, error) {
var result BuildResult
processesRes, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
Trigger: "ps-current-scale",
Args: []string{appName},
})
if err != nil {
return result, err
}
processes, err := common.ParseScaleOutput(processesRes.StdoutBytes())
if err != nil {
return result, err
}
if opts.RestartProcessType != "" {
if _, ok := processes[opts.RestartProcessType]; !ok {
return result, fmt.Errorf("Process type %s not found in the scale for app %s", opts.RestartProcessType, appName)
}
}
namespace := getComputedNamespace(appName)
image, err := common.ResolveDeployingAppImageName(appName, imageTag, "")
if err != nil {
return result, fmt.Errorf("Error getting deploying app image name: %w", err)
}
imageMetadata, err := resolveImageMetadata(appName, image, opts.FallbackImageMetadata)
if err != nil {
if !opts.AllowMissingImage {
return result, err
}
image = fmt.Sprintf("dokku/%s:not-yet-deployed", appName)
imageMetadata = ImageMetadata{SourceType: "dockerfile"}
}
deployTimeout := getComputedDeployTimeout(appName)
if _, err := strconv.Atoi(deployTimeout); err == nil {
deployTimeout = fmt.Sprintf("%ss", deployTimeout)
}
timeoutDuration, err := time.ParseDuration(deployTimeout)
if err != nil {
return result, fmt.Errorf("Error parsing deploy timeout duration: %w", err)
}
deployRollback := getComputedRollbackOnFailure(appName)
allowRollbacks, err := strconv.ParseBool(deployRollback)
if err != nil {
return result, fmt.Errorf("Error parsing rollback-on-failure value as boolean: %w", err)
}
imageSourceType := imageMetadata.SourceType
env, err := config.LoadMergedAppEnv(appName)
if err != nil {
return result, fmt.Errorf("Error loading environment for deployment: %w", err)
}
importedCertExists := false
if HasImportedTLSCert(appName) {
exists, err := TLSSecretExists(ctx, appName)
if err == nil && exists {
importedCertExists = true
}
}
tlsConfig, err := resolveAppTLSConfig(ResolveAppTLSConfigInput{
AppName: appName,
ImportedCertExists: importedCertExists,
})
if err != nil {
return result, err
}
chartDir, err := os.MkdirTemp("", "dokku-chart-")
if err != nil {
return result, fmt.Errorf("Error creating chart directory: %w", err)
}
// From this point on, any error returns require os.RemoveAll(chartDir).
cleanup := func(retErr error) (BuildResult, error) {
os.RemoveAll(chartDir)
return result, retErr
}
if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
return cleanup(fmt.Errorf("Error creating chart templates directory: %w", err))
}
deploymentId := opts.OverrideDeploymentID
if deploymentId == 0 {
deploymentId = time.Now().Unix()
}
imagePullSecrets := getComputedImagePullSecrets(appName)
if imagePullSecrets == "" {
dockerConfigPath := filepath.Join(registry.GetComputedAppRegistryConfigDir(appName), "config.json")
if fi, err := os.Stat(dockerConfigPath); err == nil && !fi.IsDir() {
imagePullSecrets = GetImagePullSecretName(appName)
}
}
globalTemplateFiles := []string{"service-account"}
for _, templateName := range globalTemplateFiles {
b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
if err != nil {
return cleanup(fmt.Errorf("Error reading %s template: %w", templateName, err))
}
filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s.yaml", templateName))
if err := os.WriteFile(filename, b, os.FileMode(0644)); err != nil {
return cleanup(fmt.Errorf("Error writing %s template: %w", templateName, err))
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(filename)
}
}
portMaps, err := getPortMaps(appName)
if err != nil {
return cleanup(fmt.Errorf("Error getting port mappings for deployment: %w", err))
}
primaryPort := int32(5000)
primaryServicePort := int32(80)
for _, portMap := range portMaps {
primaryPort = portMap.ContainerPort
primaryServicePort = portMap.HostPort
if primaryPort != 0 {
break
}
}
appJSON, err := appjson.GetAppJSON(appName)
if err != nil {
return cleanup(fmt.Errorf("Error getting app.json for deployment: %w", err))
}
workingDir := imageMetadata.WorkingDir
cronTasks, err := cron.FetchCronTasks(cron.FetchCronTasksInput{AppName: appName})
if err != nil {
return cleanup(fmt.Errorf("Error fetching cron tasks: %w", err))
}
domains := []string{}
if _, ok := processes["web"]; ok {
_, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
Trigger: "domains-vhost-enabled",
Args: []string{appName},
StreamStdio: true,
})
if err == nil {
results, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
Trigger: "domains-list",
Args: []string{appName},
})
if err != nil {
return cleanup(fmt.Errorf("Error getting domains for deployment: %w", err))
}
for _, domain := range strings.Split(results.StdoutContents(), "\n") {
domain = strings.TrimSpace(domain)
if domain != "" {
domains = append(domains, domain)
}
}
}
}
chart := &Chart{
ApiVersion: "v2",
AppVersion: "1.0.0",
Name: appName,
Icon: "https://dokku.com/assets/dokku-logo.svg",
Version: fmt.Sprintf("0.0.%d", deploymentId),
}
if err := writeYaml(WriteYamlInput{
Object: chart,
Path: filepath.Join(chartDir, "Chart.yaml"),
}); err != nil {
return cleanup(fmt.Errorf("Error writing chart: %w", err))
}
globalAnnotations, err := getGlobalAnnotations(appName)
if err != nil {
return cleanup(fmt.Errorf("Error getting global annotations: %w", err))
}
globalLabels, err := getGlobalLabel(appName)
if err != nil {
return cleanup(fmt.Errorf("Error getting global labels: %w", err))
}
clientset, err := NewKubernetesClient()
if err != nil {
return cleanup(fmt.Errorf("Error creating kubernetes client: %w", err))
}
if err := clientset.Ping(); err != nil {
return cleanup(fmt.Errorf("kubernetes api not available: %w", err))
}
kedaValues, err := getKedaValues(ctx, clientset, appName)
if err != nil {
return cleanup(fmt.Errorf("Error getting keda values: %w", err))
}
securityContext, err := getSecurityContext(appName, "deploy")
if err != nil {
return cleanup(fmt.Errorf("Error getting security context: %w", err))
}
values := &AppValues{
Global: GlobalValues{
Annotations: globalAnnotations,
AppName: appName,
DeploymentID: fmt.Sprint(deploymentId),
Keda: kedaValues,
Image: GlobalImage{
ImagePullSecrets: imagePullSecrets,
Name: image,
Type: imageSourceType,
WorkingDir: workingDir,
},
Issuer: tlsConfig.Issuer,
Labels: globalLabels,
Namespace: namespace,
Network: GlobalNetwork{
IngressClass: getComputedIngressClass(),
PrimaryPort: primaryPort,
PrimaryServicePort: primaryServicePort,
},
SecurityContext: securityContext,
},
Processes: map[string]ProcessValues{},
}
if len(kedaValues.Authentications) > 0 {
templateFiles := []string{"keda-secret", "keda-trigger-authentication"}
for _, templateName := range templateFiles {
b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
if err != nil {
return cleanup(fmt.Errorf("Error reading %s template: %w", templateName, err))
}
filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s.yaml", templateName))
if err := os.WriteFile(filename, b, os.FileMode(0644)); err != nil {
return cleanup(fmt.Errorf("Error writing %s template: %w", templateName, err))
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(filename)
}
}
}
processVolumes := []ProcessVolume{}
if shmSize := getComputedShmSize(appName); shmSize != "" {
processVolumes = append(processVolumes, ProcessVolume{
Name: "shmem",
MountPath: "/dev/shm",
EmptyDir: &ProcessVolumeEmptyDir{
Medium: "Memory",
SizeLimit: shmSize,
},
})
}
deployMountPairs, err := LoadAppMounts(appName, "deploy")
if err != nil {
return cleanup(fmt.Errorf("error loading storage app mounts: %w", err))
}
deployVolumes, err := ToProcessVolumes(deployMountPairs)
if err != nil {
return cleanup(err)
}
processVolumes = append(processVolumes, deployVolumes...)
for processType, processCount := range processes {
healthchecks, ok := appJSON.Healthchecks[processType]
if !ok {
healthchecks = []appjson.Healthcheck{}
}
processHealthchecks := getProcessHealtchecks(healthchecks, primaryPort)
startCommand, err := getStartCommand(StartCommandInput{
AppName: appName,
ProcessType: processType,
ImageSourceType: imageSourceType,
Port: primaryPort,
Env: env.Map(),
})
if err != nil {
return cleanup(fmt.Errorf("Error getting start command for deployment: %w", err))
}
args := startCommand.Command
processResources, err := getProcessResources(appName, processType)
if err != nil {
return cleanup(fmt.Errorf("Error getting process resources: %w", err))
}
annotations, err := getAnnotations(appName, processType)
if err != nil {
return cleanup(fmt.Errorf("Error getting process annotations: %w", err))
}
labels, err := getLabels(appName, processType)
if err != nil {
return cleanup(fmt.Errorf("Error getting process labels: %w", err))
}
autoscaling, err := getAutoscaling(GetAutoscalingInput{
AppName: appName,
ProcessType: processType,
Replicas: int(processCount),
KedaValues: kedaValues,
})
if err != nil {
return cleanup(fmt.Errorf("Error getting autoscaling: %w", err))
}
processValues := ProcessValues{
Annotations: annotations,
Autoscaling: autoscaling,
Args: args,
DeploymentID: resolveProcessDeploymentID(processType, deploymentId, opts.RestartProcessType, opts.PriorProcessDeploymentIDs),
Healthchecks: processHealthchecks,
Labels: labels,
ProcessType: ProcessType_Worker,
Replicas: int32(processCount),
Resources: processResources,
Volumes: processVolumes,
}
if processType == "web" {
sort.Strings(domains)
domainValues := []ProcessDomains{}
for _, domain := range domains {
domainValues = append(domainValues, ProcessDomains{
Name: domain,
Slug: domainSlug(domain),
})
}
processValues.Web = ProcessWeb{
Domains: domainValues,
PortMaps: []ProcessPortMap{},
TLS: ProcessTls{
Enabled: tlsConfig.Enabled,
IssuerKind: tlsConfig.IssuerKind,
IssuerName: tlsConfig.IssuerName,
UseImportedCert: tlsConfig.UseImportedCert,
},
}
processValues.ProcessType = ProcessType_Web
for _, portMap := range portMaps {
protocol := PortmapProtocol_TCP
if portMap.Scheme == "udp" {
protocol = PortmapProtocol_UDP
}
processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
ContainerPort: portMap.ContainerPort,
HostPort: portMap.HostPort,
Name: portMap.String(),
Protocol: protocol,
Scheme: portMap.Scheme,
})
}
for _, portMap := range processValues.Web.PortMaps {
_, httpOk := portMaps[fmt.Sprintf("http-80-%d", portMap.ContainerPort)]
_, httpsOk := portMaps[fmt.Sprintf("https-443-%d", portMap.ContainerPort)]
if portMap.Scheme == "http" && !httpsOk && tlsConfig.Enabled {
processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
ContainerPort: portMap.ContainerPort,
HostPort: 443,
Name: fmt.Sprintf("https-443-%d", portMap.ContainerPort),
Protocol: PortmapProtocol_TCP,
Scheme: "https",
})
}
if portMap.Scheme == "https" && !httpOk {
processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
ContainerPort: portMap.ContainerPort,
HostPort: 80,
Name: fmt.Sprintf("http-80-%d", portMap.ContainerPort),
Protocol: PortmapProtocol_TCP,
Scheme: "http",
})
}
}
sort.Sort(NameSorter(processValues.Web.PortMaps))
} else if appJSON.Formation[processType].Service != nil && appJSON.Formation[processType].Service.Exposed {
processValues.Web = ProcessWeb{
Domains: []ProcessDomains{},
PortMaps: []ProcessPortMap{},
TLS: ProcessTls{
Enabled: false,
},
}
processValues.Web.PortMaps = append(processValues.Web.PortMaps, ProcessPortMap{
ContainerPort: 5000,
HostPort: 5000,
Name: "http-5000-5000",
Protocol: PortmapProtocol_TCP,
Scheme: "http",
})
}
values.Processes[processType] = processValues
templateFiles := []string{"deployment", "keda-scaled-object"}
if processType == "web" {
templateFiles = append(templateFiles, "service", "certificate", "issuer", "ingress", "ingress-route", "compression-middleware", "https-redirect-middleware", "keda-http-scaled-object", "keda-interceptor-proxy-service")
}
for _, templateName := range templateFiles {
b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName))
if err != nil {
return cleanup(fmt.Errorf("Error reading %s template: %w", templateName, err))
}
filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s.yaml", templateName))
if err := os.WriteFile(filename, b, os.FileMode(0644)); err != nil {
return cleanup(fmt.Errorf("Error writing %s template: %w", templateName, err))
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(filename)
}
}
}
cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
Namespace: namespace,
})
if err != nil {
return cleanup(fmt.Errorf("Error listing cron jobs: %w", err))
}
for _, cronTask := range cronTasks {
suffix := ""
for _, cronJob := range cronJobs {
if cronJob.Annotations["dokku.com/cron-id"] == cronTask.ID {
if value, ok := cronJob.Annotations["dokku.com/job-suffix"]; ok {
suffix = value
}
}
}
if suffix == "" {
n := 5
b := make([]byte, n)
if _, err := rand.Read(b); err != nil {
return cleanup(fmt.Errorf("Error generating cron suffix: %w", err))
}
suffix = strings.ToLower(fmt.Sprintf("%X", b))
}
words, err := shellquote.Split(cronTask.Command)
if err != nil {
return cleanup(fmt.Errorf("Error parsing cron task command: %w", err))
}
processResources, err := getProcessResources(appName, cronTask.ID)
if err != nil {
return cleanup(fmt.Errorf("Error getting process resources: %w", err))
}
annotations, err := getAnnotations(appName, cronTask.ID)
if err != nil {
return cleanup(fmt.Errorf("Error getting process annotations: %w", err))
}
labels, err := getLabels(appName, cronTask.ID)
if err != nil {
return cleanup(fmt.Errorf("Error getting process labels: %w", err))
}
concurrencyPolicy := strings.ToUpper(cronTask.ConcurrencyPolicy)
switch concurrencyPolicy {
case "ALLOW":
concurrencyPolicy = "Allow"
case "FORBID":
concurrencyPolicy = "Forbid"
case "REPLACE":
concurrencyPolicy = "Replace"
default:
return cleanup(fmt.Errorf("Invalid concurrency_policy specified: %v", concurrencyPolicy))
}
processValues := ProcessValues{
Args: words,
Annotations: annotations,
Cron: ProcessCron{
ID: cronTask.ID,
Hash: cronIDLabelValue(cronTask.ID),
Schedule: cronTask.Schedule,
Suffix: suffix,
Suspend: cronTask.Maintenance,
ConcurrencyPolicy: ProcessCronConcurrencyPolicy(concurrencyPolicy),
ActiveDeadlineSeconds: cron.DefaultTTLSeconds,
},
Labels: labels,
ProcessType: ProcessType_Cron,
Replicas: 1,
Resources: processResources,
Volumes: processVolumes,
}
values.Processes[cronTask.ID] = processValues
}
if len(cronTasks) > 0 {
b, err := templates.ReadFile("templates/chart/cron-job.yaml")
if err != nil {
return cleanup(fmt.Errorf("Error reading cron job template: %w", err))
}
cronFile := filepath.Join(chartDir, "templates", "cron-job.yaml")
if err := os.WriteFile(cronFile, b, os.FileMode(0644)); err != nil {
return cleanup(fmt.Errorf("Error writing cron job template: %w", err))
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(cronFile)
}
}
helpersBytes, err := templates.ReadFile("templates/chart/_helpers.tpl")
if err != nil {
return cleanup(fmt.Errorf("Error reading _helpers template: %w", err))
}
helpersFile := filepath.Join(chartDir, "templates", "_helpers.tpl")
if err := os.WriteFile(helpersFile, helpersBytes, os.FileMode(0644)); err != nil {
return cleanup(fmt.Errorf("Error writing _helpers template: %w", err))
}
if os.Getenv("DOKKU_TRACE") == "1" {
common.CatFile(helpersFile)
}
if err := writeYaml(WriteYamlInput{
Object: values,
Path: filepath.Join(chartDir, "values.yaml"),
}); err != nil {
return cleanup(fmt.Errorf("Error writing chart: %w", err))
}
chartPath, err := filepath.Abs(chartDir)
if err != nil {
return cleanup(fmt.Errorf("Error getting chart path: %w", err))
}
kustomizeRootPath := ""
if hasKustomizeDirectory(appName) {
kustomizeRootPath = getProcessSpecificKustomizeRootPath(appName)
}
result = BuildResult{
ChartDir: chartDir,
ChartPath: chartPath,
ReleaseName: appName,
Namespace: namespace,
Timeout: timeoutDuration,
RollbackOnFailure: allowRollbacks,
KustomizeRootPath: kustomizeRootPath,
}
return result, nil
}