feat: manage node-level kernel sysctls on the k3s scheduler

Sysctls the kernel does not namespace, such as `vm.max_map_count`, cannot be set from a pod spec and previously had no answer beyond editing `/etc/sysctl.d` on each host by hand. `scheduler-k3s:node-sysctls:set` now applies them through a privileged daemonset, which reaches nodes joined later and reapplies after a reboot. Sysctls may be scoped to a node profile, with a profile scope inheriting the global values and overriding them on conflict so that every node is covered by exactly one daemonset. Clearing a sysctl stops dokku managing it but does not restore the previous value, which persists until the node reboots.

plugins/scheduler-k3s/Makefile

@@ -1,4 +1,4 @@
-SUBCOMMANDS = subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/labels:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
+SUBCOMMANDS = subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/labels:report subcommands/node-sysctls:set subcommands/node-sysctls:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
 TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write triggers/storage-create triggers/storage-destroy triggers/storage-status triggers/scheduler-storage-exec
 BUILD = commands subcommands triggers
 PLUGIN_NAME = scheduler-k3s

plugins/scheduler-k3s/functions.go

@@ -1391,6 +1391,24 @@ func getComputedKustomizeRootPath(appName string) string {
 	return kustomizeRootPath
 }
 
+func getComputedNodeSysctlsImage() string {
+	image := common.PropertyGet("scheduler-k3s", "--global", "node-sysctls-image")
+	if image == "" {
+		image = DefaultNodeSysctlsImage
+	}
+
+	return image
+}
+
+func getComputedNodeSysctlsPauseImage() string {
+	image := common.PropertyGet("scheduler-k3s", "--global", "node-sysctls-pause-image")
+	if image == "" {
+		image = DefaultNodeSysctlsPauseImage
+	}
+
+	return image
+}
+
 func getNamespace(appName string) string {
 	return common.PropertyGet("scheduler-k3s", appName, "namespace")
 }
@@ -1790,7 +1808,7 @@ func parseSysctls(values []string) ([]Sysctl, error) {
 		}
 
 		if !isNamespacedSysctl(name) {
-			return nil, fmt.Errorf("Sysctl %s is not namespaced and cannot be set on a pod, it must be applied at the node level instead", name)
+			return nil, fmt.Errorf("Sysctl %s is not namespaced and cannot be set on a pod, apply it to the nodes with 'dokku scheduler-k3s:node-sysctls:set %s <value>' instead", name, name)
 		}
 
 		sysctls = append(sysctls, Sysctl{Name: name, Value: sysctlValue})

plugins/scheduler-k3s/node_sysctls.go

@@ -0,0 +1,324 @@
+package scheduler_k3s
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/dokku/dokku/plugins/common"
+)
+
+// NodeSysctlsValues contains the values for a dokku-managed node sysctls helm chart
+type NodeSysctlsValues struct {
+	Global NodeSysctlsGlobalValues `yaml:"global"`
+}
+
+// NodeSysctlsGlobalValues contains the global values for the node sysctls chart
+type NodeSysctlsGlobalValues struct {
+	Image       string   `yaml:"image"`
+	PauseImage  string   `yaml:"pause_image"`
+	ProfileName string   `yaml:"profile_name,omitempty"`
+	ReleaseName string   `yaml:"release_name"`
+	Sysctls     []Sysctl `yaml:"sysctls"`
+}
+
+// nodeSysctlScope is a resolved set of sysctls destined for a single DaemonSet
+type nodeSysctlScope struct {
+	// ProfileName is the node profile this scope targets, empty for the global scope
+	ProfileName string
+	// ReleaseName is the helm release backing this scope
+	ReleaseName string
+	// Sysctls are the fully resolved sysctls for this scope
+	Sysctls []Sysctl
+}
+
+// getNodeSysctlsProperty returns the property name backing a node sysctls scope
+func getNodeSysctlsProperty(profileName string) string {
+	if profileName == "" {
+		return "node-sysctls.global"
+	}
+
+	return fmt.Sprintf("node-sysctls.profile.%s", profileName)
+}
+
+// getNodeSysctlsReleaseName returns the helm release name for a node sysctls scope
+func getNodeSysctlsReleaseName(profileName string) string {
+	if profileName == "" {
+		return "dokku-node-sysctls-global"
+	}
+
+	return fmt.Sprintf("dokku-node-sysctls-profile-%s", profileName)
+}
+
+// getNodeSysctls returns the sysctls stored against a single scope
+func getNodeSysctls(profileName string) (map[string]string, error) {
+	sysctls, err := common.PropertyMapGet("scheduler-k3s", "--global", getNodeSysctlsProperty(profileName))
+	if err != nil {
+		return nil, fmt.Errorf("Unable to read node sysctls: %w", err)
+	}
+
+	return sysctls, nil
+}
+
+// listNodeProfileNames returns the names of every stored node profile
+func listNodeProfileNames() ([]string, error) {
+	properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "node-profile-")
+	if err != nil {
+		return nil, fmt.Errorf("Unable to get node profiles: %w", err)
+	}
+
+	names := []string{}
+	for property, data := range properties {
+		if !strings.HasSuffix(property, ".json") {
+			continue
+		}
+
+		var profile NodeProfile
+		if err := json.Unmarshal([]byte(data), &profile); err != nil {
+			return nil, fmt.Errorf("Unable to unmarshal node profile: %w", err)
+		}
+
+		names = append(names, profile.Name)
+	}
+
+	sort.Strings(names)
+	return names, nil
+}
+
+// sortedSysctls converts a name/value map into sysctls sorted by name so the
+// rendered manifest is stable across runs
+func sortedSysctls(values map[string]string) []Sysctl {
+	sysctls := make([]Sysctl, 0, len(values))
+	for name, value := range values {
+		sysctls = append(sysctls, Sysctl{Name: name, Value: value})
+	}
+
+	sort.Slice(sysctls, func(i int, j int) bool {
+		return sysctls[i].Name < sysctls[j].Name
+	})
+
+	return sysctls
+}
+
+// mergeNodeSysctls layers a profile's sysctls over the global ones, with the profile
+// winning on conflict, and returns the result sorted by name. Profile scopes must
+// carry the global values too: profiled nodes are excluded from the global DaemonSet,
+// so anything omitted here would never reach them.
+func mergeNodeSysctls(global map[string]string, profile map[string]string) []Sysctl {
+	merged := map[string]string{}
+	for name, value := range global {
+		merged[name] = value
+	}
+	for name, value := range profile {
+		merged[name] = value
+	}
+
+	return sortedSysctls(merged)
+}
+
+// resolveNodeSysctlScopes returns one scope per DaemonSet that should exist, with
+// profile scopes carrying the global sysctls merged underneath their own. Every
+// node matches exactly one scope: profiled nodes match their profile's DaemonSet,
+// and unprofiled nodes match the global one, so no two DaemonSets ever write the
+// same sysctl on the same node.
+func resolveNodeSysctlScopes() ([]nodeSysctlScope, error) {
+	globalSysctls, err := getNodeSysctls("")
+	if err != nil {
+		return nil, err
+	}
+
+	scopes := []nodeSysctlScope{
+		{
+			ReleaseName: getNodeSysctlsReleaseName(""),
+			Sysctls:     sortedSysctls(globalSysctls),
+		},
+	}
+
+	profileNames, err := listNodeProfileNames()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, profileName := range profileNames {
+		profileSysctls, err := getNodeSysctls(profileName)
+		if err != nil {
+			return nil, err
+		}
+
+		scopes = append(scopes, nodeSysctlScope{
+			ProfileName: profileName,
+			ReleaseName: getNodeSysctlsReleaseName(profileName),
+			Sysctls:     mergeNodeSysctls(globalSysctls, profileSysctls),
+		})
+	}
+
+	return scopes, nil
+}
+
+// CreateOrUpdateNodeSysctls reconciles every node sysctls DaemonSet against the
+// stored properties. All scopes are reconciled on every call rather than only the
+// mutated one, since profile scopes inherit the global sysctls at render time and
+// would otherwise serve a stale value after a global change.
+func CreateOrUpdateNodeSysctls(ctx context.Context) error {
+	if err := isKubernetesAvailable(); err != nil {
+		common.LogDebug("kubernetes not available, skipping node sysctls sync")
+		return nil
+	}
+
+	scopes, err := resolveNodeSysctlScopes()
+	if err != nil {
+		return err
+	}
+
+	for _, scope := range scopes {
+		if len(scope.Sysctls) == 0 {
+			if err := deleteNodeSysctlsRelease(scope.ReleaseName); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if err := installNodeSysctlsChart(ctx, scope); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// installNodeSysctlsChart installs or upgrades the DaemonSet backing a single scope
+func installNodeSysctlsChart(ctx context.Context, scope nodeSysctlScope) error {
+	chartDir, err := os.MkdirTemp("", "dokku-node-sysctls-chart-")
+	if err != nil {
+		return fmt.Errorf("error creating chart directory: %w", err)
+	}
+	defer os.RemoveAll(chartDir)
+
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
+		return fmt.Errorf("error creating chart templates directory: %w", err)
+	}
+
+	chart := &Chart{
+		ApiVersion: "v2",
+		AppVersion: "1.0.0",
+		Name:       scope.ReleaseName,
+		Icon:       "https://dokku.com/assets/dokku-logo.svg",
+		Version:    "0.0.1",
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: chart,
+		Path:   filepath.Join(chartDir, "Chart.yaml"),
+	})
+	if err != nil {
+		return fmt.Errorf("error writing chart: %w", err)
+	}
+
+	b, err := templates.ReadFile("templates/node-sysctls-chart/templates/daemonset.yaml")
+	if err != nil {
+		return fmt.Errorf("error reading node-sysctls template: %w", err)
+	}
+
+	filename := filepath.Join(chartDir, "templates", "daemonset.yaml")
+	if err := os.WriteFile(filename, b, os.FileMode(0644)); err != nil {
+		return fmt.Errorf("error writing node-sysctls template: %w", err)
+	}
+
+	if os.Getenv("DOKKU_TRACE") == "1" {
+		common.CatFile(filename)
+	}
+
+	values := &NodeSysctlsValues{
+		Global: NodeSysctlsGlobalValues{
+			Image:       getComputedNodeSysctlsImage(),
+			PauseImage:  getComputedNodeSysctlsPauseImage(),
+			ProfileName: scope.ProfileName,
+			ReleaseName: scope.ReleaseName,
+			Sysctls:     scope.Sysctls,
+		},
+	}
+
+	err = writeYaml(WriteYamlInput{
+		Object: values,
+		Path:   filepath.Join(chartDir, "values.yaml"),
+	})
+	if err != nil {
+		return fmt.Errorf("error writing values: %w", err)
+	}
+
+	helmAgent, err := NewHelmAgent(NodeSysctlsNamespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("error creating helm agent: %w", err)
+	}
+
+	chartPath, err := filepath.Abs(chartDir)
+	if err != nil {
+		return fmt.Errorf("error getting chart path: %w", err)
+	}
+
+	common.LogVerboseQuiet(fmt.Sprintf("Applying node sysctls for %s", nodeSysctlsScopeLabel(scope.ProfileName)))
+	err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
+		ChartPath:   chartPath,
+		Namespace:   NodeSysctlsNamespace,
+		ReleaseName: scope.ReleaseName,
+		Wait:        false,
+	})
+	if err != nil {
+		return fmt.Errorf("error installing node sysctls chart: %w", err)
+	}
+
+	return nil
+}
+
+// deleteNodeSysctlsRelease removes a node sysctls DaemonSet when its scope resolves
+// to an empty set. The kernel values it wrote are not reverted; they persist on the
+// affected nodes until those nodes reboot.
+func deleteNodeSysctlsRelease(releaseName string) error {
+	helmAgent, err := NewHelmAgent(NodeSysctlsNamespace, DeployLogPrinter)
+	if err != nil {
+		return fmt.Errorf("error creating helm agent: %w", err)
+	}
+
+	exists, err := helmAgent.ChartExists(releaseName)
+	if err != nil {
+		return fmt.Errorf("error checking if node sysctls chart exists: %w", err)
+	}
+
+	if !exists {
+		return nil
+	}
+
+	common.LogVerboseQuiet(fmt.Sprintf("Removing node sysctls release %s", releaseName))
+	if err := helmAgent.UninstallChart(releaseName); err != nil {
+		return fmt.Errorf("error uninstalling node sysctls chart: %w", err)
+	}
+
+	return nil
+}
+
+// DeleteNodeSysctls removes the stored sysctls and DaemonSet for a single node profile
+func DeleteNodeSysctls(ctx context.Context, profileName string) error {
+	if err := common.PropertyDelete("scheduler-k3s", "--global", getNodeSysctlsProperty(profileName)); err != nil {
+		return fmt.Errorf("Unable to delete node sysctls: %w", err)
+	}
+
+	if err := isKubernetesAvailable(); err != nil {
+		common.LogDebug("kubernetes not available, skipping node sysctls deletion")
+		return nil
+	}
+
+	return deleteNodeSysctlsRelease(getNodeSysctlsReleaseName(profileName))
+}
+
+// nodeSysctlsScopeLabel returns a human readable name for a node sysctls scope
+func nodeSysctlsScopeLabel(profileName string) string {
+	if profileName == "" {
+		return "unprofiled nodes"
+	}
+
+	return fmt.Sprintf("node profile %s", profileName)
+}

plugins/scheduler-k3s/node_sysctls_test.go

@@ -0,0 +1,305 @@
+package scheduler_k3s
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+)
+
+func TestGetNodeSysctlsProperty(t *testing.T) {
+	if got := getNodeSysctlsProperty(""); got != "node-sysctls.global" {
+		t.Errorf("getNodeSysctlsProperty(\"\") = %q, want node-sysctls.global", got)
+	}
+	if got := getNodeSysctlsProperty("edge-workers"); got != "node-sysctls.profile.edge-workers" {
+		t.Errorf("getNodeSysctlsProperty(\"edge-workers\") = %q, want node-sysctls.profile.edge-workers", got)
+	}
+}
+
+// TestGetNodeSysctlsPropertyIsReserved asserts the property prefix is excluded from
+// the annotations scan. Without this a property like node-sysctls.profile.pod would
+// be mistaken for a pod annotation.
+func TestGetNodeSysctlsPropertyIsReserved(t *testing.T) {
+	reserved := false
+	for _, prefix := range reservedAnnotationPrefixes {
+		if prefix == "node-sysctls." {
+			reserved = true
+		}
+	}
+
+	if !reserved {
+		t.Error("node-sysctls. is not in reservedAnnotationPrefixes")
+	}
+}
+
+func TestGetNodeSysctlsReleaseName(t *testing.T) {
+	if got := getNodeSysctlsReleaseName(""); got != "dokku-node-sysctls-global" {
+		t.Errorf("getNodeSysctlsReleaseName(\"\") = %q, want dokku-node-sysctls-global", got)
+	}
+	if got := getNodeSysctlsReleaseName("edge-workers"); got != "dokku-node-sysctls-profile-edge-workers" {
+		t.Errorf("getNodeSysctlsReleaseName(\"edge-workers\") = %q, want dokku-node-sysctls-profile-edge-workers", got)
+	}
+}
+
+func TestSortedSysctls(t *testing.T) {
+	got := sortedSysctls(map[string]string{
+		"vm.swappiness":    "10",
+		"vm.max_map_count": "262144",
+		"fs.file-max":      "100000",
+	})
+
+	want := []Sysctl{
+		{Name: "fs.file-max", Value: "100000"},
+		{Name: "vm.max_map_count", Value: "262144"},
+		{Name: "vm.swappiness", Value: "10"},
+	}
+
+	if len(got) != len(want) {
+		t.Fatalf("sortedSysctls() = %v, want %v", got, want)
+	}
+	for i := range got {
+		if got[i] != want[i] {
+			t.Errorf("sortedSysctls()[%d] = %v, want %v", i, got[i], want[i])
+		}
+	}
+}
+
+func TestSortedSysctlsEmpty(t *testing.T) {
+	if got := sortedSysctls(map[string]string{}); len(got) != 0 {
+		t.Errorf("sortedSysctls(empty) = %v, want empty", got)
+	}
+}
+
+func TestNodeSysctlsScopeLabel(t *testing.T) {
+	if got := nodeSysctlsScopeLabel(""); got != "unprofiled nodes" {
+		t.Errorf("nodeSysctlsScopeLabel(\"\") = %q, want 'unprofiled nodes'", got)
+	}
+	if got := nodeSysctlsScopeLabel("edge"); got != "node profile edge" {
+		t.Errorf("nodeSysctlsScopeLabel(\"edge\") = %q, want 'node profile edge'", got)
+	}
+}
+
+func renderNodeSysctlsTemplate(t *testing.T, values NodeSysctlsGlobalValues) string {
+	t.Helper()
+
+	chartDir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(chartDir, "templates"), 0o755); err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+
+	chartYAML := []byte("apiVersion: v2\nname: test\nversion: 0.0.1\n")
+	if err := os.WriteFile(filepath.Join(chartDir, "Chart.yaml"), chartYAML, 0o644); err != nil {
+		t.Fatalf("write Chart.yaml: %v", err)
+	}
+
+	tpl, err := templates.ReadFile("templates/node-sysctls-chart/templates/daemonset.yaml")
+	if err != nil {
+		t.Fatalf("read daemonset template: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(chartDir, "templates", "daemonset.yaml"), tpl, 0o644); err != nil {
+		t.Fatalf("write daemonset template: %v", err)
+	}
+
+	loaded, err := loader.Load(chartDir)
+	if err != nil {
+		t.Fatalf("load chart: %v", err)
+	}
+
+	sysctls := []interface{}{}
+	for _, sysctl := range values.Sysctls {
+		sysctls = append(sysctls, map[string]interface{}{"name": sysctl.Name, "value": sysctl.Value})
+	}
+
+	renderValues, err := chartutil.ToRenderValues(loaded, map[string]interface{}{
+		"global": map[string]interface{}{
+			"image":        values.Image,
+			"pause_image":  values.PauseImage,
+			"profile_name": values.ProfileName,
+			"release_name": values.ReleaseName,
+			"sysctls":      sysctls,
+		},
+	}, chartutil.ReleaseOptions{Name: "test", Namespace: "kube-system"}, nil)
+	if err != nil {
+		t.Fatalf("ToRenderValues: %v", err)
+	}
+
+	rendered, err := engine.Render(loaded, renderValues)
+	if err != nil {
+		t.Fatalf("render: %v", err)
+	}
+
+	for name, content := range rendered {
+		if filepath.Base(name) == "daemonset.yaml" {
+			return content
+		}
+	}
+	t.Fatalf("daemonset.yaml not rendered; got: %v", rendered)
+	return ""
+}
+
+// TestNodeSysctlsDaemonSetRendering asserts each scope targets a disjoint set of
+// nodes. If both the global and a profile DaemonSet landed on the same node they
+// would race writing the same /proc/sys file, making the result depend on pod
+// scheduling order.
+func TestNodeSysctlsDaemonSetRendering(t *testing.T) {
+	base := NodeSysctlsGlobalValues{
+		Image:      DefaultNodeSysctlsImage,
+		PauseImage: DefaultNodeSysctlsPauseImage,
+		Sysctls:    []Sysctl{{Name: "vm.max_map_count", Value: "262144"}},
+	}
+
+	t.Run("global scope excludes profiled nodes", func(t *testing.T) {
+		values := base
+		values.ReleaseName = "dokku-node-sysctls-global"
+		manifest := renderNodeSysctlsTemplate(t, values)
+
+		if !strings.Contains(manifest, "operator: DoesNotExist") {
+			t.Errorf("global daemonset missing DoesNotExist affinity:\n%s", manifest)
+		}
+		if !strings.Contains(manifest, "key: dokku.com/node-profile") {
+			t.Errorf("global daemonset missing node profile affinity key:\n%s", manifest)
+		}
+		if strings.Contains(manifest, "nodeSelector:") {
+			t.Errorf("global daemonset should not use a nodeSelector:\n%s", manifest)
+		}
+	})
+
+	t.Run("profile scope targets only its own nodes", func(t *testing.T) {
+		values := base
+		values.ProfileName = "edge-workers"
+		values.ReleaseName = "dokku-node-sysctls-profile-edge-workers"
+		manifest := renderNodeSysctlsTemplate(t, values)
+
+		if !strings.Contains(manifest, "dokku.com/node-profile: \"edge-workers\"") {
+			t.Errorf("profile daemonset missing nodeSelector:\n%s", manifest)
+		}
+		if strings.Contains(manifest, "DoesNotExist") {
+			t.Errorf("profile daemonset should not carry the global affinity:\n%s", manifest)
+		}
+	})
+
+	t.Run("tolerates every taint", func(t *testing.T) {
+		values := base
+		values.ReleaseName = "dokku-node-sysctls-global"
+		manifest := renderNodeSysctlsTemplate(t, values)
+
+		if !strings.Contains(manifest, "- operator: Exists") {
+			t.Errorf("daemonset does not tolerate all taints:\n%s", manifest)
+		}
+	})
+
+	t.Run("applies each sysctl privileged", func(t *testing.T) {
+		values := base
+		values.ReleaseName = "dokku-node-sysctls-global"
+		values.Sysctls = []Sysctl{
+			{Name: "vm.max_map_count", Value: "262144"},
+			{Name: "vm.swappiness", Value: "10"},
+		}
+		manifest := renderNodeSysctlsTemplate(t, values)
+
+		if !strings.Contains(manifest, "privileged: true") {
+			t.Errorf("daemonset init container is not privileged:\n%s", manifest)
+		}
+		if !strings.Contains(manifest, `sysctl -w "vm.max_map_count=262144"`) {
+			t.Errorf("daemonset missing max_map_count write:\n%s", manifest)
+		}
+		if !strings.Contains(manifest, `sysctl -w "vm.swappiness=10"`) {
+			t.Errorf("daemonset missing swappiness write:\n%s", manifest)
+		}
+	})
+
+	t.Run("image overrides reach both containers", func(t *testing.T) {
+		values := base
+		values.ReleaseName = "dokku-node-sysctls-global"
+		values.Image = "registry.internal/busybox:1.36"
+		values.PauseImage = "registry.internal/pause:3.9"
+		manifest := renderNodeSysctlsTemplate(t, values)
+
+		if !strings.Contains(manifest, `image: "registry.internal/busybox:1.36"`) {
+			t.Errorf("daemonset did not use the sysctl image override:\n%s", manifest)
+		}
+		if !strings.Contains(manifest, `image: "registry.internal/pause:3.9"`) {
+			t.Errorf("daemonset did not use the pause image override:\n%s", manifest)
+		}
+	})
+}
+
+// TestMergeNodeSysctls asserts a profile scope inherits the global sysctls and
+// overrides them on conflict. Profiled nodes are excluded from the global
+// DaemonSet, so a global value omitted here would never reach them.
+func TestMergeNodeSysctls(t *testing.T) {
+	cases := []struct {
+		name    string
+		global  map[string]string
+		profile map[string]string
+		want    []Sysctl
+	}{
+		{
+			name:    "no sysctls at all",
+			global:  map[string]string{},
+			profile: map[string]string{},
+			want:    []Sysctl{},
+		},
+		{
+			name:    "profile inherits global values",
+			global:  map[string]string{"vm.max_map_count": "262144"},
+			profile: map[string]string{},
+			want:    []Sysctl{{Name: "vm.max_map_count", Value: "262144"}},
+		},
+		{
+			name:    "profile wins on conflict",
+			global:  map[string]string{"vm.max_map_count": "262144"},
+			profile: map[string]string{"vm.max_map_count": "524288"},
+			want:    []Sysctl{{Name: "vm.max_map_count", Value: "524288"}},
+		},
+		{
+			name:    "profile adds to global",
+			global:  map[string]string{"vm.max_map_count": "262144"},
+			profile: map[string]string{"vm.swappiness": "10"},
+			want: []Sysctl{
+				{Name: "vm.max_map_count", Value: "262144"},
+				{Name: "vm.swappiness", Value: "10"},
+			},
+		},
+		{
+			name:    "profile only",
+			global:  map[string]string{},
+			profile: map[string]string{"vm.swappiness": "10"},
+			want:    []Sysctl{{Name: "vm.swappiness", Value: "10"}},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := mergeNodeSysctls(tc.global, tc.profile)
+			if len(got) != len(tc.want) {
+				t.Fatalf("mergeNodeSysctls() = %v, want %v", got, tc.want)
+			}
+			for i := range got {
+				if got[i] != tc.want[i] {
+					t.Errorf("mergeNodeSysctls()[%d] = %v, want %v", i, got[i], tc.want[i])
+				}
+			}
+		})
+	}
+}
+
+// TestMergeNodeSysctlsDoesNotMutateInputs asserts the merge leaves the caller's maps
+// alone, since resolveNodeSysctlScopes reuses the global map across every profile.
+func TestMergeNodeSysctlsDoesNotMutateInputs(t *testing.T) {
+	global := map[string]string{"vm.max_map_count": "262144"}
+	profile := map[string]string{"vm.max_map_count": "524288", "vm.swappiness": "10"}
+
+	mergeNodeSysctls(global, profile)
+
+	if len(global) != 1 || global["vm.max_map_count"] != "262144" {
+		t.Errorf("mergeNodeSysctls() mutated the global map: %v", global)
+	}
+	if len(profile) != 2 {
+		t.Errorf("mergeNodeSysctls() mutated the profile map: %v", profile)
+	}
+}

plugins/scheduler-k3s/report.go

@@ -62,6 +62,8 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 			"--scheduler-k3s-global-namespace":                reportGlobalNamespace,
 			"--scheduler-k3s-computed-network-interface":      reportComputedNetworkInterface,
 			"--scheduler-k3s-global-network-interface":        reportGlobalNetworkInterface,
+			"--scheduler-k3s-global-node-sysctls-image":       reportGlobalNodeSysctlsImage,
+			"--scheduler-k3s-global-node-sysctls-pause-image": reportGlobalNodeSysctlsPauseImage,
 			"--scheduler-k3s-computed-rollback-on-failure":    reportComputedRollbackOnFailure,
 			"--scheduler-k3s-global-rollback-on-failure":      reportGlobalRollbackOnFailure,
 			"--scheduler-k3s-computed-shm-size":               reportComputedShmSize,
@@ -99,6 +101,8 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 			"--scheduler-k3s-global-namespace":                reportGlobalNamespace,
 			"--scheduler-k3s-computed-network-interface":      reportComputedNetworkInterface,
 			"--scheduler-k3s-global-network-interface":        reportGlobalNetworkInterface,
+			"--scheduler-k3s-global-node-sysctls-image":       reportGlobalNodeSysctlsImage,
+			"--scheduler-k3s-global-node-sysctls-pause-image": reportGlobalNodeSysctlsPauseImage,
 			"--scheduler-k3s-computed-rollback-on-failure":    reportComputedRollbackOnFailure,
 			"--scheduler-k3s-rollback-on-failure":             reportRollbackOnFailure,
 			"--scheduler-k3s-global-rollback-on-failure":      reportGlobalRollbackOnFailure,
@@ -715,6 +719,14 @@ func reportGlobalNetworkInterface(appName string) string {
 	return getGlobalNetworkInterface()
 }
 
+func reportGlobalNodeSysctlsImage(appName string) string {
+	return getComputedNodeSysctlsImage()
+}
+
+func reportGlobalNodeSysctlsPauseImage(appName string) string {
+	return getComputedNodeSysctlsPauseImage()
+}
+
 func reportComputedRollbackOnFailure(appName string) string {
 	return getComputedRollbackOnFailure(appName)
 }

plugins/scheduler-k3s/scheduler_k3s.go

@@ -31,20 +31,22 @@ var (
 
 	// GlobalProperties is a map of all valid global k3s properties
 	GlobalProperties = map[string]bool{
-		"deploy-timeout":         true,
-		"image-pull-secrets":     true,
-		"ingress-class":          true,
-		"kube-context":           true,
-		"kubeconfig-path":        true,
-		"kustomize-root-path":    true,
-		"letsencrypt-server":     true,
-		"letsencrypt-email-prod": true,
-		"letsencrypt-email-stag": true,
-		"namespace":              true,
-		"network-interface":      true,
-		"rollback-on-failure":    true,
-		"shm-size":               true,
-		"token":                  true,
+		"deploy-timeout":           true,
+		"image-pull-secrets":       true,
+		"ingress-class":            true,
+		"kube-context":             true,
+		"kubeconfig-path":          true,
+		"kustomize-root-path":      true,
+		"letsencrypt-server":       true,
+		"letsencrypt-email-prod":   true,
+		"letsencrypt-email-stag":   true,
+		"namespace":                true,
+		"network-interface":        true,
+		"node-sysctls-image":       true,
+		"node-sysctls-pause-image": true,
+		"rollback-on-failure":      true,
+		"shm-size":                 true,
+		"token":                    true,
 	}
 )
 
@@ -102,6 +104,7 @@ var reservedAnnotationPrefixes = []string{
 	"chart-overrides.",
 	"labels.",
 	"node-profile-",
+	"node-sysctls.",
 	TriggerAuthPropertyPrefix,
 }
 
@@ -234,6 +237,15 @@ type NodeProfile struct {
 // NodeProfileLabel is the node label recording the node profile a node was added with
 const NodeProfileLabel = "dokku.com/node-profile"
 
+// NodeSysctlsNamespace is the namespace the node sysctls daemonsets are installed into
+const NodeSysctlsNamespace = "kube-system"
+
+// DefaultNodeSysctlsImage is the image used to apply sysctls on each node
+const DefaultNodeSysctlsImage = "busybox:1.36"
+
+// DefaultNodeSysctlsPauseImage is the image keeping the node sysctls daemonset pods running
+const DefaultNodeSysctlsPauseImage = "registry.k8s.io/pause:3.9"
+
 // ServerLabels are the labels for a server node
 var ServerLabels = map[string]string{
 	"svccontroller.k3s.cattle.io/enablelb": "true",

plugins/scheduler-k3s/src/commands/commands.go

@@ -30,6 +30,8 @@ Additional commands:`
     scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Initializes a cluster
     scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
     scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s labels report for one or more apps
+    scheduler-k3s:node-sysctls:set <sysctl> (<value>) [--global|--profile PROFILE], Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile
+    scheduler-k3s:node-sysctls:report [--format stdout|json], Displays the node-level kernel sysctls applied to each scope
     scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app
     scheduler-k3s:profiles:add <profile> [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Adds a node profile to the k3s cluster
     scheduler-k3s:profiles:list [--format json|stdout], Lists all node profiles in the k3s cluster

plugins/scheduler-k3s/src/subcommands/subcommands.go

@@ -164,6 +164,23 @@ func main() {
 			appName = "--global"
 		}
 		err = scheduler_k3s.CommandLabelsReport(appName, *format, *processType, *resourceType, infoFlag)
+	case "node-sysctls:set":
+		args := flag.NewFlagSet("scheduler-k3s:node-sysctls:set", flag.ExitOnError)
+		global := args.Bool("global", false, "--global: scope to all nodes without a node profile")
+		profileName := args.String("profile", "", "--profile: scope to a node profile instead of all unprofiled nodes")
+		args.Parse(os.Args[2:])
+		key := args.Arg(0)
+		value := args.Arg(1)
+		if *global && *profileName != "" {
+			err = fmt.Errorf("Only one of --global and --profile may be specified")
+			break
+		}
+		err = scheduler_k3s.CommandNodeSysctlsSet(*profileName, key, value)
+	case "node-sysctls:report":
+		args := flag.NewFlagSet("scheduler-k3s:node-sysctls:report", flag.ExitOnError)
+		format := args.String("format", "stdout", "format: [ stdout | json ]")
+		args.Parse(os.Args[2:])
+		err = scheduler_k3s.CommandNodeSysctlsReport(*format)
 	case "preview":
 		args := flag.NewFlagSet("scheduler-k3s:preview", flag.ExitOnError)
 		context := args.Int("context", 3, "--context: number of unchanged lines of context around each change (-1 for full output)")

plugins/scheduler-k3s/subcommands.go

@@ -45,6 +45,95 @@ func CommandAnnotationsSet(appName string, processType string, resourceType stri
 	return nil
 }
 
+// CommandNodeSysctlsSet sets or clears a node-level kernel sysctl for a scope
+func CommandNodeSysctlsSet(profileName string, key string, value string) error {
+	if key == "" {
+		return fmt.Errorf("Missing sysctl name")
+	}
+
+	if profileName != "" {
+		if err := verifyNodeProfileExists(profileName); err != nil {
+			return err
+		}
+	}
+
+	property := getNodeSysctlsProperty(profileName)
+	if value == "" {
+		if err := common.PropertyMapDelete("scheduler-k3s", "--global", property, key); err != nil {
+			return fmt.Errorf("Unable to delete property map entry: %w", err)
+		}
+
+		common.LogWarn(fmt.Sprintf("Removing %s stops dokku managing it, but does not restore the previous value on affected nodes until they reboot", key))
+	} else {
+		if err := common.PropertyMapSet("scheduler-k3s", "--global", property, key, value); err != nil {
+			return fmt.Errorf("Unable to set property map entry: %w", err)
+		}
+	}
+
+	return CreateOrUpdateNodeSysctls(context.Background())
+}
+
+// CommandNodeSysctlsReport displays the configured node-level kernel sysctls
+func CommandNodeSysctlsReport(format string) error {
+	if format != "stdout" && format != "json" {
+		return fmt.Errorf("Invalid format: %s", format)
+	}
+
+	scopes, err := resolveNodeSysctlScopes()
+	if err != nil {
+		return err
+	}
+
+	if format == "json" {
+		output := map[string]map[string]string{}
+		for _, scope := range scopes {
+			key := scope.ProfileName
+			if key == "" {
+				key = "--global"
+			}
+
+			entries := map[string]string{}
+			for _, sysctl := range scope.Sysctls {
+				entries[sysctl.Name] = sysctl.Value
+			}
+			output[key] = entries
+		}
+
+		b, err := json.Marshal(output)
+		if err != nil {
+			return fmt.Errorf("Unable to marshal json: %w", err)
+		}
+
+		fmt.Println(string(b))
+		return nil
+	}
+
+	lines := []string{"scope|sysctl|value"}
+	for _, scope := range scopes {
+		scopeName := scope.ProfileName
+		if scopeName == "" {
+			scopeName = "--global"
+		}
+
+		for _, sysctl := range scope.Sysctls {
+			lines = append(lines, fmt.Sprintf("%s|%s|%s", scopeName, sysctl.Name, sysctl.Value))
+		}
+	}
+
+	fmt.Println(columnize.SimpleFormat(lines))
+	return nil
+}
+
+// verifyNodeProfileExists returns an error when a node profile has not been created
+func verifyNodeProfileExists(profileName string) error {
+	properties := common.PropertyGetDefault("scheduler-k3s", "--global", fmt.Sprintf("node-profile-%s.json", profileName), "")
+	if properties == "" {
+		return fmt.Errorf("Node profile %s not found", profileName)
+	}
+
+	return nil
+}
+
 // CommandAutoscalingAuthSet set or clear a scheduler-k3s autoscaling keda trigger authentication object for an app
 func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[string]string, global bool) error {
 	if global {
@@ -397,6 +486,11 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo
 		return fmt.Errorf("Unable to install helper commands: %w", err)
 	}
 
+	common.LogInfo2Quiet("Applying node sysctls")
+	if err := CreateOrUpdateNodeSysctls(ctx); err != nil {
+		return fmt.Errorf("Unable to apply node sysctls: %w", err)
+	}
+
 	common.LogVerboseQuiet("Done")
 
 	return nil
@@ -1262,6 +1356,10 @@ func CommandProfilesRemove(profileName string) error {
 		return fmt.Errorf("Unable to delete node profile: %w", err)
 	}
 
+	if err := DeleteNodeSysctls(context.Background(), profileName); err != nil {
+		return err
+	}
+
 	common.LogInfo1(fmt.Sprintf("Node profile %s removed", profileName))
 	return nil
 }

plugins/scheduler-k3s/templates/node-sysctls-chart/templates/daemonset.yaml

@@ -0,0 +1,55 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  annotations:
+    dokku.com/managed: "true"
+  labels:
+    app.kubernetes.io/name: {{ .Values.global.release_name }}
+    app.kubernetes.io/part-of: dokku
+  name: {{ .Values.global.release_name }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ .Values.global.release_name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ .Values.global.release_name }}
+        app.kubernetes.io/part-of: dokku
+    spec:
+      {{- if .Values.global.profile_name }}
+      nodeSelector:
+        dokku.com/node-profile: {{ .Values.global.profile_name | quote }}
+      {{- else }}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: dokku.com/node-profile
+                operator: DoesNotExist
+      {{- end }}
+      tolerations:
+      - operator: Exists
+      initContainers:
+      - name: apply-sysctls
+        image: {{ .Values.global.image | quote }}
+        securityContext:
+          privileged: true
+        command:
+        - sh
+        - -c
+        - |
+          set -e
+          {{- range .Values.global.sysctls }}
+          sysctl -w {{ printf "%s=%s" .name .value | quote }}
+          {{- end }}
+      containers:
+      - name: pause
+        image: {{ .Values.global.pause_image | quote }}
+        resources:
+          requests:
+            cpu: 1m
+            memory: 8Mi