feat: add --kubelet-args to scheduler-k3s:initialize

The server node created by `scheduler-k3s:initialize` had no way to receive kubelet arguments, unlike nodes joined through `scheduler-k3s:cluster:add` or configured via `scheduler-k3s:profiles:add`. This meant settings such as `allowed-unsafe-sysctls` were unreachable on a single-node install.

docs/deployment/schedulers/k3s.md

@@ -78,6 +78,20 @@ Dokku can also use Traefik on cluster initialization via the [Traefik's CRDs](ht
 dokku scheduler-k3s:initialize --ingress-class traefik
 ```
 
+Kubelet flags for the initial server node can be supplied by passing `--kubelet-args` with a comma-separated `key=value` list. This is the only way to configure the kubelet on the node created by `scheduler-k3s:initialize`, as that node never passes through `scheduler-k3s:cluster:add`.
+
+```shell
+dokku scheduler-k3s:initialize \
+  --kubelet-args allowed-unsafe-sysctls=net.ipv6.conf.all.disable_ipv6
+```
+
+Multiple kubelet arguments can be specified in the same call by separating them with commas.
+
+```shell
+dokku scheduler-k3s:initialize \
+  --kubelet-args allowed-unsafe-sysctls=net.ipv6.conf.all.disable_ipv6,max-pods=150
+```
+
 ### Adding nodes to the cluster
 
 > [!WARNING]

plugins/scheduler-k3s/functions.go

@@ -1286,6 +1286,65 @@ func resolveLetsencryptIssuer(appName string, clusterIssuerName string, appEmail
 	}
 }
 
+// InitializeInstallerArgsInput contains the inputs to initializeInstallerArgs
+type InitializeInstallerArgsInput struct {
+	// IngressClass is the ingress class the cluster is initialized with
+	IngressClass string
+	// KubeletArgs is a list of key=value kubelet arguments for the server node
+	KubeletArgs []string
+	// NodeName is the generated name of the server node
+	NodeName string
+	// TaintScheduling is whether to taint the node against app workloads
+	TaintScheduling bool
+	// Token is the cluster join token
+	Token string
+}
+
+// initializeInstallerArgs builds the argument list handed to the k3s installer
+// when creating the initial server node
+func initializeInstallerArgs(input InitializeInstallerArgsInput) []string {
+	args := []string{
+		// initialize the cluster
+		"--cluster-init",
+		// disable local-storage
+		"--disable", "local-storage",
+		// disable traefik so it can be installed separately
+		"--disable", "traefik",
+		// expose etcd metrics
+		"--etcd-expose-metrics",
+		// use wireguard for flannel
+		"--flannel-backend=wireguard-native",
+		// bind controller-manager to all interfaces
+		"--kube-controller-manager-arg", "bind-address=0.0.0.0",
+		// bind proxy metrics to all interfaces
+		"--kube-proxy-arg", "metrics-bind-address=0.0.0.0",
+		// bind scheduler to all interfaces
+		"--kube-scheduler-arg", "bind-address=0.0.0.0",
+		// gc terminated pods
+		"--kube-controller-manager-arg", "terminated-pod-gc-threshold=10",
+		// specify the node name
+		"--node-name", input.NodeName,
+		// allow access for the dokku user
+		"--write-kubeconfig-mode", "0644",
+		// specify a token
+		"--token", input.Token,
+	}
+
+	if input.TaintScheduling {
+		args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
+	}
+
+	for _, kubeletArg := range input.KubeletArgs {
+		args = append(args, "--kubelet-arg", kubeletArg)
+	}
+
+	if input.IngressClass == "nginx" {
+		args = append(args, "--disable", "traefik")
+	}
+
+	return args
+}
+
 func getKustomizeDirectory(appName string) string {
 	directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName)
 	return filepath.Join(directory, "kustomization")

plugins/scheduler-k3s/functions_test.go

@@ -91,3 +91,90 @@ func TestNeedsImagePullSecretsPrune(t *testing.T) {
 		})
 	}
 }
+
+func TestInitializeInstallerArgs(t *testing.T) {
+	cases := []struct {
+		name       string
+		input      InitializeInstallerArgsInput
+		wantPairs  [][2]string
+		wantAbsent []string
+	}{
+		{
+			name: "no kubelet args",
+			input: InitializeInstallerArgsInput{
+				IngressClass: "traefik",
+				NodeName:     "ip-10-0-0-1-abc",
+				Token:        "sometoken",
+			},
+			wantAbsent: []string{"--kubelet-arg", "--node-taint"},
+		},
+		{
+			name: "single kubelet arg",
+			input: InitializeInstallerArgsInput{
+				IngressClass: "traefik",
+				KubeletArgs:  []string{"allowed-unsafe-sysctls=net.ipv4.tcp_rmem"},
+				NodeName:     "ip-10-0-0-1-abc",
+				Token:        "sometoken",
+			},
+			wantPairs: [][2]string{
+				{"--kubelet-arg", "allowed-unsafe-sysctls=net.ipv4.tcp_rmem"},
+			},
+		},
+		{
+			name: "multiple kubelet args each get their own flag",
+			input: InitializeInstallerArgsInput{
+				IngressClass: "traefik",
+				KubeletArgs:  []string{"allowed-unsafe-sysctls=net.ipv4.tcp_rmem", "max-pods=150"},
+				NodeName:     "ip-10-0-0-1-abc",
+				Token:        "sometoken",
+			},
+			wantPairs: [][2]string{
+				{"--kubelet-arg", "allowed-unsafe-sysctls=net.ipv4.tcp_rmem"},
+				{"--kubelet-arg", "max-pods=150"},
+			},
+		},
+		{
+			name: "kubelet args coexist with taint scheduling",
+			input: InitializeInstallerArgsInput{
+				IngressClass:    "nginx",
+				KubeletArgs:     []string{"max-pods=150"},
+				NodeName:        "ip-10-0-0-1-abc",
+				TaintScheduling: true,
+				Token:           "sometoken",
+			},
+			wantPairs: [][2]string{
+				{"--node-taint", "CriticalAddonsOnly=true:NoSchedule"},
+				{"--kubelet-arg", "max-pods=150"},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := initializeInstallerArgs(tc.input)
+
+			for _, pair := range tc.wantPairs {
+				if !containsArgPair(got, pair[0], pair[1]) {
+					t.Errorf("initializeInstallerArgs() missing %q %q, got %v", pair[0], pair[1], got)
+				}
+			}
+
+			for _, absent := range tc.wantAbsent {
+				for _, arg := range got {
+					if arg == absent {
+						t.Errorf("initializeInstallerArgs() unexpectedly contains %q, got %v", absent, got)
+					}
+				}
+			}
+		})
+	}
+}
+
+func containsArgPair(args []string, flag string, value string) bool {
+	for i := 0; i < len(args)-1; i++ {
+		if args[i] == flag && args[i+1] == value {
+			return true
+		}
+	}
+	return false
+}

plugins/scheduler-k3s/src/commands/commands.go

@@ -27,7 +27,7 @@ Additional commands:`
     scheduler-k3s:cluster:list [--format json|stdout], Lists all nodes in a Dokku-managed cluster
     scheduler-k3s:cluster:remove [node-id], Removes client node to a Dokku-managed cluster
     scheduler-k3s:ensure-charts, Ensures the k3s charts are installed
-    scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling], Initializes a cluster
+    scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Initializes a cluster
     scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
     scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s labels report for one or more apps
     scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app

plugins/scheduler-k3s/src/subcommands/subcommands.go

@@ -128,8 +128,9 @@ func main() {
 		taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
 		serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node")
 		ingressClass := args.String("ingress-class", "nginx", "ingress-class: ingress-class to use for all outbound traffic")
+		kubeletArgs := args.StringSlice("kubelet-args", []string{}, "kubelet-args: repeatable key=value kubelet arguments (e.g., --kubelet-args key=value)")
 		args.Parse(os.Args[2:])
-		err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling)
+		err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling, *kubeletArgs)
 	case "labels:set":
 		args := flag.NewFlagSet("scheduler-k3s:labels:set", flag.ExitOnError)
 		global := args.Bool("global", false, "--global: set a global property")

plugins/scheduler-k3s/subcommands.go

@@ -174,7 +174,7 @@ func CommandLabelsReport(appName string, format string, processType string, reso
 }
 
 // CommandInitialize initializes a k3s cluster on the local server
-func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool) error {
+func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool, kubeletArgs []string) error {
 	if ingressClass != "nginx" && ingressClass != "traefik" {
 		return fmt.Errorf("Invalid ingress-class: %s", ingressClass)
 	}
@@ -308,40 +308,15 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo
 	}
 	nodeName = strings.ReplaceAll(strings.ToLower(fmt.Sprintf("ip-%s-%s", nodeName, fmt.Sprintf("%X", b))), ".", "-")
 
-	args := []string{
-		// initialize the cluster
-		"--cluster-init",
-		// disable local-storage
-		"--disable", "local-storage",
-		// disable traefik so it can be installed separately
-		"--disable", "traefik",
-		// expose etcd metrics
-		"--etcd-expose-metrics",
-		// use wireguard for flannel
-		"--flannel-backend=wireguard-native",
-		// bind controller-manager to all interfaces
-		"--kube-controller-manager-arg", "bind-address=0.0.0.0",
-		// bind proxy metrics to all interfaces
-		"--kube-proxy-arg", "metrics-bind-address=0.0.0.0",
-		// bind scheduler to all interfaces
-		"--kube-scheduler-arg", "bind-address=0.0.0.0",
-		// gc terminated pods
-		"--kube-controller-manager-arg", "terminated-pod-gc-threshold=10",
-		// specify the node name
-		"--node-name", nodeName,
-		// allow access for the dokku user
-		"--write-kubeconfig-mode", "0644",
-		// specify a token
-		"--token", token,
-	}
-	if taintScheduling {
-		args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
-	}
-
 	common.CommandPropertySet("scheduler-k3s", "--global", "ingress-class", ingressClass, DefaultProperties, GlobalProperties)
-	if ingressClass == "nginx" {
-		args = append(args, "--disable", "traefik")
-	}
+
+	args := initializeInstallerArgs(InitializeInstallerArgsInput{
+		IngressClass:    ingressClass,
+		KubeletArgs:     kubeletArgs,
+		NodeName:        nodeName,
+		TaintScheduling: taintScheduling,
+		Token:           token,
+	})
 
 	common.LogInfo2Quiet("Running k3s installer")
 	installerCmd, err := common.CallExecCommand(common.ExecCommandInput{

tests/unit/test_helper.bash

@@ -740,6 +740,10 @@ install_k3s() {
     args="$args --server-ip $CI_SERVER_IP"
   fi
 
+  if [[ -n "$KUBELET_ARGS" ]]; then
+    args="$args --kubelet-args $KUBELET_ARGS"
+  fi
+
   run /bin/bash -c "dokku scheduler-k3s:initialize ${args}"
   echo "output: $output"
   echo "status: $status"