diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index fd55e2adb..8fdad03bc 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -78,6 +78,20 @@ Dokku can also use Traefik on cluster initialization via the [Traefik's CRDs](ht dokku scheduler-k3s:initialize --ingress-class traefik ``` +Kubelet flags for the initial server node can be supplied by passing `--kubelet-args` with a comma-separated `key=value` list. This is the only way to configure the kubelet on the node created by `scheduler-k3s:initialize`, as that node never passes through `scheduler-k3s:cluster:add`. + +```shell +dokku scheduler-k3s:initialize \ + --kubelet-args allowed-unsafe-sysctls=net.ipv6.conf.all.disable_ipv6 +``` + +Multiple kubelet arguments can be specified in the same call by separating them with commas. + +```shell +dokku scheduler-k3s:initialize \ + --kubelet-args allowed-unsafe-sysctls=net.ipv6.conf.all.disable_ipv6,max-pods=150 +``` + ### Adding nodes to the cluster > [!WARNING] diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 08fba6bac..dcfb18acc 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1286,6 +1286,65 @@ func resolveLetsencryptIssuer(appName string, clusterIssuerName string, appEmail } } +// InitializeInstallerArgsInput contains the inputs to initializeInstallerArgs +type InitializeInstallerArgsInput struct { + // IngressClass is the ingress class the cluster is initialized with + IngressClass string + // KubeletArgs is a list of key=value kubelet arguments for the server node + KubeletArgs []string + // NodeName is the generated name of the server node + NodeName string + // TaintScheduling is whether to taint the node against app workloads + TaintScheduling bool + // Token is the cluster join token + Token string +} + +// initializeInstallerArgs builds the argument list handed to the k3s installer +// when creating the initial server node +func initializeInstallerArgs(input InitializeInstallerArgsInput) []string { + args := []string{ + // initialize the cluster + "--cluster-init", + // disable local-storage + "--disable", "local-storage", + // disable traefik so it can be installed separately + "--disable", "traefik", + // expose etcd metrics + "--etcd-expose-metrics", + // use wireguard for flannel + "--flannel-backend=wireguard-native", + // bind controller-manager to all interfaces + "--kube-controller-manager-arg", "bind-address=0.0.0.0", + // bind proxy metrics to all interfaces + "--kube-proxy-arg", "metrics-bind-address=0.0.0.0", + // bind scheduler to all interfaces + "--kube-scheduler-arg", "bind-address=0.0.0.0", + // gc terminated pods + "--kube-controller-manager-arg", "terminated-pod-gc-threshold=10", + // specify the node name + "--node-name", input.NodeName, + // allow access for the dokku user + "--write-kubeconfig-mode", "0644", + // specify a token + "--token", input.Token, + } + + if input.TaintScheduling { + args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule") + } + + for _, kubeletArg := range input.KubeletArgs { + args = append(args, "--kubelet-arg", kubeletArg) + } + + if input.IngressClass == "nginx" { + args = append(args, "--disable", "traefik") + } + + return args +} + func getKustomizeDirectory(appName string) string { directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName) return filepath.Join(directory, "kustomization") diff --git a/plugins/scheduler-k3s/functions_test.go b/plugins/scheduler-k3s/functions_test.go index 10bf58d6c..c2de4576f 100644 --- a/plugins/scheduler-k3s/functions_test.go +++ b/plugins/scheduler-k3s/functions_test.go @@ -91,3 +91,90 @@ func TestNeedsImagePullSecretsPrune(t *testing.T) { }) } } + +func TestInitializeInstallerArgs(t *testing.T) { + cases := []struct { + name string + input InitializeInstallerArgsInput + wantPairs [][2]string + wantAbsent []string + }{ + { + name: "no kubelet args", + input: InitializeInstallerArgsInput{ + IngressClass: "traefik", + NodeName: "ip-10-0-0-1-abc", + Token: "sometoken", + }, + wantAbsent: []string{"--kubelet-arg", "--node-taint"}, + }, + { + name: "single kubelet arg", + input: InitializeInstallerArgsInput{ + IngressClass: "traefik", + KubeletArgs: []string{"allowed-unsafe-sysctls=net.ipv4.tcp_rmem"}, + NodeName: "ip-10-0-0-1-abc", + Token: "sometoken", + }, + wantPairs: [][2]string{ + {"--kubelet-arg", "allowed-unsafe-sysctls=net.ipv4.tcp_rmem"}, + }, + }, + { + name: "multiple kubelet args each get their own flag", + input: InitializeInstallerArgsInput{ + IngressClass: "traefik", + KubeletArgs: []string{"allowed-unsafe-sysctls=net.ipv4.tcp_rmem", "max-pods=150"}, + NodeName: "ip-10-0-0-1-abc", + Token: "sometoken", + }, + wantPairs: [][2]string{ + {"--kubelet-arg", "allowed-unsafe-sysctls=net.ipv4.tcp_rmem"}, + {"--kubelet-arg", "max-pods=150"}, + }, + }, + { + name: "kubelet args coexist with taint scheduling", + input: InitializeInstallerArgsInput{ + IngressClass: "nginx", + KubeletArgs: []string{"max-pods=150"}, + NodeName: "ip-10-0-0-1-abc", + TaintScheduling: true, + Token: "sometoken", + }, + wantPairs: [][2]string{ + {"--node-taint", "CriticalAddonsOnly=true:NoSchedule"}, + {"--kubelet-arg", "max-pods=150"}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := initializeInstallerArgs(tc.input) + + for _, pair := range tc.wantPairs { + if !containsArgPair(got, pair[0], pair[1]) { + t.Errorf("initializeInstallerArgs() missing %q %q, got %v", pair[0], pair[1], got) + } + } + + for _, absent := range tc.wantAbsent { + for _, arg := range got { + if arg == absent { + t.Errorf("initializeInstallerArgs() unexpectedly contains %q, got %v", absent, got) + } + } + } + }) + } +} + +func containsArgPair(args []string, flag string, value string) bool { + for i := 0; i < len(args)-1; i++ { + if args[i] == flag && args[i+1] == value { + return true + } + } + return false +} diff --git a/plugins/scheduler-k3s/src/commands/commands.go b/plugins/scheduler-k3s/src/commands/commands.go index 0e93ffc3d..b0c4be449 100644 --- a/plugins/scheduler-k3s/src/commands/commands.go +++ b/plugins/scheduler-k3s/src/commands/commands.go @@ -27,7 +27,7 @@ Additional commands:` scheduler-k3s:cluster:list [--format json|stdout], Lists all nodes in a Dokku-managed cluster scheduler-k3s:cluster:remove [node-id], Removes client node to a Dokku-managed cluster scheduler-k3s:ensure-charts, Ensures the k3s charts are installed - scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling], Initializes a cluster + scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Initializes a cluster scheduler-k3s:labels:set () [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination scheduler-k3s:labels:report [|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s labels report for one or more apps scheduler-k3s:preview [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app diff --git a/plugins/scheduler-k3s/src/subcommands/subcommands.go b/plugins/scheduler-k3s/src/subcommands/subcommands.go index 7cc2f1e5b..52a58ce96 100644 --- a/plugins/scheduler-k3s/src/subcommands/subcommands.go +++ b/plugins/scheduler-k3s/src/subcommands/subcommands.go @@ -128,8 +128,9 @@ func main() { taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads") serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node") ingressClass := args.String("ingress-class", "nginx", "ingress-class: ingress-class to use for all outbound traffic") + kubeletArgs := args.StringSlice("kubelet-args", []string{}, "kubelet-args: repeatable key=value kubelet arguments (e.g., --kubelet-args key=value)") args.Parse(os.Args[2:]) - err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling) + err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling, *kubeletArgs) case "labels:set": args := flag.NewFlagSet("scheduler-k3s:labels:set", flag.ExitOnError) global := args.Bool("global", false, "--global: set a global property") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index b65844c8f..09e5094d7 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -174,7 +174,7 @@ func CommandLabelsReport(appName string, format string, processType string, reso } // CommandInitialize initializes a k3s cluster on the local server -func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool) error { +func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool, kubeletArgs []string) error { if ingressClass != "nginx" && ingressClass != "traefik" { return fmt.Errorf("Invalid ingress-class: %s", ingressClass) } @@ -308,40 +308,15 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo } nodeName = strings.ReplaceAll(strings.ToLower(fmt.Sprintf("ip-%s-%s", nodeName, fmt.Sprintf("%X", b))), ".", "-") - args := []string{ - // initialize the cluster - "--cluster-init", - // disable local-storage - "--disable", "local-storage", - // disable traefik so it can be installed separately - "--disable", "traefik", - // expose etcd metrics - "--etcd-expose-metrics", - // use wireguard for flannel - "--flannel-backend=wireguard-native", - // bind controller-manager to all interfaces - "--kube-controller-manager-arg", "bind-address=0.0.0.0", - // bind proxy metrics to all interfaces - "--kube-proxy-arg", "metrics-bind-address=0.0.0.0", - // bind scheduler to all interfaces - "--kube-scheduler-arg", "bind-address=0.0.0.0", - // gc terminated pods - "--kube-controller-manager-arg", "terminated-pod-gc-threshold=10", - // specify the node name - "--node-name", nodeName, - // allow access for the dokku user - "--write-kubeconfig-mode", "0644", - // specify a token - "--token", token, - } - if taintScheduling { - args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule") - } - common.CommandPropertySet("scheduler-k3s", "--global", "ingress-class", ingressClass, DefaultProperties, GlobalProperties) - if ingressClass == "nginx" { - args = append(args, "--disable", "traefik") - } + + args := initializeInstallerArgs(InitializeInstallerArgsInput{ + IngressClass: ingressClass, + KubeletArgs: kubeletArgs, + NodeName: nodeName, + TaintScheduling: taintScheduling, + Token: token, + }) common.LogInfo2Quiet("Running k3s installer") installerCmd, err := common.CallExecCommand(common.ExecCommandInput{ diff --git a/tests/unit/test_helper.bash b/tests/unit/test_helper.bash index fb35b626a..d24ee09d2 100644 --- a/tests/unit/test_helper.bash +++ b/tests/unit/test_helper.bash @@ -740,6 +740,10 @@ install_k3s() { args="$args --server-ip $CI_SERVER_IP" fi + if [[ -n "$KUBELET_ARGS" ]]; then + args="$args --kubelet-args $KUBELET_ARGS" + fi + run /bin/bash -c "dokku scheduler-k3s:initialize ${args}" echo "output: $output" echo "status: $status"