mirror of
https://github.com/dokku/dokku.git
synced 2026-08-29 10:08:53 +02:00
Merge pull request #8903 from dokku/scheduler-k3s-sysctls
feat: support kernel sysctls on the k3s scheduler
This commit is contained in:
@@ -38,6 +38,16 @@ More information on supported Docker options can be found [here](https://docs.do
|
||||
|
||||
Container options configured via the `docker-options` plugin are not used to modify the process a container runs. Container options are the `[OPTIONS]` portion of the following, where `[CONTAINER_COMMAND]` and `[ARG]` are the process and the arguments passed to it that are launched in the created container: `docker run [OPTIONS] [CONTAINER_COMMAND] [ARG...]`. Please see the documentation for [customizing the run command](/docs/deployment/builders/dockerfiles.md#customizing-the-run-command) or use a [Procfile](/docs/deployment/builders/dockerfiles.md#procfiles-and-multiple-processes) to modify the command used by a Dockerfile-based container.
|
||||
|
||||
#### Scheduler support
|
||||
|
||||
Docker options are written in Docker's own vocabulary and are passed verbatim to `docker run` by the `docker-local` scheduler. Other schedulers translate only the subset that has an equivalent in their own runtime, and ignore the rest.
|
||||
|
||||
The `k3s` scheduler translates `--cap-add`, `--cap-drop`, `--privileged`, and `--sysctl` into their Kubernetes equivalents. See the [k3s scheduler documentation](/docs/deployment/schedulers/k3s.md) for details, including the restriction that only namespaced sysctls can be set on a pod.
|
||||
|
||||
```shell
|
||||
dokku docker-options:add node-js-app deploy "--sysctl net.ipv4.ip_unprivileged_port_start=1024"
|
||||
```
|
||||
|
||||
#### Mounting volumes and host directories
|
||||
|
||||
Docker supports volume and host directory mounting via the `-v` or `--volume` flags. In order to simplify usage, Dokku provides a `storage` plugin as an abstraction to interact with persistent storage. In most cases, the Dokku project recommends using the persistent storage plugin over directly manipulating docker options at different phases. See the [persistent storage documentation](/docs/advanced-usage/persistent-storage.md) for more information on how to attach persistent storage to your app.
|
||||
|
||||
@@ -17,6 +17,8 @@ scheduler-k3s:ensure-charts # Ensures the k3s charts are
|
||||
scheduler-k3s:initialize # Initializes a cluster
|
||||
scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE> # Set or clear a label for a given app/process-type/resource-type combination
|
||||
scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE] # Displays a scheduler-k3s labels report for one or more apps
|
||||
scheduler-k3s:node-sysctls:set <sysctl> (<value>) [--global|--profile PROFILE] # Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile
|
||||
scheduler-k3s:node-sysctls:report [--format stdout|json] # Displays the node-level kernel sysctls applied to each scope
|
||||
scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded] # Displays a diff between the current and next deployment for an app
|
||||
scheduler-k3s:profiles:add <profile> [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS] Adds a node profile to the k3s cluster
|
||||
scheduler-k3s:profiles:list [--format json|stdout] # Lists all node profiles in the k3s cluster
|
||||
@@ -78,6 +80,20 @@ Dokku can also use Traefik on cluster initialization via the [Traefik's CRDs](ht
|
||||
dokku scheduler-k3s:initialize --ingress-class traefik
|
||||
```
|
||||
|
||||
Kubelet flags for the initial server node can be supplied by passing `--kubelet-args` with a comma-separated `key=value` list. This is the only way to configure the kubelet on the node created by `scheduler-k3s:initialize`, as that node never passes through `scheduler-k3s:cluster:add`.
|
||||
|
||||
```shell
|
||||
dokku scheduler-k3s:initialize \
|
||||
--kubelet-args allowed-unsafe-sysctls=net.ipv6.conf.all.disable_ipv6
|
||||
```
|
||||
|
||||
Multiple kubelet arguments can be specified in the same call by separating them with commas.
|
||||
|
||||
```shell
|
||||
dokku scheduler-k3s:initialize \
|
||||
--kubelet-args allowed-unsafe-sysctls=net.ipv6.conf.all.disable_ipv6,max-pods=150
|
||||
```
|
||||
|
||||
### Adding nodes to the cluster
|
||||
|
||||
> [!WARNING]
|
||||
@@ -200,6 +216,21 @@ dokku scheduler-k3s:profiles:remove edge-workers
|
||||
|
||||
Removal only deletes the stored definition; nodes that already joined the cluster keep their existing configuration.
|
||||
|
||||
#### The node profile label
|
||||
|
||||
When a node joins via `scheduler-k3s:cluster:add --profile <name>`, Dokku labels it with `dokku.com/node-profile=<name>`. This makes a profile selectable after the fact, whether via `kubectl`, a `nodeSelector`, or a node affinity rule.
|
||||
|
||||
```shell
|
||||
kubectl get nodes -L dokku.com/node-profile
|
||||
```
|
||||
|
||||
Nodes added without `--profile` are not labeled, as an empty label value would be indistinguishable from a profile literally named the empty string.
|
||||
|
||||
Two limits are worth knowing before relying on this label:
|
||||
|
||||
- The server node never carries it. That node is created by `scheduler-k3s:initialize` and never passes through `scheduler-k3s:cluster:add`, so no profile is ever associated with it.
|
||||
- Nodes that joined before this label existed are not backfilled. Use `kubectl label node <node> dokku.com/node-profile=<name>` to set it on an existing node.
|
||||
|
||||
### Changing deployment settings
|
||||
|
||||
The k3s plugin provides a number of settings that can be used to managed deployments on a per-app basis. The following table outlines ones not covered elsewhere:
|
||||
@@ -711,6 +742,74 @@ A single configured metadata key can also be queried with a flag of the form `--
|
||||
dokku scheduler-k3s:autoscaling-auth:report node-js-app --scheduler-k3s-autoscaling-auth.datadog.apiKey
|
||||
```
|
||||
|
||||
### Setting kernel sysctls
|
||||
|
||||
Kernel sysctls fall into two categories, and which one a sysctl belongs to determines how it must be set.
|
||||
|
||||
The kernel maintains a per-namespace copy of `net.*` (network namespace) as well as `kernel.shm*`, `kernel.msg*`, `kernel.sem`, and `fs.mqueue.*` (IPC namespace). These can be set on a single app's pods. Every other sysctl - including all of `vm.*`, and therefore `vm.max_map_count` - holds a single value shared by the entire machine, so it cannot be scoped to a pod and must be applied to the node itself.
|
||||
|
||||
#### Namespaced sysctls
|
||||
|
||||
Namespaced sysctls are set with the `docker-options` plugin, and are translated into the pod's `securityContext.sysctls`. A `ps:restart` is required to apply them.
|
||||
|
||||
```shell
|
||||
dokku docker-options:add node-js-app deploy "--sysctl net.ipv4.ip_unprivileged_port_start=1024"
|
||||
```
|
||||
|
||||
Passing a non-namespaced sysctl this way fails the deploy rather than silently dropping the value, since it provably cannot take effect within a pod. Note this differs from the `docker-local` scheduler, where such an option is passed straight through to `docker run`.
|
||||
|
||||
Kubernetes further splits namespaced sysctls into a *safe* list that any pod may set, and everything else. A sysctl outside the safe list - `net.core.somaxconn`, for example - is rejected at pod admission unless the node's kubelet was started with a matching `allowed-unsafe-sysctls` value, which can be supplied at cluster initialization or when joining a node.
|
||||
|
||||
```shell
|
||||
dokku scheduler-k3s:initialize --kubelet-args allowed-unsafe-sysctls=net.core.somaxconn
|
||||
```
|
||||
|
||||
Dokku does not enforce the safe list itself, as its membership changes between Kubernetes releases. Only the namespaced/non-namespaced distinction, which is a property of the kernel, is validated.
|
||||
|
||||
#### Non-namespaced sysctls
|
||||
|
||||
Non-namespaced sysctls are a property of the node, not of any app, and are managed with the `node-sysctls:set` command. Dokku applies them via a privileged DaemonSet, so they reach every node without being told which nodes exist, cover nodes joined later, and are reapplied after a node reboots.
|
||||
|
||||
```shell
|
||||
dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count 262144
|
||||
```
|
||||
|
||||
Omitting the value clears it.
|
||||
|
||||
```shell
|
||||
dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count
|
||||
```
|
||||
|
||||
Clearing a sysctl stops Dokku managing it, but does not restore whatever the node had before. The last value written stays in place until that node reboots, which is how `sysctl -w` behaves everywhere else.
|
||||
|
||||
Sysctls can also be scoped to a [node profile](#node-profiles) with `--profile`, which applies them only to nodes joined with that profile.
|
||||
|
||||
```shell
|
||||
dokku scheduler-k3s:node-sysctls:set --profile edge-workers vm.max_map_count 524288
|
||||
```
|
||||
|
||||
A profile scope inherits everything set globally and overrides it on conflict, so each node is covered by exactly one DaemonSet and no two ever write the same value. Note that the server node created by `scheduler-k3s:initialize` never carries a profile label, so only globally-scoped sysctls reach it.
|
||||
|
||||
Use `node-sysctls:report` to see the resolved set for every scope.
|
||||
|
||||
```shell
|
||||
dokku scheduler-k3s:node-sysctls:report
|
||||
```
|
||||
|
||||
```shell
|
||||
dokku scheduler-k3s:node-sysctls:report --format json
|
||||
```
|
||||
|
||||
The DaemonSet pulls `busybox` and `registry.k8s.io/pause` by default. On an air-gapped cluster or one behind a registry mirror, point them elsewhere:
|
||||
|
||||
```shell
|
||||
dokku scheduler-k3s:set --global node-sysctls-image registry.internal/busybox:1.36
|
||||
```
|
||||
|
||||
```shell
|
||||
dokku scheduler-k3s:set --global node-sysctls-pause-image registry.internal/pause:3.9
|
||||
```
|
||||
|
||||
### Integrating Kustomize
|
||||
|
||||
Dokku supports integration with [Kustomize](https://kustomize.io/) to further customize the generated helm charts for app deployments. For example, a `config/kustomize/kustomization.yaml` file with the following contents will override the scale for each process deployed to `3`:
|
||||
@@ -865,6 +964,7 @@ This plugin implements various functionality through `plugn` triggers to integra
|
||||
- `--cap-add`
|
||||
- `--cap-drop`
|
||||
- `--privileged`
|
||||
- `--sysctl` (namespaced sysctls only, see [Setting kernel sysctls](#setting-kernel-sysctls))
|
||||
- `cron`
|
||||
- `enter`
|
||||
- `deploy`
|
||||
@@ -943,6 +1043,8 @@ If unspecified for any task, the default reservation will be `.1` CPU and `128Mi
|
||||
| `letsencrypt-server` | app + global | `prod` | `--scheduler-k3s-letsencrypt-server`, `--scheduler-k3s-global-letsencrypt-server`, `--scheduler-k3s-computed-letsencrypt-server` | ACME directory (`prod` or `staging`) used for app certificates |
|
||||
| `namespace` | app + global | `default` | `--scheduler-k3s-namespace`, `--scheduler-k3s-global-namespace`, `--scheduler-k3s-computed-namespace` | Kubernetes namespace into which the app's resources are installed |
|
||||
| `network-interface` | global only | `eth0` | `--scheduler-k3s-global-network-interface`, `--scheduler-k3s-computed-network-interface` | Host network interface used by k3s |
|
||||
| `node-sysctls-image` | global only | `busybox:1.36` | `--scheduler-k3s-global-node-sysctls-image` | Image used to apply node-level sysctls, override for air-gapped clusters |
|
||||
| `node-sysctls-pause-image` | global only | `registry.k8s.io/pause:3.9` | `--scheduler-k3s-global-node-sysctls-pause-image` | Image keeping the node sysctls daemonset pods running |
|
||||
| `rollback-on-failure` | app + global | `false` | `--scheduler-k3s-rollback-on-failure`, `--scheduler-k3s-global-rollback-on-failure`, `--scheduler-k3s-computed-rollback-on-failure` | When `true`, helm rolls back the release if a deploy fails |
|
||||
| `shm-size` | app + global | none | `--scheduler-k3s-shm-size`, `--scheduler-k3s-global-shm-size`, `--scheduler-k3s-computed-shm-size` | `/dev/shm` size override applied to app containers |
|
||||
| `token` | global only | none | `--scheduler-k3s-global-token` (masked as `*******` in default stdout output; the raw value is returned when queried via `--format json` or when this flag is requested explicitly) | Cluster join token used by `scheduler-k3s:cluster-add` |
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
SUBCOMMANDS = subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/labels:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
|
||||
SUBCOMMANDS = subcommands/annotations:set subcommands/annotations:report subcommands/autoscaling-auth:set subcommands/autoscaling-auth:report subcommands/charts:report subcommands/charts:set subcommands/cluster:add subcommands/cluster:list subcommands/cluster:remove subcommands/ensure-charts subcommands/initialize subcommands/labels:set subcommands/labels:report subcommands/node-sysctls:set subcommands/node-sysctls:report subcommands/preview subcommands/profiles:add subcommands/profiles:list subcommands/profiles:remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall
|
||||
TRIGGERS = triggers/core-post-deploy triggers/core-post-extract triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-certs-update triggers/post-certs-remove triggers/post-create triggers/post-delete triggers/report triggers/scheduler-app-status triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-is-deployed triggers/scheduler-logs triggers/scheduler-proxy-config triggers/scheduler-proxy-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop triggers/scheduler-cron-write triggers/storage-create triggers/storage-destroy triggers/storage-status triggers/scheduler-storage-exec
|
||||
BUILD = commands subcommands triggers
|
||||
PLUGIN_NAME = scheduler-k3s
|
||||
|
||||
@@ -1286,6 +1286,86 @@ func resolveLetsencryptIssuer(appName string, clusterIssuerName string, appEmail
|
||||
}
|
||||
}
|
||||
|
||||
// nodeLabels returns the labels to apply to a node joining the cluster, including
|
||||
// the node profile label when the node was added with a named profile. The returned
|
||||
// map is always a fresh copy so callers cannot mutate ServerLabels or WorkerLabels.
|
||||
func nodeLabels(role string, profileName string) map[string]string {
|
||||
source := ServerLabels
|
||||
if role == "worker" {
|
||||
source = WorkerLabels
|
||||
}
|
||||
|
||||
labels := make(map[string]string, len(source)+1)
|
||||
for key, value := range source {
|
||||
labels[key] = value
|
||||
}
|
||||
|
||||
if profileName != "" {
|
||||
labels[NodeProfileLabel] = profileName
|
||||
}
|
||||
|
||||
return labels
|
||||
}
|
||||
|
||||
// InitializeInstallerArgsInput contains the inputs to initializeInstallerArgs
|
||||
type InitializeInstallerArgsInput struct {
|
||||
// IngressClass is the ingress class the cluster is initialized with
|
||||
IngressClass string
|
||||
// KubeletArgs is a list of key=value kubelet arguments for the server node
|
||||
KubeletArgs []string
|
||||
// NodeName is the generated name of the server node
|
||||
NodeName string
|
||||
// TaintScheduling is whether to taint the node against app workloads
|
||||
TaintScheduling bool
|
||||
// Token is the cluster join token
|
||||
Token string
|
||||
}
|
||||
|
||||
// initializeInstallerArgs builds the argument list handed to the k3s installer
|
||||
// when creating the initial server node
|
||||
func initializeInstallerArgs(input InitializeInstallerArgsInput) []string {
|
||||
args := []string{
|
||||
// initialize the cluster
|
||||
"--cluster-init",
|
||||
// disable local-storage
|
||||
"--disable", "local-storage",
|
||||
// disable traefik so it can be installed separately
|
||||
"--disable", "traefik",
|
||||
// expose etcd metrics
|
||||
"--etcd-expose-metrics",
|
||||
// use wireguard for flannel
|
||||
"--flannel-backend=wireguard-native",
|
||||
// bind controller-manager to all interfaces
|
||||
"--kube-controller-manager-arg", "bind-address=0.0.0.0",
|
||||
// bind proxy metrics to all interfaces
|
||||
"--kube-proxy-arg", "metrics-bind-address=0.0.0.0",
|
||||
// bind scheduler to all interfaces
|
||||
"--kube-scheduler-arg", "bind-address=0.0.0.0",
|
||||
// gc terminated pods
|
||||
"--kube-controller-manager-arg", "terminated-pod-gc-threshold=10",
|
||||
// specify the node name
|
||||
"--node-name", input.NodeName,
|
||||
// allow access for the dokku user
|
||||
"--write-kubeconfig-mode", "0644",
|
||||
// specify a token
|
||||
"--token", input.Token,
|
||||
}
|
||||
|
||||
if input.TaintScheduling {
|
||||
args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
|
||||
}
|
||||
|
||||
for _, kubeletArg := range input.KubeletArgs {
|
||||
args = append(args, "--kubelet-arg", kubeletArg)
|
||||
}
|
||||
|
||||
if input.IngressClass == "nginx" {
|
||||
args = append(args, "--disable", "traefik")
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
func getKustomizeDirectory(appName string) string {
|
||||
directory := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "scheduler-k3s", appName)
|
||||
return filepath.Join(directory, "kustomization")
|
||||
@@ -1311,6 +1391,24 @@ func getComputedKustomizeRootPath(appName string) string {
|
||||
return kustomizeRootPath
|
||||
}
|
||||
|
||||
func getComputedNodeSysctlsImage() string {
|
||||
image := common.PropertyGet("scheduler-k3s", "--global", "node-sysctls-image")
|
||||
if image == "" {
|
||||
image = DefaultNodeSysctlsImage
|
||||
}
|
||||
|
||||
return image
|
||||
}
|
||||
|
||||
func getComputedNodeSysctlsPauseImage() string {
|
||||
image := common.PropertyGet("scheduler-k3s", "--global", "node-sysctls-pause-image")
|
||||
if image == "" {
|
||||
image = DefaultNodeSysctlsPauseImage
|
||||
}
|
||||
|
||||
return image
|
||||
}
|
||||
|
||||
func getNamespace(appName string) string {
|
||||
return common.PropertyGet("scheduler-k3s", appName, "namespace")
|
||||
}
|
||||
@@ -1674,12 +1772,62 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// namespacedSysctlPrefixes are the sysctl prefixes the kernel maintains per-namespace
|
||||
var namespacedSysctlPrefixes = []string{
|
||||
"net.",
|
||||
"kernel.shm",
|
||||
"kernel.msg",
|
||||
"fs.mqueue.",
|
||||
}
|
||||
|
||||
// isNamespacedSysctl reports whether a sysctl is maintained per-namespace by the
|
||||
// kernel and can therefore be set on a pod spec. Sysctls outside these subtrees
|
||||
// hold a single value shared by the entire machine, and kubelet rejects them.
|
||||
func isNamespacedSysctl(name string) bool {
|
||||
if name == "kernel.sem" {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, prefix := range namespacedSysctlPrefixes {
|
||||
if strings.HasPrefix(name, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// parseSysctls converts docker-option key=value pairs into sysctls sorted by name,
|
||||
// rejecting any sysctl that cannot take effect within a pod's namespaces.
|
||||
func parseSysctls(values []string) ([]Sysctl, error) {
|
||||
sysctls := []Sysctl{}
|
||||
for _, value := range values {
|
||||
name, sysctlValue, found := strings.Cut(value, "=")
|
||||
if !found || name == "" {
|
||||
return nil, fmt.Errorf("Invalid --sysctl value, expected name=value: %s", value)
|
||||
}
|
||||
|
||||
if !isNamespacedSysctl(name) {
|
||||
return nil, fmt.Errorf("Sysctl %s is not namespaced and cannot be set on a pod, apply it to the nodes with 'dokku scheduler-k3s:node-sysctls:set %s <value>' instead", name, name)
|
||||
}
|
||||
|
||||
sysctls = append(sysctls, Sysctl{Name: name, Value: sysctlValue})
|
||||
}
|
||||
|
||||
sort.Slice(sysctls, func(i int, j int) bool {
|
||||
return sysctls[i].Name < sysctls[j].Name
|
||||
})
|
||||
|
||||
return sysctls, nil
|
||||
}
|
||||
|
||||
func getSecurityContext(appName string, phase string) (SecurityContext, error) {
|
||||
securityContext := SecurityContext{}
|
||||
deployOptions, err := dockeroptions.GetSpecifiedDockerOptionsForPhase(appName, phase, []string{
|
||||
"--cap-add",
|
||||
"--cap-drop",
|
||||
"--privileged",
|
||||
"--sysctl",
|
||||
})
|
||||
if err != nil {
|
||||
return SecurityContext{}, fmt.Errorf("Error getting deploy options: %w", err)
|
||||
@@ -1702,6 +1850,13 @@ func getSecurityContext(appName string, phase string) (SecurityContext, error) {
|
||||
}
|
||||
securityContext.Capabilities.Drop = capabilities
|
||||
}
|
||||
if sysctlOptions, ok := deployOptions["--sysctl"]; ok {
|
||||
sysctls, err := parseSysctls(sysctlOptions)
|
||||
if err != nil {
|
||||
return SecurityContext{}, err
|
||||
}
|
||||
securityContext.Sysctls = sysctls
|
||||
}
|
||||
return securityContext, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -91,3 +91,269 @@ func TestNeedsImagePullSecretsPrune(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeInstallerArgs(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
input InitializeInstallerArgsInput
|
||||
wantPairs [][2]string
|
||||
wantAbsent []string
|
||||
}{
|
||||
{
|
||||
name: "no kubelet args",
|
||||
input: InitializeInstallerArgsInput{
|
||||
IngressClass: "traefik",
|
||||
NodeName: "ip-10-0-0-1-abc",
|
||||
Token: "sometoken",
|
||||
},
|
||||
wantAbsent: []string{"--kubelet-arg", "--node-taint"},
|
||||
},
|
||||
{
|
||||
name: "single kubelet arg",
|
||||
input: InitializeInstallerArgsInput{
|
||||
IngressClass: "traefik",
|
||||
KubeletArgs: []string{"allowed-unsafe-sysctls=net.ipv4.tcp_rmem"},
|
||||
NodeName: "ip-10-0-0-1-abc",
|
||||
Token: "sometoken",
|
||||
},
|
||||
wantPairs: [][2]string{
|
||||
{"--kubelet-arg", "allowed-unsafe-sysctls=net.ipv4.tcp_rmem"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple kubelet args each get their own flag",
|
||||
input: InitializeInstallerArgsInput{
|
||||
IngressClass: "traefik",
|
||||
KubeletArgs: []string{"allowed-unsafe-sysctls=net.ipv4.tcp_rmem", "max-pods=150"},
|
||||
NodeName: "ip-10-0-0-1-abc",
|
||||
Token: "sometoken",
|
||||
},
|
||||
wantPairs: [][2]string{
|
||||
{"--kubelet-arg", "allowed-unsafe-sysctls=net.ipv4.tcp_rmem"},
|
||||
{"--kubelet-arg", "max-pods=150"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "kubelet args coexist with taint scheduling",
|
||||
input: InitializeInstallerArgsInput{
|
||||
IngressClass: "nginx",
|
||||
KubeletArgs: []string{"max-pods=150"},
|
||||
NodeName: "ip-10-0-0-1-abc",
|
||||
TaintScheduling: true,
|
||||
Token: "sometoken",
|
||||
},
|
||||
wantPairs: [][2]string{
|
||||
{"--node-taint", "CriticalAddonsOnly=true:NoSchedule"},
|
||||
{"--kubelet-arg", "max-pods=150"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := initializeInstallerArgs(tc.input)
|
||||
|
||||
for _, pair := range tc.wantPairs {
|
||||
if !containsArgPair(got, pair[0], pair[1]) {
|
||||
t.Errorf("initializeInstallerArgs() missing %q %q, got %v", pair[0], pair[1], got)
|
||||
}
|
||||
}
|
||||
|
||||
for _, absent := range tc.wantAbsent {
|
||||
for _, arg := range got {
|
||||
if arg == absent {
|
||||
t.Errorf("initializeInstallerArgs() unexpectedly contains %q, got %v", absent, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func containsArgPair(args []string, flag string, value string) bool {
|
||||
for i := 0; i < len(args)-1; i++ {
|
||||
if args[i] == flag && args[i+1] == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestNodeLabels(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
role string
|
||||
profileName string
|
||||
wantKey string
|
||||
wantValue string
|
||||
wantProfile bool
|
||||
}{
|
||||
{
|
||||
name: "server without a profile",
|
||||
role: "server",
|
||||
profileName: "",
|
||||
wantKey: "svccontroller.k3s.cattle.io/enablelb",
|
||||
wantValue: "true",
|
||||
wantProfile: false,
|
||||
},
|
||||
{
|
||||
name: "worker without a profile",
|
||||
role: "worker",
|
||||
profileName: "",
|
||||
wantKey: "node-role.kubernetes.io/worker",
|
||||
wantValue: "worker",
|
||||
wantProfile: false,
|
||||
},
|
||||
{
|
||||
name: "worker with a profile",
|
||||
role: "worker",
|
||||
profileName: "edge-workers",
|
||||
wantKey: "node-role.kubernetes.io/worker",
|
||||
wantValue: "worker",
|
||||
wantProfile: true,
|
||||
},
|
||||
{
|
||||
name: "server with a profile",
|
||||
role: "server",
|
||||
profileName: "control-plane",
|
||||
wantKey: "svccontroller.k3s.cattle.io/enablelb",
|
||||
wantValue: "true",
|
||||
wantProfile: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := nodeLabels(tc.role, tc.profileName)
|
||||
|
||||
if got[tc.wantKey] != tc.wantValue {
|
||||
t.Errorf("nodeLabels() role label = %q, want %q", got[tc.wantKey], tc.wantValue)
|
||||
}
|
||||
|
||||
profile, ok := got[NodeProfileLabel]
|
||||
if ok != tc.wantProfile {
|
||||
t.Errorf("nodeLabels() has %s = %v, want %v", NodeProfileLabel, ok, tc.wantProfile)
|
||||
}
|
||||
if tc.wantProfile && profile != tc.profileName {
|
||||
t.Errorf("nodeLabels() %s = %q, want %q", NodeProfileLabel, profile, tc.profileName)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeLabelsDoesNotMutatePackageLabels(t *testing.T) {
|
||||
serverBefore := len(ServerLabels)
|
||||
workerBefore := len(WorkerLabels)
|
||||
|
||||
nodeLabels("server", "control-plane")
|
||||
nodeLabels("worker", "edge-workers")
|
||||
|
||||
if len(ServerLabels) != serverBefore {
|
||||
t.Errorf("nodeLabels() mutated ServerLabels: len = %d, want %d", len(ServerLabels), serverBefore)
|
||||
}
|
||||
if len(WorkerLabels) != workerBefore {
|
||||
t.Errorf("nodeLabels() mutated WorkerLabels: len = %d, want %d", len(WorkerLabels), workerBefore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsNamespacedSysctl(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
want bool
|
||||
}{
|
||||
{name: "net.core.somaxconn", want: true},
|
||||
{name: "net.ipv4.tcp_rmem", want: true},
|
||||
{name: "kernel.shm_rmid_forced", want: true},
|
||||
{name: "kernel.shmmax", want: true},
|
||||
{name: "kernel.msgmax", want: true},
|
||||
{name: "kernel.sem", want: true},
|
||||
{name: "fs.mqueue.msg_max", want: true},
|
||||
{name: "vm.max_map_count", want: false},
|
||||
{name: "vm.swappiness", want: false},
|
||||
{name: "kernel.pid_max", want: false},
|
||||
{name: "kernel.semaphore", want: false},
|
||||
{name: "fs.file-max", want: false},
|
||||
{name: "", want: false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isNamespacedSysctl(tc.name); got != tc.want {
|
||||
t.Errorf("isNamespacedSysctl(%q) = %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSysctls(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
values []string
|
||||
want []Sysctl
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "empty input",
|
||||
values: []string{},
|
||||
want: []Sysctl{},
|
||||
},
|
||||
{
|
||||
name: "single namespaced sysctl",
|
||||
values: []string{"net.core.somaxconn=1024"},
|
||||
want: []Sysctl{{Name: "net.core.somaxconn", Value: "1024"}},
|
||||
},
|
||||
{
|
||||
name: "sorted by name regardless of input order",
|
||||
values: []string{"net.core.somaxconn=1024", "kernel.sem=250", "fs.mqueue.msg_max=20"},
|
||||
want: []Sysctl{
|
||||
{Name: "fs.mqueue.msg_max", Value: "20"},
|
||||
{Name: "kernel.sem", Value: "250"},
|
||||
{Name: "net.core.somaxconn", Value: "1024"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "value containing an equals sign is preserved",
|
||||
values: []string{"net.ipv4.tcp_rmem=4096 87380 6291456"},
|
||||
want: []Sysctl{{Name: "net.ipv4.tcp_rmem", Value: "4096 87380 6291456"}},
|
||||
},
|
||||
{
|
||||
name: "non-namespaced sysctl is rejected",
|
||||
values: []string{"vm.max_map_count=262144"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "missing equals sign is rejected",
|
||||
values: []string{"net.core.somaxconn"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "empty name is rejected",
|
||||
values: []string{"=1024"},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := parseSysctls(tc.values)
|
||||
if tc.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("parseSysctls(%v) expected an error, got %v", tc.values, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("parseSysctls(%v) unexpected error: %v", tc.values, err)
|
||||
}
|
||||
|
||||
if len(got) != len(tc.want) {
|
||||
t.Fatalf("parseSysctls(%v) = %v, want %v", tc.values, got, tc.want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tc.want[i] {
|
||||
t.Errorf("parseSysctls(%v)[%d] = %v, want %v", tc.values, i, got[i], tc.want[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
324
plugins/scheduler-k3s/node_sysctls.go
Normal file
324
plugins/scheduler-k3s/node_sysctls.go
Normal file
@@ -0,0 +1,324 @@
|
||||
package scheduler_k3s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/dokku/dokku/plugins/common"
|
||||
)
|
||||
|
||||
// NodeSysctlsValues contains the values for a dokku-managed node sysctls helm chart
|
||||
type NodeSysctlsValues struct {
|
||||
Global NodeSysctlsGlobalValues `yaml:"global"`
|
||||
}
|
||||
|
||||
// NodeSysctlsGlobalValues contains the global values for the node sysctls chart
|
||||
type NodeSysctlsGlobalValues struct {
|
||||
Image string `yaml:"image"`
|
||||
PauseImage string `yaml:"pause_image"`
|
||||
ProfileName string `yaml:"profile_name,omitempty"`
|
||||
ReleaseName string `yaml:"release_name"`
|
||||
Sysctls []Sysctl `yaml:"sysctls"`
|
||||
}
|
||||
|
||||
// nodeSysctlScope is a resolved set of sysctls destined for a single DaemonSet
|
||||
type nodeSysctlScope struct {
|
||||
// ProfileName is the node profile this scope targets, empty for the global scope
|
||||
ProfileName string
|
||||
// ReleaseName is the helm release backing this scope
|
||||
ReleaseName string
|
||||
// Sysctls are the fully resolved sysctls for this scope
|
||||
Sysctls []Sysctl
|
||||
}
|
||||
|
||||
// getNodeSysctlsProperty returns the property name backing a node sysctls scope
|
||||
func getNodeSysctlsProperty(profileName string) string {
|
||||
if profileName == "" {
|
||||
return "node-sysctls.global"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("node-sysctls.profile.%s", profileName)
|
||||
}
|
||||
|
||||
// getNodeSysctlsReleaseName returns the helm release name for a node sysctls scope
|
||||
func getNodeSysctlsReleaseName(profileName string) string {
|
||||
if profileName == "" {
|
||||
return "dokku-node-sysctls-global"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("dokku-node-sysctls-profile-%s", profileName)
|
||||
}
|
||||
|
||||
// getNodeSysctls returns the sysctls stored against a single scope
|
||||
func getNodeSysctls(profileName string) (map[string]string, error) {
|
||||
sysctls, err := common.PropertyMapGet("scheduler-k3s", "--global", getNodeSysctlsProperty(profileName))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to read node sysctls: %w", err)
|
||||
}
|
||||
|
||||
return sysctls, nil
|
||||
}
|
||||
|
||||
// listNodeProfileNames returns the names of every stored node profile
|
||||
func listNodeProfileNames() ([]string, error) {
|
||||
properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", "--global", "node-profile-")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to get node profiles: %w", err)
|
||||
}
|
||||
|
||||
names := []string{}
|
||||
for property, data := range properties {
|
||||
if !strings.HasSuffix(property, ".json") {
|
||||
continue
|
||||
}
|
||||
|
||||
var profile NodeProfile
|
||||
if err := json.Unmarshal([]byte(data), &profile); err != nil {
|
||||
return nil, fmt.Errorf("Unable to unmarshal node profile: %w", err)
|
||||
}
|
||||
|
||||
names = append(names, profile.Name)
|
||||
}
|
||||
|
||||
sort.Strings(names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// sortedSysctls converts a name/value map into sysctls sorted by name so the
|
||||
// rendered manifest is stable across runs
|
||||
func sortedSysctls(values map[string]string) []Sysctl {
|
||||
sysctls := make([]Sysctl, 0, len(values))
|
||||
for name, value := range values {
|
||||
sysctls = append(sysctls, Sysctl{Name: name, Value: value})
|
||||
}
|
||||
|
||||
sort.Slice(sysctls, func(i int, j int) bool {
|
||||
return sysctls[i].Name < sysctls[j].Name
|
||||
})
|
||||
|
||||
return sysctls
|
||||
}
|
||||
|
||||
// mergeNodeSysctls layers a profile's sysctls over the global ones, with the profile
|
||||
// winning on conflict, and returns the result sorted by name. Profile scopes must
|
||||
// carry the global values too: profiled nodes are excluded from the global DaemonSet,
|
||||
// so anything omitted here would never reach them.
|
||||
func mergeNodeSysctls(global map[string]string, profile map[string]string) []Sysctl {
|
||||
merged := map[string]string{}
|
||||
for name, value := range global {
|
||||
merged[name] = value
|
||||
}
|
||||
for name, value := range profile {
|
||||
merged[name] = value
|
||||
}
|
||||
|
||||
return sortedSysctls(merged)
|
||||
}
|
||||
|
||||
// resolveNodeSysctlScopes returns one scope per DaemonSet that should exist, with
|
||||
// profile scopes carrying the global sysctls merged underneath their own. Every
|
||||
// node matches exactly one scope: profiled nodes match their profile's DaemonSet,
|
||||
// and unprofiled nodes match the global one, so no two DaemonSets ever write the
|
||||
// same sysctl on the same node.
|
||||
func resolveNodeSysctlScopes() ([]nodeSysctlScope, error) {
|
||||
globalSysctls, err := getNodeSysctls("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
scopes := []nodeSysctlScope{
|
||||
{
|
||||
ReleaseName: getNodeSysctlsReleaseName(""),
|
||||
Sysctls: sortedSysctls(globalSysctls),
|
||||
},
|
||||
}
|
||||
|
||||
profileNames, err := listNodeProfileNames()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, profileName := range profileNames {
|
||||
profileSysctls, err := getNodeSysctls(profileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
scopes = append(scopes, nodeSysctlScope{
|
||||
ProfileName: profileName,
|
||||
ReleaseName: getNodeSysctlsReleaseName(profileName),
|
||||
Sysctls: mergeNodeSysctls(globalSysctls, profileSysctls),
|
||||
})
|
||||
}
|
||||
|
||||
return scopes, nil
|
||||
}
|
||||
|
||||
// CreateOrUpdateNodeSysctls reconciles every node sysctls DaemonSet against the
|
||||
// stored properties. All scopes are reconciled on every call rather than only the
|
||||
// mutated one, since profile scopes inherit the global sysctls at render time and
|
||||
// would otherwise serve a stale value after a global change.
|
||||
func CreateOrUpdateNodeSysctls(ctx context.Context) error {
|
||||
if err := isKubernetesAvailable(); err != nil {
|
||||
common.LogDebug("kubernetes not available, skipping node sysctls sync")
|
||||
return nil
|
||||
}
|
||||
|
||||
scopes, err := resolveNodeSysctlScopes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, scope := range scopes {
|
||||
if len(scope.Sysctls) == 0 {
|
||||
if err := deleteNodeSysctlsRelease(scope.ReleaseName); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if err := installNodeSysctlsChart(ctx, scope); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// installNodeSysctlsChart installs or upgrades the DaemonSet backing a single scope
|
||||
func installNodeSysctlsChart(ctx context.Context, scope nodeSysctlScope) error {
|
||||
chartDir, err := os.MkdirTemp("", "dokku-node-sysctls-chart-")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating chart directory: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(chartDir)
|
||||
|
||||
if err := os.MkdirAll(filepath.Join(chartDir, "templates"), os.FileMode(0755)); err != nil {
|
||||
return fmt.Errorf("error creating chart templates directory: %w", err)
|
||||
}
|
||||
|
||||
chart := &Chart{
|
||||
ApiVersion: "v2",
|
||||
AppVersion: "1.0.0",
|
||||
Name: scope.ReleaseName,
|
||||
Icon: "https://dokku.com/assets/dokku-logo.svg",
|
||||
Version: "0.0.1",
|
||||
}
|
||||
|
||||
err = writeYaml(WriteYamlInput{
|
||||
Object: chart,
|
||||
Path: filepath.Join(chartDir, "Chart.yaml"),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing chart: %w", err)
|
||||
}
|
||||
|
||||
b, err := templates.ReadFile("templates/node-sysctls-chart/templates/daemonset.yaml")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading node-sysctls template: %w", err)
|
||||
}
|
||||
|
||||
filename := filepath.Join(chartDir, "templates", "daemonset.yaml")
|
||||
if err := os.WriteFile(filename, b, os.FileMode(0644)); err != nil {
|
||||
return fmt.Errorf("error writing node-sysctls template: %w", err)
|
||||
}
|
||||
|
||||
if os.Getenv("DOKKU_TRACE") == "1" {
|
||||
common.CatFile(filename)
|
||||
}
|
||||
|
||||
values := &NodeSysctlsValues{
|
||||
Global: NodeSysctlsGlobalValues{
|
||||
Image: getComputedNodeSysctlsImage(),
|
||||
PauseImage: getComputedNodeSysctlsPauseImage(),
|
||||
ProfileName: scope.ProfileName,
|
||||
ReleaseName: scope.ReleaseName,
|
||||
Sysctls: scope.Sysctls,
|
||||
},
|
||||
}
|
||||
|
||||
err = writeYaml(WriteYamlInput{
|
||||
Object: values,
|
||||
Path: filepath.Join(chartDir, "values.yaml"),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing values: %w", err)
|
||||
}
|
||||
|
||||
helmAgent, err := NewHelmAgent(NodeSysctlsNamespace, DeployLogPrinter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating helm agent: %w", err)
|
||||
}
|
||||
|
||||
chartPath, err := filepath.Abs(chartDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting chart path: %w", err)
|
||||
}
|
||||
|
||||
common.LogVerboseQuiet(fmt.Sprintf("Applying node sysctls for %s", nodeSysctlsScopeLabel(scope.ProfileName)))
|
||||
err = helmAgent.InstallOrUpgradeChart(ctx, ChartInput{
|
||||
ChartPath: chartPath,
|
||||
Namespace: NodeSysctlsNamespace,
|
||||
ReleaseName: scope.ReleaseName,
|
||||
Wait: false,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error installing node sysctls chart: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteNodeSysctlsRelease removes a node sysctls DaemonSet when its scope resolves
|
||||
// to an empty set. The kernel values it wrote are not reverted; they persist on the
|
||||
// affected nodes until those nodes reboot.
|
||||
func deleteNodeSysctlsRelease(releaseName string) error {
|
||||
helmAgent, err := NewHelmAgent(NodeSysctlsNamespace, DeployLogPrinter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating helm agent: %w", err)
|
||||
}
|
||||
|
||||
exists, err := helmAgent.ChartExists(releaseName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error checking if node sysctls chart exists: %w", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
common.LogVerboseQuiet(fmt.Sprintf("Removing node sysctls release %s", releaseName))
|
||||
if err := helmAgent.UninstallChart(releaseName); err != nil {
|
||||
return fmt.Errorf("error uninstalling node sysctls chart: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteNodeSysctls removes the stored sysctls and DaemonSet for a single node profile
|
||||
func DeleteNodeSysctls(ctx context.Context, profileName string) error {
|
||||
if err := common.PropertyDelete("scheduler-k3s", "--global", getNodeSysctlsProperty(profileName)); err != nil {
|
||||
return fmt.Errorf("Unable to delete node sysctls: %w", err)
|
||||
}
|
||||
|
||||
if err := isKubernetesAvailable(); err != nil {
|
||||
common.LogDebug("kubernetes not available, skipping node sysctls deletion")
|
||||
return nil
|
||||
}
|
||||
|
||||
return deleteNodeSysctlsRelease(getNodeSysctlsReleaseName(profileName))
|
||||
}
|
||||
|
||||
// nodeSysctlsScopeLabel returns a human readable name for a node sysctls scope
|
||||
func nodeSysctlsScopeLabel(profileName string) string {
|
||||
if profileName == "" {
|
||||
return "unprofiled nodes"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("node profile %s", profileName)
|
||||
}
|
||||
305
plugins/scheduler-k3s/node_sysctls_test.go
Normal file
305
plugins/scheduler-k3s/node_sysctls_test.go
Normal file
@@ -0,0 +1,305 @@
|
||||
package scheduler_k3s
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"helm.sh/helm/v3/pkg/chart/loader"
|
||||
"helm.sh/helm/v3/pkg/chartutil"
|
||||
"helm.sh/helm/v3/pkg/engine"
|
||||
)
|
||||
|
||||
func TestGetNodeSysctlsProperty(t *testing.T) {
|
||||
if got := getNodeSysctlsProperty(""); got != "node-sysctls.global" {
|
||||
t.Errorf("getNodeSysctlsProperty(\"\") = %q, want node-sysctls.global", got)
|
||||
}
|
||||
if got := getNodeSysctlsProperty("edge-workers"); got != "node-sysctls.profile.edge-workers" {
|
||||
t.Errorf("getNodeSysctlsProperty(\"edge-workers\") = %q, want node-sysctls.profile.edge-workers", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetNodeSysctlsPropertyIsReserved asserts the property prefix is excluded from
|
||||
// the annotations scan. Without this a property like node-sysctls.profile.pod would
|
||||
// be mistaken for a pod annotation.
|
||||
func TestGetNodeSysctlsPropertyIsReserved(t *testing.T) {
|
||||
reserved := false
|
||||
for _, prefix := range reservedAnnotationPrefixes {
|
||||
if prefix == "node-sysctls." {
|
||||
reserved = true
|
||||
}
|
||||
}
|
||||
|
||||
if !reserved {
|
||||
t.Error("node-sysctls. is not in reservedAnnotationPrefixes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNodeSysctlsReleaseName(t *testing.T) {
|
||||
if got := getNodeSysctlsReleaseName(""); got != "dokku-node-sysctls-global" {
|
||||
t.Errorf("getNodeSysctlsReleaseName(\"\") = %q, want dokku-node-sysctls-global", got)
|
||||
}
|
||||
if got := getNodeSysctlsReleaseName("edge-workers"); got != "dokku-node-sysctls-profile-edge-workers" {
|
||||
t.Errorf("getNodeSysctlsReleaseName(\"edge-workers\") = %q, want dokku-node-sysctls-profile-edge-workers", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortedSysctls(t *testing.T) {
|
||||
got := sortedSysctls(map[string]string{
|
||||
"vm.swappiness": "10",
|
||||
"vm.max_map_count": "262144",
|
||||
"fs.file-max": "100000",
|
||||
})
|
||||
|
||||
want := []Sysctl{
|
||||
{Name: "fs.file-max", Value: "100000"},
|
||||
{Name: "vm.max_map_count", Value: "262144"},
|
||||
{Name: "vm.swappiness", Value: "10"},
|
||||
}
|
||||
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("sortedSysctls() = %v, want %v", got, want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
t.Errorf("sortedSysctls()[%d] = %v, want %v", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortedSysctlsEmpty(t *testing.T) {
|
||||
if got := sortedSysctls(map[string]string{}); len(got) != 0 {
|
||||
t.Errorf("sortedSysctls(empty) = %v, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeSysctlsScopeLabel(t *testing.T) {
|
||||
if got := nodeSysctlsScopeLabel(""); got != "unprofiled nodes" {
|
||||
t.Errorf("nodeSysctlsScopeLabel(\"\") = %q, want 'unprofiled nodes'", got)
|
||||
}
|
||||
if got := nodeSysctlsScopeLabel("edge"); got != "node profile edge" {
|
||||
t.Errorf("nodeSysctlsScopeLabel(\"edge\") = %q, want 'node profile edge'", got)
|
||||
}
|
||||
}
|
||||
|
||||
func renderNodeSysctlsTemplate(t *testing.T, values NodeSysctlsGlobalValues) string {
|
||||
t.Helper()
|
||||
|
||||
chartDir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(chartDir, "templates"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
chartYAML := []byte("apiVersion: v2\nname: test\nversion: 0.0.1\n")
|
||||
if err := os.WriteFile(filepath.Join(chartDir, "Chart.yaml"), chartYAML, 0o644); err != nil {
|
||||
t.Fatalf("write Chart.yaml: %v", err)
|
||||
}
|
||||
|
||||
tpl, err := templates.ReadFile("templates/node-sysctls-chart/templates/daemonset.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("read daemonset template: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(chartDir, "templates", "daemonset.yaml"), tpl, 0o644); err != nil {
|
||||
t.Fatalf("write daemonset template: %v", err)
|
||||
}
|
||||
|
||||
loaded, err := loader.Load(chartDir)
|
||||
if err != nil {
|
||||
t.Fatalf("load chart: %v", err)
|
||||
}
|
||||
|
||||
sysctls := []interface{}{}
|
||||
for _, sysctl := range values.Sysctls {
|
||||
sysctls = append(sysctls, map[string]interface{}{"name": sysctl.Name, "value": sysctl.Value})
|
||||
}
|
||||
|
||||
renderValues, err := chartutil.ToRenderValues(loaded, map[string]interface{}{
|
||||
"global": map[string]interface{}{
|
||||
"image": values.Image,
|
||||
"pause_image": values.PauseImage,
|
||||
"profile_name": values.ProfileName,
|
||||
"release_name": values.ReleaseName,
|
||||
"sysctls": sysctls,
|
||||
},
|
||||
}, chartutil.ReleaseOptions{Name: "test", Namespace: "kube-system"}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("ToRenderValues: %v", err)
|
||||
}
|
||||
|
||||
rendered, err := engine.Render(loaded, renderValues)
|
||||
if err != nil {
|
||||
t.Fatalf("render: %v", err)
|
||||
}
|
||||
|
||||
for name, content := range rendered {
|
||||
if filepath.Base(name) == "daemonset.yaml" {
|
||||
return content
|
||||
}
|
||||
}
|
||||
t.Fatalf("daemonset.yaml not rendered; got: %v", rendered)
|
||||
return ""
|
||||
}
|
||||
|
||||
// TestNodeSysctlsDaemonSetRendering asserts each scope targets a disjoint set of
|
||||
// nodes. If both the global and a profile DaemonSet landed on the same node they
|
||||
// would race writing the same /proc/sys file, making the result depend on pod
|
||||
// scheduling order.
|
||||
func TestNodeSysctlsDaemonSetRendering(t *testing.T) {
|
||||
base := NodeSysctlsGlobalValues{
|
||||
Image: DefaultNodeSysctlsImage,
|
||||
PauseImage: DefaultNodeSysctlsPauseImage,
|
||||
Sysctls: []Sysctl{{Name: "vm.max_map_count", Value: "262144"}},
|
||||
}
|
||||
|
||||
t.Run("global scope excludes profiled nodes", func(t *testing.T) {
|
||||
values := base
|
||||
values.ReleaseName = "dokku-node-sysctls-global"
|
||||
manifest := renderNodeSysctlsTemplate(t, values)
|
||||
|
||||
if !strings.Contains(manifest, "operator: DoesNotExist") {
|
||||
t.Errorf("global daemonset missing DoesNotExist affinity:\n%s", manifest)
|
||||
}
|
||||
if !strings.Contains(manifest, "key: dokku.com/node-profile") {
|
||||
t.Errorf("global daemonset missing node profile affinity key:\n%s", manifest)
|
||||
}
|
||||
if strings.Contains(manifest, "nodeSelector:") {
|
||||
t.Errorf("global daemonset should not use a nodeSelector:\n%s", manifest)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("profile scope targets only its own nodes", func(t *testing.T) {
|
||||
values := base
|
||||
values.ProfileName = "edge-workers"
|
||||
values.ReleaseName = "dokku-node-sysctls-profile-edge-workers"
|
||||
manifest := renderNodeSysctlsTemplate(t, values)
|
||||
|
||||
if !strings.Contains(manifest, "dokku.com/node-profile: \"edge-workers\"") {
|
||||
t.Errorf("profile daemonset missing nodeSelector:\n%s", manifest)
|
||||
}
|
||||
if strings.Contains(manifest, "DoesNotExist") {
|
||||
t.Errorf("profile daemonset should not carry the global affinity:\n%s", manifest)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("tolerates every taint", func(t *testing.T) {
|
||||
values := base
|
||||
values.ReleaseName = "dokku-node-sysctls-global"
|
||||
manifest := renderNodeSysctlsTemplate(t, values)
|
||||
|
||||
if !strings.Contains(manifest, "- operator: Exists") {
|
||||
t.Errorf("daemonset does not tolerate all taints:\n%s", manifest)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("applies each sysctl privileged", func(t *testing.T) {
|
||||
values := base
|
||||
values.ReleaseName = "dokku-node-sysctls-global"
|
||||
values.Sysctls = []Sysctl{
|
||||
{Name: "vm.max_map_count", Value: "262144"},
|
||||
{Name: "vm.swappiness", Value: "10"},
|
||||
}
|
||||
manifest := renderNodeSysctlsTemplate(t, values)
|
||||
|
||||
if !strings.Contains(manifest, "privileged: true") {
|
||||
t.Errorf("daemonset init container is not privileged:\n%s", manifest)
|
||||
}
|
||||
if !strings.Contains(manifest, `sysctl -w "vm.max_map_count=262144"`) {
|
||||
t.Errorf("daemonset missing max_map_count write:\n%s", manifest)
|
||||
}
|
||||
if !strings.Contains(manifest, `sysctl -w "vm.swappiness=10"`) {
|
||||
t.Errorf("daemonset missing swappiness write:\n%s", manifest)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("image overrides reach both containers", func(t *testing.T) {
|
||||
values := base
|
||||
values.ReleaseName = "dokku-node-sysctls-global"
|
||||
values.Image = "registry.internal/busybox:1.36"
|
||||
values.PauseImage = "registry.internal/pause:3.9"
|
||||
manifest := renderNodeSysctlsTemplate(t, values)
|
||||
|
||||
if !strings.Contains(manifest, `image: "registry.internal/busybox:1.36"`) {
|
||||
t.Errorf("daemonset did not use the sysctl image override:\n%s", manifest)
|
||||
}
|
||||
if !strings.Contains(manifest, `image: "registry.internal/pause:3.9"`) {
|
||||
t.Errorf("daemonset did not use the pause image override:\n%s", manifest)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestMergeNodeSysctls asserts a profile scope inherits the global sysctls and
|
||||
// overrides them on conflict. Profiled nodes are excluded from the global
|
||||
// DaemonSet, so a global value omitted here would never reach them.
|
||||
func TestMergeNodeSysctls(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
global map[string]string
|
||||
profile map[string]string
|
||||
want []Sysctl
|
||||
}{
|
||||
{
|
||||
name: "no sysctls at all",
|
||||
global: map[string]string{},
|
||||
profile: map[string]string{},
|
||||
want: []Sysctl{},
|
||||
},
|
||||
{
|
||||
name: "profile inherits global values",
|
||||
global: map[string]string{"vm.max_map_count": "262144"},
|
||||
profile: map[string]string{},
|
||||
want: []Sysctl{{Name: "vm.max_map_count", Value: "262144"}},
|
||||
},
|
||||
{
|
||||
name: "profile wins on conflict",
|
||||
global: map[string]string{"vm.max_map_count": "262144"},
|
||||
profile: map[string]string{"vm.max_map_count": "524288"},
|
||||
want: []Sysctl{{Name: "vm.max_map_count", Value: "524288"}},
|
||||
},
|
||||
{
|
||||
name: "profile adds to global",
|
||||
global: map[string]string{"vm.max_map_count": "262144"},
|
||||
profile: map[string]string{"vm.swappiness": "10"},
|
||||
want: []Sysctl{
|
||||
{Name: "vm.max_map_count", Value: "262144"},
|
||||
{Name: "vm.swappiness", Value: "10"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "profile only",
|
||||
global: map[string]string{},
|
||||
profile: map[string]string{"vm.swappiness": "10"},
|
||||
want: []Sysctl{{Name: "vm.swappiness", Value: "10"}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := mergeNodeSysctls(tc.global, tc.profile)
|
||||
if len(got) != len(tc.want) {
|
||||
t.Fatalf("mergeNodeSysctls() = %v, want %v", got, tc.want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tc.want[i] {
|
||||
t.Errorf("mergeNodeSysctls()[%d] = %v, want %v", i, got[i], tc.want[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMergeNodeSysctlsDoesNotMutateInputs asserts the merge leaves the caller's maps
|
||||
// alone, since resolveNodeSysctlScopes reuses the global map across every profile.
|
||||
func TestMergeNodeSysctlsDoesNotMutateInputs(t *testing.T) {
|
||||
global := map[string]string{"vm.max_map_count": "262144"}
|
||||
profile := map[string]string{"vm.max_map_count": "524288", "vm.swappiness": "10"}
|
||||
|
||||
mergeNodeSysctls(global, profile)
|
||||
|
||||
if len(global) != 1 || global["vm.max_map_count"] != "262144" {
|
||||
t.Errorf("mergeNodeSysctls() mutated the global map: %v", global)
|
||||
}
|
||||
if len(profile) != 2 {
|
||||
t.Errorf("mergeNodeSysctls() mutated the profile map: %v", profile)
|
||||
}
|
||||
}
|
||||
@@ -62,6 +62,8 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
|
||||
"--scheduler-k3s-global-namespace": reportGlobalNamespace,
|
||||
"--scheduler-k3s-computed-network-interface": reportComputedNetworkInterface,
|
||||
"--scheduler-k3s-global-network-interface": reportGlobalNetworkInterface,
|
||||
"--scheduler-k3s-global-node-sysctls-image": reportGlobalNodeSysctlsImage,
|
||||
"--scheduler-k3s-global-node-sysctls-pause-image": reportGlobalNodeSysctlsPauseImage,
|
||||
"--scheduler-k3s-computed-rollback-on-failure": reportComputedRollbackOnFailure,
|
||||
"--scheduler-k3s-global-rollback-on-failure": reportGlobalRollbackOnFailure,
|
||||
"--scheduler-k3s-computed-shm-size": reportComputedShmSize,
|
||||
@@ -99,6 +101,8 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
|
||||
"--scheduler-k3s-global-namespace": reportGlobalNamespace,
|
||||
"--scheduler-k3s-computed-network-interface": reportComputedNetworkInterface,
|
||||
"--scheduler-k3s-global-network-interface": reportGlobalNetworkInterface,
|
||||
"--scheduler-k3s-global-node-sysctls-image": reportGlobalNodeSysctlsImage,
|
||||
"--scheduler-k3s-global-node-sysctls-pause-image": reportGlobalNodeSysctlsPauseImage,
|
||||
"--scheduler-k3s-computed-rollback-on-failure": reportComputedRollbackOnFailure,
|
||||
"--scheduler-k3s-rollback-on-failure": reportRollbackOnFailure,
|
||||
"--scheduler-k3s-global-rollback-on-failure": reportGlobalRollbackOnFailure,
|
||||
@@ -715,6 +719,14 @@ func reportGlobalNetworkInterface(appName string) string {
|
||||
return getGlobalNetworkInterface()
|
||||
}
|
||||
|
||||
func reportGlobalNodeSysctlsImage(appName string) string {
|
||||
return getComputedNodeSysctlsImage()
|
||||
}
|
||||
|
||||
func reportGlobalNodeSysctlsPauseImage(appName string) string {
|
||||
return getComputedNodeSysctlsPauseImage()
|
||||
}
|
||||
|
||||
func reportComputedRollbackOnFailure(appName string) string {
|
||||
return getComputedRollbackOnFailure(appName)
|
||||
}
|
||||
|
||||
@@ -31,20 +31,22 @@ var (
|
||||
|
||||
// GlobalProperties is a map of all valid global k3s properties
|
||||
GlobalProperties = map[string]bool{
|
||||
"deploy-timeout": true,
|
||||
"image-pull-secrets": true,
|
||||
"ingress-class": true,
|
||||
"kube-context": true,
|
||||
"kubeconfig-path": true,
|
||||
"kustomize-root-path": true,
|
||||
"letsencrypt-server": true,
|
||||
"letsencrypt-email-prod": true,
|
||||
"letsencrypt-email-stag": true,
|
||||
"namespace": true,
|
||||
"network-interface": true,
|
||||
"rollback-on-failure": true,
|
||||
"shm-size": true,
|
||||
"token": true,
|
||||
"deploy-timeout": true,
|
||||
"image-pull-secrets": true,
|
||||
"ingress-class": true,
|
||||
"kube-context": true,
|
||||
"kubeconfig-path": true,
|
||||
"kustomize-root-path": true,
|
||||
"letsencrypt-server": true,
|
||||
"letsencrypt-email-prod": true,
|
||||
"letsencrypt-email-stag": true,
|
||||
"namespace": true,
|
||||
"network-interface": true,
|
||||
"node-sysctls-image": true,
|
||||
"node-sysctls-pause-image": true,
|
||||
"rollback-on-failure": true,
|
||||
"shm-size": true,
|
||||
"token": true,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -102,6 +104,7 @@ var reservedAnnotationPrefixes = []string{
|
||||
"chart-overrides.",
|
||||
"labels.",
|
||||
"node-profile-",
|
||||
"node-sysctls.",
|
||||
TriggerAuthPropertyPrefix,
|
||||
}
|
||||
|
||||
@@ -231,6 +234,18 @@ type NodeProfile struct {
|
||||
KubeletArgs []string `json:"kubelet_args,omitempty"`
|
||||
}
|
||||
|
||||
// NodeProfileLabel is the node label recording the node profile a node was added with
|
||||
const NodeProfileLabel = "dokku.com/node-profile"
|
||||
|
||||
// NodeSysctlsNamespace is the namespace the node sysctls daemonsets are installed into
|
||||
const NodeSysctlsNamespace = "kube-system"
|
||||
|
||||
// DefaultNodeSysctlsImage is the image used to apply sysctls on each node
|
||||
const DefaultNodeSysctlsImage = "busybox:1.36"
|
||||
|
||||
// DefaultNodeSysctlsPauseImage is the image keeping the node sysctls daemonset pods running
|
||||
const DefaultNodeSysctlsPauseImage = "registry.k8s.io/pause:3.9"
|
||||
|
||||
// ServerLabels are the labels for a server node
|
||||
var ServerLabels = map[string]string{
|
||||
"svccontroller.k3s.cattle.io/enablelb": "true",
|
||||
|
||||
@@ -27,9 +27,11 @@ Additional commands:`
|
||||
scheduler-k3s:cluster:list [--format json|stdout], Lists all nodes in a Dokku-managed cluster
|
||||
scheduler-k3s:cluster:remove [node-id], Removes client node to a Dokku-managed cluster
|
||||
scheduler-k3s:ensure-charts, Ensures the k3s charts are installed
|
||||
scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling], Initializes a cluster
|
||||
scheduler-k3s:initialize [--server-ip SERVER_IP] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Initializes a cluster
|
||||
scheduler-k3s:labels:set <app|--global> <property> (<value>) [--process-type PROCESS_TYPE] <--resource-type RESOURCE_TYPE>, Set or clear a label for a given app/process-type/resource-type combination
|
||||
scheduler-k3s:labels:report [<app>|--global] [--format stdout|json] [--process-type PROCESS_TYPE] [--resource-type RESOURCE_TYPE], Displays a scheduler-k3s labels report for one or more apps
|
||||
scheduler-k3s:node-sysctls:set <sysctl> (<value>) [--global|--profile PROFILE], Set or clear a node-level kernel sysctl for unprofiled nodes or a single node profile
|
||||
scheduler-k3s:node-sysctls:report [--format stdout|json], Displays the node-level kernel sysctls applied to each scope
|
||||
scheduler-k3s:preview <app> [--context N] [--show-secrets] [--show-secrets-decoded], Displays a diff between the current and next deployment for an app
|
||||
scheduler-k3s:profiles:add <profile> [--role ROLE] [--insecure-allow-unknown-hosts] [--taint-scheduling] [--kubelet-args KUBELET_ARGS], Adds a node profile to the k3s cluster
|
||||
scheduler-k3s:profiles:list [--format json|stdout], Lists all node profiles in the k3s cluster
|
||||
|
||||
@@ -128,8 +128,9 @@ func main() {
|
||||
taintScheduling := args.Bool("taint-scheduling", false, "taint-scheduling: add a taint against scheduling app workloads")
|
||||
serverIP := args.String("server-ip", "", "server-ip: IP address of the dokku server node")
|
||||
ingressClass := args.String("ingress-class", "nginx", "ingress-class: ingress-class to use for all outbound traffic")
|
||||
kubeletArgs := args.StringSlice("kubelet-args", []string{}, "kubelet-args: repeatable key=value kubelet arguments (e.g., --kubelet-args key=value)")
|
||||
args.Parse(os.Args[2:])
|
||||
err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling)
|
||||
err = scheduler_k3s.CommandInitialize(*ingressClass, *serverIP, *taintScheduling, *kubeletArgs)
|
||||
case "labels:set":
|
||||
args := flag.NewFlagSet("scheduler-k3s:labels:set", flag.ExitOnError)
|
||||
global := args.Bool("global", false, "--global: set a global property")
|
||||
@@ -163,6 +164,23 @@ func main() {
|
||||
appName = "--global"
|
||||
}
|
||||
err = scheduler_k3s.CommandLabelsReport(appName, *format, *processType, *resourceType, infoFlag)
|
||||
case "node-sysctls:set":
|
||||
args := flag.NewFlagSet("scheduler-k3s:node-sysctls:set", flag.ExitOnError)
|
||||
global := args.Bool("global", false, "--global: scope to all nodes without a node profile")
|
||||
profileName := args.String("profile", "", "--profile: scope to a node profile instead of all unprofiled nodes")
|
||||
args.Parse(os.Args[2:])
|
||||
key := args.Arg(0)
|
||||
value := args.Arg(1)
|
||||
if *global && *profileName != "" {
|
||||
err = fmt.Errorf("Only one of --global and --profile may be specified")
|
||||
break
|
||||
}
|
||||
err = scheduler_k3s.CommandNodeSysctlsSet(*profileName, key, value)
|
||||
case "node-sysctls:report":
|
||||
args := flag.NewFlagSet("scheduler-k3s:node-sysctls:report", flag.ExitOnError)
|
||||
format := args.String("format", "stdout", "format: [ stdout | json ]")
|
||||
args.Parse(os.Args[2:])
|
||||
err = scheduler_k3s.CommandNodeSysctlsReport(*format)
|
||||
case "preview":
|
||||
args := flag.NewFlagSet("scheduler-k3s:preview", flag.ExitOnError)
|
||||
context := args.Int("context", 3, "--context: number of unchanged lines of context around each change (-1 for full output)")
|
||||
|
||||
@@ -45,6 +45,95 @@ func CommandAnnotationsSet(appName string, processType string, resourceType stri
|
||||
return nil
|
||||
}
|
||||
|
||||
// CommandNodeSysctlsSet sets or clears a node-level kernel sysctl for a scope
|
||||
func CommandNodeSysctlsSet(profileName string, key string, value string) error {
|
||||
if key == "" {
|
||||
return fmt.Errorf("Missing sysctl name")
|
||||
}
|
||||
|
||||
if profileName != "" {
|
||||
if err := verifyNodeProfileExists(profileName); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
property := getNodeSysctlsProperty(profileName)
|
||||
if value == "" {
|
||||
if err := common.PropertyMapDelete("scheduler-k3s", "--global", property, key); err != nil {
|
||||
return fmt.Errorf("Unable to delete property map entry: %w", err)
|
||||
}
|
||||
|
||||
common.LogWarn(fmt.Sprintf("Removing %s stops dokku managing it, but does not restore the previous value on affected nodes until they reboot", key))
|
||||
} else {
|
||||
if err := common.PropertyMapSet("scheduler-k3s", "--global", property, key, value); err != nil {
|
||||
return fmt.Errorf("Unable to set property map entry: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return CreateOrUpdateNodeSysctls(context.Background())
|
||||
}
|
||||
|
||||
// CommandNodeSysctlsReport displays the configured node-level kernel sysctls
|
||||
func CommandNodeSysctlsReport(format string) error {
|
||||
if format != "stdout" && format != "json" {
|
||||
return fmt.Errorf("Invalid format: %s", format)
|
||||
}
|
||||
|
||||
scopes, err := resolveNodeSysctlScopes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if format == "json" {
|
||||
output := map[string]map[string]string{}
|
||||
for _, scope := range scopes {
|
||||
key := scope.ProfileName
|
||||
if key == "" {
|
||||
key = "--global"
|
||||
}
|
||||
|
||||
entries := map[string]string{}
|
||||
for _, sysctl := range scope.Sysctls {
|
||||
entries[sysctl.Name] = sysctl.Value
|
||||
}
|
||||
output[key] = entries
|
||||
}
|
||||
|
||||
b, err := json.Marshal(output)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to marshal json: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println(string(b))
|
||||
return nil
|
||||
}
|
||||
|
||||
lines := []string{"scope|sysctl|value"}
|
||||
for _, scope := range scopes {
|
||||
scopeName := scope.ProfileName
|
||||
if scopeName == "" {
|
||||
scopeName = "--global"
|
||||
}
|
||||
|
||||
for _, sysctl := range scope.Sysctls {
|
||||
lines = append(lines, fmt.Sprintf("%s|%s|%s", scopeName, sysctl.Name, sysctl.Value))
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println(columnize.SimpleFormat(lines))
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyNodeProfileExists returns an error when a node profile has not been created
|
||||
func verifyNodeProfileExists(profileName string) error {
|
||||
properties := common.PropertyGetDefault("scheduler-k3s", "--global", fmt.Sprintf("node-profile-%s.json", profileName), "")
|
||||
if properties == "" {
|
||||
return fmt.Errorf("Node profile %s not found", profileName)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CommandAutoscalingAuthSet set or clear a scheduler-k3s autoscaling keda trigger authentication object for an app
|
||||
func CommandAutoscalingAuthSet(appName string, trigger string, metadata map[string]string, global bool) error {
|
||||
if global {
|
||||
@@ -174,7 +263,7 @@ func CommandLabelsReport(appName string, format string, processType string, reso
|
||||
}
|
||||
|
||||
// CommandInitialize initializes a k3s cluster on the local server
|
||||
func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool) error {
|
||||
func CommandInitialize(ingressClass string, serverIP string, taintScheduling bool, kubeletArgs []string) error {
|
||||
if ingressClass != "nginx" && ingressClass != "traefik" {
|
||||
return fmt.Errorf("Invalid ingress-class: %s", ingressClass)
|
||||
}
|
||||
@@ -308,40 +397,15 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo
|
||||
}
|
||||
nodeName = strings.ReplaceAll(strings.ToLower(fmt.Sprintf("ip-%s-%s", nodeName, fmt.Sprintf("%X", b))), ".", "-")
|
||||
|
||||
args := []string{
|
||||
// initialize the cluster
|
||||
"--cluster-init",
|
||||
// disable local-storage
|
||||
"--disable", "local-storage",
|
||||
// disable traefik so it can be installed separately
|
||||
"--disable", "traefik",
|
||||
// expose etcd metrics
|
||||
"--etcd-expose-metrics",
|
||||
// use wireguard for flannel
|
||||
"--flannel-backend=wireguard-native",
|
||||
// bind controller-manager to all interfaces
|
||||
"--kube-controller-manager-arg", "bind-address=0.0.0.0",
|
||||
// bind proxy metrics to all interfaces
|
||||
"--kube-proxy-arg", "metrics-bind-address=0.0.0.0",
|
||||
// bind scheduler to all interfaces
|
||||
"--kube-scheduler-arg", "bind-address=0.0.0.0",
|
||||
// gc terminated pods
|
||||
"--kube-controller-manager-arg", "terminated-pod-gc-threshold=10",
|
||||
// specify the node name
|
||||
"--node-name", nodeName,
|
||||
// allow access for the dokku user
|
||||
"--write-kubeconfig-mode", "0644",
|
||||
// specify a token
|
||||
"--token", token,
|
||||
}
|
||||
if taintScheduling {
|
||||
args = append(args, "--node-taint", "CriticalAddonsOnly=true:NoSchedule")
|
||||
}
|
||||
|
||||
common.CommandPropertySet("scheduler-k3s", "--global", "ingress-class", ingressClass, DefaultProperties, GlobalProperties)
|
||||
if ingressClass == "nginx" {
|
||||
args = append(args, "--disable", "traefik")
|
||||
}
|
||||
|
||||
args := initializeInstallerArgs(InitializeInstallerArgsInput{
|
||||
IngressClass: ingressClass,
|
||||
KubeletArgs: kubeletArgs,
|
||||
NodeName: nodeName,
|
||||
TaintScheduling: taintScheduling,
|
||||
Token: token,
|
||||
})
|
||||
|
||||
common.LogInfo2Quiet("Running k3s installer")
|
||||
installerCmd, err := common.CallExecCommand(common.ExecCommandInput{
|
||||
@@ -422,6 +486,11 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo
|
||||
return fmt.Errorf("Unable to install helper commands: %w", err)
|
||||
}
|
||||
|
||||
common.LogInfo2Quiet("Applying node sysctls")
|
||||
if err := CreateOrUpdateNodeSysctls(ctx); err != nil {
|
||||
return fmt.Errorf("Unable to apply node sysctls: %w", err)
|
||||
}
|
||||
|
||||
common.LogVerboseQuiet("Done")
|
||||
|
||||
return nil
|
||||
@@ -900,10 +969,7 @@ export INSTALL_K3S_VERSION=%s
|
||||
return fmt.Errorf("Unable to find node after joining cluster, node will not be annotated/labeled appropriately access registry secrets")
|
||||
}
|
||||
|
||||
labels := ServerLabels
|
||||
if incomingProfile.Role == "worker" {
|
||||
labels = WorkerLabels
|
||||
}
|
||||
labels := nodeLabels(incomingProfile.Role, profileName)
|
||||
|
||||
for key, value := range labels {
|
||||
common.LogInfo2Quiet(fmt.Sprintf("Labeling node %s=%s", key, value))
|
||||
@@ -1290,6 +1356,10 @@ func CommandProfilesRemove(profileName string) error {
|
||||
return fmt.Errorf("Unable to delete node profile: %w", err)
|
||||
}
|
||||
|
||||
if err := DeleteNodeSysctls(context.Background(), profileName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
common.LogInfo1(fmt.Sprintf("Node profile %s removed", profileName))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -394,6 +394,32 @@ type SecurityContext struct {
|
||||
Capabilities SecurityContextCapabilities `yaml:"capabilities,omitempty"`
|
||||
// Privileged contains the privileged flag for a process
|
||||
Privileged bool `yaml:"privileged,omitempty"`
|
||||
// Sysctls contains the namespaced kernel sysctls for a process
|
||||
Sysctls []Sysctl `yaml:"sysctls,omitempty"`
|
||||
}
|
||||
|
||||
// Sysctl contains a single kernel sysctl key/value pair
|
||||
type Sysctl struct {
|
||||
// Name is the name of the sysctl
|
||||
Name string `yaml:"name"`
|
||||
// Value is the value the sysctl is set to
|
||||
Value string `yaml:"value"`
|
||||
}
|
||||
|
||||
// ToCoreV1PodSecurityContext converts the sysctls to a corev1.PodSecurityContext,
|
||||
// returning nil when no sysctls are configured so an empty security context does
|
||||
// not churn the pod template hash and trigger a spurious rollout.
|
||||
func (s SecurityContext) ToCoreV1PodSecurityContext() *corev1.PodSecurityContext {
|
||||
if len(s.Sysctls) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
sysctls := make([]corev1.Sysctl, len(s.Sysctls))
|
||||
for i, sysctl := range s.Sysctls {
|
||||
sysctls[i] = corev1.Sysctl{Name: sysctl.Name, Value: sysctl.Value}
|
||||
}
|
||||
|
||||
return &corev1.PodSecurityContext{Sysctls: sysctls}
|
||||
}
|
||||
|
||||
// ToCoreV1SecurityContext converts the security context to a corev1.SecurityContext
|
||||
@@ -533,6 +559,7 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) {
|
||||
},
|
||||
},
|
||||
RestartPolicy: corev1.RestartPolicyNever,
|
||||
SecurityContext: input.SecurityContext.ToCoreV1PodSecurityContext(),
|
||||
ServiceAccountName: input.AppName,
|
||||
},
|
||||
},
|
||||
|
||||
199
plugins/scheduler-k3s/template_test.go
Normal file
199
plugins/scheduler-k3s/template_test.go
Normal file
@@ -0,0 +1,199 @@
|
||||
package scheduler_k3s
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"helm.sh/helm/v3/pkg/chart/loader"
|
||||
"helm.sh/helm/v3/pkg/chartutil"
|
||||
"helm.sh/helm/v3/pkg/engine"
|
||||
)
|
||||
|
||||
func TestToCoreV1PodSecurityContext(t *testing.T) {
|
||||
t.Run("nil when no sysctls are configured", func(t *testing.T) {
|
||||
securityContext := SecurityContext{Privileged: true}
|
||||
if got := securityContext.ToCoreV1PodSecurityContext(); got != nil {
|
||||
t.Errorf("ToCoreV1PodSecurityContext() = %v, want nil", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil for an empty sysctl slice", func(t *testing.T) {
|
||||
securityContext := SecurityContext{Sysctls: []Sysctl{}}
|
||||
if got := securityContext.ToCoreV1PodSecurityContext(); got != nil {
|
||||
t.Errorf("ToCoreV1PodSecurityContext() = %v, want nil", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("preserves order and values", func(t *testing.T) {
|
||||
securityContext := SecurityContext{
|
||||
Sysctls: []Sysctl{
|
||||
{Name: "kernel.sem", Value: "250"},
|
||||
{Name: "net.core.somaxconn", Value: "1024"},
|
||||
},
|
||||
}
|
||||
|
||||
got := securityContext.ToCoreV1PodSecurityContext()
|
||||
if got == nil {
|
||||
t.Fatal("ToCoreV1PodSecurityContext() = nil, want a security context")
|
||||
}
|
||||
if len(got.Sysctls) != 2 {
|
||||
t.Fatalf("ToCoreV1PodSecurityContext() has %d sysctls, want 2", len(got.Sysctls))
|
||||
}
|
||||
if got.Sysctls[0].Name != "kernel.sem" || got.Sysctls[0].Value != "250" {
|
||||
t.Errorf("ToCoreV1PodSecurityContext() sysctls[0] = %v, want kernel.sem=250", got.Sysctls[0])
|
||||
}
|
||||
if got.Sysctls[1].Name != "net.core.somaxconn" || got.Sysctls[1].Value != "1024" {
|
||||
t.Errorf("ToCoreV1PodSecurityContext() sysctls[1] = %v, want net.core.somaxconn=1024", got.Sysctls[1])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func renderDeploymentTemplate(t *testing.T, globalValues map[string]interface{}) string {
|
||||
t.Helper()
|
||||
|
||||
chartDir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(chartDir, "templates"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
chartYAML := []byte("apiVersion: v2\nname: test\nversion: 0.0.1\n")
|
||||
if err := os.WriteFile(filepath.Join(chartDir, "Chart.yaml"), chartYAML, 0o644); err != nil {
|
||||
t.Fatalf("write Chart.yaml: %v", err)
|
||||
}
|
||||
|
||||
for _, name := range []string{"deployment.yaml", "_helpers.tpl"} {
|
||||
contents, err := templates.ReadFile("templates/chart/" + name)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", name, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(chartDir, "templates", name), contents, 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
loaded, err := loader.Load(chartDir)
|
||||
if err != nil {
|
||||
t.Fatalf("load chart: %v", err)
|
||||
}
|
||||
|
||||
global := map[string]interface{}{
|
||||
"app_name": "myapp",
|
||||
"deployment_id": "1",
|
||||
"namespace": "myapp",
|
||||
"image": map[string]interface{}{
|
||||
"name": "myapp:latest",
|
||||
"type": "dockerfile",
|
||||
},
|
||||
}
|
||||
for key, value := range globalValues {
|
||||
global[key] = value
|
||||
}
|
||||
|
||||
values := map[string]interface{}{
|
||||
"global": global,
|
||||
"processes": map[string]interface{}{
|
||||
"worker": map[string]interface{}{
|
||||
"args": []interface{}{"echo", "hello"},
|
||||
"replicas": 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
renderValues, err := chartutil.ToRenderValues(loaded, values, chartutil.ReleaseOptions{Name: "test", Namespace: "default"}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("ToRenderValues: %v", err)
|
||||
}
|
||||
|
||||
rendered, err := engine.Render(loaded, renderValues)
|
||||
if err != nil {
|
||||
t.Fatalf("render: %v", err)
|
||||
}
|
||||
|
||||
for name, content := range rendered {
|
||||
if filepath.Base(name) == "deployment.yaml" {
|
||||
return content
|
||||
}
|
||||
}
|
||||
t.Fatalf("deployment.yaml not rendered; got: %v", rendered)
|
||||
return ""
|
||||
}
|
||||
|
||||
// TestDeploymentSysctlsRendering asserts the pod-level securityContext.sysctls
|
||||
// block only appears when sysctls are configured, and that numeric values are
|
||||
// quoted. Kubernetes types sysctls[].value as a string, so an unquoted 1024
|
||||
// renders as a YAML integer and the API server rejects the manifest.
|
||||
func TestDeploymentSysctlsRendering(t *testing.T) {
|
||||
t.Run("absent when no security context is set", func(t *testing.T) {
|
||||
manifest := renderDeploymentTemplate(t, map[string]interface{}{})
|
||||
if strings.Contains(manifest, "sysctls:") {
|
||||
t.Errorf("rendered deployment unexpectedly contains sysctls:\n%s", manifest)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("absent when the security context has no sysctls", func(t *testing.T) {
|
||||
manifest := renderDeploymentTemplate(t, map[string]interface{}{
|
||||
"security_context": map[string]interface{}{"privileged": true},
|
||||
})
|
||||
if strings.Contains(manifest, "sysctls:") {
|
||||
t.Errorf("rendered deployment unexpectedly contains sysctls:\n%s", manifest)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("numeric values are quoted", func(t *testing.T) {
|
||||
manifest := renderDeploymentTemplate(t, map[string]interface{}{
|
||||
"security_context": map[string]interface{}{
|
||||
"sysctls": []interface{}{
|
||||
map[string]interface{}{"name": "net.core.somaxconn", "value": "1024"},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if !strings.Contains(manifest, "- name: net.core.somaxconn") {
|
||||
t.Errorf("rendered deployment missing sysctl name:\n%s", manifest)
|
||||
}
|
||||
if !strings.Contains(manifest, `value: "1024"`) {
|
||||
t.Errorf("rendered deployment did not quote the sysctl value:\n%s", manifest)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("sysctls land on the pod spec, not the container", func(t *testing.T) {
|
||||
manifest := renderDeploymentTemplate(t, map[string]interface{}{
|
||||
"security_context": map[string]interface{}{
|
||||
"sysctls": []interface{}{
|
||||
map[string]interface{}{"name": "net.core.somaxconn", "value": "1024"},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
var doc map[string]interface{}
|
||||
decoder := yaml.NewDecoder(strings.NewReader(manifest))
|
||||
for {
|
||||
if err := decoder.Decode(&doc); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
t.Fatalf("decode manifest: %v", err)
|
||||
}
|
||||
if doc == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
spec := doc["spec"].(map[string]interface{})
|
||||
template := spec["template"].(map[string]interface{})
|
||||
podSpec := template["spec"].(map[string]interface{})
|
||||
|
||||
securityContext, ok := podSpec["securityContext"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("pod spec has no securityContext:\n%s", manifest)
|
||||
}
|
||||
if _, ok := securityContext["sysctls"]; !ok {
|
||||
t.Errorf("pod securityContext has no sysctls:\n%s", manifest)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -73,6 +73,14 @@ spec:
|
||||
{{ include "print.labels" (dict "config" $.Values.global "key" "pod") | indent 12 }}
|
||||
{{ include "print.labels" (dict "config" $config "key" "pod") | indent 12 }}
|
||||
spec:
|
||||
{{- if and (hasKey $.Values.global "security_context") $.Values.global.security_context.sysctls }}
|
||||
securityContext:
|
||||
sysctls:
|
||||
{{- range $.Values.global.security_context.sysctls }}
|
||||
- name: {{ .name }}
|
||||
value: {{ .value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- args:
|
||||
{{- range $config.args }}
|
||||
|
||||
@@ -58,6 +58,14 @@ spec:
|
||||
{{ include "print.labels" (dict "config" $.Values.global "key" "pod") | indent 8 }}
|
||||
{{ include "print.labels" (dict "config" $config "key" "pod") | indent 8 }}
|
||||
spec:
|
||||
{{- if and (hasKey $.Values.global "security_context") $.Values.global.security_context.sysctls }}
|
||||
securityContext:
|
||||
sysctls:
|
||||
{{- range $.Values.global.security_context.sysctls }}
|
||||
- name: {{ .name }}
|
||||
value: {{ .value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- args:
|
||||
{{- range $config.args }}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
annotations:
|
||||
dokku.com/managed: "true"
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ .Values.global.release_name }}
|
||||
app.kubernetes.io/part-of: dokku
|
||||
name: {{ .Values.global.release_name }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ .Values.global.release_name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ .Values.global.release_name }}
|
||||
app.kubernetes.io/part-of: dokku
|
||||
spec:
|
||||
{{- if .Values.global.profile_name }}
|
||||
nodeSelector:
|
||||
dokku.com/node-profile: {{ .Values.global.profile_name | quote }}
|
||||
{{- else }}
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: dokku.com/node-profile
|
||||
operator: DoesNotExist
|
||||
{{- end }}
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
initContainers:
|
||||
- name: apply-sysctls
|
||||
image: {{ .Values.global.image | quote }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
{{- range .Values.global.sysctls }}
|
||||
sysctl -w {{ printf "%s=%s" .name .value | quote }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: pause
|
||||
image: {{ .Values.global.pause_image | quote }}
|
||||
resources:
|
||||
requests:
|
||||
cpu: 1m
|
||||
memory: 8Mi
|
||||
147
tests/unit/scheduler-k3s-5.bats
Normal file
147
tests/unit/scheduler-k3s-5.bats
Normal file
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
load test_helper
|
||||
|
||||
TEST_APP="rdmtestapp"
|
||||
|
||||
setup() {
|
||||
uninstall_k3s || true
|
||||
global_setup
|
||||
dokku nginx:stop
|
||||
export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
|
||||
}
|
||||
|
||||
teardown() {
|
||||
dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count || true
|
||||
global_teardown
|
||||
dokku nginx:start
|
||||
uninstall_k3s || true
|
||||
}
|
||||
|
||||
@test "(scheduler-k3s) docker-options sysctl" {
|
||||
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
||||
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
||||
fi
|
||||
|
||||
INGRESS_CLASS=nginx install_k3s
|
||||
|
||||
run /bin/bash -c "dokku apps:create $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.securityContext.sysctls'"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output "null"
|
||||
|
||||
run /bin/bash -c "dokku docker-options:add $TEST_APP deploy '--sysctl net.ipv4.ip_unprivileged_port_start=1024'"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku ps:restart $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.securityContext.sysctls[0].name'"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output "net.ipv4.ip_unprivileged_port_start"
|
||||
|
||||
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.securityContext.sysctls[0].value'"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output "1024"
|
||||
|
||||
run /bin/bash -c "dokku docker-options:remove $TEST_APP deploy '--sysctl net.ipv4.ip_unprivileged_port_start=1024'"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku docker-options:add $TEST_APP deploy '--sysctl vm.max_map_count=262144'"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku ps:restart $TEST_APP"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
assert_output_contains "is not namespaced" -1
|
||||
}
|
||||
|
||||
@test "(scheduler-k3s:node-sysctls) applies non-namespaced sysctls to nodes" {
|
||||
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
||||
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
||||
fi
|
||||
|
||||
INGRESS_CLASS=nginx install_k3s
|
||||
|
||||
run /bin/bash -c "kubectl get daemonset -n kube-system dokku-node-sysctls-global"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_failure
|
||||
|
||||
run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count 262144"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\".\"vm.max_map_count\"'"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output "262144"
|
||||
|
||||
run /bin/bash -c "kubectl rollout status daemonset -n kube-system dokku-node-sysctls-global --timeout=120s"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run /bin/bash -c "kubectl get daemonset -n kube-system dokku-node-sysctls-global -o json | jq -r '.status.desiredNumberScheduled'"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output "$(kubectl get nodes --no-headers | wc -l | tr -d ' ')"
|
||||
|
||||
run /bin/bash -c "cat /proc/sys/vm/max_map_count"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
assert_output "262144"
|
||||
|
||||
run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
|
||||
run wait_for_daemonset_deletion dokku-node-sysctls-global
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
assert_success
|
||||
}
|
||||
|
||||
wait_for_daemonset_deletion() {
|
||||
declare desc="waits for a daemonset to be removed from the api server"
|
||||
declare NAME="$1"
|
||||
|
||||
for _ in $(seq 1 30); do
|
||||
if ! kubectl get daemonset -n kube-system "$NAME" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "daemonset $NAME still exists after 60s"
|
||||
return 1
|
||||
}
|
||||
@@ -740,6 +740,10 @@ install_k3s() {
|
||||
args="$args --server-ip $CI_SERVER_IP"
|
||||
fi
|
||||
|
||||
if [[ -n "$KUBELET_ARGS" ]]; then
|
||||
args="$args --kubelet-args $KUBELET_ARGS"
|
||||
fi
|
||||
|
||||
run /bin/bash -c "dokku scheduler-k3s:initialize ${args}"
|
||||
echo "output: $output"
|
||||
echo "status: $status"
|
||||
|
||||
Reference in New Issue
Block a user