mirror of
https://github.com/dokku/dokku.git
synced 2026-08-29 10:08:53 +02:00
44cd566178
Sysctls the kernel does not namespace, such as `vm.max_map_count`, cannot be set from a pod spec and previously had no answer beyond editing `/etc/sysctl.d` on each host by hand. `scheduler-k3s:node-sysctls:set` now applies them through a privileged daemonset, which reaches nodes joined later and reapplies after a reboot. Sysctls may be scoped to a node profile, with a profile scope inheriting the global values and overriding them on conflict so that every node is covered by exactly one daemonset. Clearing a sysctl stops dokku managing it but does not restore the previous value, which persists until the node reboots.
272 lines
7.6 KiB
Go
272 lines
7.6 KiB
Go
package scheduler_k3s
|
|
|
|
import (
|
|
"embed"
|
|
"sync"
|
|
|
|
certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
|
|
kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
|
|
traefikv1alpha1 "github.com/traefik/traefik/v2/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
batchv1 "k8s.io/api/batch/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/runtime/serializer"
|
|
kjson "k8s.io/apimachinery/pkg/runtime/serializer/json"
|
|
)
|
|
|
|
var (
|
|
// DefaultProperties is a map of all valid k3s properties with corresponding default property values
|
|
DefaultProperties = map[string]string{
|
|
"deploy-timeout": "",
|
|
"letsencrypt-email-prod": "",
|
|
"letsencrypt-email-stag": "",
|
|
"letsencrypt-server": "",
|
|
"kustomize-root-path": "",
|
|
"image-pull-secrets": "",
|
|
"namespace": "",
|
|
"rollback-on-failure": "",
|
|
"shm-size": "",
|
|
}
|
|
|
|
// GlobalProperties is a map of all valid global k3s properties
|
|
GlobalProperties = map[string]bool{
|
|
"deploy-timeout": true,
|
|
"image-pull-secrets": true,
|
|
"ingress-class": true,
|
|
"kube-context": true,
|
|
"kubeconfig-path": true,
|
|
"kustomize-root-path": true,
|
|
"letsencrypt-server": true,
|
|
"letsencrypt-email-prod": true,
|
|
"letsencrypt-email-stag": true,
|
|
"namespace": true,
|
|
"network-interface": true,
|
|
"node-sysctls-image": true,
|
|
"node-sysctls-pause-image": true,
|
|
"rollback-on-failure": true,
|
|
"shm-size": true,
|
|
"token": true,
|
|
}
|
|
)
|
|
|
|
const DefaultIngressClass = "nginx"
|
|
const GlobalProcessType = "--global"
|
|
const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
|
|
const DefaultKubeContext = ""
|
|
const TriggerAuthPropertyPrefix = "trigger-auth."
|
|
const LetsencryptServerProd = "https://acme-v02.api.letsencrypt.org/directory"
|
|
const LetsencryptServerStag = "https://acme-staging-v02.api.letsencrypt.org/directory"
|
|
|
|
// AnnotationResourceTypes lists the kubernetes resource types that scheduler-k3s
|
|
// supports user-provided annotations for. The order here is also the iteration order
|
|
// used when rendering reports.
|
|
var AnnotationResourceTypes = []string{
|
|
"certificate",
|
|
"cronjob",
|
|
"deployment",
|
|
"ingress",
|
|
"job",
|
|
"keda_scaled_object",
|
|
"keda_secret",
|
|
"keda_trigger_authentication",
|
|
"pod",
|
|
"secret",
|
|
"service",
|
|
"serviceaccount",
|
|
"traefik_ingressroute",
|
|
"traefik_middleware",
|
|
}
|
|
|
|
// LabelResourceTypes lists the kubernetes resource types that scheduler-k3s
|
|
// supports user-provided labels for. Differs from AnnotationResourceTypes because
|
|
// keda resources do not currently flow user labels through.
|
|
var LabelResourceTypes = []string{
|
|
"certificate",
|
|
"cronjob",
|
|
"deployment",
|
|
"ingress",
|
|
"job",
|
|
"pod",
|
|
"secret",
|
|
"service",
|
|
"serviceaccount",
|
|
"traefik_ingressroute",
|
|
"traefik_middleware",
|
|
}
|
|
|
|
// reservedAnnotationPrefixes lists property name prefixes that are not annotations
|
|
// even though their trailing segment may collide with an AnnotationResourceTypes value
|
|
// (e.g. "chart.cert-manager.deployment"). The annotations report uses this list to
|
|
// exclude such properties when scanning the property store.
|
|
var reservedAnnotationPrefixes = []string{
|
|
"chart.",
|
|
"chart-overrides.",
|
|
"labels.",
|
|
"node-profile-",
|
|
"node-sysctls.",
|
|
TriggerAuthPropertyPrefix,
|
|
}
|
|
|
|
var (
|
|
runtimeScheme = runtime.NewScheme()
|
|
codecs = serializer.NewCodecFactory(runtimeScheme)
|
|
deserializer = codecs.UniversalDeserializer()
|
|
jsonSerializer = kjson.NewSerializerWithOptions(kjson.DefaultMetaFactory, runtimeScheme, runtimeScheme, kjson.SerializerOptions{})
|
|
)
|
|
|
|
var k8sNativeSchemeOnce sync.Once
|
|
|
|
type Manifest struct {
|
|
Name string
|
|
Version string
|
|
Path string
|
|
}
|
|
|
|
var KubernetesManifests = []Manifest{
|
|
{
|
|
Name: "system-upgrader",
|
|
Version: "0.13.2",
|
|
Path: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.2/system-upgrade-controller.yaml",
|
|
},
|
|
}
|
|
|
|
type HelmChart struct {
|
|
ChartPath string
|
|
CreateNamespace bool
|
|
Namespace string
|
|
Path string
|
|
ReleaseName string
|
|
RepoURL string
|
|
Version string
|
|
}
|
|
|
|
var HelmCharts = []HelmChart{
|
|
{
|
|
ChartPath: "cert-manager",
|
|
CreateNamespace: true,
|
|
Namespace: "cert-manager",
|
|
ReleaseName: "cert-manager",
|
|
RepoURL: "https://charts.jetstack.io",
|
|
Version: "v1.13.3",
|
|
},
|
|
{
|
|
ChartPath: "longhorn",
|
|
CreateNamespace: true,
|
|
Namespace: "longhorn-system",
|
|
ReleaseName: "longhorn",
|
|
RepoURL: "https://charts.longhorn.io",
|
|
Version: "1.5.3",
|
|
},
|
|
{
|
|
ChartPath: "traefik",
|
|
CreateNamespace: true,
|
|
Namespace: "traefik",
|
|
ReleaseName: "traefik",
|
|
RepoURL: "https://helm.traefik.io/traefik",
|
|
Version: "26.0.0",
|
|
},
|
|
{
|
|
ChartPath: "ingress-nginx",
|
|
CreateNamespace: true,
|
|
Namespace: "ingress-nginx",
|
|
ReleaseName: "ingress-nginx",
|
|
RepoURL: "https://kubernetes.github.io/ingress-nginx",
|
|
Version: "4.15.1",
|
|
},
|
|
{
|
|
ChartPath: "keda",
|
|
CreateNamespace: true,
|
|
Namespace: "keda",
|
|
ReleaseName: "keda",
|
|
RepoURL: "https://kedacore.github.io/charts",
|
|
Version: "2.19.0",
|
|
},
|
|
{
|
|
ChartPath: "keda-add-ons-http",
|
|
CreateNamespace: true,
|
|
Namespace: "keda",
|
|
ReleaseName: "keda-add-ons-http",
|
|
RepoURL: "https://kedacore.github.io/charts",
|
|
Version: "0.12.2",
|
|
},
|
|
{
|
|
ChartPath: "vector",
|
|
CreateNamespace: true,
|
|
Namespace: "vector",
|
|
ReleaseName: "vector",
|
|
RepoURL: "https://helm.vector.dev",
|
|
Version: "0.52.0",
|
|
},
|
|
}
|
|
|
|
type HelmRepository struct {
|
|
Name string
|
|
URL string
|
|
}
|
|
|
|
var HelmRepositories = []HelmRepository{
|
|
{
|
|
Name: "jetstack",
|
|
URL: "https://charts.jetstack.io",
|
|
},
|
|
{
|
|
Name: "longhorn",
|
|
URL: "https://charts.longhorn.io",
|
|
},
|
|
{
|
|
Name: "traefik",
|
|
URL: "https://helm.traefik.io/traefik",
|
|
},
|
|
}
|
|
|
|
// NodeProfile is a profile for a node in the k3s cluster
|
|
type NodeProfile struct {
|
|
// Name is the name of the node profile
|
|
Name string `json:"name"`
|
|
// Role is the role of the node
|
|
Role string `json:"role"`
|
|
// AllowUknownHosts is whether to allow unknown hosts
|
|
AllowUknownHosts bool `json:"allow_unknown_hosts,omitempty"`
|
|
// TaintScheduling is whether to taint the node for scheduling
|
|
TaintScheduling bool `json:"taint_scheduling,omitempty"`
|
|
// KubeletArgs is a list of kubelet arguments
|
|
KubeletArgs []string `json:"kubelet_args,omitempty"`
|
|
}
|
|
|
|
// NodeProfileLabel is the node label recording the node profile a node was added with
|
|
const NodeProfileLabel = "dokku.com/node-profile"
|
|
|
|
// NodeSysctlsNamespace is the namespace the node sysctls daemonsets are installed into
|
|
const NodeSysctlsNamespace = "kube-system"
|
|
|
|
// DefaultNodeSysctlsImage is the image used to apply sysctls on each node
|
|
const DefaultNodeSysctlsImage = "busybox:1.36"
|
|
|
|
// DefaultNodeSysctlsPauseImage is the image keeping the node sysctls daemonset pods running
|
|
const DefaultNodeSysctlsPauseImage = "registry.k8s.io/pause:3.9"
|
|
|
|
// ServerLabels are the labels for a server node
|
|
var ServerLabels = map[string]string{
|
|
"svccontroller.k3s.cattle.io/enablelb": "true",
|
|
}
|
|
|
|
// WorkerLabels are the labels for a worker node
|
|
var WorkerLabels = map[string]string{
|
|
"node-role.kubernetes.io/worker": "worker",
|
|
}
|
|
|
|
//go:embed all:templates
|
|
var templates embed.FS
|
|
|
|
func init() {
|
|
k8sNativeSchemeOnce.Do(func() {
|
|
_ = appsv1.AddToScheme(runtimeScheme)
|
|
_ = batchv1.AddToScheme(runtimeScheme)
|
|
_ = certmanagerv1.AddToScheme(runtimeScheme)
|
|
_ = corev1.AddToScheme(runtimeScheme)
|
|
_ = traefikv1alpha1.AddToScheme(runtimeScheme)
|
|
_ = kedav1alpha1.AddToScheme(runtimeScheme)
|
|
})
|
|
}
|