feat: add ability for users to specify alternative kubeconfig and kubecontext

This will provide the possibility for users to talk to existing kubernetes clusters, thereby removing one of the biggest reasons for having the old scheduler-kubernetes plugin around.

docs/deployment/schedulers/k3s.md

@@ -320,6 +320,38 @@ By default, Dokku assumes that all it controls all actions on the cluster, and t
 dokku scheduler-k3s:show-kubeconfig
 ```
 
+### Interacting with an external Kubernetes cluster
+
+While the k3s scheduler plugin is designed to work with a Dokku-managed k3s cluster, Dokku can be configured to interact with any Kubernetes cluster by setting the global `kubeconfig-path` to a path to a custom kubeconfig on the Dokku server. This property is only available at a global level.
+
+```shell
+dokku scheduler-k3s:set --global kubeconfig-path /path/to/custom/kubeconfig
+```
+
+To set the default value, omit the value from the `scheduler-k3s:set` call:
+
+```shell
+dokku scheduler-k3s:set --global kubeconfig-path
+```
+
+The default value for the `kubeconfig-path` is the k3s kubeconfig located at `/etc/rancher/k3s/k3s.yaml`.
+
+### Customizing the Kubernetes context
+
+When interacting with a custom Kubeconfig, the `kube-context` property can be set to specify a specific context within the kubeconfig to use. This property is available only at the global leve.
+
+```shell
+dokku scheduler-k3s:set --global kube-context lollipop
+```
+
+To set the default value, omit the value from the `scheduler-k3s:set` call:
+
+```shell
+dokku scheduler-k3s:set --global kube-context
+```
+
+The default value for the `kube-context` is an empty string, and will result in Dokku using the current context within the kubeconfig.
+
 ## Scheduler Interface
 
 The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler.

plugins/scheduler-k3s/functions.go

@@ -1142,12 +1142,25 @@ func installHelm(ctx context.Context) error {
 	return nil
 }
 
+func isKubernetesAvailable() error {
+	client, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Error creating kubernetes client: %w", err)
+	}
+
+	if err := client.Ping(); err != nil {
+		return fmt.Errorf("Error pinging kubernetes: %w", err)
+	}
+
+	return nil
+}
+
 func isK3sInstalled() error {
 	if !common.FileExists("/usr/local/bin/k3s") {
 		return fmt.Errorf("k3s binary is not available")
 	}
 
-	if !common.FileExists(KubeConfigPath) {
+	if !common.FileExists(getKubeconfigPath()) {
 		return fmt.Errorf("k3s kubeconfig is not available")
 	}
 

plugins/scheduler-k3s/helm.go

@@ -75,7 +75,9 @@ func NewHelmAgent(namespace string, logger action.DebugLog) (*HelmAgent, error)
 		helmDriver = "secrets"
 	}
 
-	kubeConfig := kube.GetConfig(KubeConfigPath, "", namespace)
+	kubeconfigPath := getKubeconfigPath()
+	kubeContext := getKubeContext()
+	kubeConfig := kube.GetConfig(kubeconfigPath, kubeContext, namespace)
 	if err := actionConfig.Init(kubeConfig, namespace, helmDriver, logger); err != nil {
 		return nil, err
 	}

plugins/scheduler-k3s/k8s.go

@@ -22,11 +22,22 @@ import (
 	"k8s.io/utils/ptr"
 )
 
+func getKubeconfigPath() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "kubeconfig-path", KubeConfigPath)
+}
+
+func getKubeContext() string {
+	return common.PropertyGetDefault("scheduler-k3s", "--global", "kube-context", DefaultKubeContext)
+}
+
 // KubernetesClient is a wrapper around the Kubernetes client
 type KubernetesClient struct {
 	// Client is the Kubernetes client
 	Client kubernetes.Clientset
 
+	// KubeConfigPath is the path to the Kubernetes config
+	KubeConfigPath string
+
 	// RestClient is the Kubernetes REST client
 	RestClient rest.Interface
 
@@ -36,7 +47,9 @@ type KubernetesClient struct {
 
 // NewKubernetesClient creates a new Kubernetes client
 func NewKubernetesClient() (KubernetesClient, error) {
-	clientConfig := KubernetesClientConfig()
+	kubeconfigPath := getKubeconfigPath()
+	kubeContext := getKubeContext()
+	clientConfig := KubernetesClientConfig(kubeconfigPath, kubeContext)
 	restConf, err := clientConfig.ClientConfig()
 	if err != nil {
 		return KubernetesClient{}, err
@@ -60,17 +73,29 @@ func NewKubernetesClient() (KubernetesClient, error) {
 	}
 
 	return KubernetesClient{
-		Client:     *client,
-		RestConfig: *restConf,
-		RestClient: restClient,
+		Client:         *client,
+		KubeConfigPath: kubeconfigPath,
+		RestConfig:     *restConf,
+		RestClient:     restClient,
 	}, nil
 }
 
 // KubernetesClientConfig returns a Kubernetes client config
-func KubernetesClientConfig() clientcmd.ClientConfig {
+func KubernetesClientConfig(kubeconfigPath string, kubecontext string) clientcmd.ClientConfig {
+	configOverrides := clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}}
+	if kubecontext != "" {
+		configOverrides.CurrentContext = kubecontext
+	}
+
 	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-		&clientcmd.ClientConfigLoadingRules{ExplicitPath: KubeConfigPath},
-		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}})
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
+		&configOverrides,
+	)
+}
+
+func (k KubernetesClient) Ping() error {
+	_, err := k.Client.Discovery().ServerVersion()
+	return err
 }
 
 // AnnotateNodeInput contains all the information needed to annotates a Kubernetes node
@@ -110,16 +135,23 @@ type ApplyKubernetesManifestInput struct {
 }
 
 func (k KubernetesClient) ApplyKubernetesManifest(ctx context.Context, input ApplyKubernetesManifestInput) error {
+	args := []string{
+		"apply",
+		"-f",
+		input.Manifest,
+	}
+
+	if kubeContext := getKubeContext(); kubeContext != "" {
+		args = append([]string{"--context", kubeContext}, args...)
+	}
+
+	if kubeconfigPath := getKubeconfigPath(); kubeconfigPath != "" {
+		args = append([]string{"--kubeconfig", kubeconfigPath}, args...)
+	}
+
 	upgradeCmd, err := common.CallExecCommand(common.ExecCommandInput{
-		Command: "kubectl",
-		Args: []string{
-			"apply",
-			"-f",
-			input.Manifest,
-		},
-		Env: map[string]string{
-			"KUBECONFIG": KubeConfigPath,
-		},
+		Command:     "kubectl",
+		Args:        args,
 		StreamStdio: true,
 	})
 	if err != nil {

plugins/scheduler-k3s/report.go

@@ -17,6 +17,8 @@ func ReportSingleApp(appName string, format string, infoFlag string) error {
 		"--scheduler-k3s-computed-image-pull-secrets":   reportComputedImagePullSecrets,
 		"--scheduler-k3s-image-pull-secrets":            reportImagePullSecrets,
 		"--scheduler-k3s-global-image-pull-secrets":     reportGlobalImagePullSecrets,
+		"--scheduler-k3s-global-kubeconfig-path":        reportGlobalKubeconfigPath,
+		"--scheduler-k3s-global-kube-context":           reportGlobalKubeContext,
 		"--scheduler-k3s-computed-letsencrypt-server":   reportComputedLetsencryptServer,
 		"--scheduler-k3s-letsencrypt-server":            reportLetsencryptServer,
 		"--scheduler-k3s-global-letsencrypt-server":     reportGlobalLetsencryptServer,
@@ -71,6 +73,13 @@ func reportGlobalIngressClass(appName string) string {
 	return getGlobalIngressClass()
 }
 
+func reportGlobalKubeconfigPath(appName string) string {
+	return getKubeconfigPath()
+}
+
+func reportGlobalKubeContext(appName string) string {
+	return getKubeContext()
+}
 func reportComputedLetsencryptServer(appName string) string {
 	return getComputedLetsencryptServer(appName)
 }

plugins/scheduler-k3s/scheduler_k3s.go

@@ -29,6 +29,8 @@ var (
 		"deploy-timeout":         true,
 		"image-pull-secrets":     true,
 		"ingress-class":          true,
+		"kube-context":           true,
+		"kubeconfig-path":        true,
 		"letsencrypt-server":     true,
 		"letsencrypt-email-prod": true,
 		"letsencrypt-email-stag": true,
@@ -42,6 +44,7 @@ var (
 const DefaultIngressClass = "traefik"
 const GlobalProcessType = "--global"
 const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml"
+const DefaultKubeContext = ""
 
 var (
 	runtimeScheme  = runtime.NewScheme()

plugins/scheduler-k3s/subcommands.go

@@ -315,7 +315,16 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo
 // CommandClusterAdd adds a server to the k3s cluster
 func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUknownHosts bool, taintScheduling bool) error {
 	if err := isK3sInstalled(); err != nil {
-		return fmt.Errorf("k3s not installed, cannot join cluster")
+		return fmt.Errorf("k3s not installed, cannot add node to cluster: %w", err)
+	}
+
+	clientset, err := NewKubernetesClient()
+	if err != nil {
+		return fmt.Errorf("Unable to create kubernetes client: %w", err)
+	}
+
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available, cannot add node to cluster: %w", err)
 	}
 
 	if role != "server" && role != "worker" {
@@ -530,11 +539,6 @@ func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUkn
 		return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode)
 	}
 
-	clientset, err := NewKubernetesClient()
-	if err != nil {
-		return fmt.Errorf("Unable to create kubernetes client: %w", err)
-	}
-
 	common.LogInfo2Quiet("Waiting for node to exist")
 	nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{
 		Clientset:  clientset,
@@ -588,9 +592,6 @@ func CommandClusterList(format string) error {
 	if format != "stdout" && format != "json" {
 		return fmt.Errorf("Invalid format: %s", format)
 	}
-	if err := isK3sInstalled(); err != nil {
-		return fmt.Errorf("k3s not installed, cannot list cluster nodes")
-	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
@@ -608,6 +609,10 @@ func CommandClusterList(format string) error {
 		return fmt.Errorf("Unable to create kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available, cannot list cluster nodes: %w", err)
+	}
+
 	nodes, err := clientset.ListNodes(ctx, ListNodesInput{})
 	if err != nil {
 		return fmt.Errorf("Unable to list nodes: %w", err)
@@ -641,7 +646,7 @@ func CommandClusterList(format string) error {
 // CommandClusterRemove removes a node from the k3s cluster
 func CommandClusterRemove(nodeName string) error {
 	if err := isK3sInstalled(); err != nil {
-		return fmt.Errorf("k3s not installed, cannot remove node")
+		return fmt.Errorf("k3s not installed, cannot remove node from cluster: %w", err)
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -661,6 +666,10 @@ func CommandClusterRemove(nodeName string) error {
 		return fmt.Errorf("Unable to create kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	common.LogVerboseQuiet("Getting node remote connection information")
 	node, err := clientset.GetNode(ctx, GetNodeInput{
 		Name: nodeName,
@@ -730,11 +739,12 @@ func CommandSet(appName string, property string, value string) error {
 
 // CommandShowKubeconfig displays the kubeconfig file contents
 func CommandShowKubeconfig() error {
-	if !common.FileExists(KubeConfigPath) {
-		return fmt.Errorf("Kubeconfig file does not exist: %s", KubeConfigPath)
+	kubeconfigPath := getKubeconfigPath()
+	if !common.FileExists(kubeconfigPath) {
+		return fmt.Errorf("Kubeconfig file does not exist: %s", kubeconfigPath)
 	}
 
-	b, err := os.ReadFile(KubeConfigPath)
+	b, err := os.ReadFile(kubeconfigPath)
 	if err != nil {
 		return fmt.Errorf("Unable to read kubeconfig file: %w", err)
 	}
@@ -746,7 +756,7 @@ func CommandShowKubeconfig() error {
 
 func CommandUninstall() error {
 	if err := isK3sInstalled(); err != nil {
-		return fmt.Errorf("k3s not installed, cannot uninstall")
+		return fmt.Errorf("k3s not installed, cannot uninstall: %w", err)
 	}
 
 	common.LogInfo1("Uninstalling k3s")

plugins/scheduler-k3s/triggers.go

@@ -414,6 +414,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
 		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
 		Namespace:     namespace,
@@ -573,6 +577,10 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string,
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	namespace := getComputedNamespace(appName)
 	labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)}
 	processIndex := 1
@@ -664,6 +672,10 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string,
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)}
 	processIndex := 0
 	if processType != "" {
@@ -926,6 +938,10 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args []
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
@@ -1078,6 +1094,10 @@ func TriggerSchedulerRunList(scheduler string, appName string, format string) er
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	namespace := getComputedNamespace(appName)
 	cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{
 		LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName),
@@ -1139,9 +1159,8 @@ func TriggerSchedulerPostDelete(scheduler string, appName string) error {
 		return nil
 	}
 
-	if err := isK3sInstalled(); err != nil {
-		common.LogWarn(fmt.Sprintf("Skipping app deletion: %s", err.Error()))
-		return nil
+	if err := isKubernetesAvailable(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
 	}
 
 	namespace := getComputedNamespace(appName)
@@ -1164,11 +1183,6 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 		return nil
 	}
 
-	if err := isK3sInstalled(); err != nil {
-		common.LogWarn(fmt.Sprintf("Skipping app stop: %s", err.Error()))
-		return nil
-	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
@@ -1185,6 +1199,10 @@ func TriggerSchedulerStop(scheduler string, appName string) error {
 		return fmt.Errorf("Error creating kubernetes client: %w", err)
 	}
 
+	if err := clientset.Ping(); err != nil {
+		return fmt.Errorf("kubernetes api not available: %w", err)
+	}
+
 	namespace := getComputedNamespace(appName)
 	deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{
 		Namespace:     namespace,