diff --git a/debian/control b/debian/control index 6741050b8..c207c0b56 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.33.5 Section: web Priority: optional Architecture: amd64 -Depends: apache2-utils, acl, locales, git, cpio, curl, man-db, netcat, sshcommand, docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-compose-plugin | moby-compose, docker-buildx-plugin | moby-buildx, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, software-properties-common, parallel, procfile-util, python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip, util-linux +Depends: apache2-utils, locales, git, cpio, curl, man-db, netcat, sshcommand, docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-compose-plugin | moby-compose, docker-buildx-plugin | moby-buildx, docker-container-healthchecker, docker-image-labeler, lambda-builder, net-tools, netrc, software-properties-common, parallel, procfile-util, python-software-properties | python3-software-properties, rsync, rsyslog, dos2unix, jq, unzip, util-linux Recommends: herokuish, bash-completion, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn, sudo, python3, debconf Maintainer: Jose Diaz-Gonzalez diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index e17a25fc7..f21315f53 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -219,7 +219,7 @@ The default value may be set by passing an empty value for the option: dokku scheduler-k3s:set node-js-app image-pull-secrets ``` -The `image-pull-secrets` property can also be set globally. The global default is empty string, and k3s will use the local `registries.yaml` for any private registry pulls. +The `image-pull-secrets` property can also be set globally. The global default is empty string, and k3s will use Dokku's locally configured `~/.docker/config.json` for any private registry pulls. ```shell dokku scheduler-k3s:set --global image-pull-secrets 60s diff --git a/plugins/scheduler-k3s/Makefile b/plugins/scheduler-k3s/Makefile index 2a41b159a..50a1184af 100644 --- a/plugins/scheduler-k3s/Makefile +++ b/plugins/scheduler-k3s/Makefile @@ -1,5 +1,5 @@ SUBCOMMANDS = subcommands/initialize subcommands/cluster-add subcommands/cluster-list subcommands/cluster-remove subcommands/report subcommands/set subcommands/show-kubeconfig subcommands/uninstall -TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/post-registry-login triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop +TRIGGERS = triggers/install triggers/post-app-clone-setup triggers/post-app-rename-setup triggers/post-delete triggers/report triggers/scheduler-deploy triggers/scheduler-enter triggers/scheduler-logs triggers/scheduler-post-delete triggers/scheduler-run triggers/scheduler-run-list triggers/scheduler-stop BUILD = commands subcommands triggers PLUGIN_NAME = scheduler-k3s diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 540d922a3..90b6e617e 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1120,10 +1120,6 @@ func isK3sInstalled() error { return fmt.Errorf("k3s binary is not available") } - if !common.FileExists(RegistryConfigPath) { - return fmt.Errorf("k3s registry config is not available") - } - if !common.FileExists(KubeConfigPath) { return fmt.Errorf("k3s kubeconfig is not available") } diff --git a/plugins/scheduler-k3s/registry.go b/plugins/scheduler-k3s/registry.go deleted file mode 100644 index 824e1b7b4..000000000 --- a/plugins/scheduler-k3s/registry.go +++ /dev/null @@ -1,78 +0,0 @@ -package scheduler_k3s - -import ( - "context" - "fmt" - - "github.com/dokku/dokku/plugins/common" -) - -func copyRegistryToNode(ctx context.Context, remoteHost string) error { - common.LogInfo1Quiet(fmt.Sprintf("Updating k3s registry configuration on %s", remoteHost)) - sftpCmd, err := common.CallSftpCopy(common.SftpCopyInput{ - AllowUknownHosts: true, - DestinationPath: "/tmp/registries.yaml", - RemoteHost: remoteHost, - SourcePath: RegistryConfigPath, - }) - if err != nil { - return fmt.Errorf("Error copying registries.yaml to remote host: %w", err) - } - - if sftpCmd.ExitErr != nil { - return fmt.Errorf("Error copying registries.yaml to remote host: %s", sftpCmd.ExitErr.Error()) - } - - common.LogVerboseQuiet("Moving k3s registry configuration into place") - mvCmd, err := common.CallSshCommand(common.SshCommandInput{ - AllowUknownHosts: true, - Args: []string{"/tmp/registries.yaml", RegistryConfigPath}, - Command: "mv", - RemoteHost: remoteHost, - StreamStdio: true, - Sudo: true, - }) - if err != nil { - return fmt.Errorf("Unable to call mv command over ssh: %w", err) - } - - if mvCmd.ExitCode != 0 { - return fmt.Errorf("Invalid exit code from mv command over ssh: %d", mvCmd.ExitCode) - } - - common.LogVerboseQuiet("Updating k3s registry permissions") - chmodCmd, err := common.CallSshCommand(common.SshCommandInput{ - AllowUknownHosts: true, - Args: []string{"0644", RegistryConfigPath}, - Command: "chmod", - RemoteHost: remoteHost, - StreamStdio: true, - Sudo: true, - }) - if err != nil { - return fmt.Errorf("Unable to call chmod command over ssh: %w", err) - } - - if chmodCmd.ExitCode != 0 { - return fmt.Errorf("Invalid exit code from chmod command over ssh: %d", chmodCmd.ExitCode) - } - - common.LogVerboseQuiet("Updating k3s registry ower") - chownCmd, err := common.CallSshCommand(common.SshCommandInput{ - AllowUknownHosts: true, - Args: []string{"root:root", RegistryConfigPath}, - Command: "chown", - RemoteHost: remoteHost, - StreamStdio: true, - Sudo: true, - }) - if err != nil { - return fmt.Errorf("Unable to call chown command over ssh: %w", err) - } - - if chownCmd.ExitCode != 0 { - return fmt.Errorf("Invalid exit code from chown command over ssh: %d", chownCmd.ExitCode) - } - - return nil -} diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 78a857fa4..36bd33e35 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -42,7 +42,6 @@ var ( const DefaultIngressClass = "traefik" const GlobalProcessType = "--global" const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml" -const RegistryConfigPath = "/etc/rancher/k3s/registries.yaml" var ( runtimeScheme = runtime.NewScheme() diff --git a/plugins/scheduler-k3s/src/triggers/triggers.go b/plugins/scheduler-k3s/src/triggers/triggers.go index 4e99e14d8..ee171e767 100644 --- a/plugins/scheduler-k3s/src/triggers/triggers.go +++ b/plugins/scheduler-k3s/src/triggers/triggers.go @@ -35,10 +35,6 @@ func main() { case "post-delete": appName := flag.Arg(0) err = scheduler_k3s.TriggerPostDelete(appName) - case "post-registry-login": - server := flag.Arg(0) - username := flag.Arg(1) - err = scheduler_k3s.TriggerPostRegistryLogin(server, username) case "report": appName := flag.Arg(0) err = scheduler_k3s.ReportSingleApp(appName, "", "") diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 5a72476d9..5e5bd930e 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -241,27 +241,6 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo return fmt.Errorf("Invalid exit code from k3s installer command: %d", installerCmd.ExitCode) } - if err := common.TouchFile(RegistryConfigPath); err != nil { - return fmt.Errorf("Error creating initial registries.yaml file") - } - - common.LogInfo2Quiet("Setting registries.yaml permissions") - registryAclCmd, err := common.CallExecCommand(common.ExecCommandInput{ - Command: "setfacl", - Args: []string{ - "-m", - "user:dokku:rwx", - RegistryConfigPath, - }, - StreamStdio: true, - }) - if err != nil { - return fmt.Errorf("Unable to call setfacl command: %w", err) - } - if registryAclCmd.ExitCode != 0 { - return fmt.Errorf("Invalid exit code from setfacl command: %d", registryAclCmd.ExitCode) - } - clientset, err := NewKubernetesClient() if err != nil { return fmt.Errorf("Unable to create kubernetes client: %w", err) @@ -572,11 +551,6 @@ func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUkn return fmt.Errorf("Unable to find node after joining cluster, node will not be annotated/labeled appropriately access registry secrets") } - err = copyRegistryToNode(ctx, remoteHost) - if err != nil { - return fmt.Errorf("Error copying registries.yaml to remote host: %w", err) - } - labels := ServerLabels if role == "worker" { labels = WorkerLabels diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index 3addd48d3..5003a3965 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -45,6 +45,7 @@ type GlobalValues struct { type GlobalImage struct { ImagePullSecrets string `yaml:"image_pull_secrets"` Name string `yaml:"name"` + PullSecretBase64 string `yaml:"pull_secret_base64"` Type string `yaml:"type"` WorkingDir string `yaml:"working_dir"` } diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml index 73cceec23..ddd5cbefb 100644 --- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml +++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml @@ -111,7 +111,8 @@ spec: workingDir: {{ $.Values.global.image.working_dir }} {{- end }} {{- if $.Values.global.image.image_pull_secrets }} - imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }} + imagePullSecrets: + - name: {{ $.Values.global.image.image_pull_secrets }} {{- end }} restartPolicy: Never serviceAccountName: {{ $.Values.global.app_name }} diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index db957375e..38d1eea83 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -125,6 +125,7 @@ spec: workingDir: {{ $.Values.global.image.working_dir }} {{- end }} {{- if $.Values.global.image.image_pull_secrets }} - imagePullSecrets: {{ $.Values.global.image.image_pull_secrets }} + imagePullSecrets: + - name: {{ $.Values.global.image.image_pull_secrets }} {{- end }} serviceAccountName: {{ $.Values.global.app_name }} diff --git a/plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml b/plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml new file mode 100644 index 000000000..3ffec7186 --- /dev/null +++ b/plugins/scheduler-k3s/templates/chart/image-pull-secret.yaml @@ -0,0 +1,23 @@ +{{- if $.Values.global.image.pull_secret_base64 }} +--- +apiVersion: v1 +kind: Secret +metadata: + annotations: + app.kubernetes.io/version: {{ $.Values.global.deploment_id | quote }} + dokku.com/managed: "true" + {{- if and $.Values.global.annotations $.Values.global.annotations.secret }} + {{- range $k, $v := $.Values.global.annotations.secret }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- end }} + labels: + app.kubernetes.io/instance: image-pull-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} + app.kubernetes.io/name: image-pull-{{ $.Values.global.app_name }} + app.kubernetes.io/part-of: {{ $.Values.global.app_name }} + name: image-pull-{{ $.Values.global.app_name }}.{{ $.Values.global.deploment_id }} + namespace: {{ $.Values.global.namespace }} +data: + .dockerconfigjson: "{{ $.Values.global.image.pull_secret_base64 }}" +type: kubernetes.io/dockerconfigjson +{{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/secret.yaml b/plugins/scheduler-k3s/templates/chart/secret.yaml index af4f8131c..f8ac42d21 100644 --- a/plugins/scheduler-k3s/templates/chart/secret.yaml +++ b/plugins/scheduler-k3s/templates/chart/secret.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Secret metadata: diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index f566e32c8..3ff7a52ab 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -23,11 +23,8 @@ import ( "github.com/dokku/dokku/plugins/config" "github.com/dokku/dokku/plugins/cron" "github.com/fatih/color" - "github.com/hashicorp/go-multierror" "github.com/kballard/go-shellquote" - "github.com/rancher/wharfie/pkg/registries" "github.com/ryanuber/columnize" - "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/client/conditions" @@ -78,88 +75,6 @@ func TriggerPostDelete(appName string) error { return propertyErr } -// TriggerPostRegistryLogin updates the `/etc/rancher/k3s/registries.yaml` to include -// auth information for the registry. Note that if the file does not exist, it won't be updated. -func TriggerPostRegistryLogin(server string, username string) error { - if !common.FileExists("/usr/local/bin/k3s") { - return nil - } - - password := os.Getenv("DOCKER_REGISTRY_PASS") - - yamlFile, err := os.ReadFile(RegistryConfigPath) - if err != nil { - return fmt.Errorf("Unable to read existing registries.yaml: %w", err) - } - - var registry registries.Registry - err = yaml.Unmarshal(yamlFile, ®istry) - if err != nil { - return fmt.Errorf("Unable to unmarshal registry configuration from yaml: %w", err) - } - - common.LogInfo1("Updating k3s configuration") - if registry.Auths == nil { - registry.Auths = map[string]registries.AuthConfig{} - } - - if server == "docker.io" { - server = "registry-1.docker.io" - } - - registry.Auths[server] = registries.AuthConfig{ - Username: username, - Password: password, - } - - data, err := yaml.Marshal(®istry) - if err != nil { - return fmt.Errorf("Unable to marshal registry configuration to yaml: %w", err) - } - - if err := os.WriteFile(RegistryConfigPath, data, os.FileMode(0644)); err != nil { - return fmt.Errorf("Unable to write registry configuration to file: %w", err) - } - - ctx, cancel := context.WithCancel(context.Background()) - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt, syscall.SIGHUP, - syscall.SIGINT, - syscall.SIGQUIT, - syscall.SIGTERM) - go func() { - <-signals - cancel() - }() - - clientset, err := NewKubernetesClient() - if err != nil { - return fmt.Errorf("Error creating kubernetes client: %w", err) - } - - nodes, err := clientset.ListNodes(ctx, ListNodesInput{}) - if err != nil { - return fmt.Errorf("Error listing nodes: %w", err) - } - - var result error - for _, node := range nodes { - remoteHost, ok := node.Annotations["dokku.com/remote-host"] - if !ok { - continue - } - - err := copyRegistryToNode(ctx, remoteHost) - if err != nil { - wrappedErr := fmt.Errorf("Error copying registry to node: %w", err) - result = multierror.Append(result, wrappedErr) - common.LogWarn(wrappedErr.Error()) - } - } - - return result -} - // TriggerSchedulerDeploy deploys an image tag for a given application func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) error { if scheduler != "k3s" { @@ -208,8 +123,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error parsing rollback-on-failure value as boolean: %w", err) } - imagePullSecrets := getComputedImagePullSecrets(appName) - imageSourceType := "dockerfile" if common.IsImageCnbBased(image) { imageSourceType = "pack" @@ -256,7 +169,22 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e } deploymentId := time.Now().Unix() - globalTemplateFiles := []string{"service-account", "secret"} + pullSecretBase64 := base64.StdEncoding.EncodeToString([]byte("")) + imagePullSecrets := getComputedImagePullSecrets(appName) + if imagePullSecrets == "" { + dockerConfigPath := filepath.Join(os.Getenv("DOKKU_ROOT"), ".docker/config.json") + if fi, err := os.Stat(dockerConfigPath); err == nil && !fi.IsDir() { + b, err := os.ReadFile(dockerConfigPath) + if err != nil { + return fmt.Errorf("Error reading docker config: %w", err) + } + + imagePullSecrets = fmt.Sprintf("image-pull-%s.%d", appName, deploymentId) + pullSecretBase64 = base64.StdEncoding.EncodeToString(b) + } + } + + globalTemplateFiles := []string{"service-account", "secret", "image-pull-secret"} for _, templateName := range globalTemplateFiles { b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName)) if err != nil { @@ -345,6 +273,7 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e DeploymentID: fmt.Sprint(deploymentId), Image: GlobalImage{ ImagePullSecrets: imagePullSecrets, + PullSecretBase64: pullSecretBase64, Name: image, Type: imageSourceType, WorkingDir: workingDir,