diff --git a/plugins/common/properties.go b/plugins/common/properties.go index 3b131b4f3..f0f1c23d2 100644 --- a/plugins/common/properties.go +++ b/plugins/common/properties.go @@ -138,6 +138,40 @@ func PropertyGetAll(pluginName string, appName string) (map[string]string, error return properties, nil } +// PropertyGetAllByPrefix returns a map of all properties for a given app with a specified prefix +func PropertyGetAllByPrefix(pluginName string, appName string, prefix string) (map[string]string, error) { + properties := make(map[string]string) + pluginAppConfigRoot := getPluginAppPropertyPath(pluginName, appName) + + fi, err := os.Stat(pluginAppConfigRoot) + if err != nil { + return properties, nil + } + + if !fi.IsDir() { + return properties, errors.New("Specified property path is not a directory") + } + + files, err := os.ReadDir(pluginAppConfigRoot) + if err != nil { + return properties, err + } + + for _, file := range files { + if file.IsDir() { + continue + } + property := file.Name() + if !strings.HasPrefix(property, prefix) { + continue + } + + properties[property] = PropertyGet(pluginName, appName, property) + } + + return properties, nil +} + // PropertyGetDefault returns the value for a given property with a specified default value func PropertyGetDefault(pluginName, appName, property, defaultValue string) (val string) { if !PropertyExists(pluginName, appName, property) { diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 5efa8947e..eba23b449 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -3,6 +3,7 @@ package scheduler_k3s import ( "bytes" "context" + "encoding/base64" "errors" "fmt" "net" @@ -510,6 +511,41 @@ func getAutoscaling(appName string, processType string, replicas int) (ProcessAu return autoscaling, nil } +// getKedaValues retrieves keda values for a given app and process type +func getKedaValues(appName string) (GlobalKedaValues, error) { + properties, err := common.PropertyGetAllByPrefix("scheduler-k3s", appName, "trigger-auth-") + if err != nil { + return GlobalKedaValues{}, fmt.Errorf("Error getting trigger-auth properties: %w", err) + } + + auths := map[string]KedaAuthentication{} + for key, value := range properties { + parts := strings.SplitN(strings.TrimPrefix(key, "trigger-auth-"), ":", 2) + if len(parts) != 2 { + return GlobalKedaValues{}, fmt.Errorf("Invalid trigger-auth property format: %s", key) + } + + authType := parts[0] + secretKey := parts[1] + if len(secretKey) == 0 { + return GlobalKedaValues{}, fmt.Errorf("Invalid trigger-auth property format: %s", key) + } + + if _, ok := auths[authType]; !ok { + auths[authType] = KedaAuthentication{ + Type: authType, + Secrets: make(map[string]string), + } + } + + auths[authType].Secrets[secretKey] = base64.StdEncoding.EncodeToString([]byte(value)) + } + + return GlobalKedaValues{ + Authentications: auths, + }, nil +} + // getGlobalAnnotations retrieves global annotations for a given app func getGlobalAnnotations(appName string) (ProcessAnnotations, error) { return getAnnotations(appName, GlobalProcessType) diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index b3aea7001..3711f631f 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -38,6 +38,7 @@ type GlobalValues struct { DeploymentID string `yaml:"deploment_id"` Image GlobalImage `yaml:"image"` Labels ProcessLabels `yaml:"labels,omitempty"` + Keda GlobalKedaValues `yaml:"keda"` Namespace string `yaml:"namespace"` Network GlobalNetwork `yaml:"network"` Secrets map[string]string `yaml:"secrets,omitempty"` @@ -56,6 +57,21 @@ type GlobalNetwork struct { PrimaryPort int32 `yaml:"primary_port"` } +// GlobalKedaValues contains the global keda configuration +type GlobalKedaValues struct { + // Authentications is a map of authentication objects to use for keda + Authentications map[string]KedaAuthentication `yaml:"authentications"` +} + +// KedaAuthentication contains the authentication configuration for keda +type KedaAuthentication struct { + // Type is the type of authentication to use + Type string `yaml:"type"` + + // Secrets is a map of secrets to use for authentication + Secrets map[string]string `yaml:"secrets"` +} + type ProcessValues struct { Annotations ProcessAnnotations `yaml:"annotations,omitempty"` Args []string `yaml:"args,omitempty"` @@ -120,6 +136,15 @@ type ProcessAutoscalingTrigger struct { // Metadata is a map of key-value pairs that can be used to store arbitrary trigger data Metadata map[string]string `yaml:"metadata,omitempty"` + + // AuthenticationRef is a reference to an authentication object + AuthenticationRef *ProcessAutoscalingTriggerAuthenticationRef `yaml:"authenticationRef,omitempty"` +} + +// ProcessAutoscalingTriggerAuthenticationRef is a reference to an authentication object +type ProcessAutoscalingTriggerAuthenticationRef struct { + // Name is the name of the authentication object + Name string `yaml:"name"` } type ProcessHealthchecks struct { diff --git a/plugins/scheduler-k3s/templates/chart/keda-secret.yaml b/plugins/scheduler-k3s/templates/chart/keda-secret.yaml index 3809df019..29531b479 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-secret.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-secret.yaml @@ -1,5 +1,5 @@ {{- if .Capabilities.APIVersions.Has "keda.sh/v1alpha1" -}} -{{- range $type, $config := .Values.global.keda.auth }} +{{- $config := index .Values.global.keda.authentications "AUTH_NAME" }} --- apiVersion: v1 kind: Secret @@ -12,16 +12,11 @@ metadata: {{ $k }}: {{ $v | quote }} {{- end }} {{- end }} - {{- if and $config.annotations $config.annotations.keda_secret }} - {{- range $k, $v := $config.annotations.keda_secret }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} labels: - app.kubernetes.io/instance: {{ $.Values.global.app_name }}-trigger-auth-secret - app.kubernetes.io/name: {{ $type }} + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $config.type }} + app.kubernetes.io/name: {{ $config.type }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} - name: app-auth-{{ $type }} + name: keda-app-auth-{{ $.Values.global.app_name }}-{{ $config.type }} namespace: {{ $.Values.global.namespace }} data: {{- with $config.secrets }} @@ -29,4 +24,3 @@ data: {{- toYaml . | nindent 2 }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file diff --git a/plugins/scheduler-k3s/templates/chart/keda-trigger-authentication.yaml b/plugins/scheduler-k3s/templates/chart/keda-trigger-authentication.yaml index 73807a9a8..a78cdabc2 100644 --- a/plugins/scheduler-k3s/templates/chart/keda-trigger-authentication.yaml +++ b/plugins/scheduler-k3s/templates/chart/keda-trigger-authentication.yaml @@ -1,5 +1,5 @@ {{- if .Capabilities.APIVersions.Has "keda.sh/v1alpha1" -}} -{{- range $type, $config := .Values.global.keda.auth }} +{{- $config := index .Values.global.keda.authentications "AUTH_NAME" }} --- apiVersion: keda.sh/v1alpha1 kind: TriggerAuthentication @@ -12,23 +12,17 @@ metadata: {{ $k }}: {{ $v | quote }} {{- end }} {{- end }} - {{- if and $config.annotations $config.annotations.keda_trigger_authentication }} - {{- range $k, $v := $config.annotations.keda_trigger_authentication }} - {{ $k }}: {{ $v | quote }} - {{- end }} - {{- end }} labels: - app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $type }} - app.kubernetes.io/name: {{ $type }} + app.kubernetes.io/instance: {{ $.Values.global.app_name }}-{{ $config.type }} + app.kubernetes.io/name: {{ $config.type }} app.kubernetes.io/part-of: {{ $.Values.global.app_name }} - name: {{ $.Values.global.app_name }}-{{ $type }} + name: {{ $.Values.global.app_name }}-{{ $config.type }} namespace: {{ $.Values.global.namespace }} spec: secretTargetRef: - {{- range $key := $config.secrets }} - - parameter: {{ $key }} - name: app-auth-{{ $type }} - key: {{ $key }} + {{- range $k, $v := $config.secrets }} + - key: {{ $k }} + name: app-auth-{{ $config.type }} + parameter: {{ $k }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 4818639ef..764300714 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -266,11 +266,17 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error getting global labels: %w", err) } + kedaValues, err := getKedaValues(appName) + if err != nil { + return fmt.Errorf("Error getting keda values: %w", err) + } + values := &AppValues{ Global: GlobalValues{ Annotations: globalAnnotations, AppName: appName, DeploymentID: fmt.Sprint(deploymentId), + Keda: kedaValues, Image: GlobalImage{ ImagePullSecrets: imagePullSecrets, PullSecretBase64: pullSecretBase64, @@ -289,6 +295,27 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e Processes: map[string]ProcessValues{}, } + for authName := range kedaValues.Authentications { + templateFiles := []string{"keda-secret", "keda-trigger-authentication"} + for _, templateName := range templateFiles { + b, err := templates.ReadFile(fmt.Sprintf("templates/chart/%s.yaml", templateName)) + if err != nil { + return fmt.Errorf("Error reading %s template: %w", templateName, err) + } + + filename := filepath.Join(chartDir, "templates", fmt.Sprintf("%s-%s.yaml", templateName, authName)) + contents := strings.ReplaceAll(string(b), "AUTH_NAME", authName) + err = os.WriteFile(filename, []byte(contents), os.FileMode(0644)) + if err != nil { + return fmt.Errorf("Error writing %s template: %w", templateName, err) + } + + if os.Getenv("DOKKU_TRACE") == "1" { + common.CatFile(filename) + } + } + } + for processType, processCount := range processes { // todo: implement deployment annotations // todo: implement pod annotations @@ -420,7 +447,6 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e common.CatFile(filename) } } - } clientset, err := NewKubernetesClient()