diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index dad1ee926..585acd72d 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -343,6 +343,20 @@ In the above example, the `internal-web` process is exposed as a service. The `P > [!NOTE] > It is not possible to modify the port mapping, nor is it possible to assign domains or SSL to a non-web process. +### Wildcard domains + +Both the `nginx` and `traefik` ingress classes route wildcard domains. Add the wildcard as a domain on the app: + +```shell +dokku domains:add node-js-app '*.node-js-app.com' +``` + +A wildcard matches exactly one label, matching DNS itself, so `*.node-js-app.com` covers `api.node-js-app.com` but not `node-js-app.com` or `api.staging.node-js-app.com`. Add the apex as a separate domain if it should also be served. + +An exact domain always takes precedence over a wildcard, including across apps. If one app serves `*.node-js-app.com` and another serves `api.node-js-app.com`, requests for `api.node-js-app.com` are routed to the second app. + +Only the leading label may be wildcarded. A domain such as `api.*.node-js-app.com` is treated as a literal hostname and will not match anything. + ### SSL Certificates #### Enabling letsencrypt integration @@ -485,10 +499,7 @@ A `dns01` issuer can issue wildcard certificates. Add the wildcard as a domain o dokku domains:add node-js-app '*.node-js-app.com' ``` -> [!WARNING] -> Wildcard domains require the `nginx` ingress class. The Traefik integration matches hosts exactly and will not route requests for a wildcard domain. - -Note that a Kubernetes wildcard host matches exactly one label, so `*.node-js-app.com` does not cover `node-js-app.com`. Add both domains if the apex should also be served. +See [wildcard domains](#wildcard-domains) for how a wildcard is matched against incoming requests. #### Using imported SSL certificates diff --git a/plugins/scheduler-k3s/ingress_route_template_test.go b/plugins/scheduler-k3s/ingress_route_template_test.go index 5faf5bc92..bf859e3b3 100644 --- a/plugins/scheduler-k3s/ingress_route_template_test.go +++ b/plugins/scheduler-k3s/ingress_route_template_test.go @@ -281,6 +281,83 @@ func TestIngressRouteTemplateMultipleDomainsRenderOneRoutePerDomain(t *testing.T } } +func TestIngressRouteTemplateWildcardDomainUsesHostRegexp(t *testing.T) { + values := testIngressRouteValues(true) + web := values["processes"].(map[string]interface{})["web"].(map[string]interface{})["web"].(map[string]interface{}) + web["domains"] = []interface{}{ + map[string]interface{}{"name": "*.example.com"}, + } + + docs := renderIngressRouteTemplate(t, values) + if len(docs) != 2 { + t.Fatalf("expected 2 ingress routes when tls enabled, got %d", len(docs)) + } + + for _, name := range []string{"myapp-web-http-80-5000", "myapp-web-http-80-5000-websecure"} { + routes := findDocByName(t, docs, name)["spec"].(map[string]interface{})["routes"].([]interface{}) + if len(routes) != 1 { + t.Fatalf("expected %s to have 1 route, got %d", name, len(routes)) + } + + route := routes[0].(map[string]interface{}) + if got := route["match"]; got != "HostRegexp(`{subdomain:[^.]+}.example.com`)" { + t.Fatalf("expected %s match HostRegexp(`{subdomain:[^.]+}.example.com`), got %#v", name, got) + } + if got := route["priority"]; got != 1 { + t.Fatalf("expected %s priority 1, got %#v", name, got) + } + } +} + +func TestIngressRouteTemplateWildcardRouteYieldsToExactDomain(t *testing.T) { + values := testIngressRouteValues(false) + web := values["processes"].(map[string]interface{})["web"].(map[string]interface{})["web"].(map[string]interface{}) + web["domains"] = []interface{}{ + map[string]interface{}{"name": "*.example.com"}, + map[string]interface{}{"name": "app.example.com"}, + } + + docs := renderIngressRouteTemplate(t, values) + routes := findDocByName(t, docs, "myapp-web-http-80-5000")["spec"].(map[string]interface{})["routes"].([]interface{}) + if len(routes) != 2 { + t.Fatalf("expected 2 routes (one per domain), got %d", len(routes)) + } + + wildcard := routes[0].(map[string]interface{}) + if got := wildcard["match"]; got != "HostRegexp(`{subdomain:[^.]+}.example.com`)" { + t.Fatalf("expected wildcard match HostRegexp(`{subdomain:[^.]+}.example.com`), got %#v", got) + } + if got := wildcard["priority"]; got != 1 { + t.Fatalf("expected wildcard priority 1, got %#v", got) + } + + exact := routes[1].(map[string]interface{}) + if got := exact["match"]; got != "Host(`app.example.com`)" { + t.Fatalf("expected exact match Host(`app.example.com`), got %#v", got) + } + if got, ok := exact["priority"]; ok { + t.Fatalf("expected exact route to omit priority so traefik defaults it above the wildcard, got %#v", got) + } +} + +func TestIngressRouteTemplateNonLeadingWildcardStaysExactHost(t *testing.T) { + values := testIngressRouteValues(false) + web := values["processes"].(map[string]interface{})["web"].(map[string]interface{})["web"].(map[string]interface{}) + web["domains"] = []interface{}{ + map[string]interface{}{"name": "app.*.example.com"}, + } + + docs := renderIngressRouteTemplate(t, values) + routes := findDocByName(t, docs, "myapp-web-http-80-5000")["spec"].(map[string]interface{})["routes"].([]interface{}) + route := routes[0].(map[string]interface{}) + if got := route["match"]; got != "Host(`app.*.example.com`)" { + t.Fatalf("expected non-leading wildcard to stay Host(`app.*.example.com`), got %#v", got) + } + if got, ok := route["priority"]; ok { + t.Fatalf("expected non-leading wildcard to omit priority, got %#v", got) + } +} + func TestIngressRouteTemplateNonTraefikIngressClassRendersNothing(t *testing.T) { values := testIngressRouteValues(true) values["global"].(map[string]interface{})["network"].(map[string]interface{})["ingress_class"] = "nginx" diff --git a/plugins/scheduler-k3s/ingress_template_test.go b/plugins/scheduler-k3s/ingress_template_test.go new file mode 100644 index 000000000..e46ea5ba9 --- /dev/null +++ b/plugins/scheduler-k3s/ingress_template_test.go @@ -0,0 +1,205 @@ +package scheduler_k3s + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/engine" +) + +func renderIngressTemplate(t *testing.T, values map[string]interface{}) []map[string]interface{} { + t.Helper() + + chartDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(chartDir, "templates"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + + chartYAML := []byte("apiVersion: v2\nname: test\nversion: 0.0.1\n") + if err := os.WriteFile(filepath.Join(chartDir, "Chart.yaml"), chartYAML, 0o644); err != nil { + t.Fatalf("write Chart.yaml: %v", err) + } + + ingressTpl, err := templates.ReadFile("templates/chart/ingress.yaml") + if err != nil { + t.Fatalf("read ingress template: %v", err) + } + if err := os.WriteFile(filepath.Join(chartDir, "templates", "ingress.yaml"), ingressTpl, 0o644); err != nil { + t.Fatalf("write ingress template: %v", err) + } + + helpersTpl, err := templates.ReadFile("templates/chart/_helpers.tpl") + if err != nil { + t.Fatalf("read _helpers: %v", err) + } + if err := os.WriteFile(filepath.Join(chartDir, "templates", "_helpers.tpl"), helpersTpl, 0o644); err != nil { + t.Fatalf("write _helpers: %v", err) + } + + loaded, err := loader.Load(chartDir) + if err != nil { + t.Fatalf("load chart: %v", err) + } + + renderValues, err := chartutil.ToRenderValues(loaded, values, chartutil.ReleaseOptions{Name: "test", Namespace: "default"}, nil) + if err != nil { + t.Fatalf("ToRenderValues: %v", err) + } + + rendered, err := engine.Render(loaded, renderValues) + if err != nil { + t.Fatalf("render: %v", err) + } + + manifest := rendered["test/templates/ingress.yaml"] + + var docs []map[string]interface{} + decoder := yaml.NewDecoder(strings.NewReader(manifest)) + for { + var doc map[string]interface{} + if err := decoder.Decode(&doc); err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("yaml decode failed: %v\nrendered:\n%s", err, manifest) + } + if doc != nil { + docs = append(docs, doc) + } + } + + return docs +} + +func testIngressValues(tlsEnabled bool, domains ...map[string]interface{}) map[string]interface{} { + domainValues := []interface{}{} + for _, domain := range domains { + domainValues = append(domainValues, domain) + } + + web := map[string]interface{}{ + "domains": domainValues, + "port_maps": []interface{}{ + map[string]interface{}{ + "name": "http-80-5000", + "scheme": "http", + "container_port": 5000, + }, + }, + "tls": map[string]interface{}{ + "enabled": tlsEnabled, + "use_imported_cert": false, + }, + } + + return map[string]interface{}{ + "global": map[string]interface{}{ + "app_name": "myapp", + "namespace": "myns", + "network": map[string]interface{}{ + "ingress_class": "nginx", + }, + }, + "processes": map[string]interface{}{ + "web": map[string]interface{}{ + "web": web, + }, + }, + } +} + +func ingressHost(t *testing.T, doc map[string]interface{}) string { + t.Helper() + + rules, ok := doc["spec"].(map[string]interface{})["rules"].([]interface{}) + if !ok || len(rules) != 1 { + t.Fatalf("expected ingress to contain a single rule, got %#v", doc["spec"]) + } + + host, ok := rules[0].(map[string]interface{})["host"].(string) + if !ok { + t.Fatalf("expected rule to contain a string host, got %#v", rules[0]) + } + + return host +} + +func TestIngressTemplateExactDomainUsesSlugForObjectName(t *testing.T) { + values := testIngressValues(false, map[string]interface{}{"name": "app.example.com", "slug": "app-example-com"}) + + docs := renderIngressTemplate(t, values) + if len(docs) != 1 { + t.Fatalf("expected 1 ingress, got %d", len(docs)) + } + + doc := findDocByName(t, docs, "myapp-web-app-example-com") + if got := ingressHost(t, doc); got != "app.example.com" { + t.Fatalf("expected host app.example.com, got %q", got) + } +} + +func TestIngressTemplateWildcardDomainRendersWildcardHost(t *testing.T) { + values := testIngressValues(true, map[string]interface{}{"name": "*.example.com", "slug": "wildcard-example-com"}) + + docs := renderIngressTemplate(t, values) + if len(docs) != 1 { + t.Fatalf("expected 1 ingress, got %d", len(docs)) + } + + doc := findDocByName(t, docs, "myapp-web-wildcard-example-com") + if got := ingressHost(t, doc); got != "*.example.com" { + t.Fatalf("expected host *.example.com, got %q", got) + } + + tls, ok := doc["spec"].(map[string]interface{})["tls"].([]interface{}) + if !ok || len(tls) != 1 { + t.Fatalf("expected ingress to contain a single tls entry, got %#v", doc["spec"]) + } + + hosts, ok := tls[0].(map[string]interface{})["hosts"].([]interface{}) + if !ok || len(hosts) != 1 || hosts[0] != "*.example.com" { + t.Fatalf("expected tls hosts [*.example.com], got %#v", tls[0]) + } + if got := tls[0].(map[string]interface{})["secretName"]; got != "tls-myapp-web" { + t.Fatalf("expected tls secretName tls-myapp-web, got %#v", got) + } +} + +func TestIngressTemplateWildcardAndApexRenderDistinctObjects(t *testing.T) { + values := testIngressValues(false, + map[string]interface{}{"name": "*.example.com", "slug": "wildcard-example-com"}, + map[string]interface{}{"name": "example.com", "slug": "example-com"}, + ) + + docs := renderIngressTemplate(t, values) + if len(docs) != 2 { + t.Fatalf("expected 2 ingresses (one per domain), got %d", len(docs)) + } + + wildcard := findDocByName(t, docs, "myapp-web-wildcard-example-com") + if got := ingressHost(t, wildcard); got != "*.example.com" { + t.Fatalf("expected wildcard host *.example.com, got %q", got) + } + + apex := findDocByName(t, docs, "myapp-web-example-com") + if got := ingressHost(t, apex); got != "example.com" { + t.Fatalf("expected apex host example.com, got %q", got) + } +} + +func TestIngressTemplateNonNginxIngressClassRendersNothing(t *testing.T) { + values := testIngressValues(false, map[string]interface{}{"name": "app.example.com", "slug": "app-example-com"}) + values["global"].(map[string]interface{})["network"].(map[string]interface{})["ingress_class"] = "traefik" + + docs := renderIngressTemplate(t, values) + if len(docs) != 0 { + t.Fatalf("expected 0 ingresses when ingress_class is not nginx, got %d: %#v", len(docs), docs) + } +} diff --git a/plugins/scheduler-k3s/templates/chart/_helpers.tpl b/plugins/scheduler-k3s/templates/chart/_helpers.tpl index 6fb5ee787..eb3c3b468 100644 --- a/plugins/scheduler-k3s/templates/chart/_helpers.tpl +++ b/plugins/scheduler-k3s/templates/chart/_helpers.tpl @@ -16,6 +16,20 @@ {{- end }} {{- end }} +{{/* +traefik.host.match renders a domain as a Traefik router rule. Traefik v2 resolves HostRegexp +through gorilla/mux, where the pattern is a {name:regexp} template and every literal segment is +quoted, so a wildcard label is spelled out as a named group rather than a raw regexp. The group +matches a single label, mirroring both DNS and Kubernetes Ingress wildcard hosts. +*/}} +{{- define "traefik.host.match" -}} +{{- if hasPrefix "*." . -}} +HostRegexp(`{subdomain:[^.]+}.{{ trimPrefix "*." . }}`) +{{- else -}} +Host(`{{ . }}`) +{{- end -}} +{{- end -}} + {{- define "primary.port" -}} {{- $found := dict -}} {{- range $idx, $port_map := . -}} diff --git a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml index c9f0d407f..8fbb76c7f 100644 --- a/plugins/scheduler-k3s/templates/chart/ingress-route.yaml +++ b/plugins/scheduler-k3s/templates/chart/ingress-route.yaml @@ -39,7 +39,10 @@ spec: routes: {{- range $ddx, $domain := $config.web.domains }} - kind: Rule - match: Host(`{{ $domain.name }}`) + match: {{ include "traefik.host.match" $domain.name }} + {{- if hasPrefix "*." $domain.name }} + priority: 1 + {{- end }} middlewares: - name: {{ $.Values.global.app_name}}-{{ $processName }}-compression namespace: {{ $.Values.global.namespace }} diff --git a/tests/unit/scheduler-k3s-wildcard-1.bats b/tests/unit/scheduler-k3s-wildcard-1.bats new file mode 100644 index 000000000..7ea20781a --- /dev/null +++ b/tests/unit/scheduler-k3s-wildcard-1.bats @@ -0,0 +1,93 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" +EXACT_APP="rdmtestapp2" + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true +} + +@test "(scheduler-k3s) [ingress] traefik routes wildcard domains and prefers exact domains" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=traefik install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP '*.dokku.me'" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $TEST_APP HELLO=wildcard" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku apps:create $EXACT_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $EXACT_APP exact.dokku.me" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku config:set $EXACT_APP HELLO=exact" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$EXACT_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 30" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.routes[0].match}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output 'HostRegexp(`{subdomain:[^.]+}.dokku.me`)' + + run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.routes[0].priority}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "1" + + run /bin/bash -c "kubectl get ingressroutes.traefik.io ${EXACT_APP}-web-http-80-5000 -n default -o jsonpath='{.spec.routes[0].priority}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "" + + assert_http_localhost_response "http" "anything.dokku.me" "80" "/hello" "wildcard" + assert_http_localhost_response "http" "exact.dokku.me" "80" "/hello" "exact" +} diff --git a/tests/unit/scheduler-k3s-wildcard-2.bats b/tests/unit/scheduler-k3s-wildcard-2.bats new file mode 100644 index 000000000..4f7a5ed24 --- /dev/null +++ b/tests/unit/scheduler-k3s-wildcard-2.bats @@ -0,0 +1,80 @@ +#!/usr/bin/env bats + +load test_helper + +TEST_APP="rdmtestapp" + +setup_wildcard_tls() { + TLS=$BATS_TMPDIR/tls + mkdir -p $TLS + tar xf $BATS_TEST_DIRNAME/server_ssl_wildcard.tar -C $TLS + sudo chown -R dokku:dokku $TLS +} + +teardown_wildcard_tls() { + TLS=$BATS_TMPDIR/tls + rm -R $TLS +} + +setup() { + uninstall_k3s || true + global_setup + dokku nginx:stop + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" + setup_wildcard_tls +} + +teardown() { + global_teardown + dokku nginx:start + uninstall_k3s || true + teardown_wildcard_tls +} + +@test "(scheduler-k3s) [ingress] traefik serves a wildcard domain over https" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=traefik install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku domains:set $TEST_APP '*.dokku.me'" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku certs:add $TEST_APP $BATS_TMPDIR/tls/server.crt $BATS_TMPDIR/tls/server.key" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "sleep 30" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000-websecure -n default -o jsonpath='{.spec.routes[0].match}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output 'HostRegexp(`{subdomain:[^.]+}.dokku.me`)' + + run /bin/bash -c "kubectl get ingressroutes.traefik.io ${TEST_APP}-web-http-80-5000-websecure -n default -o jsonpath='{.spec.tls.secretName}'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "tls-${TEST_APP}" + + assert_http_redirect "http://wild.dokku.me" "https://wild.dokku.me/" + assert_http_localhost_response "https" "wild.dokku.me" "443" "" "python/http.server" +}