diff --git a/docs/deployment/logs.md b/docs/deployment/logs.md index 8cbf451db..b29ac6e0a 100644 --- a/docs/deployment/logs.md +++ b/docs/deployment/logs.md @@ -356,26 +356,26 @@ dokku logs:set node-js-app vector-cron-sink "file://?path=/var/log/dokku/apps/no ##### Configuring the app label -Logs shipped by vector include the label `com.dokku.app-name`, which is an alias for the app name. This can be changed via the `app-label-alias` logs property with the `logs:set` command. Specifying a new alias will reload any running vector container. +Dokku labels every app container with `com.dokku.app-name`, and events shipped by vector carry that label as the field `label."com.dokku.app-name"`. Some sinks cannot use a field named that way - Loki label names, for instance, may only contain letters, digits and underscores - so the field can be renamed on the way to the sink via the `app-label-alias` logs property. Specifying a new alias will reload any running vector container. ```shell -# setting the sink value in quotes is encouraged to avoid -# issues with ampersand encoding in shell commands -dokku logs:set node-js-app app-label-alias "app-name" +dokku logs:set node-js-app app-label-alias "app_name" ``` +Events for `node-js-app` then carry `label.app_name` and no longer carry `label."com.dokku.app-name"`. + An alias may be removed by setting an empty value, which will also reload the running vector container. ```shell dokku logs:set node-js-app app-label-alias ``` -Only one alias may be specified on a per-app basis at a given time. +Only one alias may be specified on a per-app basis at a given time. Valid values start with a letter or number and may otherwise contain letters, numbers, underscores, periods and hyphens. App label aliases can also be specified globally by specifying the `--global` flag to `logs:set` with no app name specified: ```shell -dokku logs:set --global app-label-alias "app-name" +dokku logs:set --global app-label-alias "app_name" ``` As with app-specific label alias settings, the global value may also be cleared by setting no value. @@ -384,6 +384,10 @@ As with app-specific label alias settings, the global value may also be cleared dokku logs:set --global app-label-alias ``` +An app-specific value takes precedence over the global one, and is applied to that app's events whether they are shipped by the app's own `vector-sink` or by the global one. + +The alias only changes the shipped event. Containers are always discovered by the `com.dokku.app-name` label, so changing this property never affects which logs are collected, and a change takes effect on the next vector reload without redeploying the app. Cron events are unaffected in another respect too: `dokku_app` is read from the container label before the rename, so it holds the app name regardless of the configured alias. + ## Properties ### Settable properties @@ -393,7 +397,7 @@ dokku logs:set --global app-label-alias | Property | Scope | Default | Report flags | Description | |---|---|---|---|---| -| `app-label-alias` | app + global | `com.dokku.app-name` | `--logs-app-label-alias`, `--logs-global-app-label-alias`, `--logs-computed-app-label-alias` | Docker label key whose value is used to identify the app when shipping logs | +| `app-label-alias` | app + global | `com.dokku.app-name` | `--logs-app-label-alias`, `--logs-global-app-label-alias`, `--logs-computed-app-label-alias` | Field name the app name is shipped under, renamed from `com.dokku.app-name` on the event | | `max-size` | app + global | `10m` | `--logs-max-size`, `--logs-global-max-size`, `--logs-computed-max-size` | Maximum size of an individual log file before rotation | | `vector-image` | global only | _parsed from `plugins/logs/Dockerfile`_ | `--logs-global-vector-image`, `--logs-computed-vector-image` | Docker image used to run the vector log-shipper container | | `vector-networks` | global only | none | `--logs-global-vector-networks`, `--logs-computed-vector-networks` | Comma-separated list of docker networks the vector container is attached to | diff --git a/plugins/logs/functions.go b/plugins/logs/functions.go index c85775508..6495eb4c0 100644 --- a/plugins/logs/functions.go +++ b/plugins/logs/functions.go @@ -8,6 +8,7 @@ import ( "html/template" "os" "path/filepath" + "sort" "strings" "github.com/dokku/dokku/plugins/common" @@ -232,6 +233,19 @@ type vectorAppSinks struct { // CronRemapID is the component id for the remap transform on the cron branch CronRemapID string + // RelabelID is the component id for the remap transform renaming the app + // label on the non-cron branch + RelabelID string + + // LabelAlias is the label key the app name is shipped under for this scope. + // It is the AppLabelAlias constant unless the app-label-alias property is set + LabelAlias string + + // LabelAliasOverrides holds the apps within this scope whose own alias + // differs from LabelAlias. Only the global scope carries these, since a + // per-app source only ever covers one app + LabelAliasOverrides []vectorLabelAliasOverride + // Sink is the DSN for non-cron logs, empty when unset Sink string @@ -239,6 +253,16 @@ type vectorAppSinks struct { CronSink string } +// vectorLabelAliasOverride is the alias a single app ships its name under when +// it differs from the alias of the scope collecting it +type vectorLabelAliasOverride struct { + // AppName is the app the override applies to + AppName string + + // Alias is the label key the app name is shipped under + Alias string +} + // cronRouteTransforms returns the route and remap pair that splits a source // into cron and non-cron branches. The remap flattens the cron labels into // top-level fields because vector drops any event whose sink template @@ -247,12 +271,22 @@ type vectorAppSinks struct { // The route carries a single condition, so an event either matches it or falls // through to the reserved _unmatched output. A second route added later would // need a mutually exclusive condition, since route fans out to every match. -func cronRouteTransforms(routerID string, remapID string, sourceID string, hasSink bool) map[string]any { +// +// Any label rename is appended to the remap rather than given a component of +// its own, so that dokku_app is captured from the literal label before the +// rename runs and keeps its value regardless of the configured alias. +func cronRouteTransforms(scope vectorAppSinks, relabel string) map[string]any { + source := fmt.Sprintf(".dokku_app = to_string(%s) ?? \"\"\n.dokku_cron_id = to_string(%s) ?? \"\"", + vrlLabelPath(AppLabelAlias), vrlLabelPath(CronIDLabel)) + if relabel != "" { + source = fmt.Sprintf("%s\n%s", source, relabel) + } + return map[string]any{ - routerID: vectorRouteTransform{ + scope.RouterID: vectorRouteTransform{ Type: "route", - Inputs: []string{sourceID}, - RerouteUnmatched: hasSink, + Inputs: []string{scope.SourceID}, + RerouteUnmatched: scope.Sink != "", Route: map[string]vectorCondition{ CronRouteName: { Type: "vrl", @@ -260,11 +294,10 @@ func cronRouteTransforms(routerID string, remapID string, sourceID string, hasSi }, }, }, - remapID: vectorRemapTransform{ + scope.CronRemapID: vectorRemapTransform{ Type: "remap", - Inputs: []string{fmt.Sprintf("%s.%s", routerID, CronRouteName)}, - Source: fmt.Sprintf(".dokku_app = to_string(%s) ?? \"\"\n.dokku_cron_id = to_string(%s) ?? \"\"", - vrlLabelPath(AppLabelAlias), vrlLabelPath(CronIDLabel)), + Inputs: []string{fmt.Sprintf("%s.%s", scope.RouterID, CronRouteName)}, + Source: source, }, } } @@ -275,6 +308,92 @@ func vrlLabelPath(label string) string { return fmt.Sprintf(".label.%q", label) } +// vrlRenameLabel renders the assignment moving the dokku app label onto the +// supplied alias +func vrlRenameLabel(alias string) string { + return fmt.Sprintf("%s = del(%s)", vrlLabelPath(alias), vrlLabelPath(AppLabelAlias)) +} + +// vrlClause is one branch of a generated VRL conditional. An empty Condition +// renders as a trailing else +type vrlClause struct { + Condition string + Statement string +} + +// vrlIfChain renders clauses as a single if/else if/else statement +func vrlIfChain(clauses []vrlClause) string { + if len(clauses) == 1 && clauses[0].Condition == "" { + return clauses[0].Statement + } + + var chain strings.Builder + for i, clause := range clauses { + if i > 0 { + chain.WriteString(" else ") + } + if clause.Condition != "" { + chain.WriteString(fmt.Sprintf("if %s ", clause.Condition)) + } + chain.WriteString(fmt.Sprintf("{\n %s\n}", clause.Statement)) + } + + return chain.String() +} + +// relabelVRL renders the program renaming the dokku app label to the alias +// configured for the scope, or an empty string when there is nothing to rename. +// +// The rename happens on the event rather than on the container because dokku +// only ever labels containers com.dokku.app-name: pointing the source filter at +// any other label - which is what this property used to do - matches nothing at +// all and silently collects no logs. +// +// The global scope collects every app, so an app whose own alias differs from +// the global one gets a branch of its own here. Without that, a per-app alias +// would be silently ignored for any app shipping through the global sink. +func relabelVRL(scope vectorAppSinks) string { + alias := scope.LabelAlias + if alias == "" { + alias = AppLabelAlias + } + + if len(scope.LabelAliasOverrides) == 0 { + if alias == AppLabelAlias { + return "" + } + + return vrlRenameLabel(alias) + } + + clauses := []vrlClause{} + retained := []string{} + for _, override := range scope.LabelAliasOverrides { + if override.Alias == AppLabelAlias { + retained = append(retained, fmt.Sprintf("app != %q", override.AppName)) + continue + } + + clauses = append(clauses, vrlClause{ + Condition: fmt.Sprintf("app == %q", override.AppName), + Statement: vrlRenameLabel(override.Alias), + }) + } + + if alias != AppLabelAlias { + clauses = append(clauses, vrlClause{ + Condition: strings.Join(retained, " && "), + Statement: vrlRenameLabel(alias), + }) + } + + if len(clauses) == 0 { + return "" + } + + return fmt.Sprintf("app = to_string(%s) ?? \"\"\n%s", vrlLabelPath(AppLabelAlias), vrlIfChain(clauses)) +} + // buildVectorConfig assembles the vector configuration for the supplied scopes. // It performs no IO so that the generated shape can be asserted directly. func buildVectorConfig(scopes []vectorAppSinks) (vectorConfig, error) { @@ -293,12 +412,14 @@ func buildVectorConfig(scopes []vectorAppSinks) (vectorConfig, error) { IncludeLabels: scope.IncludeLabels, } + relabel := relabelVRL(scope) + sinkInputs := []string{scope.SourceID} if scope.CronSink != "" { if data.Transforms == nil { data.Transforms = map[string]any{} } - for id, transform := range cronRouteTransforms(scope.RouterID, scope.CronRemapID, scope.SourceID, scope.Sink != "") { + for id, transform := range cronRouteTransforms(scope, relabel) { data.Transforms[id] = transform } @@ -316,6 +437,21 @@ func buildVectorConfig(scopes []vectorAppSinks) (vectorConfig, error) { } if scope.Sink != "" { + // the cron branch renames within its own remap, so this transform + // only exists when there is a non-cron sink downstream to consume it + if relabel != "" { + if data.Transforms == nil { + data.Transforms = map[string]any{} + } + data.Transforms[scope.RelabelID] = vectorRemapTransform{ + Type: "remap", + Inputs: sinkInputs, + Source: relabel, + } + + sinkInputs = []string{scope.RelabelID} + } + sink, err := SinkValueToConfig(SinkValueToConfigInput{ SinkValue: scope.Sink, Inputs: sinkInputs, @@ -355,30 +491,50 @@ func buildVectorConfig(scopes []vectorAppSinks) (vectorConfig, error) { // vectorScopes collects the sink configuration for every app plus the global scope func vectorScopes() []vectorAppSinks { apps, _ := common.UnfilteredDokkuApps() + globalAlias := reportComputedAppLabelAlias("--global") + scopes := []vectorAppSinks{} + overrides := []vectorLabelAliasOverride{} for _, appName := range apps { inflectedAppName := strings.ReplaceAll(appName, ".", "-") + appAlias := reportComputedAppLabelAlias(appName) + if appAlias != globalAlias { + overrides = append(overrides, vectorLabelAliasOverride{ + AppName: appName, + Alias: appAlias, + }) + } + scopes = append(scopes, vectorAppSinks{ SourceID: fmt.Sprintf("docker-source:%s", inflectedAppName), - IncludeLabels: []string{fmt.Sprintf("%s=%s", reportComputedAppLabelAlias(appName), appName)}, + IncludeLabels: []string{fmt.Sprintf("%s=%s", AppLabelAlias, appName)}, SinkID: fmt.Sprintf("docker-sink:%s", inflectedAppName), CronSinkID: fmt.Sprintf("docker-cron-sink:%s", inflectedAppName), RouterID: fmt.Sprintf("docker-router:%s", inflectedAppName), CronRemapID: fmt.Sprintf("docker-cron-remap:%s", inflectedAppName), + RelabelID: fmt.Sprintf("docker-relabel:%s", inflectedAppName), + LabelAlias: appAlias, Sink: common.PropertyGet("logs", appName, "vector-sink"), CronSink: common.PropertyGet("logs", appName, "vector-cron-sink"), }) } + sort.Slice(overrides, func(i int, j int) bool { + return overrides[i].AppName < overrides[j].AppName + }) + return append(scopes, vectorAppSinks{ - SourceID: "docker-global-source", - IncludeLabels: []string{reportComputedAppLabelAlias("global")}, - SinkID: "docker-global-sink", - CronSinkID: "docker-global-cron-sink", - RouterID: "docker-global-router", - CronRemapID: "docker-global-cron-remap", - Sink: common.PropertyGet("logs", "--global", "vector-sink"), - CronSink: common.PropertyGet("logs", "--global", "vector-cron-sink"), + SourceID: "docker-global-source", + IncludeLabels: []string{AppLabelAlias}, + SinkID: "docker-global-sink", + CronSinkID: "docker-global-cron-sink", + RouterID: "docker-global-router", + CronRemapID: "docker-global-cron-remap", + RelabelID: "docker-global-relabel", + LabelAlias: globalAlias, + LabelAliasOverrides: overrides, + Sink: common.PropertyGet("logs", "--global", "vector-sink"), + CronSink: common.PropertyGet("logs", "--global", "vector-cron-sink"), }) } diff --git a/plugins/logs/functions_test.go b/plugins/logs/functions_test.go index 583228579..a5e1bd61d 100644 --- a/plugins/logs/functions_test.go +++ b/plugins/logs/functions_test.go @@ -2,8 +2,12 @@ package logs import ( "encoding/json" + "os" + "path/filepath" "strings" "testing" + + "github.com/dokku/dokku/plugins/common" ) func appScope(sink string, cronSink string) vectorAppSinks { @@ -14,6 +18,27 @@ func appScope(sink string, cronSink string) vectorAppSinks { CronSinkID: "docker-cron-sink:myapp", RouterID: "docker-router:myapp", CronRemapID: "docker-cron-remap:myapp", + RelabelID: "docker-relabel:myapp", + Sink: sink, + CronSink: cronSink, + } +} + +func aliasScope(sink string, cronSink string, alias string) vectorAppSinks { + scope := appScope(sink, cronSink) + scope.LabelAlias = alias + return scope +} + +func globalScope(sink string, cronSink string) vectorAppSinks { + return vectorAppSinks{ + SourceID: "docker-global-source", + IncludeLabels: []string{"com.dokku.app-name"}, + SinkID: "docker-global-sink", + CronSinkID: "docker-global-cron-sink", + RouterID: "docker-global-router", + CronRemapID: "docker-global-cron-remap", + RelabelID: "docker-global-relabel", Sink: sink, CronSink: cronSink, } @@ -131,16 +156,9 @@ func TestBuildVectorConfigBothSinks(t *testing.T) { } func TestBuildVectorConfigGlobalScope(t *testing.T) { - _, decoded := marshalConfig(t, []vectorAppSinks{{ - SourceID: "docker-global-source", - IncludeLabels: []string{"com.dokku.app-name"}, - SinkID: "docker-global-sink", - CronSinkID: "docker-global-cron-sink", - RouterID: "docker-global-router", - CronRemapID: "docker-global-cron-remap", - Sink: "console://?encoding[codec]=json", - CronSink: "console://?encoding[codec]=text", - }}) + _, decoded := marshalConfig(t, []vectorAppSinks{ + globalScope("console://?encoding[codec]=json", "console://?encoding[codec]=text"), + }) lookup(t, decoded, "transforms", "docker-global-router") lookup(t, decoded, "transforms", "docker-global-cron-remap") @@ -172,3 +190,215 @@ func TestBuildVectorConfigInvalidSink(t *testing.T) { t.Fatal("buildVectorConfig() expected an error for an invalid sink DSN") } } + +// TestBuildVectorConfigAliasKeepsSourceFilter is the guard for the bug this +// alias handling replaced: filtering the source on the alias matched no +// container at all, because dokku only ever labels containers with the literal +// key, so setting the property silently collected nothing. +func TestBuildVectorConfigAliasKeepsSourceFilter(t *testing.T) { + _, decoded := marshalConfig(t, []vectorAppSinks{ + aliasScope("console://?encoding[codec]=json", "", "app_name"), + }) + + labels := lookup(t, decoded, "sources", "docker-source:myapp", "include_labels") + if got := labels.([]interface{})[0]; got != "com.dokku.app-name=myapp" { + t.Errorf("include_labels[0] = %v, want com.dokku.app-name=myapp", got) + } +} + +func TestBuildVectorConfigAliasRelabelsPlainBranch(t *testing.T) { + _, decoded := marshalConfig(t, []vectorAppSinks{ + aliasScope("console://?encoding[codec]=json", "", "app_name"), + }) + + if got := lookup(t, decoded, "transforms", "docker-relabel:myapp", "type"); got != "remap" { + t.Errorf("relabel type = %v, want remap", got) + } + + inputs := lookup(t, decoded, "transforms", "docker-relabel:myapp", "inputs") + if got := inputs.([]interface{})[0]; got != "docker-source:myapp" { + t.Errorf("relabel inputs[0] = %v, want docker-source:myapp", got) + } + + source := lookup(t, decoded, "transforms", "docker-relabel:myapp", "source") + want := `.label."app_name" = del(.label."com.dokku.app-name")` + if source != want { + t.Errorf("relabel source = %v, want %v", source, want) + } + + sinkInputs := lookup(t, decoded, "sinks", "docker-sink:myapp", "inputs") + if got := sinkInputs.([]interface{})[0]; got != "docker-relabel:myapp" { + t.Errorf("sink inputs[0] = %v, want docker-relabel:myapp", got) + } +} + +// TestBuildVectorConfigAliasRelabelsBothBranches pins the rename onto the tail +// of the cron remap. dokku_app is captured from the literal label first, so it +// keeps its value no matter which alias the event is shipped under. +func TestBuildVectorConfigAliasRelabelsBothBranches(t *testing.T) { + _, decoded := marshalConfig(t, []vectorAppSinks{ + aliasScope("console://?encoding[codec]=json", "console://?encoding[codec]=text", "app_name"), + }) + + remapSource := lookup(t, decoded, "transforms", "docker-cron-remap:myapp", "source") + want := ".dokku_app = to_string(.label.\"com.dokku.app-name\") ?? \"\"\n" + + ".dokku_cron_id = to_string(.label.\"com.dokku.cron-id\") ?? \"\"\n" + + ".label.\"app_name\" = del(.label.\"com.dokku.app-name\")" + if remapSource != want { + t.Errorf("cron remap source = %v, want %v", remapSource, want) + } + + inputs := lookup(t, decoded, "transforms", "docker-relabel:myapp", "inputs") + if got := inputs.([]interface{})[0]; got != "docker-router:myapp._unmatched" { + t.Errorf("relabel inputs[0] = %v, want docker-router:myapp._unmatched", got) + } + + sinkInputs := lookup(t, decoded, "sinks", "docker-sink:myapp", "inputs") + if got := sinkInputs.([]interface{})[0]; got != "docker-relabel:myapp" { + t.Errorf("sink inputs[0] = %v, want docker-relabel:myapp", got) + } +} + +// TestBuildVectorConfigAliasCronSinkOnly covers the branch with nothing +// downstream to consume a relabel component: vector rejects a transform whose +// output no sink reads, so the rename has to stay inside the cron remap. +func TestBuildVectorConfigAliasCronSinkOnly(t *testing.T) { + _, decoded := marshalConfig(t, []vectorAppSinks{ + aliasScope("", "console://?encoding[codec]=text", "app_name"), + }) + + transforms := lookup(t, decoded, "transforms").(map[string]interface{}) + if _, ok := transforms["docker-relabel:myapp"]; ok { + t.Error("relabel transform should not exist without a plain sink") + } + + remapSource := lookup(t, decoded, "transforms", "docker-cron-remap:myapp", "source").(string) + if !strings.Contains(remapSource, `.label."app_name" = del(.label."com.dokku.app-name")`) { + t.Errorf("cron remap source %q missing the rename", remapSource) + } +} + +func setupScopesTest(t *testing.T, apps []string) { + t.Helper() + t.Setenv("PLUGIN_PATH", "/var/lib/dokku/plugins") + t.Setenv("PLUGIN_ENABLED_PATH", "/var/lib/dokku/plugins/enabled") + t.Setenv("DOKKU_LIB_ROOT", t.TempDir()) + t.Setenv("DOKKU_ROOT", t.TempDir()) + t.Setenv("DOKKU_SYSTEM_USER", "root") + t.Setenv("DOKKU_SYSTEM_GROUP", "root") + + if err := common.PropertySetup("logs"); err != nil { + t.Fatalf("PropertySetup: %v", err) + } + + for _, appName := range apps { + if err := os.MkdirAll(filepath.Join(os.Getenv("DOKKU_ROOT"), appName), 0755); err != nil { + t.Fatalf("MkdirAll %s: %v", appName, err) + } + } +} + +func scopeByID(t *testing.T, scopes []vectorAppSinks, sourceID string) vectorAppSinks { + t.Helper() + + for _, scope := range scopes { + if scope.SourceID == sourceID { + return scope + } + } + + t.Fatalf("no scope with source id %q", sourceID) + return vectorAppSinks{} +} + +// TestVectorScopesGlobalAliasIgnoresAppNamedGlobal pins the global scope to the +// --global property. It used to resolve against an app literally named global, +// which would have let that app's own alias drive every other app's shipping. +func TestVectorScopesGlobalAliasIgnoresAppNamedGlobal(t *testing.T) { + setupScopesTest(t, []string{"global", "myapp"}) + + if err := common.PropertyWrite("logs", "--global", "app-label-alias", "gname"); err != nil { + t.Fatalf("PropertyWrite --global: %v", err) + } + if err := common.PropertyWrite("logs", "global", "app-label-alias", "hijack"); err != nil { + t.Fatalf("PropertyWrite global: %v", err) + } + + scopes := vectorScopes() + + globalScope := scopeByID(t, scopes, "docker-global-source") + if globalScope.LabelAlias != "gname" { + t.Errorf("global LabelAlias = %q, want gname", globalScope.LabelAlias) + } + + // the app named global differs from the global alias, so it is the only + // scope that should be listed as an override + want := []vectorLabelAliasOverride{{AppName: "global", Alias: "hijack"}} + if len(globalScope.LabelAliasOverrides) != 1 || globalScope.LabelAliasOverrides[0] != want[0] { + t.Errorf("global LabelAliasOverrides = %v, want %v", globalScope.LabelAliasOverrides, want) + } + + appScope := scopeByID(t, scopes, "docker-source:myapp") + if appScope.LabelAlias != "gname" { + t.Errorf("myapp LabelAlias = %q, want gname", appScope.LabelAlias) + } + if got := appScope.IncludeLabels[0]; got != "com.dokku.app-name=myapp" { + t.Errorf("myapp include_labels[0] = %q, want com.dokku.app-name=myapp", got) + } +} + +func TestRelabelVRLDefaultAlias(t *testing.T) { + for _, alias := range []string{"", AppLabelAlias} { + if got := relabelVRL(aliasScope("console://", "", alias)); got != "" { + t.Errorf("relabelVRL(%q) = %q, want an empty string", alias, got) + } + } +} + +// TestRelabelVRLGlobalOverrides covers the global scope, which collects every +// app: an app whose own alias differs from the global one needs a branch here, +// or its alias would be silently dropped whenever it ships through the global +// sink. +func TestRelabelVRLGlobalOverrides(t *testing.T) { + prelude := "app = to_string(.label.\"com.dokku.app-name\") ?? \"\"\n" + + tests := []struct { + name string + alias string + overrides []vectorLabelAliasOverride + want string + }{ + { + name: "renaming override under a default global alias", + alias: AppLabelAlias, + overrides: []vectorLabelAliasOverride{{AppName: "appa", Alias: "foo"}}, + want: prelude + "if app == \"appa\" {\n .label.\"foo\" = del(.label.\"com.dokku.app-name\")\n}", + }, + { + name: "override pinned back to the default label", + alias: "gname", + overrides: []vectorLabelAliasOverride{{AppName: "appb", Alias: AppLabelAlias}}, + want: prelude + "if app != \"appb\" {\n .label.\"gname\" = del(.label.\"com.dokku.app-name\")\n}", + }, + { + name: "both kinds of override at once", + alias: "gname", + overrides: []vectorLabelAliasOverride{ + {AppName: "appa", Alias: "foo"}, + {AppName: "appb", Alias: AppLabelAlias}, + }, + want: prelude + "if app == \"appa\" {\n .label.\"foo\" = del(.label.\"com.dokku.app-name\")\n}" + + " else if app != \"appb\" {\n .label.\"gname\" = del(.label.\"com.dokku.app-name\")\n}", + }, + } + + for _, test := range tests { + scope := globalScope("console://", "") + scope.LabelAlias = test.alias + scope.LabelAliasOverrides = test.overrides + + if got := relabelVRL(scope); got != test.want { + t.Errorf("%s: relabelVRL() = %q, want %q", test.name, got, test.want) + } + } +} diff --git a/plugins/logs/set.go b/plugins/logs/set.go index 31f711130..848c8484d 100644 --- a/plugins/logs/set.go +++ b/plugins/logs/set.go @@ -3,6 +3,7 @@ package logs import ( "errors" "fmt" + "regexp" "strconv" "strings" @@ -10,6 +11,10 @@ import ( ) func validateSetValue(appName string, key string, value string) error { + if key == "app-label-alias" { + return validateAppLabelAlias(appName, value) + } + if key == "max-size" { return validateMaxSize(appName, value) } @@ -29,6 +34,22 @@ func validateSetValue(appName string, key string, value string) error { return nil } +// appLabelAliasPattern matches label keys that are safe to both use as a docker +// label and quote into the generated vector remap program +var appLabelAliasPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`) + +func validateAppLabelAlias(appName string, value string) error { + if value == "" { + return nil + } + + if !appLabelAliasPattern.MatchString(value) { + return errors.New("Invalid app-label-alias value, must start with a letter or number and contain only letters, numbers, and any of [_, ., -]") + } + + return nil +} + func validateMaxSize(appName string, value string) error { if value == "" { return nil diff --git a/tests.mk b/tests.mk index be7eb8a6b..7d07723f7 100644 --- a/tests.mk +++ b/tests.mk @@ -182,6 +182,7 @@ go-tests: @$(MAKE) go-test-plugin PLUGIN_NAME=common @$(MAKE) go-test-plugin PLUGIN_NAME=config @$(MAKE) go-test-plugin PLUGIN_NAME=docker-options + @$(MAKE) go-test-plugin PLUGIN_NAME=logs @$(MAKE) go-test-plugin PLUGIN_NAME=network @$(MAKE) go-test-plugin PLUGIN_NAME=buildpacks @$(MAKE) go-test-plugin PLUGIN_NAME=scheduler-k3s diff --git a/tests/unit/logs.bats b/tests/unit/logs.bats index 77f1c43fe..e3151e478 100644 --- a/tests/unit/logs.bats +++ b/tests/unit/logs.bats @@ -713,9 +713,34 @@ teardown() { echo "status: $status" assert_success + # the relabel branches are generated VRL too, and a syntax error there would + # take the whole config down rather than just the rename + run /bin/bash -c "dokku logs:set --global vector-sink console://?encoding[codec]=json" + assert_success + + run /bin/bash -c "dokku logs:set --global app-label-alias global_alt_name" + assert_success + + run /bin/bash -c "dokku logs:set $TEST_APP app-label-alias app_alt_name" + assert_success + + run /bin/bash -c "docker exec vector-vector-1 vector validate --no-environment /etc/vector/vector.json" + echo "output: $output" + echo "status: $status" + assert_success + run /bin/bash -c "dokku logs:vector-stop 2>&1" assert_success + run /bin/bash -c "dokku logs:set $TEST_APP app-label-alias" + assert_success + + run /bin/bash -c "dokku logs:set --global app-label-alias" + assert_success + + run /bin/bash -c "dokku logs:set --global vector-sink" + assert_success + run /bin/bash -c "dokku logs:set $TEST_APP vector-cron-sink" assert_success @@ -723,6 +748,48 @@ teardown() { assert_success } +# the regression test for the alias silently disabling collection: the source +# has to keep filtering on the label dokku applies, while the event that comes +# out the other end carries the alias instead +@test "(logs) a non-default app-label-alias still ships logs" { + run create_app + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs:set $TEST_APP vector-sink 'console://?encoding[codec]=json'" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs:set --global app-label-alias alt_name" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs:vector-start 2>&1" + echo "output: $output" + echo "status: $status" + assert_success + + run start_vector_probe VECTOR_ALIAS_OK + echo "output: $output" + echo "status: $status" + assert_success + + run wait_for_vector_alias_event VECTOR_ALIAS_OK alt_name + echo "output: $output" + echo "status: $status" + dump_vector_diagnostics "$TEST_APP" + assert_success + + run /bin/bash -c "docker container rm --force vector-alias-probe" + assert_success + + run /bin/bash -c "dokku logs:vector-stop 2>&1" + assert_success +} + @test "(logs:report) global-vector-image and global-vector-networks raw" { run create_app assert_success @@ -927,7 +994,13 @@ teardown() { assert_success assert_output "com.dokku.app-name=$TEST_APP" - run /bin/bash -c "dokku logs:set --global app-label-alias global-alt-name" 2>&1 + run /bin/bash -c "jq -r '.transforms' /var/lib/dokku/data/logs/vector.json" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "null" + + run /bin/bash -c "dokku logs:set --global app-label-alias global_alt_name" 2>&1 echo "output: $output" echo "status: $status" assert_success @@ -937,20 +1010,34 @@ teardown() { echo "output: $output" echo "status: $status" assert_success - assert_output_contains "global-alt-name" + assert_output_contains "global_alt_name" run /bin/bash -c "cat /var/lib/dokku/data/logs/vector.json" echo "output: $output" echo "status: $status" assert_success + # the source filter never moves off the label dokku actually applies, or the + # source would match no container at all and silently collect nothing run /bin/bash -c "jq -r '.sources[\"docker-source:$TEST_APP\"].include_labels[0]' /var/lib/dokku/data/logs/vector.json" echo "output: $output" echo "status: $status" assert_success - assert_output "global-alt-name=$TEST_APP" + assert_output "com.dokku.app-name=$TEST_APP" - run /bin/bash -c "dokku logs:set --global app-label-alias alt-name" 2>&1 + run /bin/bash -c "jq -r '.transforms[\"docker-relabel:$TEST_APP\"].source' /var/lib/dokku/data/logs/vector.json" + echo "output: $output" + echo "status: $status" + assert_success + assert_output '.label."global_alt_name" = del(.label."com.dokku.app-name")' + + run /bin/bash -c "jq -r '.sinks[\"docker-sink:$TEST_APP\"].inputs[0]' /var/lib/dokku/data/logs/vector.json" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "docker-relabel:$TEST_APP" + + run /bin/bash -c "dokku logs:set $TEST_APP app-label-alias alt_name" 2>&1 echo "output: $output" echo "status: $status" assert_success @@ -960,13 +1047,94 @@ teardown() { echo "output: $output" echo "status: $status" assert_success - assert_output_contains "alt-name" + assert_output_contains "alt_name" run /bin/bash -c "jq -r '.sources[\"docker-source:$TEST_APP\"].include_labels[0]' /var/lib/dokku/data/logs/vector.json" echo "output: $output" echo "status: $status" assert_success - assert_output "alt-name=$TEST_APP" + assert_output "com.dokku.app-name=$TEST_APP" + + run /bin/bash -c "jq -r '.transforms[\"docker-relabel:$TEST_APP\"].source' /var/lib/dokku/data/logs/vector.json" + echo "output: $output" + echo "status: $status" + assert_success + assert_output '.label."alt_name" = del(.label."com.dokku.app-name")' + + run /bin/bash -c "dokku logs:set $TEST_APP app-label-alias" 2>&1 + echo "output: $output" + echo "status: $status" + assert_success +} + +@test "(logs:set) app-label-alias rejects an unusable label key" { + run create_app + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs:set $TEST_APP app-label-alias 'not a label' 2>&1" + echo "output: $output" + echo "status: $status" + assert_failure + assert_output_contains "Invalid app-label-alias value" + + run /bin/bash -c "dokku logs:set --global app-label-alias '_leading_underscore' 2>&1" + echo "output: $output" + echo "status: $status" + assert_failure + assert_output_contains "Invalid app-label-alias value" +} + +# an app whose own alias differs from the global one is collected by the global +# source too, so the global pipeline carries a branch for it +@test "(logs) vector.json global relabel honors a per-app alias" { + run create_app + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku logs:set --global vector-sink console://?encoding[codec]=json" + assert_success + + run /bin/bash -c "dokku logs:set --global app-label-alias global_alt_name" + assert_success + + run /bin/bash -c "dokku logs:set $TEST_APP app-label-alias app_alt_name" + assert_success + + run /bin/bash -c "jq -r '.transforms[\"docker-global-relabel\"].source' /var/lib/dokku/data/logs/vector.json" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "if app == \"$TEST_APP\"" + assert_output_contains '.label."app_alt_name" = del(.label."com.dokku.app-name")' + assert_output_contains '.label."global_alt_name" = del(.label."com.dokku.app-name")' + + run /bin/bash -c "jq -r '.sinks[\"docker-global-sink\"].inputs[0]' /var/lib/dokku/data/logs/vector.json" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "docker-global-relabel" + + # a per-app alias matching the global one needs no branch of its own + run /bin/bash -c "dokku logs:set $TEST_APP app-label-alias global_alt_name" + assert_success + + run /bin/bash -c "jq -r '.transforms[\"docker-global-relabel\"].source' /var/lib/dokku/data/logs/vector.json" + echo "output: $output" + echo "status: $status" + assert_success + assert_output '.label."global_alt_name" = del(.label."com.dokku.app-name")' + + run /bin/bash -c "dokku logs:set $TEST_APP app-label-alias" + assert_success + + run /bin/bash -c "dokku logs:set --global app-label-alias" + assert_success + + run /bin/bash -c "dokku logs:set --global vector-sink" + assert_success } @test "(logs) logs:set max-size with alternate log-driver daemon" { @@ -1256,6 +1424,34 @@ wait_for_vector_route() { return 1 } +start_vector_probe() { + declare desc="runs a container carrying only the label dokku applies, emitting a marker for long enough for vector to attach" + declare MARKER="$1" + + docker container run --detach --name vector-alias-probe \ + --label "com.dokku.app-name=$TEST_APP" \ + gliderlabs/herokuish \ + bash -c "for i in \$(seq 1 30); do echo $MARKER; sleep 1; done" +} + +wait_for_vector_alias_event() { + declare desc="waits for a marker to arrive carrying the configured alias in place of the default label" + declare MARKER="$1" ALIAS="$2" + local i + + for i in $(seq 1 60); do + if docker logs vector-vector-1 2>/dev/null | grep "$MARKER" \ + | jq -e --arg alias "$ALIAS" \ + 'select(.label[$alias] != null and .label["com.dokku.app-name"] == null)' >/dev/null 2>/dev/null; then + return 0 + fi + sleep 1 + done + + echo "timed out waiting for a $MARKER event labelled $ALIAS" + return 1 +} + wait_for_vector_cron_event() { declare desc="waits for a marker to arrive on the cron branch, carrying the fields the remap adds" declare MARKER="$1" CRON_ID="$2"