From 0431e4bf271f69df4edc87b30fd8b8440cdfaa1f Mon Sep 17 00:00:00 2001 From: Valentin Maerten Date: Sun, 19 Apr 2026 22:55:23 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=85=20test(failfast):=20use=20duration=20?= =?UTF-8?q?assertion=20instead=20of=20stdout=20to=20fix=20flake?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- executor_test.go | 70 +++++++++++++++---- testdata/failfast/default/Taskfile.yaml | 18 +++-- ...TestFailfast-Option-default-err-run.golden | 2 +- .../TestFailfast-Option-default.golden | 1 - testdata/failfast/task/Taskfile.yaml | 6 +- .../TestFailfast-Task-task-err-run.golden | 2 +- .../testdata/TestFailfast-Task-task.golden | 1 - 7 files changed, 74 insertions(+), 26 deletions(-) delete mode 100644 testdata/failfast/default/testdata/TestFailfast-Option-default.golden delete mode 100644 testdata/failfast/task/testdata/TestFailfast-Task-task.golden diff --git a/executor_test.go b/executor_test.go index d963fa6d..e11ef2d8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/sebdah/goldie/v2" "github.com/stretchr/testify/require" @@ -30,13 +31,15 @@ type ( // gen:fixtures`. ExecutorTest struct { TaskTest - task string - vars map[string]any - input string - executorOpts []task.ExecutorOption - wantSetupError bool - wantRunError bool - wantStatusError bool + task string + vars map[string]any + input string + executorOpts []task.ExecutorOption + wantSetupError bool + wantRunError bool + wantStatusError bool + skipOutputFixture bool + maxDuration time.Duration } ) @@ -113,6 +116,32 @@ func (opt *statusErrorTestOption) applyToExecutorTest(t *ExecutorTest) { t.wantStatusError = true } +// WithoutOutputFixture disables the stdout/stderr golden fixture comparison. +// Use for tasks with non-deterministic output by design (e.g. parallel deps +// cancelled mid-execution) where only the run error or timing matters. +func WithoutOutputFixture() ExecutorTestOption { + return &withoutOutputFixtureTestOption{} +} + +type withoutOutputFixtureTestOption struct{} + +func (opt *withoutOutputFixtureTestOption) applyToExecutorTest(t *ExecutorTest) { + t.skipOutputFixture = true +} + +// WithMaxDuration asserts the run phase completes within d. Use to verify +// that failfast/cancellation kicks in promptly instead of waiting for deps +// to finish naturally. +func WithMaxDuration(d time.Duration) ExecutorTestOption { + return &maxDurationTestOption{d: d} +} + +type maxDurationTestOption struct{ d time.Duration } + +func (opt *maxDurationTestOption) applyToExecutorTest(t *ExecutorTest) { + t.maxDuration = opt.d +} + // Helpers // writeFixtureErrRun is a wrapper for writing the output of an error during the @@ -172,7 +201,9 @@ func (tt *ExecutorTest) run(t *testing.T) { if err := e.Setup(); tt.wantSetupError { require.Error(t, err) tt.writeFixtureErrSetup(t, g, err) - tt.writeFixtureBuffer(t, g, buffer.buf) + if !tt.skipOutputFixture { + tt.writeFixtureBuffer(t, g, buffer.buf) + } return } else { require.NoError(t, err) @@ -190,10 +221,18 @@ func (tt *ExecutorTest) run(t *testing.T) { // Run the task and check for errors ctx := t.Context() - if err := e.Run(ctx, call); tt.wantRunError { + start := time.Now() + err := e.Run(ctx, call) + if tt.maxDuration > 0 { + require.Less(t, time.Since(start), tt.maxDuration, + "task took too long — failfast/cancellation likely did not trigger") + } + if tt.wantRunError { require.Error(t, err) tt.writeFixtureErrRun(t, g, err) - tt.writeFixtureBuffer(t, g, buffer.buf) + if !tt.skipOutputFixture { + tt.writeFixtureBuffer(t, g, buffer.buf) + } return } else { require.NoError(t, err) @@ -206,7 +245,9 @@ func (tt *ExecutorTest) run(t *testing.T) { } } - tt.writeFixtureBuffer(t, g, buffer.buf) + if !tt.skipOutputFixture { + tt.writeFixtureBuffer(t, g, buffer.buf) + } } // Run the test (with a name if it has one) @@ -1130,12 +1171,14 @@ func TestFailfast(t *testing.T) { NewExecutorTest(t, WithName("default"), + WithVar("SLEEP", "sleep 5 && "), WithExecutorOptions( task.WithDir("testdata/failfast/default"), task.WithSilent(true), task.WithFailfast(true), ), - WithPostProcessFn(PPSortedLines), + WithoutOutputFixture(), + WithMaxDuration(4*time.Second), WithRunError(), ) }) @@ -1149,7 +1192,8 @@ func TestFailfast(t *testing.T) { task.WithDir("testdata/failfast/task"), task.WithSilent(true), ), - WithPostProcessFn(PPSortedLines), + WithoutOutputFixture(), + WithMaxDuration(4*time.Second), WithRunError(), ) }) diff --git a/testdata/failfast/default/Taskfile.yaml b/testdata/failfast/default/Taskfile.yaml index 079ad05a..c7a85392 100644 --- a/testdata/failfast/default/Taskfile.yaml +++ b/testdata/failfast/default/Taskfile.yaml @@ -1,14 +1,20 @@ version: '3' +vars: + SLEEP: '' + tasks: default: deps: - - dep1 - - dep2 - - dep3 + - task: dep1 + vars: { SLEEP: '{{.SLEEP}}' } + - task: dep2 + vars: { SLEEP: '{{.SLEEP}}' } + - task: dep3 + vars: { SLEEP: '{{.SLEEP}}' } - dep4 - dep1: sleep 0.1 && echo 'dep1' - dep2: sleep 0.2 && echo 'dep2' - dep3: sleep 0.3 && echo 'dep3' + dep1: '{{.SLEEP}}echo ''dep1''' + dep2: '{{.SLEEP}}echo ''dep2''' + dep3: '{{.SLEEP}}echo ''dep3''' dep4: exit 1 diff --git a/testdata/failfast/default/testdata/TestFailfast-Option-default-err-run.golden b/testdata/failfast/default/testdata/TestFailfast-Option-default-err-run.golden index fccb62ab..cc6283cb 100644 --- a/testdata/failfast/default/testdata/TestFailfast-Option-default-err-run.golden +++ b/testdata/failfast/default/testdata/TestFailfast-Option-default-err-run.golden @@ -1 +1 @@ -task: Failed to run task "default": task: Failed to run task "dep4": exit status 1 +task: Failed to run task "default": task: Failed to run task "dep4": exit status 1 \ No newline at end of file diff --git a/testdata/failfast/default/testdata/TestFailfast-Option-default.golden b/testdata/failfast/default/testdata/TestFailfast-Option-default.golden deleted file mode 100644 index 8b137891..00000000 --- a/testdata/failfast/default/testdata/TestFailfast-Option-default.golden +++ /dev/null @@ -1 +0,0 @@ - diff --git a/testdata/failfast/task/Taskfile.yaml b/testdata/failfast/task/Taskfile.yaml index 078ed17d..42ddb697 100644 --- a/testdata/failfast/task/Taskfile.yaml +++ b/testdata/failfast/task/Taskfile.yaml @@ -9,7 +9,7 @@ tasks: - dep4 failfast: true - dep1: sleep 0.1 && echo 'dep1' - dep2: sleep 0.2 && echo 'dep2' - dep3: sleep 0.3 && echo 'dep3' + dep1: sleep 5 && echo 'dep1' + dep2: sleep 6 && echo 'dep2' + dep3: sleep 7 && echo 'dep3' dep4: exit 1 diff --git a/testdata/failfast/task/testdata/TestFailfast-Task-task-err-run.golden b/testdata/failfast/task/testdata/TestFailfast-Task-task-err-run.golden index fccb62ab..cc6283cb 100644 --- a/testdata/failfast/task/testdata/TestFailfast-Task-task-err-run.golden +++ b/testdata/failfast/task/testdata/TestFailfast-Task-task-err-run.golden @@ -1 +1 @@ -task: Failed to run task "default": task: Failed to run task "dep4": exit status 1 +task: Failed to run task "default": task: Failed to run task "dep4": exit status 1 \ No newline at end of file diff --git a/testdata/failfast/task/testdata/TestFailfast-Task-task.golden b/testdata/failfast/task/testdata/TestFailfast-Task-task.golden deleted file mode 100644 index 8b137891..00000000 --- a/testdata/failfast/task/testdata/TestFailfast-Task-task.golden +++ /dev/null @@ -1 +0,0 @@ -