From 3556942516786e17f313a59fed7fb3e1a1a280bd Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Sun, 18 Feb 2018 09:50:39 -0300 Subject: [PATCH] Improve nested variables support Closes #76 #89 #77 #83 --- .gitignore | 2 + Taskvars.yml | 1 + internal/compiler/v1/compiler_v1.go | 11 +- internal/compiler/v2/compiler_v2.go | 104 ++++++++++++++++++ task.go | 79 +++++++------ task_test.go | 71 +++++++++--- testdata/vars/{ => v1}/.gitignore | 0 testdata/vars/{ => v1}/Taskfile.yml | 0 testdata/vars/{ => v1}/Taskvars.yml | 0 testdata/vars/{ => v1}/multiline/Taskfile.yml | 0 testdata/vars/v2/.gitignore | 1 + testdata/vars/v2/Taskfile.yml | 50 +++++++++ testdata/vars/v2/Taskvars.yml | 12 ++ testdata/vars/v2/multiline/Taskfile.yml | 45 ++++++++ 14 files changed, 319 insertions(+), 57 deletions(-) create mode 100644 internal/compiler/v2/compiler_v2.go rename testdata/vars/{ => v1}/.gitignore (100%) rename testdata/vars/{ => v1}/Taskfile.yml (100%) rename testdata/vars/{ => v1}/Taskvars.yml (100%) rename testdata/vars/{ => v1}/multiline/Taskfile.yml (100%) create mode 100644 testdata/vars/v2/.gitignore create mode 100644 testdata/vars/v2/Taskfile.yml create mode 100644 testdata/vars/v2/Taskvars.yml create mode 100644 testdata/vars/v2/multiline/Taskfile.yml diff --git a/.gitignore b/.gitignore index 40625d50..15c634ca 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ ./task dist/ + +.DS_Store diff --git a/Taskvars.yml b/Taskvars.yml index 0dda4d06..7470fbd0 100644 --- a/Taskvars.yml +++ b/Taskvars.yml @@ -7,6 +7,7 @@ GO_PACKAGES: ./internal/args ./internal/compiler ./internal/compiler/v1 + ./internal/compiler/v2 ./internal/execext ./internal/logger ./internal/status diff --git a/internal/compiler/v1/compiler_v1.go b/internal/compiler/v1/compiler_v1.go index 5edd6a9b..7c56c57e 100644 --- a/internal/compiler/v1/compiler_v1.go +++ b/internal/compiler/v1/compiler_v1.go @@ -122,7 +122,7 @@ func (c *CompilerV1) HandleDynamicVar(v taskfile.Var) (string, error) { Stderr: c.Logger.Stderr, } if err := execext.RunCommand(opts); err != nil { - return "", &dynamicVarError{cause: err, cmd: opts.Command} + return "", fmt.Errorf(`task: Command "%s" in taskvars file failed: %s`, opts.Command, err) } // Trim a single trailing newline from the result to make most command @@ -134,12 +134,3 @@ func (c *CompilerV1) HandleDynamicVar(v taskfile.Var) (string, error) { return result, nil } - -type dynamicVarError struct { - cause error - cmd string -} - -func (err *dynamicVarError) Error() string { - return fmt.Sprintf(`task: Command "%s" in taskvars file failed: %s`, err.cmd, err.cause) -} diff --git a/internal/compiler/v2/compiler_v2.go b/internal/compiler/v2/compiler_v2.go new file mode 100644 index 00000000..0b67f7a6 --- /dev/null +++ b/internal/compiler/v2/compiler_v2.go @@ -0,0 +1,104 @@ +package v2 + +import ( + "bytes" + "fmt" + "strings" + "sync" + + "github.com/go-task/task/internal/compiler" + "github.com/go-task/task/internal/execext" + "github.com/go-task/task/internal/logger" + "github.com/go-task/task/internal/taskfile" + "github.com/go-task/task/internal/templater" +) + +var _ compiler.Compiler = &CompilerV2{} + +type CompilerV2 struct { + Dir string + Vars taskfile.Vars + + Logger *logger.Logger + + dynamicCache map[string]string + muDynamicCache sync.Mutex +} + +// GetVariables returns fully resolved variables following the priority order: +// 1. Task variables +// 2. Call variables +// 3. Taskvars file variables +// 4. Environment variables +func (c *CompilerV2) GetVariables(t *taskfile.Task, call taskfile.Call) (taskfile.Vars, error) { + vr := varResolver{c: c, vars: compiler.GetEnviron()} + vr.merge(c.Vars) + vr.merge(c.Vars) + vr.merge(call.Vars) + vr.merge(call.Vars) + vr.merge(t.Vars) + vr.merge(t.Vars) + return vr.vars, vr.err +} + +type varResolver struct { + c *CompilerV2 + vars taskfile.Vars + err error +} + +func (vr *varResolver) merge(vars taskfile.Vars) { + if vr.err != nil { + return + } + tr := templater.Templater{Vars: vr.vars} + for k, v := range vars { + v = taskfile.Var{ + Static: tr.Replace(v.Static), + Sh: tr.Replace(v.Sh), + } + static, err := vr.c.HandleDynamicVar(v) + if err != nil { + vr.err = err + return + } + vr.vars[k] = taskfile.Var{Static: static} + } + vr.err = tr.Err() +} + +func (c *CompilerV2) HandleDynamicVar(v taskfile.Var) (string, error) { + if v.Static != "" || v.Sh == "" { + return v.Static, nil + } + + c.muDynamicCache.Lock() + defer c.muDynamicCache.Unlock() + + if c.dynamicCache == nil { + c.dynamicCache = make(map[string]string, 30) + } + if result, ok := c.dynamicCache[v.Sh]; ok { + return result, nil + } + + var stdout bytes.Buffer + opts := &execext.RunCommandOptions{ + Command: v.Sh, + Dir: c.Dir, + Stdout: &stdout, + Stderr: c.Logger.Stderr, + } + if err := execext.RunCommand(opts); err != nil { + return "", fmt.Errorf(`task: Command "%s" in taskvars file failed: %s`, opts.Command, err) + } + + // Trim a single trailing newline from the result to make most command + // output easier to use in shell commands. + result := strings.TrimSuffix(stdout.String(), "\n") + + c.dynamicCache[v.Sh] = result + c.Logger.VerboseErrf(`task: dynamic variable: '%s' result: '%s'`, v.Sh, result) + + return result, nil +} diff --git a/task.go b/task.go index ba374f3a..c725864b 100644 --- a/task.go +++ b/task.go @@ -9,6 +9,7 @@ import ( "github.com/go-task/task/internal/compiler" compilerv1 "github.com/go-task/task/internal/compiler/v1" + compilerv2 "github.com/go-task/task/internal/compiler/v2" "github.com/go-task/task/internal/execext" "github.com/go-task/task/internal/logger" "github.com/go-task/task/internal/taskfile" @@ -49,37 +50,8 @@ type Executor struct { // Run runs Task func (e *Executor) Run(calls ...taskfile.Call) error { - if e.Context == nil { - e.Context = context.Background() - } - if e.Stdin == nil { - e.Stdin = os.Stdin - } - if e.Stdout == nil { - e.Stdout = os.Stdout - } - if e.Stderr == nil { - e.Stderr = os.Stderr - } - if e.Logger == nil { - e.Logger = &logger.Logger{ - Stdout: e.Stdout, - Stderr: e.Stderr, - Verbose: e.Verbose, - } - } - // TODO: Add version 2 - if e.Compiler == nil { - e.Compiler = &compilerv1.CompilerV1{ - Dir: e.Dir, - Vars: e.taskvars, - Logger: e.Logger, - } - } - - e.taskCallCount = make(map[string]*int32, len(e.Taskfile.Tasks)) - for k := range e.Taskfile.Tasks { - e.taskCallCount[k] = new(int32) + if err := e.setup(); err != nil { + return err } // check if given tasks exist @@ -103,6 +75,51 @@ func (e *Executor) Run(calls ...taskfile.Call) error { return nil } +func (e *Executor) setup() error { + if e.Taskfile.Version == 0 { + e.Taskfile.Version = 1 + } + if e.Context == nil { + e.Context = context.Background() + } + if e.Stdin == nil { + e.Stdin = os.Stdin + } + if e.Stdout == nil { + e.Stdout = os.Stdout + } + if e.Stderr == nil { + e.Stderr = os.Stderr + } + e.Logger = &logger.Logger{ + Stdout: e.Stdout, + Stderr: e.Stderr, + Verbose: e.Verbose, + } + switch e.Taskfile.Version { + case 1: + e.Compiler = &compilerv1.CompilerV1{ + Dir: e.Dir, + Vars: e.taskvars, + Logger: e.Logger, + } + case 2: + e.Compiler = &compilerv2.CompilerV2{ + Dir: e.Dir, + Vars: e.taskvars, + Logger: e.Logger, + } + default: + return fmt.Errorf(`task: Unrecognized Taskfile version "%d"`, e.Taskfile.Version) + } + + e.taskCallCount = make(map[string]*int32, len(e.Taskfile.Tasks)) + for k := range e.Taskfile.Tasks { + e.taskCallCount[k] = new(int32) + } + return nil +} + // RunTask runs a task by its name func (e *Executor) RunTask(ctx context.Context, call taskfile.Call) error { t, err := e.CompiledTask(call) diff --git a/task_test.go b/task_test.go index f1931f68..78bfb1c1 100644 --- a/task_test.go +++ b/task_test.go @@ -67,9 +67,9 @@ func TestEnv(t *testing.T) { tt.Run(t) } -func TestVars(t *testing.T) { +func TestVarsV1(t *testing.T) { tt := fileContentTest{ - Dir: "testdata/vars", + Dir: "testdata/vars/v1", Target: "default", TrimSpace: true, Files: map[string]string{ @@ -103,30 +103,69 @@ func TestVars(t *testing.T) { tt.Target = "hello" tt.Run(t) } -func TestMultilineVars(t *testing.T) { + +func TestVarsV2(t *testing.T) { tt := fileContentTest{ - Dir: "testdata/vars/multiline", + Dir: "testdata/vars/v2", Target: "default", - TrimSpace: false, + TrimSpace: true, Files: map[string]string{ - // Note: - // - task does not strip a trailing newline from var entries - // - task strips one trailing newline from shell output - // - the cat command adds a trailing newline - "echo_foobar.txt": "foo\nbar\n", - "echo_n_foobar.txt": "foo\nbar\n", - "echo_n_multiline.txt": "\n\nfoo\n bar\nfoobar\n\nbaz\n\n", - "var_multiline.txt": "\n\nfoo\n bar\nfoobar\n\nbaz\n\n\n", - "var_catlines.txt": " foo bar foobar baz \n", - "var_enumfile.txt": "0:\n1:\n2:foo\n3: bar\n4:foobar\n5:\n6:baz\n7:\n8:\n", + "foo.txt": "foo", + "bar.txt": "bar", + "baz.txt": "baz", + "tmpl_foo.txt": "foo", + "tmpl_bar.txt": "bar", + "tmpl_foo2.txt": "foo2", + "tmpl_bar2.txt": "bar2", + "shtmpl_foo.txt": "foo", + "shtmpl_foo2.txt": "foo2", + "nestedtmpl_foo.txt": "", + "nestedtmpl_foo2.txt": "foo2", + "foo2.txt": "foo2", + "bar2.txt": "bar2", + "baz2.txt": "baz2", + "tmpl2_foo.txt": "", + "tmpl2_foo2.txt": "foo2", + "tmpl2_bar.txt": "", + "tmpl2_bar2.txt": "bar2", + "shtmpl2_foo.txt": "", + "shtmpl2_foo2.txt": "foo2", + "nestedtmpl2_foo2.txt": "", + "override.txt": "bar", }, } tt.Run(t) + // Ensure identical results when running hello task directly. + tt.Target = "hello" + tt.Run(t) +} + +func TestMultilineVars(t *testing.T) { + for _, dir := range []string{"testdata/vars/v1/multiline", "testdata/vars/v2/multiline"} { + tt := fileContentTest{ + Dir: dir, + Target: "default", + TrimSpace: false, + Files: map[string]string{ + // Note: + // - task does not strip a trailing newline from var entries + // - task strips one trailing newline from shell output + // - the cat command adds a trailing newline + "echo_foobar.txt": "foo\nbar\n", + "echo_n_foobar.txt": "foo\nbar\n", + "echo_n_multiline.txt": "\n\nfoo\n bar\nfoobar\n\nbaz\n\n", + "var_multiline.txt": "\n\nfoo\n bar\nfoobar\n\nbaz\n\n\n", + "var_catlines.txt": " foo bar foobar baz \n", + "var_enumfile.txt": "0:\n1:\n2:foo\n3: bar\n4:foobar\n5:\n6:baz\n7:\n8:\n", + }, + } + tt.Run(t) + } } func TestVarsInvalidTmpl(t *testing.T) { const ( - dir = "testdata/vars" + dir = "testdata/vars/v1" target = "invalid-var-tmpl" expectError = "template: :1: unexpected EOF" ) diff --git a/testdata/vars/.gitignore b/testdata/vars/v1/.gitignore similarity index 100% rename from testdata/vars/.gitignore rename to testdata/vars/v1/.gitignore diff --git a/testdata/vars/Taskfile.yml b/testdata/vars/v1/Taskfile.yml similarity index 100% rename from testdata/vars/Taskfile.yml rename to testdata/vars/v1/Taskfile.yml diff --git a/testdata/vars/Taskvars.yml b/testdata/vars/v1/Taskvars.yml similarity index 100% rename from testdata/vars/Taskvars.yml rename to testdata/vars/v1/Taskvars.yml diff --git a/testdata/vars/multiline/Taskfile.yml b/testdata/vars/v1/multiline/Taskfile.yml similarity index 100% rename from testdata/vars/multiline/Taskfile.yml rename to testdata/vars/v1/multiline/Taskfile.yml diff --git a/testdata/vars/v2/.gitignore b/testdata/vars/v2/.gitignore new file mode 100644 index 00000000..2211df63 --- /dev/null +++ b/testdata/vars/v2/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/testdata/vars/v2/Taskfile.yml b/testdata/vars/v2/Taskfile.yml new file mode 100644 index 00000000..8f2324eb --- /dev/null +++ b/testdata/vars/v2/Taskfile.yml @@ -0,0 +1,50 @@ +version: 2 +tasks: + default: + deps: [hello] + + hello: + cmds: + - echo {{.FOO}} > foo.txt + - echo {{.BAR}} > bar.txt + - echo {{.BAZ}} > baz.txt + - echo '{{.TMPL_FOO}}' > tmpl_foo.txt + - echo '{{.TMPL_BAR}}' > tmpl_bar.txt + - echo '{{.TMPL_FOO2}}' > tmpl_foo2.txt + - echo '{{.TMPL_BAR2}}' > tmpl_bar2.txt + - echo '{{.SHTMPL_FOO}}' > shtmpl_foo.txt + - echo '{{.SHTMPL_FOO2}}' > shtmpl_foo2.txt + - echo '{{.NESTEDTMPL_FOO}}' > nestedtmpl_foo.txt + - echo '{{.NESTEDTMPL_FOO2}}' > nestedtmpl_foo2.txt + - echo {{.FOO2}} > foo2.txt + - echo {{.BAR2}} > bar2.txt + - echo {{.BAZ2}} > baz2.txt + - echo '{{.TMPL2_FOO}}' > tmpl2_foo.txt + - echo '{{.TMPL2_BAR}}' > tmpl2_bar.txt + - echo '{{.TMPL2_FOO2}}' > tmpl2_foo2.txt + - echo '{{.TMPL2_BAR2}}' > tmpl2_bar2.txt + - echo '{{.SHTMPL2_FOO}}' > shtmpl2_foo.txt + - echo '{{.SHTMPL2_FOO2}}' > shtmpl2_foo2.txt + - echo '{{.NESTEDTMPL2_FOO2}}' > nestedtmpl2_foo2.txt + - echo {{.OVERRIDE}} > override.txt + vars: + FOO: foo + BAR: $echo bar + BAZ: + sh: echo baz + TMPL_FOO: "{{.FOO}}" + TMPL_BAR: "{{.BAR}}" + TMPL_FOO2: "{{.FOO2}}" + TMPL_BAR2: "{{.BAR2}}" + SHTMPL_FOO: + sh: "echo '{{.FOO}}'" + SHTMPL_FOO2: + sh: "echo '{{.FOO2}}'" + NESTEDTMPL_FOO: "{{.TMPL_FOO}}" + NESTEDTMPL_FOO2: "{{.TMPL2_FOO2}}" + OVERRIDE: "bar" + + invalid-var-tmpl: + vars: + CHARS: "abcd" + INVALID: "{{range .CHARS}}no end" diff --git a/testdata/vars/v2/Taskvars.yml b/testdata/vars/v2/Taskvars.yml new file mode 100644 index 00000000..7b5bfb33 --- /dev/null +++ b/testdata/vars/v2/Taskvars.yml @@ -0,0 +1,12 @@ +FOO2: foo2 +BAR2: $echo bar2 +BAZ2: + sh: echo baz2 +TMPL2_FOO: "{{.FOO}}" +TMPL2_BAR: "{{.BAR}}" +TMPL2_FOO2: "{{.FOO2}}" +TMPL2_BAR2: "{{.BAR2}}" +SHTMPL2_FOO2: + sh: "echo '{{.FOO2}}'" +NESTEDTMPL2_FOO2: "{{.TMPL2_FOO2}}" +OVERRIDE: "foo" diff --git a/testdata/vars/v2/multiline/Taskfile.yml b/testdata/vars/v2/multiline/Taskfile.yml new file mode 100644 index 00000000..8587a367 --- /dev/null +++ b/testdata/vars/v2/multiline/Taskfile.yml @@ -0,0 +1,45 @@ +version: 2 +tasks: + default: + vars: + MULTILINE: "\n\nfoo\n bar\nfoobar\n\nbaz\n\n" + cmds: + - task: file + vars: + CONTENT: + sh: "echo 'foo\nbar'" + FILE: "echo_foobar.txt" + - task: file + vars: + CONTENT: + sh: "echo -n 'foo\nbar'" + FILE: "echo_n_foobar.txt" + - task: file + vars: + CONTENT: + sh: echo -n "{{.MULTILINE}}" + FILE: "echo_n_multiline.txt" + - task: file + vars: + CONTENT: "{{.MULTILINE}}" + FILE: "var_multiline.txt" + - task: file + vars: + CONTENT: "{{.MULTILINE | catLines}}" + FILE: "var_catlines.txt" + - task: enumfile + vars: + LINES: "{{.MULTILINE}}" + FILE: "var_enumfile.txt" + file: + cmds: + - | + cat << EOF > '{{.FILE}}' + {{.CONTENT}} + EOF + enumfile: + cmds: + - | + cat << EOF > '{{.FILE}}' + {{range $i, $line := .LINES| splitLines}}{{$i}}:{{$line}} + {{end}}EOF