diff --git a/README.md b/README.md index 97cb4e4..051e278 100644 --- a/README.md +++ b/README.md @@ -26,17 +26,26 @@ All `gdt` scenarios have the following fields: * `fixtures`: (optional) list of strings indicating named fixtures that will be started before any of the tests in the file are run * `depends`: (optional) list of [`Dependency`][dependency] objects that - describe a program or package that should be available in the host's `PATH` + describe a program binary that should be available in the host's `PATH` that the test scenario depends on. -* `depends.name`: string name of the program or package the test scenario - depends on. +* `depends.name`: string name of the program the test scenario depends on. * `depends.when`: (optional) object describing any constraints/conditions that should apply to the evaluation of the dependency. * `depends.when.os`: (optional) string operating system. if set, the dependency is only checked for that OS. -* `depends.when.version`: (optional) string version constraint. if set, the +* `depends.version`: (optional) struct containing version constraint and + selector instructions. +* `depends.version.constraint`: optional string version constraint. if set, the program or package must be available on the host `PATH` and must satisfy the version constraint. +* `depends.version.selector`: (optional) struct containing selector + instructions for gdt to determine the version of a dependent binary. +* `depends.version.selector.args`: (optional) set of string arguments to call + the dependency binary with in order to get the program's version information. + If empty, we default to []string{`-v`}. +* `depends.version.selector.filter`: (optional) regular expression to run + against the returned output from executing the dependency binary with the + `selector.args` arguments. If empty, we use a loose semver matching regex. * `skip-if`: (optional) list of [`Spec`][basespec] specializations that will be evaluated *before* running any test in the scenario. If any of these conditions evaluates successfully, the test scenario will be skipped. @@ -249,6 +258,54 @@ $ gdt run myapp.yaml Error: runtime error: exec: "myapp": executable file not found in $PATH ``` +You may also specify a particular version constraint that must pass for a +dependent binary with the `depends.version.constraint` field. For example, +let's assume I want to declare my test scenario requires that at least version +`1.2.3` of `myapp` must be present on the host machine, I would do this: + +```yaml +depends: + - name: myapp + version: + constraint: ">=1.2.3" +``` + +The `depends.version.constraint` field should be a valid Semantic Versioning +constraint. Read more about [Semantic Version constraints][semver-constraints]. + +By default to determine a binary's version, we pass a `-v` flag to the binary +itself. If you know that a binary uses a different way of returning its version +information, you can use the `depends.version.selector.args` field. As an +example, the `ls` command line utility on Linux returns its version information +when you pass the `--version` CLI flag, as shown here: + +``` +> ls --version +ls (GNU coreutils) 9.4 +Copyright (C) 2023 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later . +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. + +Written by Richard M. Stallman and David MacKenzie. +``` + +If you wanted to require that, say, version 9.1 and later of the `ls` +command-line utility was present on the host machine, you would do the +following: + +```yaml +depends: + - name: ls + version: + constraint: ">=9.1" + selector: + args: + - "--version" +``` + +[semver-constraints]: https://github.com/Masterminds/semver/blob/master/README.md#checking-version-constraints + ### Passing variables to subsequent test specs A `gdt` test scenario is comprised of a list of test specs. These test specs diff --git a/api/dependency.go b/api/dependency.go index 614f848..4b89832 100644 --- a/api/dependency.go +++ b/api/dependency.go @@ -1,23 +1,220 @@ package api -// Dependency describes a prerequisite binary or package that must be present. +import ( + "regexp" + + "github.com/Masterminds/semver/v3" + "github.com/samber/lo" + "gopkg.in/yaml.v3" + + "github.com/gdt-dev/core/parse" +) + +var ( + ValidOSs = []string{ + "linux", + "darwin", + "windows", + } +) + +// Dependency describes a prerequisite binary that must be present. type Dependency struct { - // Name is the name of the binary or package + // Name is the name of the binary that must be present. Name string `yaml:"name"` // When describes any constraining conditions that apply to this // Dependency. - When *DependencyConstraints `yaml:"when,omitempty"` + When *DependencyConditions `yaml:"when,omitempty"` + // Version contains instructions for constraining and selecting the + // dependency's version. + Version *DependencyVersion `yaml:"version,omitempty"` } -// DependencyConstraints describes constraining conditions that apply to a +func (d *Dependency) UnmarshalYAML(node *yaml.Node) error { + if node.Kind != yaml.MappingNode { + return parse.ExpectedMapAt(node) + } + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + if keyNode.Kind != yaml.ScalarNode { + return parse.ExpectedScalarAt(keyNode) + } + key := keyNode.Value + valNode := node.Content[i+1] + switch key { + case "name": + if valNode.Kind != yaml.ScalarNode { + return parse.ExpectedScalarAt(valNode) + } + d.Name = valNode.Value + case "when": + if valNode.Kind != yaml.MappingNode { + return parse.ExpectedMapAt(valNode) + } + var when DependencyConditions + if err := valNode.Decode(&when); err != nil { + return err + } + d.When = &when + case "version": + if valNode.Kind != yaml.MappingNode { + return parse.ExpectedMapAt(valNode) + } + var dv DependencyVersion + if err := valNode.Decode(&dv); err != nil { + return err + } + d.Version = &dv + default: + return parse.UnknownFieldAt(key, keyNode) + } + } + return nil +} + +// DependencyConditions describes constraining conditions that apply to a // Dependency, for instance whether the dependency is only required on a -// particular OS or whether a particular version constraint applies to the -// dependency. -type DependencyConstraints struct { +// particular OS. +type DependencyConditions struct { // OS indicates that the dependency only applies when the tests are run on // a particular operating system. OS string `yaml:"os,omitempty"` - // Version indicates a version constraint to apply to the dependency, e.g. - // >= 1.2.3 - Version string `yaml:"version,omitempty"` +} + +func (c *DependencyConditions) UnmarshalYAML(node *yaml.Node) error { + if node.Kind != yaml.MappingNode { + return parse.ExpectedMapAt(node) + } + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + if keyNode.Kind != yaml.ScalarNode { + return parse.ExpectedScalarAt(keyNode) + } + key := keyNode.Value + valNode := node.Content[i+1] + switch key { + case "os": + if valNode.Kind != yaml.ScalarNode { + return parse.ExpectedScalarAt(valNode) + } + os := valNode.Value + if os != "" { + if !lo.Contains(ValidOSs, os) { + return parse.InvalidOSAt(valNode, os, ValidOSs) + } + c.OS = os + } + default: + return parse.UnknownFieldAt(key, keyNode) + } + } + return nil +} + +// DependencyVersion expresses a version constraint that must be met for a +// particular dependency and instructs gdt how to get the version for a +// dependency from a binary or package manager. +type DependencyVersion struct { + // Constraint indicates a version constraint to apply to the dependency, + // e.g. '>= 1.2.3' would indicate that a version of the dependency binary + // after and including 1.2.3 must be present on the host. + Constraint string `yaml:"constraint"` + SemVerConstraints *semver.Constraints `yaml:"-"` + // Selector provides instructions to select the version from the binary. + Selector *DependencyVersionSelector `yaml:"selector,omitempty"` +} + +func (v *DependencyVersion) UnmarshalYAML(node *yaml.Node) error { + if node.Kind != yaml.MappingNode { + return parse.ExpectedMapAt(node) + } + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + if keyNode.Kind != yaml.ScalarNode { + return parse.ExpectedScalarAt(keyNode) + } + key := keyNode.Value + valNode := node.Content[i+1] + switch key { + case "constraint": + if valNode.Kind != yaml.ScalarNode { + return parse.ExpectedScalarAt(valNode) + } + conStr := valNode.Value + if conStr != "" { + con, err := semver.NewConstraint(conStr) + if err != nil { + return parse.InvalidVersionConstraintAt( + valNode, conStr, err, + ) + } + v.Constraint = conStr + v.SemVerConstraints = con + } + case "selector": + if valNode.Kind != yaml.MappingNode { + return parse.ExpectedMapAt(valNode) + } + var selector DependencyVersionSelector + if err := valNode.Decode(&selector); err != nil { + return err + } + v.Selector = &selector + default: + return parse.UnknownFieldAt(key, keyNode) + } + } + return nil +} + +// DependencyVersionSelector instructs gdt how to get the version of a binary. +type DependencyVersionSelector struct { + // Args is the command-line to execute the dependency binary to output + // version information, e.g. '-v' or '--version-json'. + Args []string `yaml:"args,omitempty"` + // Filter is an optional regex to run against the output returned by + // Command, e.g. 'v?(\d)+\.(\d+)(\.(\d)+)?'. + Filter string `yaml:"filter,omitempty"` + FilterRegex *regexp.Regexp `yaml:"-"` +} + +func (s *DependencyVersionSelector) UnmarshalYAML(node *yaml.Node) error { + if node.Kind != yaml.MappingNode { + return parse.ExpectedMapAt(node) + } + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + if keyNode.Kind != yaml.ScalarNode { + return parse.ExpectedScalarAt(keyNode) + } + key := keyNode.Value + valNode := node.Content[i+1] + switch key { + case "args": + if valNode.Kind != yaml.SequenceNode { + return parse.ExpectedSequenceAt(valNode) + } + var args []string + if err := valNode.Decode(&args); err != nil { + return err + } + s.Args = args + case "filter": + if valNode.Kind != yaml.ScalarNode { + return parse.ExpectedMapAt(valNode) + } + filter := valNode.Value + if filter != "" { + re, err := regexp.Compile(filter) + if err != nil { + return parse.InvalidRegexAt(valNode, filter, err) + } + s.Filter = filter + s.FilterRegex = re + } + default: + return parse.UnknownFieldAt(key, keyNode) + } + } + return nil } diff --git a/api/error.go b/api/error.go index 11e7c83..5ff9444 100644 --- a/api/error.go +++ b/api/error.go @@ -147,19 +147,29 @@ var ( // DependencyNotSatified returns an ErrDependencyNotSatisfied with the supplied // dependency name and optional constraints. func DependencyNotSatisfied(dep *Dependency) error { - constraintsStr := "" - constraints := []string{} + conditionsStr := "" + conditions := []string{} progName := dep.Name if dep.When != nil { if dep.When.OS != "" { - constraints = append(constraints, "OS:"+dep.When.OS) + conditions = append(conditions, "OS:"+dep.When.OS) } - if dep.When.Version != "" { - constraints = append(constraints, "VERSION:"+dep.When.Version) - } - constraintsStr = fmt.Sprintf(" (%s)", strings.Join(constraints, ",")) } - return fmt.Errorf("%w: %s%s", ErrDependencyNotSatisfied, progName, constraintsStr) + conditionsStr = fmt.Sprintf(" (%s)", strings.Join(conditions, ",")) + return fmt.Errorf("%w: %s%s", ErrDependencyNotSatisfied, progName, conditionsStr) +} + +// DependencyNotSatifiedVersionConstraint returns an ErrDependencyNotSatisfied with the supplied +// dependency name and version constraint failure. +func DependencyNotSatisfiedVersionConstraint( + dep *Dependency, + constraintStr string, +) error { + progName := dep.Name + return fmt.Errorf( + "%w: %q failed version constraint %q", + ErrDependencyNotSatisfied, progName, constraintStr, + ) } // RequiredFixtureMissing returns an ErrRequiredFixture with the supplied diff --git a/go.mod b/go.mod index 2ee7486..edb4793 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/gdt-dev/core go 1.24.3 require ( + github.com/Masterminds/semver/v3 v3.4.0 github.com/cenkalti/backoff v2.2.1+incompatible github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 239edbd..6bc3590 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= diff --git a/parse/error.go b/parse/error.go index 7c2cde7..840ba29 100644 --- a/parse/error.go +++ b/parse/error.go @@ -237,3 +237,55 @@ func FileNotFoundAt(path string, node *yaml.Node) error { Message: fmt.Sprintf("file not found: %q", path), } } + +// InvalidOSAt returns an error indicating an invalid operating system was +// specified, annotated with the line/column of the supplied YAML node. +func InvalidOSAt( + node *yaml.Node, + os string, + valid []string, +) error { + return &Error{ + Line: node.Line, + Column: node.Column, + Message: fmt.Sprintf( + "invalid OS specified: %s. valid values are %v", + os, valid, + ), + } +} + +// InvalidVersionConstraint returns an error indicating an invalid version +// constraint was specified, annotated with the line/column of the supplied +// YAML node. +func InvalidVersionConstraintAt( + node *yaml.Node, + constraint string, + err error, +) error { + return &Error{ + Line: node.Line, + Column: node.Column, + Message: fmt.Sprintf( + "invalid version constraint specified: %s: %s", + constraint, err, + ), + } +} + +// InvalidRegex returns an error indicating an invalid regular expression was +// specified, annotated with the line/column of the supplied YAML node. +func InvalidRegexAt( + node *yaml.Node, + re string, + err error, +) error { + return &Error{ + Line: node.Line, + Column: node.Column, + Message: fmt.Sprintf( + "invalid regular expression specified: %s: %s", + re, err, + ), + } +} diff --git a/scenario/depends.go b/scenario/depends.go index 98bba0d..937eb03 100644 --- a/scenario/depends.go +++ b/scenario/depends.go @@ -4,14 +4,32 @@ import ( "context" "fmt" "os/exec" + "regexp" "runtime" "strings" + "github.com/Masterminds/semver/v3" + "github.com/gdt-dev/core/api" gdtcontext "github.com/gdt-dev/core/context" "github.com/gdt-dev/core/debug" ) +var defaultVersionSelectorArgs = []string{"-v"} + +// looseSemVerRegex is a regular expression that lets invalid semver +// expressions through. Taken from semver library. +const defaultVersionSelectorFilter string = `v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?` + + `(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` + + `(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` + +var ( + defaultVersionSelector = &api.DependencyVersionSelector{ + Args: defaultVersionSelectorArgs, + Filter: defaultVersionSelectorFilter, + } +) + // checkDependencies examines the scenario's set of dependencies and returns a // runtime error if any dependency isn't satisfied. func (s *Scenario) checkDependencies( @@ -51,7 +69,7 @@ func (s *Scenario) checkDependency( } } - _, err := exec.LookPath(dep.Name) + binPath, err := exec.LookPath(dep.Name) if err != nil { execErr, ok := err.(*exec.Error) if ok && execErr.Err == exec.ErrNotFound { @@ -64,12 +82,57 @@ func (s *Scenario) checkDependency( } } - if when != nil && when.Version != "" { - // TODO(jaypipes): do some robust version checking for - // applications/packages here. - _ = when.Version + dv := dep.Version + if dv != nil { + vc := dv.SemVerConstraints + if vc != nil { + verStr, err := versionStringFromDependency(binPath, dv.Selector) + if err != nil { + return err + } + ver, err := semver.NewVersion(verStr) + if err != nil { + return api.DependencyNotSatisfied(dep) + } + if !vc.Check(ver) { + return api.DependencyNotSatisfiedVersionConstraint( + dep, dv.Constraint, + ) + } + } } debug.Printf(ctx, "dependency %q satisfied", dep.Name) return nil } + +// versionStringFromDependency returns a version string from the supplied +// dependency binary path and an optional version selector struct that +// instructs us how to get the version from the binary. +func versionStringFromDependency( + binPath string, + selector *api.DependencyVersionSelector, +) (string, error) { + if selector == nil { + selector = defaultVersionSelector + } + if selector.Filter == "" { + selector.Filter = defaultVersionSelectorFilter + selector.FilterRegex = regexp.MustCompile(defaultVersionSelectorFilter) + } + args := selector.Args + out, err := exec.CommandContext(context.TODO(), binPath, args...).Output() + if err != nil { + return "", err + } + if selector.FilterRegex != nil { + if !selector.FilterRegex.MatchString(string(out)) { + return "", fmt.Errorf( + "unable to determine version string from %q using regex %q", + string(out), selector.FilterRegex.String(), + ) + } + return selector.FilterRegex.FindString(string(out)), nil + } + return string(out), nil +} diff --git a/scenario/parse.go b/scenario/parse.go index 1c9e52d..aecf094 100644 --- a/scenario/parse.go +++ b/scenario/parse.go @@ -54,7 +54,7 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error { } var deps []*api.Dependency if err := valNode.Decode(&deps); err != nil { - return parse.ExpectedSequenceAt(valNode) + return err } s.Depends = deps case "fixtures": diff --git a/scenario/parse_test.go b/scenario/parse_test.go index b9d5c6b..7cf3684 100644 --- a/scenario/parse_test.go +++ b/scenario/parse_test.go @@ -22,7 +22,6 @@ import ( ) func TestFailingDefaults(t *testing.T) { - assert := assert.New(t) require := require.New(t) fp := filepath.Join("testdata", "parse", "fail", "bad-defaults.yaml") @@ -30,9 +29,61 @@ func TestFailingDefaults(t *testing.T) { require.Nil(err) s, err := scenario.FromReader(f, scenario.WithPath(fp)) - assert.NotNil(err) - assert.ErrorContains(err, "defaults parsing failed") - assert.Nil(s) + require.NotNil(err) + require.ErrorContains(err, "defaults parsing failed") + require.Nil(s) +} + +func TestFailingDependsUnknownField(t *testing.T) { + require := require.New(t) + + fp := filepath.Join("testdata", "parse", "fail", "depends-unknown-field.yaml") + f, err := os.Open(fp) + require.Nil(err) + + s, err := scenario.FromReader(f, scenario.WithPath(fp)) + require.NotNil(err) + require.ErrorContains(err, "unknown field") + require.Nil(s) +} + +func TestFailingDependsInvalidOS(t *testing.T) { + require := require.New(t) + + fp := filepath.Join("testdata", "parse", "fail", "depends-invalid-os.yaml") + f, err := os.Open(fp) + require.Nil(err) + + s, err := scenario.FromReader(f, scenario.WithPath(fp)) + require.NotNil(err) + require.ErrorContains(err, "invalid OS specified") + require.Nil(s) +} + +func TestFailingDependsVersionInvalidConstraint(t *testing.T) { + require := require.New(t) + + fp := filepath.Join("testdata", "parse", "fail", "depends-version-invalid-constraint.yaml") + f, err := os.Open(fp) + require.Nil(err) + + s, err := scenario.FromReader(f, scenario.WithPath(fp)) + require.NotNil(err) + require.ErrorContains(err, "invalid version constraint specified") + require.Nil(s) +} + +func TestFailingDependsVersionFilterInvalidRegex(t *testing.T) { + require := require.New(t) + + fp := filepath.Join("testdata", "parse", "fail", "depends-version-filter-invalid-regex.yaml") + f, err := os.Open(fp) + require.Nil(err) + + s, err := scenario.FromReader(f, scenario.WithPath(fp)) + require.NotNil(err) + require.ErrorContains(err, "invalid regular expression specified") + require.Nil(s) } func TestNoTests(t *testing.T) { diff --git a/scenario/run_test.go b/scenario/run_test.go index 4defb01..92e4c66 100644 --- a/scenario/run_test.go +++ b/scenario/run_test.go @@ -115,6 +115,27 @@ func TestDependsNotSatisfiedOS(t *testing.T) { } } +func TestDependsNotSatisfiedVersionConstraint(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("skipping non-linux host") + } + require := require.New(t) + assert := assert.New(t) + + fp := filepath.Join("testdata", "depends-not-satisfied-version-constraint.yaml") + f, err := os.Open(fp) + require.Nil(err) + + s, err := scenario.FromReader(f, scenario.WithPath(fp)) + require.Nil(err) + require.NotNil(s) + + err = s.Run(context.TODO(), t) + assert.NotNil(err) + assert.ErrorIs(err, api.ErrDependencyNotSatisfied) + assert.ErrorIs(err, api.RuntimeError) +} + func TestTimeoutConflictTotalWait(t *testing.T) { require := require.New(t) assert := assert.New(t) diff --git a/scenario/testdata/depends-not-satisfied-version-constraint.yaml b/scenario/testdata/depends-not-satisfied-version-constraint.yaml new file mode 100644 index 0000000..ac42118 --- /dev/null +++ b/scenario/testdata/depends-not-satisfied-version-constraint.yaml @@ -0,0 +1,18 @@ +name: depends-not-satisfied-version-constraint +description: a scenario with unsatisfied dependency version constraint +depends: +# ยท> ls --version +# ls (GNU coreutils) 9.4 +# Copyright (C) 2023 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later . +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. + - name: ls + version: + constraint: ">99.9.9" + selector: + args: + - "--version" +tests: + - name: should-not-get-here + foo: baz diff --git a/scenario/testdata/parse/fail/depends-invalid-os.yaml b/scenario/testdata/parse/fail/depends-invalid-os.yaml new file mode 100644 index 0000000..1d282d8 --- /dev/null +++ b/scenario/testdata/parse/fail/depends-invalid-os.yaml @@ -0,0 +1,9 @@ +name: depends-invalid-os +description: a scenario with invalid OS specified in dependencies +depends: + - name: ls + when: + os: nonexisting +tests: + - name: should-not-get-here + foo: baz diff --git a/scenario/testdata/parse/fail/depends-unknown-field.yaml b/scenario/testdata/parse/fail/depends-unknown-field.yaml new file mode 100644 index 0000000..9cb0542 --- /dev/null +++ b/scenario/testdata/parse/fail/depends-unknown-field.yaml @@ -0,0 +1,10 @@ +name: depends-unknown-field +description: a scenario with unknown field specified for depends +depends: + - name: ls + version: + # This isn't correct. It should be version.selector.filter... + filter: t%a)invalidregex +tests: + - name: should-not-get-here + foo: baz diff --git a/scenario/testdata/parse/fail/depends-version-filter-invalid-regex.yaml b/scenario/testdata/parse/fail/depends-version-filter-invalid-regex.yaml new file mode 100644 index 0000000..42bd54c --- /dev/null +++ b/scenario/testdata/parse/fail/depends-version-filter-invalid-regex.yaml @@ -0,0 +1,10 @@ +name: depends-version-filter-invalid-regex +description: a scenario with invalid regular expression specified in dependencies version selector filter +depends: + - name: ls + version: + selector: + filter: t%a)invalidregex +tests: + - name: should-not-get-here + foo: baz diff --git a/scenario/testdata/parse/fail/depends-version-invalid-constraint.yaml b/scenario/testdata/parse/fail/depends-version-invalid-constraint.yaml new file mode 100644 index 0000000..5bde8a4 --- /dev/null +++ b/scenario/testdata/parse/fail/depends-version-invalid-constraint.yaml @@ -0,0 +1,9 @@ +name: depends-version-invalid-constraint +description: a scenario with invalid constraint specified in dependencies version +depends: + - name: ls + version: + constraint: invalidconstraint +tests: + - name: should-not-get-here + foo: baz