add XUnit and JSON output for run results

api/xunit.go

@@ -0,0 +1,55 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package api
+
+import (
+	"encoding/xml"
+	"time"
+)
+
+// XUnitResults is a wrapper struct allowing the output of well-formed
+// JUnit/XUnit XML and JSON serialized scenario results
+type XUnitResults struct {
+	XMLName    xml.Name         `json:"-" xml:"testsuites"`
+	TestSuites []XUnitTestSuite `json:"testsuites" xml:"testsuite"`
+}
+
+type XUnitTestSuite struct {
+	Name       string          `json:"name" xml:"name,attr"`
+	Time       string          `json:"time,omitempty" xml:"time,attr"`
+	Skipped    bool            `json:"skipped,omitempty" xml:"skipped,omitempty"`
+	Failures   int             `json:"failures" xml:"failures,attr"`
+	Errors     int             `json:"errors" xml:"errors,attr"`
+	Tests      int             `json:"tests" xml:"tests,attr"`
+	Timestamp  time.Time       `json:"timestamp" xml:"timestamp,attr"`
+	Properties []XUnitProperty `json:"properties,omitempty" xml:"properties,omitempty"`
+	TestCases  []XUnitTestCase `json:"testcases,omitempty" xml:"testcases,omitempty"`
+}
+
+type XUnitProperty struct {
+	Name  string `json:"name" xml:"name,attr"`
+	Value string `json:"value" xml:"value,attr"`
+}
+
+type XUnitTestCase struct {
+	Name       string          `json:"name" xml:"name,attr"`
+	Time       string          `json:"time,omitempty" xml:"time,attr"`
+	Status     string          `json:"status" xml:"status,attr"`
+	Assertions int             `json:"assertions" xml:"assertions,attr,omitempty"`
+	Skipped    bool            `json:"skipped,omitempty" xml:"skipped,omitempty"`
+	Failure    *XUnitMessage   `json:"failure,omitempty" xml:"failure,omitempty"`
+	Error      *XUnitMessage   `json:"error,omitempty" xml:"error,omitempty"`
+	SystemOut  *XUnitTextBlock `json:"system-out,omitempty" xml:"system-out,omitempty"`
+	SystemErr  *XUnitTextBlock `json:"system-err,omitempty" xml:"system-err,omitempty"`
+}
+
+type XUnitTextBlock struct {
+	Text string `xml:",cdata"`
+}
+
+type XUnitMessage struct {
+	Message string `json:"message,omitempty" xml:"message,attr"`
+	Type    string `json:"type,omitempty" xml:"type,attr"`
+}

run/new.go

@@ -4,12 +4,14 @@
 
 package run
 
+import "github.com/gdt-dev/core/testunit"
+
 type Option func(*Run)
 
 // New returns a new Run object that stores test run state.
 func New(opts ...Option) *Run {
 	r := &Run{
-		scenarioResults: map[string][]TestUnitResult{},
+		scenarioResults: map[string][]testunit.Result{},
 	}
 	for _, opt := range opts {
 		opt(r)

run/run.go

@@ -1,31 +1,36 @@
 package run
 
 import (
+	"path/filepath"
 	"slices"
 	"time"
 
+	"github.com/samber/lo"
+
 	"github.com/gdt-dev/core/api"
 	"github.com/gdt-dev/core/testunit"
-	"github.com/samber/lo"
 )
 
 // Run stores state of a test run when tests are executed with the `gdt` CLI
 // tool.
 type Run struct {
 	// scenarioResults is a map, keyed by the Scenario path, of slices of
-	// TestUnitResult structs corresponding to the test specs in the scenario.
-	// There is guaranteed to be exactly the same number of TestUnitResults in
+	// testunit.Result structs corresponding to the test specs in the scenario.
+	// There is guaranteed to be exactly the same number of testunit.Results in
 	// the slice as scenarios in the scenario.
-	scenarioResults map[string][]TestUnitResult
+	scenarioResults map[string][]testunit.Result
 }
 
 // OK returns true if all Scenarios in the Run had all successful test units.
 func (r *Run) OK() bool {
-	return lo.EveryBy(lo.Values(r.scenarioResults), func(results []TestUnitResult) bool {
-		return lo.EveryBy(results, func(tur TestUnitResult) bool {
-			return tur.OK()
-		})
-	})
+	return lo.EveryBy(
+		lo.Values(r.scenarioResults),
+		func(results []testunit.Result) bool {
+			return lo.EveryBy(results, func(tur testunit.Result) bool {
+				return tur.OK()
+			})
+		},
+	)
 }
 
 // ScenarioPaths returns a sorted list of Scenario Paths.
@@ -35,9 +40,9 @@ func (r *Run) ScenarioPaths() []string {
 	return paths
 }
 
-// ScenarioResults returns the set of TestUnitResults for a Scenario with the
+// ScenarioResults returns the set of testunit.Results for a Scenario with the
 // supplied path.
-func (r *Run) ScenarioResults(path string) []TestUnitResult {
+func (r *Run) ScenarioResults(path string) []testunit.Result {
 	return r.scenarioResults[path]
 }
 
@@ -49,63 +54,52 @@ func (r *Run) StoreResult(
 	res *api.Result,
 ) {
 	if _, ok := r.scenarioResults[path]; !ok {
-		r.scenarioResults[path] = []TestUnitResult{}
+		r.scenarioResults[path] = []testunit.Result{}
 	}
 	r.scenarioResults[path] = append(
 		r.scenarioResults[path],
-		TestUnitResult{
-			index:    index,
-			name:     tu.Name(),
-			elapsed:  tu.Elapsed(),
-			skipped:  tu.Skipped(),
-			failures: res.Failures(),
-			detail:   tu.Detail(),
-		},
+		testunit.NewResult(index, tu, res),
 	)
 }
 
-// TestUnitResult stores a summary of the test execution of a single test unit.
-type TestUnitResult struct {
-	// index is the 0-based index of the test unit within the test scenario.
-	index int
-	// name is the short name of the test unit
-	name string
-	// skipped is true if the test unit was skipped
-	skipped bool
-	// failures is the collection of assertion failures for the test spec that
-	// occurred during the run. this will NOT include RuntimeErrors.
-	failures []error
-	// elapsed is the time take to execute the test unit
-	elapsed time.Duration
-	// detail is a buffer holding any log entries made during the run of the
-	// test spec.
-	detail string
-}
+// XUnit returns the Run's scenario results as a slice of structs that can be
+// serialized to either XML (JUnit/XUnit-style) or JSON.
+func (r *Run) XUnit() []api.XUnitTestSuite {
+	suites := []api.XUnitTestSuite{}
+	paths := r.ScenarioPaths()
+	for _, path := range paths {
+		shortPath := filepath.Base(path)
+		suite := api.XUnitTestSuite{
+			Name: shortPath,
+			Properties: []api.XUnitProperty{
+				{
+					Name:  "path",
+					Value: path,
+				},
+			},
+			Timestamp: time.Now(),
+		}
 
-func (u TestUnitResult) OK() bool {
-	return len(u.failures) == 0
-}
+		var scenElapsed time.Duration
 
-func (u TestUnitResult) Name() string {
-	return u.name
-}
+		unitResults := r.ScenarioResults(path)
 
-func (u TestUnitResult) Index() int {
-	return u.index
-}
-
-func (u TestUnitResult) Failures() []error {
-	return u.failures
-}
-
-func (u TestUnitResult) Skipped() bool {
-	return u.skipped
-}
-
-func (u TestUnitResult) Detail() string {
-	return u.detail
-}
+		testcases := make([]api.XUnitTestCase, len(unitResults))
+		tcFails := 0
 
-func (u TestUnitResult) Elapsed() time.Duration {
-	return u.elapsed
+		for x, res := range unitResults {
+			tc := res.XUnit()
+			if res.Failed() {
+				tcFails++
+			}
+			scenElapsed += res.Elapsed()
+			testcases[x] = tc
+		}
+		suite.Failures = tcFails
+		suite.Tests = len(testcases)
+		suite.Time = scenElapsed.String()
+		suite.TestCases = testcases
+		suites = append(suites, suite)
+	}
+	return suites
 }

testunit/result.go

@@ -0,0 +1,107 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package testunit
+
+import (
+	"time"
+
+	"github.com/gdt-dev/core/api"
+)
+
+func NewResult(
+	index int,
+	tu *TestUnit,
+	res *api.Result,
+) Result {
+	return Result{
+		index:    index,
+		name:     tu.Name(),
+		elapsed:  tu.Elapsed(),
+		skipped:  tu.Skipped(),
+		failures: res.Failures(),
+		detail:   tu.Detail(),
+	}
+}
+
+// Result stores a summary of the test execution of a single test unit.
+type Result struct {
+	// index is the 0-based index of the test unit within the test scenario.
+	index int
+	// name is the short name of the test unit
+	name string
+	// skipped is true if the test unit was skipped
+	skipped bool
+	// failures is the collection of assertion failures for the test spec that
+	// occurred during the run. this will NOT include RuntimeErrors.
+	failures []error
+	// elapsed is the time take to execute the test unit
+	elapsed time.Duration
+	// detail is a buffer holding any log entries made during the run of the
+	// test spec.
+	detail string
+}
+
+// OK returns whether the test unit executed without any failed assertions.
+func (r Result) OK() bool {
+	return len(r.failures) == 0
+}
+
+// Name returns the name of the test unit.
+func (r Result) Name() string {
+	return r.name
+}
+
+// Index returns the 0-based index of the test unit within its containing test
+// scenario.
+func (r Result) Index() int {
+	return r.index
+}
+
+// Failures returns a slice of error messages indicating the test unit
+// assertion failures.
+func (r Result) Failures() []error {
+	return r.failures
+}
+
+// Failed returns whether the test unit failed any test assertions.
+func (r Result) Failed() bool {
+	return !r.skipped && len(r.failures) > 0
+}
+
+// Skipped returns whether the test unit was skipped during execution.
+func (r Result) Skipped() bool {
+	return r.skipped
+}
+
+// Detail returns the collected details/output of the test unit.
+func (r Result) Detail() string {
+	return r.detail
+}
+
+// Elapsed returns the elapsed time of the test unit.
+func (r Result) Elapsed() time.Duration {
+	return r.elapsed
+}
+
+// XUnit returns the testunit.Result as a struct that can be serialized to
+// JUnit/XUnit XML or JSON.
+func (r Result) XUnit() api.XUnitTestCase {
+	tc := api.XUnitTestCase{
+		Name: r.Name(),
+	}
+	if r.Skipped() {
+		tc.Skipped = true
+		tc.Status = "skip"
+	} else if r.OK() {
+		tc.Status = "ok"
+	} else {
+		tc.Status = "fail"
+	}
+	tc.Time = r.Elapsed().String()
+	tc.SystemOut = &api.XUnitTextBlock{
+		Text: r.Detail(),
+	}
+	return tc
+}