diff --git a/test/converters/junitxml/junitxml.go b/test/converters/junitxml/junitxml.go index 196d4dd0..7e6cfd85 100644 --- a/test/converters/junitxml/junitxml.go +++ b/test/converters/junitxml/junitxml.go @@ -76,22 +76,104 @@ func convertTestReport(report TestReport) testreport.TestReport { func convertTestSuite(testSuite TestSuite) testreport.TestSuite { convertedTestSuite := testreport.TestSuite{ - XMLName: testSuite.XMLName, - Name: testSuite.Name, - Tests: testSuite.Tests, - Failures: testSuite.Failures + testSuite.Errors, - Skipped: testSuite.Skipped, - Time: testSuite.Time, + XMLName: testSuite.XMLName, + Name: testSuite.Name, + Time: testSuite.Time, } - for _, testCase := range testSuite.TestCases { + tests := 0 + failures := 0 + skipped := 0 + + flattenedTestCases := flattenGroupedTestCases(testSuite.TestCases) + for _, testCase := range flattenedTestCases { convertedTestCase := convertTestCase(testCase) convertedTestSuite.TestCases = append(convertedTestSuite.TestCases, convertedTestCase) + + if convertedTestCase.Failure != nil { + failures++ + } + if convertedTestCase.Skipped != nil { + skipped++ + } + tests++ } + convertedTestSuite.Tests = tests + convertedTestSuite.Failures = failures + convertedTestSuite.Skipped = skipped + return convertedTestSuite } +func flattenGroupedTestCases(testCases []TestCase) []TestCase { + var flattenedTestCases []TestCase + for _, testCase := range testCases { + flattenedTestCases = append(flattenedTestCases, testCase) + + if len(testCase.FlakyFailures) == 0 && len(testCase.FlakyErrors) == 0 && + len(testCase.RerunFailures) == 0 && len(testCase.RerunErrors) == 0 { + continue + } + + flattenedTestCase := TestCase{ + XMLName: testCase.XMLName, + ConfigurationHash: testCase.ConfigurationHash, + Name: testCase.Name, + ClassName: testCase.ClassName, + } + + for _, flakyFailure := range testCase.FlakyFailures { + flattenedTestCase.Failure = convertToFailure(flakyFailure.Type, flakyFailure.Message, flakyFailure.SystemErr) + flattenedTestCases = append(flattenedTestCases, flattenedTestCase) + } + + for _, flakyError := range testCase.FlakyErrors { + flattenedTestCase.Failure = convertToFailure(flakyError.Type, flakyError.Message, flakyError.SystemErr) + flattenedTestCases = append(flattenedTestCases, flattenedTestCase) + } + + for _, rerunfailure := range testCase.RerunFailures { + flattenedTestCase.Failure = convertToFailure(rerunfailure.Type, rerunfailure.Message, rerunfailure.SystemErr) + flattenedTestCases = append(flattenedTestCases, flattenedTestCase) + } + + for _, rerunError := range testCase.RerunErrors { + flattenedTestCase.Failure = convertToFailure(rerunError.Type, rerunError.Message, rerunError.SystemErr) + flattenedTestCases = append(flattenedTestCases, flattenedTestCase) + } + + } + return flattenedTestCases +} + +func convertToFailure(itemType, failureMessage, systemErr string) *Failure { + var message string + if len(strings.TrimSpace(itemType)) > 0 { + message = itemType + } + if len(strings.TrimSpace(failureMessage)) > 0 { + if len(message) > 0 { + message += ": " + } + message += failureMessage + } + + if len(strings.TrimSpace(systemErr)) > 0 { + if len(message) > 0 { + message += "\n\n" + } + message += "System error:\n" + systemErr + } + + if len(message) > 0 { + return &Failure{ + Value: message, + } + } + return nil +} + func convertTestCase(testCase TestCase) testreport.TestCase { convertedTestCase := testreport.TestCase{ XMLName: testCase.XMLName, @@ -141,7 +223,8 @@ func convertErrorsToFailure(failure *Failure, error *Error, systemErr string) *t if len(messages) > 0 { return &testreport.Failure{ - Value: strings.Join(messages, "\n\n"), + XMLName: xml.Name{Local: "failure"}, + Value: strings.Join(messages, "\n\n"), } } return nil diff --git a/test/converters/junitxml/junitxml_test.go b/test/converters/junitxml/junitxml_test.go index 88f47f2c..b607c6ed 100644 --- a/test/converters/junitxml/junitxml_test.go +++ b/test/converters/junitxml/junitxml_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" ) -func Test_parseTestSuites(t *testing.T) { +func Test_parseTestReport(t *testing.T) { tests := []struct { name string path string @@ -29,7 +29,7 @@ func Test_parseTestSuites(t *testing.T) { } } -func Test_regroupErrors(t *testing.T) { +func Test_convertTestReport(t *testing.T) { tests := []struct { name string suites []TestSuite @@ -44,10 +44,11 @@ func Test_regroupErrors(t *testing.T) { }, }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "Error message:\nerror message", + XMLName: xml.Name{Local: "failure"}, + Value: "Error message:\nerror message", }, }, }}}, @@ -61,10 +62,11 @@ func Test_regroupErrors(t *testing.T) { }, }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "Error value:\nerror message", + XMLName: xml.Name{Local: "failure"}, + Value: "Error value:\nerror message", }, }, }}}, @@ -76,10 +78,11 @@ func Test_regroupErrors(t *testing.T) { SystemErr: "error message", }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "System error:\nerror message", + XMLName: xml.Name{Local: "failure"}, + Value: "System error:\nerror message", }, }, }}}, @@ -99,15 +102,17 @@ func Test_regroupErrors(t *testing.T) { }, }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 2, Failures: 2, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "Error message:\nerror message", + XMLName: xml.Name{Local: "failure"}, + Value: "Error message:\nerror message", }, }, { Failure: &testreport.Failure{ - Value: "Error message:\nerror message2", + XMLName: xml.Name{Local: "failure"}, + Value: "Error message:\nerror message2", }, }, }}}, @@ -126,15 +131,17 @@ func Test_regroupErrors(t *testing.T) { }, }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 2, Failures: 2, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "Error value:\nerror message", + XMLName: xml.Name{Local: "failure"}, + Value: "Error value:\nerror message", }, }, { Failure: &testreport.Failure{ - Value: "Error value:\nerror message2", + XMLName: xml.Name{Local: "failure"}, + Value: "Error value:\nerror message2", }, }, }}}, @@ -149,15 +156,17 @@ func Test_regroupErrors(t *testing.T) { SystemErr: "error message2", }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 2, Failures: 2, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "System error:\nerror message", + XMLName: xml.Name{Local: "failure"}, + Value: "System error:\nerror message", }, }, { Failure: &testreport.Failure{ - Value: "System error:\nerror message2", + XMLName: xml.Name{Local: "failure"}, + Value: "System error:\nerror message2", }, }, }}}, @@ -171,10 +180,11 @@ func Test_regroupErrors(t *testing.T) { }, }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "error message", + XMLName: xml.Name{Local: "failure"}, + Value: "error message", }, }, }}}, @@ -191,10 +201,11 @@ func Test_regroupErrors(t *testing.T) { }, }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "failure message\n\nError value:\nerror value", + XMLName: xml.Name{Local: "failure"}, + Value: "failure message\n\nError value:\nerror value", }, }, }}}, @@ -211,10 +222,11 @@ func Test_regroupErrors(t *testing.T) { }, }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "Failure message\n\nError message:\nerror value", + XMLName: xml.Name{Local: "failure"}, + Value: "Failure message\n\nError message:\nerror value", }, }, }}}, @@ -229,10 +241,11 @@ func Test_regroupErrors(t *testing.T) { SystemErr: "error value", }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "failure message\n\nSystem error:\nerror value", + XMLName: xml.Name{Local: "failure"}, + Value: "failure message\n\nSystem error:\nerror value", }, }, }}}, @@ -247,10 +260,11 @@ func Test_regroupErrors(t *testing.T) { SystemErr: "error value", }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "failure message\n\nSystem error:\nerror value", + XMLName: xml.Name{Local: "failure"}, + Value: "failure message\n\nSystem error:\nerror value", }, }, }}}, @@ -270,10 +284,11 @@ func Test_regroupErrors(t *testing.T) { }, }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "failure message\n\nfailure content\n\nError message:\nmessage\n\nError value:\nvalue\n\nSystem error:\nerror value", + XMLName: xml.Name{Local: "failure"}, + Value: "failure message\n\nfailure content\n\nError message:\nmessage\n\nError value:\nvalue\n\nSystem error:\nerror value", }, }, }}}, @@ -287,10 +302,11 @@ func Test_regroupErrors(t *testing.T) { }, }, }}}, - want: []testreport.TestSuite{{TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Failure: &testreport.Failure{ - Value: "ErrorMsg", + XMLName: xml.Name{Local: "failure"}, + Value: "ErrorMsg", }, }, }}}, @@ -304,7 +320,7 @@ func Test_regroupErrors(t *testing.T) { } } -func TestConverter_XML(t *testing.T) { +func TestConverter_Convert(t *testing.T) { tests := []struct { name string results []resultReader @@ -312,7 +328,7 @@ func TestConverter_XML(t *testing.T) { wantErr bool }{ { - name: "Error message in Message atttribute of Failure element", + name: "Error message in Message attribute of Failure element", results: []resultReader{&stringReader{ Contents: ` @@ -332,7 +348,7 @@ func TestConverter_XML(t *testing.T) { XMLName: xml.Name{Local: "testsuite"}, Name: "MyApp-Unit-Tests", Tests: 2, - Failures: 0, + Failures: 1, Time: 0.398617148399353, TestCases: []testreport.TestCase{ testreport.TestCase{ @@ -348,7 +364,8 @@ func TestConverter_XML(t *testing.T) { ClassName: "PaymentContextTests", Time: 0.17543494701385498, Failure: &testreport.Failure{ - Value: "XCTAssertTrue failed", + XMLName: xml.Name{Local: "failure"}, + Value: "XCTAssertTrue failed", }, }, }, @@ -371,3 +388,64 @@ func TestConverter_XML(t *testing.T) { }) } } + +func TestConverter_Convert_Grouped_report(t *testing.T) { + tests := []struct { + name string + results []resultReader + want testreport.TestReport + wantErr bool + }{ + { + name: "Error message in Message attribute of Failure element", + results: []resultReader{&fileReader{Filename: "./testdata/flaky_test.xml"}}, + want: testreport.TestReport{ + XMLName: xml.Name{Space: "", Local: ""}, + TestSuites: []testreport.TestSuite{ + { + XMLName: xml.Name{Space: "", Local: "testsuite"}, + Name: "My Test Suite", + Tests: 7, Failures: 6, Skipped: 1, + Time: 28.844, + TestCases: []testreport.TestCase{ + {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 1", ClassName: "example.exampleTest", Time: 0.764, + Failure: nil, + Skipped: &testreport.Skipped{XMLName: xml.Name{Space: "", Local: "skipped"}}}, + {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 2", ClassName: "example.exampleTest", Time: 0.164, + Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nSome error message 2"}, + Skipped: nil}, + {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0.445, + Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "Failure message\n\nError value:\nError\n\nSystem error:\nSome error message 3"}, + Skipped: nil}, + {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0, + Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nFlaky failure system error"}, + Skipped: nil}, + {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0, + Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nFlaky error system error"}, + Skipped: nil}, + {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0, + Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nRerun failure system error"}, + Skipped: nil}, + {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0, + Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nRerun error system error"}, + Skipped: nil}, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := &Converter{ + results: tt.results, + } + got, err := h.Convert() + if tt.wantErr { + require.Error(t, err) + } else { + require.EqualValues(t, tt.want, got) + } + }) + } +} diff --git a/test/converters/junitxml/models.go b/test/converters/junitxml/models.go index f1f9cf7d..5188802a 100644 --- a/test/converters/junitxml/models.go +++ b/test/converters/junitxml/models.go @@ -33,6 +33,44 @@ type TestCase struct { Skipped *Skipped `xml:"skipped,omitempty"` Error *Error `xml:"error,omitempty"` SystemErr string `xml:"system-err,omitempty"` + SystemOut string `xml:"system-out,omitempty"` + + FlakyFailures []FlakyFailure `xml:"flakyFailure,omitempty"` + FlakyErrors []FlakyError `xml:"flakyError,omitempty"` + RerunFailures []RerunFailure `xml:"rerunFailure,omitempty"` + RerunErrors []RerunError `xml:"rerunError,omitempty"` +} + +type FlakyFailure struct { + XMLName xml.Name `xml:"flakyFailure"` + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` + SystemErr string `xml:"system-err,omitempty"` + SystemOut string `xml:"system-out,omitempty"` +} + +type FlakyError struct { + XMLName xml.Name `xml:"flakyError"` + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` + SystemErr string `xml:"system-err,omitempty"` + SystemOut string `xml:"system-out,omitempty"` +} + +type RerunFailure struct { + XMLName xml.Name `xml:"rerunFailure"` + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` + SystemErr string `xml:"system-err,omitempty"` + SystemOut string `xml:"system-out,omitempty"` +} + +type RerunError struct { + XMLName xml.Name `xml:"rerunError"` + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` + SystemErr string `xml:"system-err,omitempty"` + SystemOut string `xml:"system-out,omitempty"` } // Failure ... diff --git a/test/converters/junitxml/testdata/flaky_test.xml b/test/converters/junitxml/testdata/flaky_test.xml new file mode 100644 index 00000000..5e3a14a5 --- /dev/null +++ b/test/converters/junitxml/testdata/flaky_test.xml @@ -0,0 +1,55 @@ + + + + + + + + + + Skipped + + + + + + + + + Failure message + + Rerun failure stack trace + Rerun failure system out + Rerun failure system error + + + Flaky failure stack trace + Flaky failure system out + Flaky failure system error + + + Rerun error stack trace + Rerun error system out + Rerun error system error + + + Flaky error stack trace + Flaky error system out + Flaky error system error + + Error + + \ No newline at end of file