From 8a9af99175f1f7aa44f6a7d7732dcb65273fcc67 Mon Sep 17 00:00:00 2001 From: Allain Magyar Date: Wed, 21 May 2025 16:41:41 -0300 Subject: [PATCH 1/3] test: adds allure report for e2e Signed-off-by: Allain Magyar --- .../Assertion/AssertionError.swift | 44 -- E2E/TestFramework/BDD/Feature.swift | 60 ++- .../BDD/ParameterizedScenario.swift | 10 +- E2E/TestFramework/BDD/Scenario.swift | 13 +- .../Configuration/ITestConfiguration.swift | 8 +- .../Configuration/TestConfiguration.swift | 159 ++++-- E2E/TestFramework/Errors/ActorError.swift | 20 + E2E/TestFramework/Errors/AssertionError.swift | 29 ++ E2E/TestFramework/Errors/BaseError.swift | 49 ++ .../Errors/ConfigurationError.swift | 20 + E2E/TestFramework/Errors/StepError.swift | 31 ++ E2E/TestFramework/Outcome/ActionOutcome.swift | 25 +- .../Outcome/FeatureOutcome.swift | 71 ++- E2E/TestFramework/Outcome/ResultOutcome.swift | 5 - .../Outcome/ScenarioOutcome.swift | 25 +- E2E/TestFramework/Outcome/StepOutcome.swift | 22 +- E2E/TestFramework/Outcome/SuiteOutcome.swift | 33 ++ E2E/TestFramework/Outcome/TestStatus.swift | 7 + E2E/TestFramework/Report/AllureReporter.swift | 448 +++++++++++++++++ .../Report/ConsoleReporter.swift | 8 +- E2E/TestFramework/Report/DebugReporter.swift | 4 +- E2E/TestFramework/Report/DotReporter.swift | 4 +- E2E/TestFramework/Report/HtmlReporter.swift | 20 +- E2E/TestFramework/Report/JunitReporter.swift | 2 +- .../Report/SummaryReporter.swift | 142 ++++++ E2E/TestFramework/Resources/html_report.html | 26 +- E2E/TestFramework/Runner/StepRegistry.swift | 2 +- E2E/TestFramework/Runner/StepRunner.swift | 4 +- E2E/TestFramework/Screenplay/Actor.swift | 83 ++-- E2E/Tests/Source/Api.swift | 9 +- E2E/Tests/Source/Config.swift | 22 +- E2E/Tests/Source/Features/Backup.swift | 2 +- .../Source/Features/CreateConnection.swift | 4 +- .../Features/VerifyAnoncredCredential.swift | 4 +- E2E/Tests/Source/Steps/CloudAgentSteps.swift | 79 +++ E2E/Tests/Source/Steps/EdgeAgentSteps.swift | 32 +- .../Source/Workflows/CloudAgentWorkflow.swift | 75 ++- .../Source/Workflows/EdgeAgentWorkflow.swift | 461 +++++++++--------- 38 files changed, 1539 insertions(+), 523 deletions(-) delete mode 100644 E2E/TestFramework/Assertion/AssertionError.swift create mode 100644 E2E/TestFramework/Errors/ActorError.swift create mode 100644 E2E/TestFramework/Errors/AssertionError.swift create mode 100644 E2E/TestFramework/Errors/BaseError.swift create mode 100644 E2E/TestFramework/Errors/ConfigurationError.swift create mode 100644 E2E/TestFramework/Errors/StepError.swift delete mode 100644 E2E/TestFramework/Outcome/ResultOutcome.swift create mode 100644 E2E/TestFramework/Outcome/SuiteOutcome.swift create mode 100644 E2E/TestFramework/Outcome/TestStatus.swift create mode 100644 E2E/TestFramework/Report/AllureReporter.swift create mode 100644 E2E/TestFramework/Report/SummaryReporter.swift diff --git a/E2E/TestFramework/Assertion/AssertionError.swift b/E2E/TestFramework/Assertion/AssertionError.swift deleted file mode 100644 index 02746659..00000000 --- a/E2E/TestFramework/Assertion/AssertionError.swift +++ /dev/null @@ -1,44 +0,0 @@ -import Foundation - -class BaseError: Error, CustomStringConvertible { - var description: String - - let message: String - let error: String - let file: StaticString - let line: UInt - - fileprivate init(message: String, error: String, file: StaticString = #file, line: UInt = #line) { - self.message = message - self.error = error - self.file = file - self.line = line - - let fileName = URL(fileURLWithPath: String(describing: file)).lastPathComponent - self.description = error + ": " + message + " (at \(fileName):\(line))" - } -} - -class Assertion { - class AssertionError: BaseError { - init(message: String, file: StaticString = #file, line: UInt = #line) { - super.init(message: message, - error: "Assertion failure", - file: file, - line: line) - } - } - - class TimeoutError: BaseError { - let timeout: Int - - init(timeout: Int, message: String = "time limit exceeded", file: StaticString = #file, line: UInt = #line) { - self.timeout = timeout - super.init(message: message, - error: "Timeout reached (\(timeout))s", - file: file, - line: line) - } - } -} - diff --git a/E2E/TestFramework/BDD/Feature.swift b/E2E/TestFramework/BDD/Feature.swift index c0f55640..75aa93dd 100644 --- a/E2E/TestFramework/BDD/Feature.swift +++ b/E2E/TestFramework/BDD/Feature.swift @@ -5,7 +5,7 @@ import SwiftHamcrest open class Feature: XCTestCase { let id: String = UUID().uuidString open var currentScenario: Scenario? = nil - + open func title() -> String { fatalError("Set feature title") } @@ -16,9 +16,23 @@ open class Feature: XCTestCase { /// our lifecycle starts after xctest is ending public override func tearDown() async throws { - try await run() + var errorFromRun: Error? + do { + try await run() + } catch { + errorFromRun = error + } self.currentScenario = nil - try await super.tearDown() + var superTeardownError: Error? + do { + try await super.tearDown() + } catch { + superTeardownError = error + } + + if let errorToThrow = errorFromRun ?? superTeardownError { + throw errorToThrow + } } public override class func tearDown() { @@ -34,30 +48,36 @@ open class Feature: XCTestCase { } func run() async throws { - // check if we have the scenario - if (currentScenario == nil) { - throw XCTSkip(""" - To run the feature you have to setup the scenario for each test case. - Usage: - func testMyScenario() async throws { - scenario = Scenario("description") - .given // ... + let currentTestMethodName = self.name + if currentScenario == nil { + let rawMethodName = currentTestMethodName.split(separator: " ").last?.dropLast() ?? "yourTestMethod" + + let errorMessage = """ + ‼️ SCENARIO NOT DEFINED in test method: \(currentTestMethodName) + Each 'func test...()' method within a 'Feature' class must assign a 'Scenario' to 'self.currentScenario'. + + Example: + func \(rawMethodName)() async throws { + currentScenario = Scenario("A brief scenario description", file: #file, line: #line) + .given("some precondition") + .when("some action") + .then("some expected outcome") } - """) + """ + throw ConfigurationError.missingScenario(errorMessage) } - - if (currentScenario!.disabled) { - throw XCTSkip("Scenario [\(currentScenario!.title)] is disabled") + if currentScenario!.disabled { + throw XCTSkip("Scenario '\(currentScenario!.name)' in test method \(currentTestMethodName) is disabled.") } - try await TestConfiguration.setUpInstance() - if (currentScenario! is ParameterizedScenario) { - let parameterizedScenario = currentScenario! as! ParameterizedScenario - for scenario in parameterizedScenario.build() { - try await TestConfiguration.shared().run(self, scenario) + if let parameterizedScenario = currentScenario as? ParameterizedScenario { + for scenarioInstance in parameterizedScenario.build() { + scenarioInstance.feature = self + try await TestConfiguration.shared().run(self, scenarioInstance) } } else { + currentScenario?.feature = self try await TestConfiguration.shared().run(self, currentScenario!) } } diff --git a/E2E/TestFramework/BDD/ParameterizedScenario.swift b/E2E/TestFramework/BDD/ParameterizedScenario.swift index 5f48342f..cea7013f 100644 --- a/E2E/TestFramework/BDD/ParameterizedScenario.swift +++ b/E2E/TestFramework/BDD/ParameterizedScenario.swift @@ -2,18 +2,18 @@ import Foundation import XCTest public class ParameterizedScenario: Scenario { - public var parameters: [[String: String]] = [[:]] + public var table: [[String: String]] = [[:]] - public func parameters(_ parameters: [[String: String]]) -> Scenario { - self.parameters = parameters + public func table(_ table: [[String: String]]) -> Scenario { + self.table = table return self } public func build() -> [Scenario] { var scenarios: [Scenario] = [] - parameters.forEach { parameters in - let scenario = Scenario(replace(line: self.title, parameters: parameters)) + table.forEach { parameters in + let scenario = Scenario(replace(line: self.name, parameters: parameters), parameters: parameters) scenario.steps = self.steps.map { step in let newStep = ConcreteStep() newStep.context = step.context diff --git a/E2E/TestFramework/BDD/Scenario.swift b/E2E/TestFramework/BDD/Scenario.swift index a96e04a6..2c25ab77 100644 --- a/E2E/TestFramework/BDD/Scenario.swift +++ b/E2E/TestFramework/BDD/Scenario.swift @@ -3,16 +3,17 @@ import XCTest public class Scenario { let id = UUID().uuidString - var title: String + var name: String var steps: [ConcreteStep] = [] - var pass: Bool = false - var error: Error? = nil var disabled: Bool = false - + var feature: Feature? + var parameters: [String: String]? + private var lastContext: String = "" - public init(_ title: String) { - self.title = title + public init(_ title: String, parameters: [String: String] = [:]) { + self.name = title + self.parameters = parameters } public func fail(file: StaticString?, line: UInt?, message: String) { diff --git a/E2E/TestFramework/Configuration/ITestConfiguration.swift b/E2E/TestFramework/Configuration/ITestConfiguration.swift index b3768a34..cd392f68 100644 --- a/E2E/TestFramework/Configuration/ITestConfiguration.swift +++ b/E2E/TestFramework/Configuration/ITestConfiguration.swift @@ -1,10 +1,10 @@ import Foundation public protocol ITestConfiguration { - static var shared: () -> ITestConfiguration {get} - static func createInstance() -> ITestConfiguration - - func run(_ feature: Feature, _ currentScenario: Scenario) async throws + static var shared: () -> TestConfiguration {get} + static func createInstance() -> TestConfiguration + + func run(_ feature: Feature, _ currentScenario: Scenario?) async throws /// setup func setUp() async throws diff --git a/E2E/TestFramework/Configuration/TestConfiguration.swift b/E2E/TestFramework/Configuration/TestConfiguration.swift index 38b0d72a..d2ef1cfa 100644 --- a/E2E/TestFramework/Configuration/TestConfiguration.swift +++ b/E2E/TestFramework/Configuration/TestConfiguration.swift @@ -7,13 +7,14 @@ open class TestConfiguration: ITestConfiguration { public static var shared = { instance! } public var environment: [String: String] = [:] - private static var instance: ITestConfiguration? = nil + private static var instance: TestConfiguration? = nil private static var actors: [String: Actor] = [:] private var assertionFailure: (String, StaticString, UInt)? = nil + private var actionFailure: (String, Error, StaticString, UInt)? = nil private var reporters: [Reporter] = [] - private var result: ResultOutcome = ResultOutcome() + internal var suiteOutcome: SuiteOutcome = SuiteOutcome() private var features: [Feature.Type] = [] private var steps: [Steps] = [] @@ -24,7 +25,7 @@ open class TestConfiguration: ITestConfiguration { self.environment = readEnvironmentVariables(bundlePath: bundlePath) } - open class func createInstance() -> ITestConfiguration { + open class func createInstance() -> TestConfiguration { fatalError("Configuration must implement createInstance method") } @@ -66,11 +67,11 @@ open class TestConfiguration: ITestConfiguration { } /// Main function that runs feature, scenario and steps - public func run(_ feature: Feature, _ scenario: Scenario) async throws { + public func run(_ feature: Feature, _ scenario: Scenario?) async throws { currentScenario = scenario try await beforeFeature(feature) - try await beforeScenario(scenario) - let scenarioOutcome = try await runSteps(scenario) + try await beforeScenario(scenario!) + let scenarioOutcome = try await runSteps(scenario!) try await afterScenario(scenarioOutcome) } @@ -82,8 +83,9 @@ open class TestConfiguration: ITestConfiguration { } features.append(type) - currentFeatureOutcome = FeatureOutcome(feature) - result.featuresOutcome.append(currentFeatureOutcome!) + currentFeatureOutcome = FeatureOutcome(feature: feature) + currentFeatureOutcome!.start() + suiteOutcome.featureOutcomes.append(currentFeatureOutcome!) try await report(.BEFORE_FEATURE, feature) } @@ -94,61 +96,134 @@ open class TestConfiguration: ITestConfiguration { } public func beforeStep(_ step: ConcreteStep) async throws { - try await report(.BEFORE_STEP, step.action) + try await report(.BEFORE_STEP, step) } func runSteps(_ scenario: Scenario) async throws -> ScenarioOutcome { + if scenario.disabled { + return ScenarioOutcome(scenario) + } + let scenarioOutcome = ScenarioOutcome(scenario) + scenarioOutcome.start() for step in scenario.steps { - let stepOutcome: StepOutcome + var determinedStepStatus: TestStatus + var stepError: Error? = nil + let stepOutcome = StepOutcome(step) try await report(.BEFORE_STEP, step) - do { + stepOutcome.start() try await StepRegistry.run(step) - if (assertionFailure != nil) { - let message = assertionFailure!.0 - let file = assertionFailure!.1 - let line = assertionFailure!.2 + stepOutcome.end() + if let currentAssertionFailure = assertionFailure { + self.assertionFailure = nil + let message = currentAssertionFailure.0 + let file = currentAssertionFailure.1 + let line = currentAssertionFailure.2 XCTFail(message, file: file, line: line) - throw Assertion.AssertionError( + stepError = Assertion.AssertionError( message: message, file: file, line: line ) + determinedStepStatus = .failed + } else if let currentActionFailure = actionFailure { + self.actionFailure = nil + let message = currentActionFailure.0 + let error = currentActionFailure.1 + let file = currentActionFailure.2 + let line = currentActionFailure.3 + XCTFail(message, file: file, line: line) + stepError = ActorError.actionError( + message: message, + error: error, + file: file, + line: line + ) + determinedStepStatus = .broken + } else { + determinedStepStatus = .passed } - stepOutcome = StepOutcome(step) + } catch let assertionErr as Assertion.AssertionError { + stepError = assertionErr + determinedStepStatus = .failed + } catch let error as BaseError { + XCTFail(error.localizedDescription, file: error.file, line: error.line) + stepError = error + determinedStepStatus = .failed } catch { - stepOutcome = StepOutcome(step, error) currentScenario!.fail(file: step.file, line: step.line, message: String(describing: error)) + stepError = error + determinedStepStatus = .broken } + stepOutcome.status = determinedStepStatus + stepOutcome.error = stepError scenarioOutcome.steps.append(stepOutcome) try await report(.AFTER_STEP, stepOutcome) - assertionFailure = nil - if (stepOutcome.error != nil) { - scenarioOutcome.failedStep = stepOutcome + if stepOutcome.status == .failed || stepOutcome.status == .broken { + scenarioOutcome.status = stepOutcome.status + scenarioOutcome.error = stepOutcome.error + break + } + + if stepOutcome.status == .pending { + scenarioOutcome.status = stepOutcome.status break } } + scenarioOutcome.end() return scenarioOutcome + } + func executeAction( + _ message: String, + _ file: StaticString, + _ line: UInt, + _ closure: () async throws -> T + ) async throws -> T { + /// skip if any previous action failed + if (actionFailure != nil) { + throw actionFailure!.1 + } + + let actionOutcome = ActionOutcome(action: message) + actionOutcome.start() + do { + let result = try await closure() + actionOutcome.status = .passed + actionOutcome.end() + try await TestConfiguration.shared().report(.ACTION, actionOutcome) + return result + } catch { + actionOutcome.status = .failed + actionOutcome.error = error + actionOutcome.end() + try await TestConfiguration.shared().report(.ACTION, actionOutcome) + actionFailure = (message: message, error: error, file: file, line: line) + throw error + } + } + public func afterStep(_ stepOutcome: StepOutcome) async throws { - try await report(.AFTER_STEP, stepOutcome.step.action) + try await report(.AFTER_STEP, stepOutcome) } public func afterScenario(_ scenarioOutcome: ScenarioOutcome) async throws { - currentFeatureOutcome!.scenarios.append(scenarioOutcome) - if (scenarioOutcome.failedStep != nil) { - currentFeatureOutcome!.failedScenarios.append(scenarioOutcome) + guard let currentFeatureOut = self.currentFeatureOutcome else { + print("TestConfiguration Error: currentFeatureOutcome is nil in afterScenario for scenario: \(scenarioOutcome.scenario.name)") + return } + currentFeatureOut.scenarioOutcomes.append(scenarioOutcome) try await report(.AFTER_SCENARIO, scenarioOutcome) try await tearDownActors() } public func afterFeature(_ featureOutcome: FeatureOutcome) async throws { + currentFeatureOutcome!.end() try await report(.AFTER_FEATURE, featureOutcome) } @@ -157,14 +232,27 @@ open class TestConfiguration: ITestConfiguration { } public func endCurrentFeature() async throws { - try await self.afterFeature(self.currentFeatureOutcome!) + guard let featureOutcomeToFinalize = self.currentFeatureOutcome else { + print("TestConfiguration Error: currentFeatureOutcome is nil in endCurrentFeature.") + return + } + var possibleFeatureError: Error? = nil + do { + try await self.tearDownInstance() + } catch { + possibleFeatureError = error + } + + featureOutcomeToFinalize.finalizeOutcome(featureLevelError: possibleFeatureError) + try await self.afterFeature(featureOutcomeToFinalize) } /// signals the suite has ended public func end() { let semaphore = DispatchSemaphore(value: 0) Task.detached { - try await self.afterFeatures(self.result.featuresOutcome) + self.suiteOutcome.end() + try await self.afterFeatures(self.suiteOutcome.featureOutcomes) try await self.tearDownInstance() semaphore.signal() } @@ -242,25 +330,22 @@ open class TestConfiguration: ITestConfiguration { // force as own instance let instance = instanceType.createInstance() as! TestConfiguration + instance.suiteOutcome.start() + self.instance = instance do { try await instance.setUp() try await instance.setUpReporters() try await instance.setUpSteps() } catch { - print("Error setting up configuration: \(error)") - fflush(stdout) - fflush(stderr) - exit(1) + throw ConfigurationError.setup(message: error.localizedDescription) } - + /// setup hamcrest to update variable if failed HamcrestReportFunction = { message, file, line in instance.assertionFailure = (message, file, line) } - self.instance = instance - let fileManager = FileManager.default /// delete target folder do { @@ -320,10 +405,4 @@ open class TestConfiguration: ITestConfiguration { var intParser = { (int: String) in return Int(int)! } - - enum Failure: Error { - case StepParameterDoesNotMatch(step: String, expected: String, actual: String) - case StepNotFound(step: String) - case ParameterTypeNotFound - } } diff --git a/E2E/TestFramework/Errors/ActorError.swift b/E2E/TestFramework/Errors/ActorError.swift new file mode 100644 index 00000000..2c6d5f82 --- /dev/null +++ b/E2E/TestFramework/Errors/ActorError.swift @@ -0,0 +1,20 @@ +import Foundation + +class ActorError { + class actionError: BaseError { + init(message: String, error: Error, file: StaticString, line: UInt) { + super.init(message: message, error: String(describing: error), file: file, line: line) + } + } + + class cantUseAbility: BaseError { + init(_ message: String, file: StaticString = #file, line: UInt = #line) { + super.init(message: message, error: "Actor cannot use ability", file: file, line: line) + } + } + class cantFindNote: BaseError { + init(_ message: String, file: StaticString = #file, line: UInt = #line) { + super.init(message: message, error: "Actor cannot find note", file: file, line: line) + } + } +} diff --git a/E2E/TestFramework/Errors/AssertionError.swift b/E2E/TestFramework/Errors/AssertionError.swift new file mode 100644 index 00000000..68cac089 --- /dev/null +++ b/E2E/TestFramework/Errors/AssertionError.swift @@ -0,0 +1,29 @@ +import Foundation + +class Assertion { + class AssertionError: BaseError { + init(message: String, file: StaticString = #file, line: UInt = #line) { + super.init( + message: message, + error: "Assertion failure", + file: file, + line: line + ) + } + } + + class TimeoutError: BaseError { + let timeout: Int + + init(timeout: Int, message: String = "time limit exceeded", file: StaticString = #file, line: UInt = #line) { + self.timeout = timeout + super.init( + message: message, + error: "Timeout reached (\(timeout))s", + file: file, + line: line + ) + } + } +} + diff --git a/E2E/TestFramework/Errors/BaseError.swift b/E2E/TestFramework/Errors/BaseError.swift new file mode 100644 index 00000000..ab896c4c --- /dev/null +++ b/E2E/TestFramework/Errors/BaseError.swift @@ -0,0 +1,49 @@ +import Foundation + +open class BaseError: Error, LocalizedError, CustomStringConvertible { + public let file: StaticString + public let line: UInt + + // Properties for LocalizedError conformance + public var errorDescription: String? + public var failureReason: String? + public var recoverySuggestion: String? // Optional + public var helpAnchor: String? // Optional + + // Store the original components for constructing descriptions + private let providedMessage: String + private let errorTypeString: String // This was the 'error' parameter in your init + + // This is for CustomStringConvertible. + // XCTest might use this when an error is thrown out of a test method. + public var description: String { + // We'll make this the same as errorDescription for consistency, + // ensuring a detailed message is available. + return self.errorDescription ?? "Undefined error: \(self.errorTypeString) - \(self.providedMessage)" + } + + // 'error' parameter here is a string describing the category or type of the error. + public init(message: String, error: String, file: StaticString = #file, line: UInt = #line) { + self.file = file + self.line = line + self.providedMessage = message + self.errorTypeString = error + + let fileName = URL(fileURLWithPath: String(describing: file)).lastPathComponent + + // Populate LocalizedError properties + // This self.errorDescription will be used by error.localizedDescription + self.errorDescription = "\(self.errorTypeString): \(self.providedMessage) (at \(fileName):\(line))" + + self.failureReason = "An issue occurred related to '\(self.errorTypeString)'." + } +} + +// Your ConfigurationError.setup class remains the same: +// open class ConfigurationError { +// public final class setup: BaseError { +// public init(message: String, file: StaticString = #file, line: UInt = #line) { +// super.init(message: message, error: "Configuration error", file: file, line: line) +// } +// } +// } diff --git a/E2E/TestFramework/Errors/ConfigurationError.swift b/E2E/TestFramework/Errors/ConfigurationError.swift new file mode 100644 index 00000000..3e16be50 --- /dev/null +++ b/E2E/TestFramework/Errors/ConfigurationError.swift @@ -0,0 +1,20 @@ +open class ConfigurationError { + public final class setup: BaseError { + public init(message: String, file: StaticString = #file, line: UInt = #line) { + super.init(message: message, error: "Configuration error", file: file, line: line) + } + } + + public final class missingScenario: Error, CustomStringConvertible { + public var errorDescription: String + public var failureReason: String = "Missing scenario" + + public init(_ message: String) { + self.errorDescription = message + } + + public var description: String { + return self.errorDescription + } + } +} diff --git a/E2E/TestFramework/Errors/StepError.swift b/E2E/TestFramework/Errors/StepError.swift new file mode 100644 index 00000000..2eff3504 --- /dev/null +++ b/E2E/TestFramework/Errors/StepError.swift @@ -0,0 +1,31 @@ +import Foundation + +enum StepError { + class parameterTypeDoesNotMatch: BaseError { + init(_ step: String, expected: String, actual: String, file: StaticString = #file, line: UInt = #line) { + let message = "Parameter mismatch in step '\(step)': Expected a parameter matching '\(expected)', but found '\(actual)'." + super.init(message: message, error: "Parameter doesn't match", file: file, line: line) + } + } + + class notFound: BaseError { + init(_ stepPattern: String, file: StaticString = #file, line: UInt = #line) { + let message = "No step definition found matching the pattern: '\(stepPattern)'. Please ensure a corresponding step implementation exists." + super.init(message: message, error: "Step definition not found", file: file, line: line) + } + } + + class typeNotFound: BaseError { + init(typeName: String? = nil, forStep: String? = nil, file: StaticString = #file, line: UInt = #line) { + var message = "A suitable parameter parser or type was not found" + if let type = typeName { + message += " for type '\(type)'" + } + if let step = forStep { + message += " in step '\(step)'" + } + message += "." + super.init(message: message, error: "Parameter type not found", file: file, line: line) + } + } +} diff --git a/E2E/TestFramework/Outcome/ActionOutcome.swift b/E2E/TestFramework/Outcome/ActionOutcome.swift index 72233238..ba95f610 100644 --- a/E2E/TestFramework/Outcome/ActionOutcome.swift +++ b/E2E/TestFramework/Outcome/ActionOutcome.swift @@ -1,7 +1,28 @@ import Foundation public class ActionOutcome { - var action = "" - var executed = false + var action: String = "" + var status: TestStatus var error: Error? = nil + public var startTime: Date? + public var endTime: Date? +// var attachments: [AttachmentData]? + + init(action: String) { + self.action = action + self.status = .passed + } + + func start() { + startTime = Date() + } + + func end() { + endTime = Date() + } + + public var duration: TimeInterval? { + guard let start = startTime, let end = endTime else { return nil } + return end.timeIntervalSince(start) + } } diff --git a/E2E/TestFramework/Outcome/FeatureOutcome.swift b/E2E/TestFramework/Outcome/FeatureOutcome.swift index 24e79488..ab3c592d 100644 --- a/E2E/TestFramework/Outcome/FeatureOutcome.swift +++ b/E2E/TestFramework/Outcome/FeatureOutcome.swift @@ -1,11 +1,74 @@ import Foundation public class FeatureOutcome { - let feature: Feature - var scenarios: [ScenarioOutcome] = [] - var failedScenarios: [ScenarioOutcome] = [] + public let feature: Feature + public var scenarioOutcomes: [ScenarioOutcome] = [] + public var status: TestStatus + public var error: Error? + public var startTime: Date? + public var endTime: Date? + + func start() { + startTime = Date() + } + + func end() { + endTime = Date() + } - init(_ feature: Feature) { + public var duration: TimeInterval? { + guard let start = startTime, let end = endTime else { return nil } + return end.timeIntervalSince(start) + } + + public var passedScenarios: [ScenarioOutcome] { + scenarioOutcomes.filter { $0.status == .passed } + } + public var failedScenarios: [ScenarioOutcome] { + scenarioOutcomes.filter { $0.status == .failed } + } + public var brokenScenarios: [ScenarioOutcome] { + scenarioOutcomes.filter { $0.status == .broken } + } + public var skippedScenarios: [ScenarioOutcome] { + scenarioOutcomes.filter { $0.status == .skipped } + } + + public init(feature: Feature, startTime: Date = Date()) { self.feature = feature + self.startTime = startTime + self.status = .passed + } + + public func finalizeOutcome(featureLevelError: Error? = nil) { + self.endTime = Date() // Set end time when finalizing + + if let explicitError = featureLevelError { + self.error = explicitError + } + + if self.error != nil { + self.status = .broken + return + } + + if scenarioOutcomes.isEmpty { + return + } + + if scenarioOutcomes.contains(where: { $0.status == .broken }) { + self.status = .broken + } else if scenarioOutcomes.contains(where: { $0.status == .failed }) { + self.status = .failed + } else if scenarioOutcomes.allSatisfy({ $0.status == .skipped }) { + self.status = .skipped + } else if scenarioOutcomes.allSatisfy({ $0.status == .passed || $0.status == .skipped }) { + self.status = .passed + } else { + // This case should ideally not be reached if the above logic is complete. + // Could default to .broken if there's an unexpected mix. + print("Warning: FeatureOutcome for '\(feature.title())' has an undetermined status mix.") + self.status = .broken // Default for unexpected mixed states + } } } diff --git a/E2E/TestFramework/Outcome/ResultOutcome.swift b/E2E/TestFramework/Outcome/ResultOutcome.swift deleted file mode 100644 index 8c8dee0b..00000000 --- a/E2E/TestFramework/Outcome/ResultOutcome.swift +++ /dev/null @@ -1,5 +0,0 @@ -import Foundation - -public class ResultOutcome { - var featuresOutcome: [FeatureOutcome] = [] -} diff --git a/E2E/TestFramework/Outcome/ScenarioOutcome.swift b/E2E/TestFramework/Outcome/ScenarioOutcome.swift index 05b03f11..c63561ba 100644 --- a/E2E/TestFramework/Outcome/ScenarioOutcome.swift +++ b/E2E/TestFramework/Outcome/ScenarioOutcome.swift @@ -3,9 +3,30 @@ import Foundation public class ScenarioOutcome { let scenario: Scenario var steps: [StepOutcome] = [] - var failedStep: StepOutcome? = nil - + var status: TestStatus + var error: Error? + public var startTime: Date? + public var endTime: Date? + + var failedStep: StepOutcome? { + return steps.first(where: { $0.status == .failed || $0.status == .broken }) + } + init(_ scenario: Scenario) { self.scenario = scenario + self.status = .passed + } + + func start() { + startTime = Date() + } + + func end() { + endTime = Date() + } + + public var duration: TimeInterval? { + guard let start = startTime, let end = endTime else { return nil } + return end.timeIntervalSince(start) } } diff --git a/E2E/TestFramework/Outcome/StepOutcome.swift b/E2E/TestFramework/Outcome/StepOutcome.swift index 4e86cfc3..bb87a995 100644 --- a/E2E/TestFramework/Outcome/StepOutcome.swift +++ b/E2E/TestFramework/Outcome/StepOutcome.swift @@ -2,10 +2,26 @@ import Foundation public class StepOutcome { let step: ConcreteStep + var status: TestStatus var error: Error? - - init(_ step: ConcreteStep, _ error: Error? = nil) { + public var startTime: Date? + public var endTime: Date? + + init(_ step: ConcreteStep) { self.step = step - self.error = error + self.status = .passed + } + + func start() { + startTime = Date() + } + + func end() { + endTime = Date() + } + + public var duration: TimeInterval? { + guard let start = startTime, let end = endTime else { return nil } + return end.timeIntervalSince(start) } } diff --git a/E2E/TestFramework/Outcome/SuiteOutcome.swift b/E2E/TestFramework/Outcome/SuiteOutcome.swift new file mode 100644 index 00000000..ba39fb50 --- /dev/null +++ b/E2E/TestFramework/Outcome/SuiteOutcome.swift @@ -0,0 +1,33 @@ +import Foundation + +public class SuiteOutcome { + var featureOutcomes: [FeatureOutcome] = [] + public var startTime: Date? + public var endTime: Date? + + public var passedFeatures: [FeatureOutcome] { + featureOutcomes.filter { $0.status == .passed } + } + public var failedFeatures: [FeatureOutcome] { + featureOutcomes.filter { $0.status == .failed } + } + public var brokenFeatures: [FeatureOutcome] { + featureOutcomes.filter { $0.status == .broken } + } + public var skippedFeatures: [FeatureOutcome] { + featureOutcomes.filter { $0.status == .skipped } + } + + func start() { + startTime = Date() + } + + func end() { + endTime = Date() + } + + public var duration: TimeInterval? { + guard let start = startTime, let end = endTime else { return nil } + return end.timeIntervalSince(start) + } +} diff --git a/E2E/TestFramework/Outcome/TestStatus.swift b/E2E/TestFramework/Outcome/TestStatus.swift new file mode 100644 index 00000000..34599960 --- /dev/null +++ b/E2E/TestFramework/Outcome/TestStatus.swift @@ -0,0 +1,7 @@ +public enum TestStatus: String, Equatable { + case passed + case failed + case broken + case skipped + case pending +} diff --git a/E2E/TestFramework/Report/AllureReporter.swift b/E2E/TestFramework/Report/AllureReporter.swift new file mode 100644 index 00000000..57bc348d --- /dev/null +++ b/E2E/TestFramework/Report/AllureReporter.swift @@ -0,0 +1,448 @@ +import Foundation +import XCTest +import CryptoKit + +// MARK: - Allure Data Model Structs (for JSON serialization) + +public enum AllureStatus: String, Codable { + case passed + case failed + case broken + case skipped + case unknown +} + +public enum AllureStage: String, Codable { + case scheduled + case running + case finished + case pending + case interrupted +} + +public struct AllureLabel: Codable { + public let name: String + public let value: String +} + +public struct AllureLink: Codable { + public let name: String? + public let url: String + public let type: String? +} + +public struct AllureParameter: Codable { + public let name: String + public let value: String + // public var mode: String? // e.g., "masked", "hidden" (optional) + // public var excluded: Bool? // (optional) +} + +public struct AllureStatusDetails: Codable { + public var known: Bool? = false + public var muted: Bool? = false + public var flaky: Bool? = false + public var message: String? + public var trace: String? +} + +public struct AllureAttachment: Codable { + public let name: String + public let source: String + public let type: String +} + +public struct AllureStepResult: Codable { + public var uuid: String = UUID().uuidString + + public var name: String + public var status: AllureStatus? + public var statusDetails: AllureStatusDetails? + public var stage: AllureStage? = .finished // Most steps are reported once finished + public var description: String? + public var descriptionHtml: String? + public var steps: [AllureStepResult] = [] // For nested steps (from ActionOutcome) + public var attachments: [AllureAttachment] = [] + public var parameters: [AllureParameter] = [] + public var start: Int64? + public var stop: Int64? +} + +public struct AllureTestResult: Codable { + /// identifiers + public let uuid: String + public var historyId: String? + public var testCaseId: String? + /// metadata + public var name: String? + public var fullName: String? + public var description: String? + public var descriptionHtml: String? + public var links: [AllureLink] = [] + public var labels: [AllureLabel] = [] + public var parameters: [AllureParameter] = [] // Scenario-level parameters (e.g., from examples table) + public var attachments: [AllureAttachment] = [] + /// execution + public var status: AllureStatus? + public var statusDetails: AllureStatusDetails? + public var stage: AllureStage? = .finished + public var start: Int64? + public var stop: Int64? + public var steps: [AllureStepResult] = [] +} + +public struct AllureTestResultContainer: Codable { // Represents a Feature + public let uuid: String // Unique ID for this container + public var start: Int64? + public var stop: Int64? + public var children: [String] = [] // UUIDs of AllureTestResult (scenario) objects + public var befores: [AllureFixtureResult] = [] // For setup fixtures + public var afters: [AllureFixtureResult] = [] // For teardown fixtures +} + +public struct AllureFixtureResult: Codable { // For feature-level setup/teardown issues + public var name: String? + public var status: AllureStatus? + public var statusDetails: AllureStatusDetails? + public var stage: AllureStage? = .finished + public var description: String? + public var descriptionHtml: String? + public var steps: [AllureStepResult] = [] // Fixtures can also have steps + public var attachments: [AllureAttachment] = [] + public var parameters: [AllureParameter] = [] + public var start: Int64? + public var stop: Int64? +} + + +// MARK: - Allure Reporter Implementation + +public class AllureReporter: Reporter { + /// helpers + private let allureResultsPath: URL + private let fileManager: FileManager + private let jsonEncoder: JSONEncoder + + /// state + private var currentAllureFeatureContainer: AllureTestResultContainer? + private var currentAllureTestCase: AllureTestResult? + private var allureStepStack: [AllureStepResult] = [] + + public required init() { + self.fileManager = FileManager.default + + let targetDir: URL + if let configProvider = TestConfiguration.shared() as? TestConfiguration { + targetDir = configProvider.targetDirectory() + } else { + print("AllureReporter: CRITICAL - Could not determine targetDirectory from TestConfiguration.shared(). Defaulting to current directory for allure-results. THIS IS LIKELY WRONG.") + targetDir = URL(fileURLWithPath: fileManager.currentDirectoryPath) + } + self.allureResultsPath = targetDir.appendingPathComponent("allure-results") + self.jsonEncoder = JSONEncoder() + self.jsonEncoder.outputFormatting = .prettyPrinted + + do { + if !fileManager.fileExists(atPath: allureResultsPath.path) { + try fileManager.createDirectory(at: allureResultsPath, + withIntermediateDirectories: true, + attributes: nil) + } + print("AllureReporter: Results directory initialized at: \(allureResultsPath.path)") + } catch { + fatalError("AllureReporter: Could not create Allure results directory at '\(allureResultsPath.path)': \(error)") + } + } + + private func md5Hash(from string: String) -> String { + guard let data = string.data(using: .utf8) else { + // Fallback if string can't be UTF-8 encoded, though unlikely for typical identifiers + return UUID().uuidString // Or some other default + } + if #available(iOS 13.0, macOS 10.15, watchOS 6.0, tvOS 13.0, *) { + let digest = Insecure.MD5.hash(data: data) + return digest.map { String(format: "%02hhx", $0) }.joined() + } else { + print("AllureReporter: MD5 not available on this OS version. Using UUID substring for hash.") + return String(UUID().uuidString.prefix(32)) + } + } + + private func getCurrentPid() -> String { + let pid = ProcessInfo.processInfo.processIdentifier + return "pid-\(pid)" + } + + private func generatePackageName(fromFeatureType featureType: Feature.Type) -> String { + var className = String(describing: featureType) + if let dotIndex = className.lastIndex(of: ".") { + className = String(className.suffix(from: className.index(after: dotIndex))) + } + var processedName = className + if processedName.hasSuffix("Feature") { + processedName = String(processedName.dropLast("Feature".count)) + } + return "features.\(processedName.lowercased()).feature" + } + + private func millisecondsSince1970(from date: Date?) -> Int64? { + guard let date = date else { return nil } + return Int64(date.timeIntervalSince1970 * 1000) + } + + private func mapFrameworkStatusToAllureStatus(_ status: TestStatus) -> AllureStatus { // Using your TestStatus + switch status { + case .passed: return .passed + case .failed: return .failed + case .broken: return .broken + case .skipped: return .skipped + case .pending: return .skipped // Allure convention: pending often maps to skipped + } + } + + private func allureStatusDetails(from error: Error?) -> AllureStatusDetails? { + guard let err = error else { return nil } + + var message: String + if let localizedError = err as? LocalizedError, let errDescription = localizedError.errorDescription { + message = errDescription + } else { + message = err.localizedDescription + } + if message.isEmpty { + message = String(describing: err) + } + let trace: String = "\(err)" + return AllureStatusDetails(message: message, trace: trace) + } + + private func ensureResultsDirectoryExists() throws { + if !fileManager.fileExists(atPath: allureResultsPath.path) { + print("AllureReporter: Results directory \(allureResultsPath.path) not found before write. Attempting to create.") + try fileManager.createDirectory(at: allureResultsPath, + withIntermediateDirectories: true, + attributes: nil) + } + } + + private func writeJSON(_ data: T, fileName: String) { + do { + try ensureResultsDirectoryExists() + let filePath = allureResultsPath.appendingPathComponent(fileName) + let jsonData = try jsonEncoder.encode(data) + try jsonData.write(to: filePath) + } catch { + // Provide more context in the error print if possible + print("AllureReporter: Error writing Allure JSON for '\(fileName)' to '\(allureResultsPath.appendingPathComponent(fileName).path)': \(error). Underlying POSIX error (if any): \(String(describing: (error as NSError).userInfo[NSUnderlyingErrorKey]))") + } + } + + // MARK: - Reporter Protocol Implementation + + public func beforeFeature(_ feature: Feature) async throws { + let containerUUID = UUID().uuidString + + self.currentAllureFeatureContainer = AllureTestResultContainer( + uuid: containerUUID, + ) + } + + public func beforeScenario(_ scenario: Scenario) async throws { + let testCaseUUID = UUID().uuidString + let scenarioUniqueName = "\(scenario.feature!.name)#\(scenario.name)" + let calculatedTestCaseId = md5Hash(from: scenarioUniqueName) + + var parameterString = "" + if let params = scenario.parameters, !params.isEmpty { + let sortedParams = params.sorted { $0.key < $1.key } + parameterString = sortedParams.map { "\($0.key)=\($0.value)" }.joined(separator: ";") + } + let parameterHash = md5Hash(from: parameterString) + let calculatedHistoryId = "\(calculatedTestCaseId):\(parameterHash)" + + var initialStatus: AllureStatus? = nil + var initialStage: AllureStage = .running + if scenario.disabled { + initialStatus = .skipped + initialStage = .finished + } + + self.currentAllureTestCase = AllureTestResult( + uuid: testCaseUUID, + historyId: calculatedHistoryId, + testCaseId: calculatedTestCaseId, + name: scenario.name, + fullName: scenarioUniqueName, + description: scenario.feature?.description(), + labels: [ + AllureLabel(name: "host", value: ProcessInfo.processInfo.hostName), + AllureLabel(name: "thread", value: getCurrentPid()), + AllureLabel(name: "package", value: generatePackageName(fromFeatureType: type(of: scenario.feature!))), + AllureLabel(name: "language", value: "swift"), + AllureLabel(name: "framework", value: "identus-e2e-framework"), + AllureLabel(name: "feature", value: scenario.feature!.title()), + //AllureLabel(name: "suite", value: "suite"), // FIXME: property? config? + //AllureLabel(name: "epic", value: "suite"), // FIXME: property? config? + //AllureLabel(name: "story", value: scenario.name) + ], + status: initialStatus, + stage: initialStage + ) + allureStepStack.removeAll() + } + + public func beforeStep(_ step: ConcreteStep) async throws { + guard self.currentAllureTestCase != nil else { + print("AllureReporter: Warning - beforeStep called without an active scenario.") + return + } + + let allureStep = AllureStepResult( + name: "\(step.context) \(step.action)", + stage: .running, + ) + + if var parentStep = allureStepStack.last { + parentStep.steps.append(allureStep) + allureStepStack[allureStepStack.count - 1] = parentStep + } else { + self.currentAllureTestCase?.steps.append(allureStep) + } + allureStepStack.append(allureStep) + } + + public func action(_ actionOutcome: ActionOutcome) async throws { + var parentAllureStep = allureStepStack.removeLast() + let subStep = AllureStepResult( + name: actionOutcome.action, + status: mapFrameworkStatusToAllureStatus(actionOutcome.status), + statusDetails: allureStatusDetails(from: actionOutcome.error), + stage: .finished, + start: millisecondsSince1970(from: actionOutcome.startTime), + stop: millisecondsSince1970(from: actionOutcome.endTime) + ) + parentAllureStep.steps.append(subStep) + allureStepStack.append(parentAllureStep) + } + + public func afterStep(_ stepOutcome: StepOutcome) async throws { + guard var completedAllureStep = allureStepStack.popLast() else { + print("AllureReporter: Warning - afterStep called with no matching Allure step on stack. Step: \(stepOutcome.step.context) \(stepOutcome.step.action)") + return + } + + completedAllureStep.status = mapFrameworkStatusToAllureStatus(stepOutcome.status) + completedAllureStep.statusDetails = allureStatusDetails(from: stepOutcome.error) + completedAllureStep.stage = .finished + completedAllureStep.start = millisecondsSince1970(from: stepOutcome.startTime) + completedAllureStep.stop = millisecondsSince1970(from: stepOutcome.endTime) + + if var parentStep = allureStepStack.last { + if let index = parentStep.steps.firstIndex(where: { $0.uuid == completedAllureStep.uuid }) { + parentStep.steps[index] = completedAllureStep + allureStepStack[allureStepStack.count - 1] = parentStep + } else { + print("AllureReporter: Warning - Could not find step \(completedAllureStep.name) in parent step \(parentStep.name) to update.") + } + } else if self.currentAllureTestCase != nil { + if let index = self.currentAllureTestCase!.steps.firstIndex(where: { $0.uuid == completedAllureStep.uuid }) { + self.currentAllureTestCase!.steps[index] = completedAllureStep + } else { + print("AllureReporter: Warning - Could not find step \(completedAllureStep.name) in current test case to update.") + } + } + } + + public func afterScenario(_ scenarioOutcome: ScenarioOutcome) async throws { + guard var testCase = self.currentAllureTestCase else { + print("AllureReporter: Warning - afterScenario called with no active Allure test case. Scenario: \(scenarioOutcome.scenario.name)") + return + } + + testCase.status = mapFrameworkStatusToAllureStatus(scenarioOutcome.status) + let relevantError = scenarioOutcome.error ?? scenarioOutcome.failedStep?.error + testCase.statusDetails = allureStatusDetails(from: relevantError) + testCase.stage = .finished + testCase.start = millisecondsSince1970(from: scenarioOutcome.startTime) + testCase.stop = millisecondsSince1970(from: scenarioOutcome.endTime) + + // If the scenario was disabled and not caught by beforeScenario (e.g. if status was updated later) + if scenarioOutcome.scenario.disabled && testCase.status != .skipped { + testCase.status = .skipped + if testCase.statusDetails == nil { + testCase.statusDetails = AllureStatusDetails(message: "Scenario was marked as disabled.") + } + } + + if self.currentAllureFeatureContainer != nil { + self.currentAllureFeatureContainer!.children.append(testCase.uuid) + } + + writeJSON(testCase, fileName: "\(testCase.uuid)-result.json") + + self.currentAllureTestCase = nil + if !allureStepStack.isEmpty { + print("AllureReporter: Warning - Step stack not empty after scenario \(scenarioOutcome.scenario.name). This may indicate mismatched beforeStep/afterStep calls. Clearing stack.") + allureStepStack.removeAll() + } + } + + public func afterFeature(_ featureOutcome: FeatureOutcome) async throws { + var container = self.currentAllureFeatureContainer! + container.start = millisecondsSince1970(from: featureOutcome.startTime) + container.stop = millisecondsSince1970(from: featureOutcome.endTime) + if (featureOutcome.status == .broken || featureOutcome.status == .failed), + let featureErr = featureOutcome.error { // From your FeatureOutcome model + let fixtureName = "Feature Level Issue: \(featureOutcome.feature.title())" + let problemFixture = AllureFixtureResult( + name: fixtureName, + status: mapFrameworkStatusToAllureStatus(featureOutcome.status), + statusDetails: allureStatusDetails(from: featureErr), + stage: .finished, + start: container.start, + stop: container.stop + ) + container.befores.append(problemFixture) + } + writeJSON(container, fileName: "\(container.uuid)-container.json") + self.currentAllureFeatureContainer = nil + } + + public func afterFeatures(_ featuresOutcome: [FeatureOutcome]) async throws { +// writeEnvironmentProperties() // Using your TestConfiguration.environment + print("AllureReporter: All features processed. Allure JSON generation finished. Results in: \(allureResultsPath.path)") + } + + private func writeEnvironmentProperties() { + let environmentDict = TestConfiguration.shared().environment + guard !environmentDict.isEmpty else { + print("AllureReporter: No environment properties to write (environment dictionary is empty).") + return + } + + var propertiesContent = "" + for (key, value) in environmentDict.sorted(by: { $0.key < $1.key }) { + let escapedKey = key + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: " ", with: "\\ ") + .replacingOccurrences(of: "=", with: "\\=") + .replacingOccurrences(of: ":", with: "\\:") + + let escapedValue = value + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\n", with: "\\n") + .replacingOccurrences(of: "\r", with: "\\r") + + propertiesContent += "\(escapedKey)=\(escapedValue)\n" + } + + if !propertiesContent.isEmpty { + let filePath = allureResultsPath.appendingPathComponent("environment.properties") + do { + try propertiesContent.write(to: filePath, atomically: true, encoding: .utf8) + } catch { + print("AllureReporter: Error writing environment.properties: \(error)") + } + } + } +} diff --git a/E2E/TestFramework/Report/ConsoleReporter.swift b/E2E/TestFramework/Report/ConsoleReporter.swift index 00194c96..762a8528 100644 --- a/E2E/TestFramework/Report/ConsoleReporter.swift +++ b/E2E/TestFramework/Report/ConsoleReporter.swift @@ -11,11 +11,12 @@ public class ConsoleReporter: Reporter { public func beforeFeature(_ feature: Feature) async throws { print() + print("---") print("Feature:", feature.title()) } public func beforeScenario(_ scenario: Scenario) async throws { - print(" ", scenario.title) + print(" ", scenario.name) } public func beforeStep(_ step: ConcreteStep) async throws { @@ -35,12 +36,11 @@ public class ConsoleReporter: Reporter { } public func afterScenario(_ scenarioOutcome: ScenarioOutcome) async throws { - let result = scenarioOutcome.failedStep != nil ? "FAIL" : "PASS" - print(" ", "Result:", result) + print(" ", "Result:", scenarioOutcome.status.rawValue.uppercased()) } public func afterFeature(_ featureOutcome: FeatureOutcome) async throws { - print() + print("Feature result", featureOutcome.status.rawValue.uppercased()) } public func afterFeatures(_ featuresOutcome: [FeatureOutcome]) async throws { diff --git a/E2E/TestFramework/Report/DebugReporter.swift b/E2E/TestFramework/Report/DebugReporter.swift index 0cdcb489..d7dcbc95 100644 --- a/E2E/TestFramework/Report/DebugReporter.swift +++ b/E2E/TestFramework/Report/DebugReporter.swift @@ -11,7 +11,7 @@ public class DebugReporter: Reporter { } public func beforeScenario(_ scenario: Scenario) async throws { - if debug { print("Before Scenario:", scenario.title) } + if debug { print("Before Scenario:", scenario.name) } } public func beforeStep(_ step: ConcreteStep) async throws { @@ -31,7 +31,7 @@ public class DebugReporter: Reporter { } public func afterScenario(_ scenarioOutcome: ScenarioOutcome) async throws { - print("After Scenario", scenarioOutcome.scenario.title) + print("After Scenario", scenarioOutcome.scenario.name) } public func afterFeature(_ featureOutcome: FeatureOutcome) async throws { diff --git a/E2E/TestFramework/Report/DotReporter.swift b/E2E/TestFramework/Report/DotReporter.swift index 06a1e9d7..aa62c797 100644 --- a/E2E/TestFramework/Report/DotReporter.swift +++ b/E2E/TestFramework/Report/DotReporter.swift @@ -39,11 +39,11 @@ public class DotReporter: Reporter { print("Executed", featuresOutcome.count, "features") for featureOutcome in featuresOutcome { print(" ", "Feature:", featureOutcome.feature.title()) - for scenarioOutcome in featureOutcome.scenarios { + for scenarioOutcome in featureOutcome.scenarioOutcomes { print( " ", scenarioOutcome.failedStep != nil ? "(fail)" : "(pass)", - scenarioOutcome.scenario.title + scenarioOutcome.scenario.name ) if (scenarioOutcome.failedStep != nil) { let failedStep = scenarioOutcome.failedStep! diff --git a/E2E/TestFramework/Report/HtmlReporter.swift b/E2E/TestFramework/Report/HtmlReporter.swift index c2745681..3c56e690 100644 --- a/E2E/TestFramework/Report/HtmlReporter.swift +++ b/E2E/TestFramework/Report/HtmlReporter.swift @@ -26,11 +26,11 @@ public class HtmlReporter: Reporter { currentId = currentFeature!.id + currentScenario!.id + step.id } - public func action(_ action: ActionOutcome) async throws { + public func action(_ actionOutcome: ActionOutcome) async throws { if (actions[currentId!] == nil) { actions[currentId!] = [] } - actions[currentId!]!.append(action) + actions[currentId!]!.append(actionOutcome) } public func afterStep(_ stepOutcome: StepOutcome) async throws { @@ -52,11 +52,12 @@ public class HtmlReporter: Reporter { featureReport.name = featureOutcome.feature.title() htmlReport.data.append(featureReport) - for scenarioOutcome in featureOutcome.scenarios { + for scenarioOutcome in featureOutcome.scenarioOutcomes { let scenarioReport = ScenarioReport() - scenarioReport.name = scenarioOutcome.scenario.title + scenarioReport.name = scenarioOutcome.scenario.name featureReport.scenarios.append(scenarioReport) - + scenarioReport.status = scenarioOutcome.status.rawValue + for stepOutcome in scenarioOutcome.steps { let stepReport = StepReport() stepReport.name = stepOutcome.step.action @@ -67,8 +68,7 @@ public class HtmlReporter: Reporter { for actionOutcome in stepActions { let actionReport = ActionReport() actionReport.action = actionOutcome.action - actionReport.passed = actionOutcome.error == nil - actionReport.executed = actionOutcome.executed + actionReport.status = actionOutcome.status.rawValue stepReport.actions.append(actionReport) if(actionOutcome.error != nil) { break @@ -76,7 +76,6 @@ public class HtmlReporter: Reporter { } } if (stepOutcome.error != nil) { - scenarioReport.passed = false stepReport.passed = false stepReport.error = String(describing: scenarioOutcome.failedStep!.error!) break @@ -114,7 +113,7 @@ private class FeatureReport: Codable { private class ScenarioReport: Codable { var name: String = "" - var passed: Bool = true + var status: String = "" var steps: [StepReport] = [] } @@ -127,6 +126,5 @@ private class StepReport: Codable { private class ActionReport: Codable { var action: String = "" - var passed: Bool = true - var executed: Bool = false + var status: String = "" } diff --git a/E2E/TestFramework/Report/JunitReporter.swift b/E2E/TestFramework/Report/JunitReporter.swift index 432dec88..7719416e 100644 --- a/E2E/TestFramework/Report/JunitReporter.swift +++ b/E2E/TestFramework/Report/JunitReporter.swift @@ -54,7 +54,7 @@ public class JunitReporter: Reporter { currentScenario = XMLElement(name: "testcase") let id = XMLNode.attribute(withName: "id", stringValue: scenario.id) as! XMLNode - let name = XMLNode.attribute(withName: "name", stringValue: scenario.title) as! XMLNode + let name = XMLNode.attribute(withName: "name", stringValue: scenario.name) as! XMLNode currentScenario.addAttribute(id) currentScenario.addAttribute(name) diff --git a/E2E/TestFramework/Report/SummaryReporter.swift b/E2E/TestFramework/Report/SummaryReporter.swift new file mode 100644 index 00000000..31a6ebbf --- /dev/null +++ b/E2E/TestFramework/Report/SummaryReporter.swift @@ -0,0 +1,142 @@ +import Foundation + +public class SummaryReporter: Reporter { + required public init () { + } + + public func beforeFeature(_ feature: Feature) async throws { + } + + public func beforeScenario(_ scenario: Scenario) async throws { + } + + public func beforeStep(_ step: ConcreteStep) async throws { + } + + public func action(_ action: ActionOutcome) async throws { + } + + public func afterStep(_ stepOutcome: StepOutcome) async throws { + } + + public func afterScenario(_ scenarioOutcome: ScenarioOutcome) async throws { + } + + public func afterFeature(_ featureOutcome: FeatureOutcome) async throws { + } + + public func afterFeatures(_ featuresOutcome: [FeatureOutcome]) async throws { + var totalFeaturesExecuted = 0 + var featuresPassed = 0 + var featuresFailed = 0 + var featuresBroken = 0 + var featuresSkipped = 0 + var featuresPending = 0 + + var totalScenariosExecuted = 0 + var scenariosPassed = 0 + var scenariosFailed = 0 + var scenariosBroken = 0 + var scenariosSkipped = 0 + var scenariosPending = 0 + + let suiteObservedStartTime = featuresOutcome.first?.startTime + + print("\n\n===================================") + print(" TEST EXECUTION SUMMARY") + print("===================================\n") + + for featureOutcome in featuresOutcome { // Corrected Swift loop syntax + totalFeaturesExecuted += 1 + let featureTitle = featureOutcome.feature.title() + let featureStatusString = featureOutcome.status.rawValue.uppercased() + let featureDurationString = String(format: "%.2fs", featureOutcome.duration ?? 0.0) + + print("FEATURE: \(featureTitle)") + print(" Status: \(featureStatusString)") + print(" Duration: \(featureDurationString)") + + if featureOutcome.status == .failed || featureOutcome.status == .broken { + if let error = featureOutcome.error { // Feature-level error + print(" Error: \(error.localizedDescription)") + } + } + + switch featureOutcome.status { + case .passed: featuresPassed += 1 + case .failed: featuresFailed += 1 + case .broken: featuresBroken += 1 + case .skipped: featuresSkipped += 1 + case .pending: featuresPending += 1 + } + + let scenariosInFeature = featureOutcome.scenarioOutcomes.count + let passedInFeature = featureOutcome.passedScenarios.count + let failedInFeature = featureOutcome.failedScenarios.count + let brokenInFeature = featureOutcome.brokenScenarios.count + let skippedInFeature = featureOutcome.skippedScenarios.count + let pendingInFeature = featureOutcome.scenarioOutcomes.filter { $0.status == .pending }.count + + totalScenariosExecuted += scenariosInFeature + scenariosPassed += passedInFeature + scenariosFailed += failedInFeature + scenariosBroken += brokenInFeature + scenariosSkipped += skippedInFeature + scenariosPending += pendingInFeature + + print(" Scenarios (\(scenariosInFeature) total):") + print(" Passed: \(passedInFeature), Failed: \(failedInFeature), Broken: \(brokenInFeature), Skipped: \(skippedInFeature), Pending: \(pendingInFeature)") + + let unsuccessfulScenarios = featureOutcome.scenarioOutcomes.filter { + $0.status == .failed || $0.status == .broken + } + + if !unsuccessfulScenarios.isEmpty { + print(" Unsuccessful Scenarios Details:") + for scenarioOutcome in unsuccessfulScenarios { + let scenarioStatusStr = scenarioOutcome.status.rawValue.uppercased() + let scenarioDurationStr = String(format: "%.2fs", scenarioOutcome.duration ?? 0.0) + print(" [\(scenarioStatusStr)] \(scenarioOutcome.scenario.name) (\(scenarioDurationStr)), caused by") + if let error = scenarioOutcome.error ?? scenarioOutcome.failedStep?.error { + if (!error.localizedDescription.starts(with: "The operation couldn’t be completed.")) { + print(" \(error.localizedDescription)") + } else { + print(" \(error)") + } + } + } + } + print("-----------------------------------") + } + + print("\n====== OVERALL SUITE RESULTS ======\n") + print("Total Features Executed: \(totalFeaturesExecuted)") + print(" Features Passed: \(featuresPassed)") + print(" Features Failed: \(featuresFailed)") + print(" Features Broken: \(featuresBroken)") + print(" Features Skipped: \(featuresSkipped)") + print(" Features Pending: \(featuresPending)") + print("---") + print("Total Scenarios Executed: \(totalScenariosExecuted)") + print(" Scenarios Passed: \(scenariosPassed)") + print(" Scenarios Failed: \(scenariosFailed)") + print(" Scenarios Broken: \(scenariosBroken)") + print(" Scenarios Skipped: \(scenariosSkipped)") + print(" Scenarios Pending: \(scenariosPending)") + print("---") + + if let startTime = suiteObservedStartTime, let lastFeature = featuresOutcome.last, let endTime = lastFeature.endTime { + let suiteDuration = endTime.timeIntervalSince(startTime) + print(String(format: "Approx. Total Suite Duration: %.2fs", suiteDuration)) + } else if totalFeaturesExecuted > 0 { // Fallback if precise start/end is not available for the whole suite + let sumOfFeatureDurations = featuresOutcome.reduce(0.0) { $0 + ($1.duration ?? 0.0) } + print(String(format: "Approx. Total Suite Duration (Sum of Features): %.2fs", sumOfFeatureDurations)) + } else { + print("Approx. Total Suite Duration: N/A (No features run or timing unavailable)") + } + + print("\n===================================") + print(" END OF SUMMARY") + print("===================================\n") + } +} diff --git a/E2E/TestFramework/Resources/html_report.html b/E2E/TestFramework/Resources/html_report.html index f8fd73a4..3d90b140 100644 --- a/E2E/TestFramework/Resources/html_report.html +++ b/E2E/TestFramework/Resources/html_report.html @@ -61,6 +61,22 @@ .scenario-failed h4 { color: #f44336 } + + .scenario-skipped { + border: 1px solid grey; + } + + .scenario-skipped h4 { + color: grey + } + + .scenario-broken { + border: 1px solid purple; + } + + .scenario-broken h4 { + color: purple + } .steps-block { padding-right: 20px; @@ -91,6 +107,10 @@ .action-pending { color: grey } + + .action-broken { + color: purple + } .test-container { padding: 10px; @@ -202,7 +222,7 @@

Scenarios

Object.keys(feature.scenarios).forEach((key) => { let scenario = feature.scenarios[key] scenario.passed ? passedScenarios++ : failedScenarios++ - var scenarioClass = scenario.passed ? 'scenario-passed' : 'scenario-failed'; + var scenarioClass = 'scenario-' + scenario.status; reportHTML += `