diff --git a/Sources/Classes/Internal/SLTestState.h b/Sources/Classes/Internal/SLTestState.h new file mode 100644 index 0000000..669f4bf --- /dev/null +++ b/Sources/Classes/Internal/SLTestState.h @@ -0,0 +1,37 @@ +// +// SLTestState.h +// Subliminal +// +// Created by Jacob Relkin on 8/22/14. +// Copyright (c) 2014 Inkling. All rights reserved. +// + +#import + +@class SLTestFailure; + +/** + SLTestState objects define the state of a Subliminal test or test case. + */ + +@interface SLTestState : NSObject + +/** + Denotes whether the test or test case failed + */ +@property (nonatomic, readonly) BOOL failed; + +/** + If the test or test case failure was expected. + */ +@property (nonatomic, readonly) BOOL failureWasExpected; + +/** + Sets the properties above, from the failure's description. + The value of `failureWasExpected` is determined by the first failure that was encountered. + + @param failure The test failure to record. + */ +- (void)recordFailure:(SLTestFailure *)failure; + +@end diff --git a/Sources/Classes/Internal/SLTestState.m b/Sources/Classes/Internal/SLTestState.m new file mode 100644 index 0000000..7c6cc90 --- /dev/null +++ b/Sources/Classes/Internal/SLTestState.m @@ -0,0 +1,31 @@ +// +// SLTestState.m +// Subliminal +// +// Created by Jacob Relkin on 8/22/14. +// Copyright (c) 2014 Inkling. All rights reserved. +// + +#import "SLTestState.h" +#import "SLTestFailure.h" + +@interface SLTestState () + +@property (nonatomic, readwrite) BOOL failed; +@property (nonatomic, readwrite) BOOL failureWasExpected; + +@end + +@implementation SLTestState + +- (void)recordFailure:(SLTestFailure *)failure { + NSParameterAssert(failure); + + if (!self.failed) { + self.failureWasExpected = failure.isExpected; + } + + self.failed = YES; +} + +@end diff --git a/Sources/Classes/SLTest.h b/Sources/Classes/SLTest.h index 4ea5414..1fd44f3 100644 --- a/Sources/Classes/SLTest.h +++ b/Sources/Classes/SLTest.h @@ -25,6 +25,8 @@ #import "SLTestController+AppHooks.h" #import "SLStringUtilities.h" +@class SLTestFailure; + /** `SLTest` is the abstract superclass of Subliminal integration tests. @@ -366,6 +368,15 @@ */ + (NSUInteger)runGroup; +/** + If overridden, provides a hook into test and test case failures. + + @param failure An object which describes the failure. + @warning This method will be invoked for each exception that is handled by the test framework. + @see -[SLTestFailure failureWithException:phase:testCaseSelector:] + */ +- (void)testDidEncounterFailure:(SLTestFailure *)failure; + @end diff --git a/Sources/Classes/SLTest.m b/Sources/Classes/SLTest.m index ba64d00..9e992c6 100644 --- a/Sources/Classes/SLTest.m +++ b/Sources/Classes/SLTest.m @@ -29,6 +29,8 @@ #import #import +#import "SLTestFailure.h" +#import "SLTestState.h" // All exceptions thrown by SLTest must have names beginning with this prefix // so that `-[SLTest exceptionByAddingFileInfo:]` can determine whether to attach @@ -36,7 +38,6 @@ static NSString *const SLTestExceptionNamePrefix = @"SLTest"; - @implementation SLTest static NSString *__lastKnownFilename; @@ -330,26 +331,38 @@ + (NSString *)unfocusedTestCaseName:(NSString *)testCase { return testCase; } +- (void)reportFailureInPhase:(SLTestFailurePhase)phase toState:(SLTestState *)state exception:(NSException *)exception testCaseSelector:(SEL)testCaseSelector { + SLTestFailure *failure = [SLTestFailure failureWithException:exception phase:phase testCaseSelector:testCaseSelector]; + NSException *exceptionToLog = [self exceptionByAddingFileInfo:exception]; + [[SLLogger sharedLogger] logException:exceptionToLog + expected:[failure isExpected]]; + [state recordFailure:failure]; + [self testDidEncounterFailure:failure]; +} + - (BOOL)runAndReportNumExecuted:(NSUInteger *)numCasesExecuted failed:(NSUInteger *)numCasesFailed failedUnexpectedly:(NSUInteger *)numCasesFailedUnexpectedly { NSUInteger numberOfCasesExecuted = 0, numberOfCasesFailed = 0, numberOfCasesFailedUnexpectedly = 0; + SLTestState *testState = [SLTestState new]; - BOOL testDidFailInSetUpOrTearDown = NO; @try { [self setUpTest]; } @catch (NSException *exception) { - [[SLLogger sharedLogger] logException:[self exceptionByAddingFileInfo:exception] - expected:[[self class] exceptionWasExpected:exception]]; - testDidFailInSetUpOrTearDown = YES; + [self reportFailureInPhase:SLTestFailurePhaseTestSetup + toState:testState + exception:exception + testCaseSelector:NULL]; } // if setUpTest failed, skip the test cases - if (!testDidFailInSetUpOrTearDown) { + if (!testState.failed) { NSString *test = NSStringFromClass([self class]); for (NSString *testCaseName in [[self class] testCasesToRun]) { @autoreleasepool { + SLTestState *testCaseState = [SLTestState new]; + // all logs below use the focused name, so that the logs are consistent // with what's actually running [[SLLogger sharedLogger] logTest:test caseStart:testCaseName]; @@ -362,29 +375,29 @@ - (BOOL)runAndReportNumExecuted:(NSUInteger *)numCasesExecuted // (though we can't guarantee it won't be reused within a test case) [SLTest clearLastKnownCallSite]; - BOOL caseFailed = NO, failureWasExpected = NO; @try { [self setUpTestCaseWithSelector:unfocusedTestCaseSelector]; } @catch (NSException *exception) { - caseFailed = YES; - failureWasExpected = [[self class] exceptionWasExpected:exception]; - [[SLLogger sharedLogger] logException:[self exceptionByAddingFileInfo:exception] - expected:failureWasExpected]; + [self reportFailureInPhase:SLTestFailurePhaseTestCaseSetup + toState:testCaseState + exception:exception + testCaseSelector:unfocusedTestCaseSelector]; } // Only execute the test case if set-up succeeded. - if (!caseFailed) { + if (!testCaseState.failed) { @try { // We use objc_msgSend so that Clang won't complain about performSelector leaks // Make sure to send the actual test case selector ((void(*)(id, SEL))objc_msgSend)(self, NSSelectorFromString(testCaseName)); } @catch (NSException *exception) { - caseFailed = YES; - failureWasExpected = [[self class] exceptionWasExpected:exception]; - [[SLLogger sharedLogger] logException:[self exceptionByAddingFileInfo:exception] - expected:failureWasExpected]; + [self reportFailureInPhase:SLTestFailurePhaseTestCaseExecution + toState:testCaseState + exception:exception + testCaseSelector:unfocusedTestCaseSelector]; + } } @@ -394,22 +407,22 @@ - (BOOL)runAndReportNumExecuted:(NSUInteger *)numCasesExecuted [self tearDownTestCaseWithSelector:unfocusedTestCaseSelector]; } @catch (NSException *exception) { - BOOL caseHadFailed = caseFailed; - caseFailed = YES; - // don't override `failureWasExpected` if we had already failed - BOOL exceptionWasExpected = [[self class] exceptionWasExpected:exception]; - if (!caseHadFailed) failureWasExpected = exceptionWasExpected; - [[SLLogger sharedLogger] logException:[self exceptionByAddingFileInfo:exception] - expected:exceptionWasExpected]; + [self reportFailureInPhase:SLTestFailurePhaseTestCaseTeardown + toState:testCaseState + exception:exception + testCaseSelector:unfocusedTestCaseSelector]; } - if (caseFailed) { - [[SLLogger sharedLogger] logTest:test caseFail:testCaseName expected:failureWasExpected]; + if (testCaseState.failed) { + [[SLLogger sharedLogger] logTest:test caseFail:testCaseName expected:testCaseState.failureWasExpected]; numberOfCasesFailed++; - if (!failureWasExpected) numberOfCasesFailedUnexpectedly++; + if (!testCaseState.failureWasExpected) { + numberOfCasesFailedUnexpectedly++; + } } else { [[SLLogger sharedLogger] logTest:test casePass:testCaseName]; } + numberOfCasesExecuted++; } } @@ -420,16 +433,17 @@ - (BOOL)runAndReportNumExecuted:(NSUInteger *)numCasesExecuted [self tearDownTest]; } @catch (NSException *exception) { - [[SLLogger sharedLogger] logException:[self exceptionByAddingFileInfo:exception] - expected:[[self class] exceptionWasExpected:exception]]; - testDidFailInSetUpOrTearDown = YES; + [self reportFailureInPhase:SLTestFailurePhaseTestTeardown + toState:testState + exception:exception + testCaseSelector:NULL]; } if (numCasesExecuted) *numCasesExecuted = numberOfCasesExecuted; if (numCasesFailed) *numCasesFailed = numberOfCasesFailed; if (numCasesFailedUnexpectedly) *numCasesFailedUnexpectedly = numberOfCasesFailedUnexpectedly; - return !testDidFailInSetUpOrTearDown; + return !testState.failed; } - (void)wait:(NSTimeInterval)interval { @@ -468,9 +482,9 @@ - (NSException *)exceptionByAddingFileInfo:(NSException *)exception { return exception; } -+ (BOOL)exceptionWasExpected:(NSException *)exception { - return [[exception name] isEqualToString:SLTestAssertionFailedException]; -} +// Abstract +- (void)testDidEncounterFailure:(SLTestFailure *)failure {} + @end diff --git a/Sources/Classes/SLTestFailure.h b/Sources/Classes/SLTestFailure.h new file mode 100644 index 0000000..6ef10b1 --- /dev/null +++ b/Sources/Classes/SLTestFailure.h @@ -0,0 +1,53 @@ +// +// SLTestCaseFailure.h +// Subliminal +// +// Created by Jacob Relkin on 6/20/14. +// Copyright (c) 2014 Inkling. All rights reserved. +// + +#import + +typedef NS_ENUM(NSUInteger, SLTestFailurePhase) { + SLTestFailurePhaseTestSetup, + SLTestFailurePhaseTestCaseSetup, + SLTestFailurePhaseTestCaseExecution, + SLTestFailurePhaseTestCaseTeardown, + SLTestFailurePhaseTestTeardown +}; + +/** + SLTestFailure objects hold failure information for Subliminal test and test cases. + */ + +@interface SLTestFailure : NSObject + +/** + @param exception The exception that was thrown to cause this failure. + @param phase The phase in the test lifecycle in which the failure happened. + @param testCaseSelector The failed test case's selector. (can be NULL) + @return A new SLTestFailure object. + */ ++ (instancetype)failureWithException:(NSException *)exception phase:(SLTestFailurePhase)phase testCaseSelector:(SEL)testCaseSelector; + +/** + The phase in the test lifecycle in which the failure happened. + */ +@property (nonatomic, readonly, assign) SLTestFailurePhase phase; + +/** + The failed test case's selector. (can be NULL) + */ +@property (nonatomic, readonly, assign) SEL testCaseSelector; + +/** + The exception that was thrown to cause this failure. + */ +@property (nonatomic, readonly, strong) NSException *exception; + +/** + If the failure was expected. + */ +@property (nonatomic, readonly, getter = isExpected) BOOL expected; + +@end diff --git a/Sources/Classes/SLTestFailure.m b/Sources/Classes/SLTestFailure.m new file mode 100644 index 0000000..26a6068 --- /dev/null +++ b/Sources/Classes/SLTestFailure.m @@ -0,0 +1,47 @@ +// +// SLTestCaseFailure.m +// Subliminal +// +// Created by Jacob Relkin on 6/20/14. +// Copyright (c) 2014 Inkling. All rights reserved. +// + +#import "SLTestFailure.h" +#import "SLTest.h" + +@interface SLTestFailure () + +@property (nonatomic, readwrite, strong) NSException *exception; +@property (nonatomic, readwrite, assign) SEL testCaseSelector; +@property (nonatomic, readwrite, assign) SLTestFailurePhase phase; + +@end + +@implementation SLTestFailure + ++ (instancetype)failureWithException:(NSException *)exception phase:(SLTestFailurePhase)phase testCaseSelector:(SEL)testCaseSelector { + SLTestFailure *failure = [self new]; + failure.phase = phase; + failure.exception = exception; + failure.testCaseSelector = testCaseSelector; + return failure; +} + +- (BOOL)isEqual:(id)object { + if (![object isKindOfClass:[SLTestFailure class]]) { + return NO; + } + + SLTestFailure *otherObject = object; + return (otherObject.phase == self.phase && + otherObject.testCaseSelector == self.testCaseSelector && + otherObject.isExpected == self.isExpected && + [otherObject.exception.name isEqualToString:self.exception.name] && + [otherObject.exception.reason isEqualToString:self.exception.reason]); +} + +- (BOOL)isExpected { + return [self.exception.name isEqualToString:SLTestAssertionFailedException]; +} + +@end diff --git a/Sources/Subliminal.h b/Sources/Subliminal.h index 33fcb34..ad5df45 100644 --- a/Sources/Subliminal.h +++ b/Sources/Subliminal.h @@ -24,6 +24,7 @@ #import "SLTestController+AppHooks.h" #import "SLTest.h" #import "SLTestAssertions.h" +#import "SLTestFailure.h" #import "SLDevice.h" #import "SLElement.h" diff --git a/Subliminal.xcodeproj/project.pbxproj b/Subliminal.xcodeproj/project.pbxproj index 264b91e..e458b55 100644 --- a/Subliminal.xcodeproj/project.pbxproj +++ b/Subliminal.xcodeproj/project.pbxproj @@ -60,6 +60,10 @@ 627B5FD4194FACA50059B692 /* SLTestMatchingElementsWithinTableHeaderView.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 627B5FD0194FA9D70059B692 /* SLTestMatchingElementsWithinTableHeaderView.storyboard */; }; 62E7A633193EF84C00CB11AB /* SLStaticText.h in Headers */ = {isa = PBXBuildFile; fileRef = 62E7A631193EF84C00CB11AB /* SLStaticText.h */; settings = {ATTRIBUTES = (Public, ); }; }; 62E7A634193EF84C00CB11AB /* SLStaticText.m in Sources */ = {isa = PBXBuildFile; fileRef = 62E7A632193EF84C00CB11AB /* SLStaticText.m */; }; + C7C0445019A7CE14001F82B9 /* SLTestState.h in Headers */ = {isa = PBXBuildFile; fileRef = C7C0444E19A7CE14001F82B9 /* SLTestState.h */; }; + C7C0445119A7CE14001F82B9 /* SLTestState.m in Sources */ = {isa = PBXBuildFile; fileRef = C7C0444F19A7CE14001F82B9 /* SLTestState.m */; }; + C7FD55131954CAD700E99FB7 /* SLTestFailure.h in Headers */ = {isa = PBXBuildFile; fileRef = C7FD55111954CAD700E99FB7 /* SLTestFailure.h */; settings = {ATTRIBUTES = (Public, ); }; }; + C7FD55141954CAD700E99FB7 /* SLTestFailure.m in Sources */ = {isa = PBXBuildFile; fileRef = C7FD55121954CAD700E99FB7 /* SLTestFailure.m */; }; CA75E78216697A1200D57E92 /* SLDevice.h in Headers */ = {isa = PBXBuildFile; fileRef = CA75E78016697A1200D57E92 /* SLDevice.h */; settings = {ATTRIBUTES = (Public, ); }; }; CA75E78516697C0000D57E92 /* SLDevice.m in Sources */ = {isa = PBXBuildFile; fileRef = CA75E78116697A1200D57E92 /* SLDevice.m */; }; CAC388051641CD7500F995F9 /* SLStringUtilities.h in Headers */ = {isa = PBXBuildFile; fileRef = CAC388031641CD7500F995F9 /* SLStringUtilities.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -313,6 +317,10 @@ 627B5FD0194FA9D70059B692 /* SLTestMatchingElementsWithinTableHeaderView.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; path = SLTestMatchingElementsWithinTableHeaderView.storyboard; sourceTree = ""; }; 62E7A631193EF84C00CB11AB /* SLStaticText.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLStaticText.h; sourceTree = ""; }; 62E7A632193EF84C00CB11AB /* SLStaticText.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SLStaticText.m; sourceTree = ""; }; + C7C0444E19A7CE14001F82B9 /* SLTestState.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLTestState.h; sourceTree = ""; }; + C7C0444F19A7CE14001F82B9 /* SLTestState.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SLTestState.m; sourceTree = ""; }; + C7FD55111954CAD700E99FB7 /* SLTestFailure.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SLTestFailure.h; path = ../SLTestFailure.h; sourceTree = ""; }; + C7FD55121954CAD700E99FB7 /* SLTestFailure.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = SLTestFailure.m; path = ../SLTestFailure.m; sourceTree = ""; }; CA75E78016697A1200D57E92 /* SLDevice.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLDevice.h; sourceTree = ""; }; CA75E78116697A1200D57E92 /* SLDevice.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SLDevice.m; sourceTree = ""; }; CAC388031641CD7500F995F9 /* SLStringUtilities.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SLStringUtilities.h; sourceTree = ""; }; @@ -602,6 +610,10 @@ F05C51E4171C8AE000A381BC /* SLMainThreadRef.m */, F02DF30617EC064F00BE28BF /* UIScrollView+SLProgrammaticScrolling.h */, F02DF30717EC064F00BE28BF /* UIScrollView+SLProgrammaticScrolling.m */, + C7FD55111954CAD700E99FB7 /* SLTestFailure.h */, + C7FD55121954CAD700E99FB7 /* SLTestFailure.m */, + C7C0444E19A7CE14001F82B9 /* SLTestState.h */, + C7C0444F19A7CE14001F82B9 /* SLTestState.m */, ); path = Internal; sourceTree = ""; @@ -1109,6 +1121,7 @@ F0C07A531704011400C93F93 /* SLKeyboard.h in Headers */, F0C07A57170401E500C93F93 /* SLWebView.h in Headers */, F05C4F90171406EF00A381BC /* SLTerminal+ConvenienceFunctions.h in Headers */, + C7C0445019A7CE14001F82B9 /* SLTestState.h in Headers */, F05C51E5171C8AE000A381BC /* SLMainThreadRef.h in Headers */, F0A04E1D1749F70F002C7520 /* SLElement.h in Headers */, 2CE9AA4C17E3A747007EF0B5 /* SLSwitch.h in Headers */, @@ -1116,6 +1129,7 @@ F02DF30817EC064F00BE28BF /* UIScrollView+SLProgrammaticScrolling.h in Headers */, F00800CE174C1C64001927AC /* SLPopover.h in Headers */, 50F3E18C1783A5CB00C6BD1B /* SLGeometry.h in Headers */, + C7FD55131954CAD700E99FB7 /* SLTestFailure.h in Headers */, F043469F175ACE3A00D91F7F /* NSObject+SLAccessibilityDescription.h in Headers */, 62009F8A196CB30E00419585 /* SLTestAssertions.h in Headers */, F04346AF175AD63E00D91F7F /* SLAccessibilityPath.h in Headers */, @@ -1385,12 +1399,14 @@ 62E7A634193EF84C00CB11AB /* SLStaticText.m in Sources */, 50F3E18E1783A60100C6BD1B /* SLGeometry.m in Sources */, F0695DE4160138DF000B05D0 /* SLUIAElement.m in Sources */, + C7C0445119A7CE14001F82B9 /* SLTestState.m in Sources */, F0695DE5160138DF000B05D0 /* SLLogger.m in Sources */, F0695DE7160138DF000B05D0 /* SLTerminal.m in Sources */, F0695DE8160138DF000B05D0 /* SLTest.m in Sources */, 622DA0BB194E2CB900EFFE05 /* SLDatePicker.m in Sources */, F0695DE9160138DF000B05D0 /* SLTestController.m in Sources */, F0271B00162E0B950098F5F2 /* SLTestController+AppHooks.m in Sources */, + C7FD55141954CAD700E99FB7 /* SLTestFailure.m in Sources */, F02DF30917EC064F00BE28BF /* UIScrollView+SLProgrammaticScrolling.m in Sources */, CAC388061641CD7500F995F9 /* SLStringUtilities.m in Sources */, 622DA090194AF1E200EFFE05 /* SLPickerView.m in Sources */, diff --git a/Unit Tests/SLTestTests.m b/Unit Tests/SLTestTests.m index 41e8306..8aa8c79 100644 --- a/Unit Tests/SLTestTests.m +++ b/Unit Tests/SLTestTests.m @@ -704,7 +704,8 @@ - (void)testAllTestCasesRunByDefault { [[testMock expect] testOne]; [[testMock expect] testTwo]; [[testMock expect] testThree]; - + [[testMock reject] testDidEncounterFailure:OCMOCK_ANY]; + SLRunTestsAndWaitUntilFinished([NSSet setWithObject:testWithSomeTestCasesTest], nil); STAssertNoThrow([testMock verify], @"Test cases did not run as expected."); } @@ -724,6 +725,15 @@ - (void)testInvalidTestCasesAreNotRun { STAssertNoThrow([testMock verify], @"Invalid test cases were unexpectedly run."); } +- (void)testSuccesfulTestDoesntInvokeTestFailureHook { + Class testWithSomeTestCasesTest = [TestWithSomeTestCases class]; + id testMock = [OCMockObject partialMockForClass:testWithSomeTestCasesTest]; + [[testMock reject] testDidEncounterFailure:OCMOCK_ANY]; + + SLRunTestsAndWaitUntilFinished([NSSet setWithObject:testWithSomeTestCasesTest], nil); + SLAssertNoThrow([testMock verify], @"Test failure hook was unexpectedly invoked."); +} + // this test verifies the complete order in which testing normally executes, // but is mostly for illustration--it makes too many assertions // traditional "unit" tests follow @@ -840,22 +850,31 @@ - (void)runWithTestFailingInTestSetupOrTeardownToTestAnErrorAndTestAbortAreLogge // If either setup or teardown fails... NSException *exception; + SLTestFailure *failureToExpect = nil; + if (failInSetUp) { exception = [NSException exceptionWithName:SLTestAssertionFailedException reason:@"Test setup failed." userInfo:nil]; + failureToExpect = [SLTestFailure failureWithException:exception phase:SLTestFailurePhaseTestSetup testCaseSelector:NULL]; + [[[failingTestMock expect] andThrow:exception] setUpTest]; } else { exception = [NSException exceptionWithName:SLTestAssertionFailedException reason:@"Test teardown failed." userInfo:nil]; + failureToExpect = [SLTestFailure failureWithException:exception phase:SLTestFailurePhaseTestTeardown testCaseSelector:NULL]; + [[[failingTestMock expect] andThrow:exception] tearDownTest]; } // ...the test controller logs an error... [[_loggerMock expect] logError:[OCMArg any]]; - // ...and the test controller logs the test as aborted (rather than finishing)... + // ...the test encounters the expected failure... + [[failingTestMock expect] testDidEncounterFailure:failureToExpect]; + + // ...and the test controller logs the test as aborte/d (rather than finishing)... [[_loggerMock expect] logTestAbort:NSStringFromClass(failingTestClass)]; // ...and the test controller logs testing as finishing with one test executed, one test failing. @@ -957,27 +976,37 @@ - (void)runWithTestFailingInTestCaseSetupOrTeardownToTestAnErrorAndTestCaseFailA Class failingTestClass = [TestWithSomeTestCases class]; SEL failingTestCase = @selector(testOne); id failingTestMock = [OCMockObject partialMockForClass:failingTestClass]; + [failingTestMock setExpectationOrderMatters:YES]; OCMExpectationSequencer *failingTestSequencer = [OCMExpectationSequencer sequencerWithMocks:@[ failingTestMock, _loggerMock ]]; // *** Begin expected test run // If either test case setup or teardown fails... NSException *exception; + SLTestFailure *failureToExpect; + if (failInSetUp) { exception = [NSException exceptionWithName:SLTestAssertionFailedException reason:@"Test case setup failed." userInfo:nil]; + failureToExpect = [SLTestFailure failureWithException:exception phase:SLTestFailurePhaseTestCaseSetup testCaseSelector:failingTestCase]; + [[[failingTestMock expect] andThrow:exception] setUpTestCaseWithSelector:failingTestCase]; } else { exception = [NSException exceptionWithName:SLTestAssertionFailedException reason:@"Test case teardown failed." userInfo:nil]; + failureToExpect = [SLTestFailure failureWithException:exception phase:SLTestFailurePhaseTestCaseTeardown testCaseSelector:failingTestCase]; + [[[failingTestMock expect] andThrow:exception] tearDownTestCaseWithSelector:failingTestCase]; } // ...the test catches the exception and logs an error... [[_loggerMock expect] logError:[OCMArg any]]; + // ...the test encounters the expected failure... + [[failingTestMock expect] testDidEncounterFailure:failureToExpect]; + // ...and the test controller reports the test finishing with one test case having failed... // (and that failure was "expected" because it was due to an assertion failing) // (these values will need to be updated if the test class' definition changes) @@ -1044,6 +1073,9 @@ - (void)runWithTestFailingInTestCaseSetupWithExpectedFailure:(BOOL)expectedFailu // ...the test catches and logs the exception... [[_loggerMock expect] logError:[OCMArg any]]; + // ...the test calls -testDidEncounterFailure: with the expected failure. + [[failingTestMock expect] testDidEncounterFailure:[SLTestFailure failureWithException:setUpException phase:SLTestFailurePhaseTestCaseSetup testCaseSelector:failingTestCase]]; + // ...and then if test case teardown fails... NSException *tearDownException = exceptionWithReason(expectedFailureInTeardown, @"Test case teardown failed."); [[[failingTestMock expect] andThrow:tearDownException] tearDownTestCaseWithSelector:failingTestCase]; @@ -1051,6 +1083,9 @@ - (void)runWithTestFailingInTestCaseSetupWithExpectedFailure:(BOOL)expectedFailu // ...the test again catches and logs the exception... [[_loggerMock expect] logError:[OCMArg any]]; + // ...the test again calls -testDidEncounterFailure: with the expected failure. + [[failingTestMock expect] testDidEncounterFailure:[SLTestFailure failureWithException:tearDownException phase:SLTestFailurePhaseTestCaseTeardown testCaseSelector:failingTestCase]]; + // ...but when the test logs the test case as failing, // that failure is reported as "expected" or not depending on the first exception (setup) thrown... [[_loggerMock expect] logTest:NSStringFromClass(failingTestClass) @@ -1191,11 +1226,15 @@ - (void)runWithFailureInTestCaseDueToAssertionException:(BOOL)failWithAssertionE reason:@"Test case failed because element was not tappable." userInfo:nil]; } + [[[failingTestMock expect] andThrow:exception] testOne]; // ...the test catches the exception and logs an error... [[_loggerMock expect] logError:[OCMArg any]]; + // ...the test encounters a failure with the correct state... + [[failingTestMock expect] testDidEncounterFailure:[SLTestFailure failureWithException:exception phase:SLTestFailurePhaseTestCaseExecution testCaseSelector:failingTestCase]]; + // ...and logs the test case failing... [[_loggerMock expect] logTest:NSStringFromClass(failingTestClass) caseFail:NSStringFromSelector(failingTestCase)