Fix stubbed test recording, improve flaky tests (#897)

.circleci/config.yml

@@ -239,7 +239,10 @@ jobs:
       - prep_environment
       - prep_clone
       - prep_bundler_carthage
-      - run: sudo bundle exec xcversion simulators --install="iOS 12.4" --no-progress || true
+      - run:
+          name: "Install old simulator"
+          no_output_timeout: 20m
+          command: sudo bundle exec xcversion simulators --install="iOS 12.4" --no-progress || true
       - run: "bundle exec fastlane legacy_tests_12"
       - archive_logs
 
@@ -250,7 +253,10 @@ jobs:
       - prep_environment
       - prep_clone
       - prep_bundler_carthage
-      - run: sudo bundle exec xcversion simulators --install="iOS 13.7" --no-progress || true
+      - run:
+          name: "Install old simulator"
+          no_output_timeout: 20m
+          command: sudo bundle exec xcversion simulators --install="iOS 13.7" --no-progress || true
       - run: "bundle exec fastlane legacy_tests_13"
       - archive_logs
 

StripeCore/StripeCoreTestUtils/APIStubbedTestCase.swift

@@ -13,6 +13,17 @@ import OHHTTPStubs
 
 /* A test case offering a custom STPAPIClient with manual JSON stubbing. */
 open class APIStubbedTestCase: XCTestCase {
+    open override func setUp() {
+        super.setUp()
+
+        // Stubs are evaluated in the reverse order that they are added, so if the network is hit and no other stub is matched, raise an exception
+        stub(condition: { request in
+            return true
+        }) { request in
+            XCTFail("Attempted to hit the live network at \(request.url?.path ?? "")")
+            return HTTPStubsResponse()
+        }
+    }
     public override func tearDown() {
         super.tearDown()
         HTTPStubs.removeAllStubs()

Tests/Tests/STPCustomerContextTest.swift

@@ -120,6 +120,7 @@ class STPCustomerContextTests: APIStubbedTestCase {
         // apiClient.retrieveCustomer should be called once, when the context is initialized.
         // When sut.retrieveCustomer is called below, the cached customer will be used.
         stubRetrieveCustomers(key: customerKey, returningCustomerJSON: expectedCustomerJSON, expectedCount: 1, apiClient: apiClient)
+        stubListPaymentMethods(key: customerKey, paymentMethodJSONs: [], expectedCount: 1, apiClient: apiClient)
         let ekm = MockEphemeralKeyManager(key: customerKey, error: nil)
         let sut = STPCustomerContext(keyManager: ekm, apiClient: apiClient)
         // Give the mocked API request a little time to complete and cache the customer, then check cache
@@ -144,6 +145,8 @@ class STPCustomerContextTests: APIStubbedTestCase {
         // - when the context is initialized,
         // - when sut.retrieveCustomer is called below, as the cached customer has expired.
         stubRetrieveCustomers(key: customerKey, returningCustomerJSON: expectedCustomerJSON, expectedCount: 2, apiClient: apiClient)
+        stubListPaymentMethods(key: customerKey, paymentMethodJSONs: [], expectedCount: 1, apiClient: apiClient)
+
         let ekm = MockEphemeralKeyManager(key: customerKey, error: nil)
         let sut = STPCustomerContext(keyManager: ekm, apiClient: apiClient)
         // Give the mocked API request a little time to complete and cache the customer, then reset and check cache
@@ -168,6 +171,8 @@ class STPCustomerContextTests: APIStubbedTestCase {
         // - when the context is initialized,
         // - when sut.retrieveCustomer is called below, as the cached customer has been cleared.
         stubRetrieveCustomers(key: customerKey, returningCustomerJSON: expectedCustomerJSON, expectedCount: 2, apiClient: apiClient)
+        stubListPaymentMethods(key: customerKey, paymentMethodJSONs: [], expectedCount: 1, apiClient: apiClient)
+
         let ekm = MockEphemeralKeyManager(key: customerKey, error: nil)
         let sut = STPCustomerContext(keyManager: ekm, apiClient: apiClient)
         // Give the mocked API request a little time to complete and cache the customer, then reset and check cache
@@ -268,6 +273,7 @@ class STPCustomerContextTests: APIStubbedTestCase {
         let apiClient = stubbedAPIClient()
 
         stubRetrieveCustomers(key: customerKey, returningCustomerJSON: expectedCustomerJSON, expectedCount: 1, apiClient: apiClient)
+        stubListPaymentMethods(key: customerKey, paymentMethodJSONs: [], expectedCount: 1, apiClient: apiClient)
 
         let exp = expectation(description: "updateCustomer")
         stub { urlRequest in
@@ -303,7 +309,8 @@ class STPCustomerContextTests: APIStubbedTestCase {
         let expectedPaymentMethods = [STPFixtures.paymentMethod()]
 
         stubRetrieveCustomers(key: customerKey, returningCustomerJSON: expectedCustomerJSON, expectedCount: 1, apiClient: apiClient)
-
+        stubListPaymentMethods(key: customerKey, paymentMethodJSONs: [], expectedCount: 1, apiClient: apiClient)
+
         let exp = expectation(description: "payment method attach")
         // We're attaching 2 payment methods:
         exp.expectedFulfillmentCount = 2
@@ -343,7 +350,8 @@ class STPCustomerContextTests: APIStubbedTestCase {
         let expectedPaymentMethods = [STPFixtures.paymentMethod()]
 
         stubRetrieveCustomers(key: customerKey, returningCustomerJSON: expectedCustomerJSON, expectedCount: 1, apiClient: apiClient)
-
+        stubListPaymentMethods(key: customerKey, paymentMethodJSONs: [], expectedCount: 1, apiClient: apiClient)
+
         let exp = expectation(description: "payment method detach")
         // We're detaching 2 payment methods:
         exp.expectedFulfillmentCount = 2

Tests/Tests/STPNetworkStubbingTestCase.swift

@@ -19,7 +19,10 @@ class STPNetworkStubbingTestCase: XCTestCase {
 
     override func setUp() {
         super.setUp()
-
+
+        // Set the STPTestingAPIClient to use the sharedURLSessionConfig so that we can intercept requests from it too
+        STPTestingAPIClient.shared().sessionConfig = StripeAPIConfiguration.sharedUrlSessionConfiguration
+
         // [self name] returns a string like `-[STPMyTestCase testThing]` - this transforms it into the recorded path `recorded_network_traffic/STPMyTestCase/testThing`.
         let rawComponents = name.components(separatedBy: " ")
         assert(rawComponents.count == 2, "Invalid format received from XCTest#name: \(name)")
@@ -52,7 +55,7 @@ class STPNetworkStubbingTestCase: XCTestCase {
                 let method = request!.httpMethod?.lowercased()
                 let urlPath = request!.url?.path.replacingOccurrences(of: "/", with: "_")
                 var fileName = "\(method ?? "")\(urlPath ?? "")_\(count)"
-                fileName = URL(fileURLWithPath: fileName).appendingPathExtension("tail").path
+                fileName = URL(fileURLWithPath: fileName).appendingPathExtension("tail").lastPathComponent
                 count += 1
                 return fileName
             }

Tests/Tests/STPPaymentHandlerTests.swift

@@ -10,11 +10,15 @@ import XCTest
 import Foundation
 import StripeCoreTestUtils
 @testable import Stripe
-@testable import StripeCore
+@testable @_spi(STP) import StripeCore
 @testable import Stripe3DS2
 import OHHTTPStubs
 
-class STPPaymentHandlerTests: APIStubbedTestCase {
+class STPPaymentHandlerStubbedTests: STPNetworkStubbingTestCase {
+    override func setUp() {
+        self.recordingMode = false;
+        super.setUp()
+    }
 
     func testCanPresentErrorsAreReported() {
         let createPaymentIntentExpectation = expectation(
@@ -72,6 +76,9 @@ class STPPaymentHandlerTests: APIStubbedTestCase {
         // test in addition to fetching the payment intent
         wait(for: [paymentHandlerExpectation], timeout: 2*8)
     }
+}
+
+class STPPaymentHandlerTests: APIStubbedTestCase {
 
     func testPaymentHandlerRetriesWithBackoff() {
         STPPaymentHandler.sharedHandler.apiClient = stubbedAPIClient()
@@ -170,6 +177,10 @@ extension STPPaymentHandlerTests: STPAuthenticationContext {
     func authenticationPresentingViewController() -> UIViewController {
         return UIViewController()
     }
-
-
+}
+
+extension STPPaymentHandlerStubbedTests: STPAuthenticationContext {
+    func authenticationPresentingViewController() -> UIViewController {
+        return UIViewController()
+    }
 }

Tests/Tests/STPTestingAPIClient.h

@@ -34,6 +34,9 @@ static NSString * const STPTestingBRPublishableKey = @"pk_test_51JYFFjJQVROkWvqT
 
 + (instancetype)sharedClient;
 
+// Set this to the Stripe SDK session for SWHTTPRecorder recording to work correctly
+@property (nonatomic, readwrite) NSURLSessionConfiguration *sessionConfig;
+
 - (void)createPaymentIntentWithParams:(nullable NSDictionary *)params
                            completion:(void (^)(NSString * _Nullable, NSError * _Nullable))completion;
 

Tests/Tests/STPTestingAPIClient.m

@@ -26,6 +26,12 @@ + (instancetype)sharedClient {
     return sharedClient;
 }
 
+- (instancetype)init {
+    self = [super init];
+    self.sessionConfig = [[NSURLSession sharedSession] configuration];
+    return self;
+}
+
 - (void)createPaymentIntentWithParams:(nullable NSDictionary *)params
                            completion:(void (^)(NSString * _Nullable, NSError * _Nullable))completion {
     [self createPaymentIntentWithParams:params
@@ -46,7 +52,7 @@ - (void)createPaymentIntentWithParams:(nullable NSDictionary *)params
                               account:(nullable NSString *)account
                            apiVersion:(nullable NSString *)apiVersion
                            completion:(void (^)(NSString * _Nullable, NSError * _Nullable))completion {
-    NSURLSession *session = [NSURLSession sharedSession];
+    NSURLSession *session = [NSURLSession sessionWithConfiguration:self.sessionConfig];
     NSURL *url = [NSURL URLWithString:[STPTestingBackendURL stringByAppendingString:@"create_payment_intent"]];
 
     NSMutableURLRequest *request = [[NSMutableURLRequest alloc] initWithURL:url];
@@ -114,7 +120,7 @@ - (void)createSetupIntentWithParams:(nullable NSDictionary *)params
                             account:(nullable NSString *)account
                          apiVersion:(nullable NSString *)apiVersion
                          completion:(void (^)(NSString *_Nullable, NSError * _Nullable))completion {
-    NSURLSession *session = [NSURLSession sharedSession];
+    NSURLSession *session = [NSURLSession sessionWithConfiguration:self.sessionConfig];
     NSURL *url = [NSURL URLWithString:[STPTestingBackendURL stringByAppendingString:@"create_setup_intent"]];
 
     NSMutableURLRequest *request = [[NSMutableURLRequest alloc] initWithURL:url];

Tests/recorded_network_traffic/STPPaymentHandlerStubbedTests/testCanPresentErrorsAreReported/post_create_payment_intent_0.tail

@@ -0,0 +1,18 @@
+POST
+/create_payment_intent$
+200
+text/html
+Content-Type: text/html;charset=utf-8
+Alt-Svc: clear
+Set-Cookie: rack.session=7mmwWRPQtNDztyFI7vcbHHpfb8Kg%2FGwJsi3yjFhDBPc3p%2BUXtXSrShd9xGHLKAcTVyYSsKSiFqPPBxi6xIFSMMSL4%2B42nLw3XOMtvFKncxuLuqvbGD1n3sZNjMSqqHT3r%2B%2FrsWQN6KU%2Bk%2B51dxHhZ9WmczQyqhXOZdMrC2fbjl8FdeaBohIIr59FPd3GemuC0c8D%2Ba3pnw7sxGzl277UKBMaxA7c68OlhQYdLRMhAVA%3D; path=/
+Server: Google Frontend
+X-Cloud-Trace-Context: 1b98cc26fedf5037b09e133679db4bfc;o=1
+Via: 1.1 google
+x-xss-protection: 1; mode=block
+Date: Tue, 22 Mar 2022 22:17:55 GMT
+X-Robots-Tag: noindex, nofollow
+Content-Length: 147
+x-content-type-options: nosniff
+x-frame-options: SAMEORIGIN
+
+{"intent":"pi_3KgG1XFY0qyl6XeW1TLgmwD3","secret":"pi_3KgG1XFY0qyl6XeW1TLgmwD3_secret_VEKa074h9bDuCe2ArEs0Mpcr7","status":"requires_payment_method"}

Tests/recorded_network_traffic/STPPaymentHandlerStubbedTests/testCanPresentErrorsAreReported/post_v1_3ds2_authenticate_2.tail

@@ -0,0 +1,47 @@
+POST
+/v1/3ds2/authenticate$
+200
+application/json
+Content-Type: application/json
+Access-Control-Allow-Origin: *
+idempotency-key: 8f01ab53-7fbb-4305-8298-ca08ac71acaa
+original-request: req_DINGjWXMtxQmA8
+stripe-should-retry: false
+Server: nginx
+access-control-allow-methods: GET, POST, HEAD, OPTIONS, DELETE
+access-control-expose-headers: Request-Id, Stripe-Manage-Version, X-Stripe-External-Auth-Required, X-Stripe-Privileged-Session-Required
+access-control-max-age: 300
+Cache-Control: no-cache, no-store
+Date: Tue, 22 Mar 2022 22:17:58 GMT
+stripe-version: 2020-08-27
+access-control-allow-credentials: true
+Content-Length: 1699
+Connection: keep-alive
+Strict-Transport-Security: max-age=31556926; includeSubDomains; preload
+request-id: req_DINGjWXMtxQmA8
+
+{
+  "object" : "three_d_secure_2",
+  "fallback_redirect_url" : null,
+  "source" : "src_1KgG1ZFY0qyl6XeWK2TSv8yQ",
+  "id" : "threeds2_1KgG1aFY0qyl6XeWupDexdTO",
+  "livemode" : false,
+  "creq" : "eyJ0aHJlZURTU2VydmVyVHJhbnNJRCI6ImQzZDdjYTQxLTRhMzUtNDcyMS04MGZkLTg5NGEwNTM3ZmNhMCIsImFjc1RyYW5zSUQiOiI1NjY0OWE0Mi1iODY5LTQ0YzgtYTA0Yi01ZmRlYmY2ZWVhMzgiLCJjaGFsbGVuZ2VXaW5kb3dTaXplIjoiMDUiLCJtZXNzYWdlVHlwZSI6IkNSZXEiLCJtZXNzYWdlVmVyc2lvbiI6IjIuMS4wIn0=",
+  "created" : 1647987478,
+  "ares" : {
+    "threeDSServerTransID" : "d3d7ca41-4a35-4721-80fd-894a0537fca0",
+    "acsSignedContent" : "eyJhbGciOiJFUzI1NiJ9.eyJhY3NFcGhlbVB1YktleSI6eyJjcnYiOiJQLTI1NiIsImt0eSI6IkVDIiwieCI6IjV3dV9yWHhVX19SQTVqQzF2dWpPOWRLZWtnVW41V1dPSHRnSF9nek1kZkkiLCJ5IjoieG4xcjlmaHVrRlpTMjRkZDN4OFFaalJJTzRCYUFTelJuQ2N5ZU1QNTRpUSJ9LCJzZGtFcGhlbVB1YktleSI6eyJ5IjoiTlcwUVcyOE5acGJUYjMwWnZxS2pfQmRDT0tfVlBocWNxYjZIeEFWUE54dyIsIngiOiJ1UlhZVGFTMU1FcV94R1JYd29xc3NsV24xZnVFMzRDVnBYSndqbWNVRVFFIiwia3R5IjoiRUMiLCJjcnYiOiJQLTI1NiJ9LCJhY3NVUkwiOiJodHRwczovL3Rlc3Rtb2RlLWFjcy5zdHJpcGUuY29tLzNkX3NlY3VyZV8yX3Rlc3QvYWNjdF8xRzZtMXBGWTBxeWw2WGVXL3RocmVlZHMyXzFLZ0cxYUZZMHF5bDZYZVd1cERleGRUTy9hcHBfY2hhbGxlbmdlL0RCcWNtaGNNVUotWHgySW5Ld0UxbmpsaVJVdk1uN2xQZkJWbkdCVnFOWEE9In0.tfD-Qynyhh6QZtBQbpffb-TE1ziFyacELVwnNAO1xbqpfCIryg_ZIvS8VOv3hzQKTXXjbTAHLTbFngzqR0wuCA",
+    "acsTransID" : "56649a42-b869-44c8-a04b-5fdebf6eea38",
+    "transStatus" : "C",
+    "messageType" : "ARes",
+    "messageVersion" : "2.1.0",
+    "acsURL" : null,
+    "messageExtension" : null,
+    "cardholderInfo" : null,
+    "authenticationType" : "02",
+    "acsChallengeMandated" : "Y",
+    "sdkTransID" : "999c2ad5-6cd7-48fd-932e-8275649f477e"
+  },
+  "error" : null,
+  "state" : "challenge_required"
+}