RUST-1064 Retry on handshake failure (#598)

mongodb · Mar 25, 2022 · 2f12793 · 2f12793
1 parent d55c079
commit 2f12793
Show file tree

Hide file tree

Showing 98 changed files with 915 additions and 62 deletions.
diff --git a/src/client/auth/sasl.rs b/src/client/auth/sasl.rs
@@ -4,6 +4,7 @@ use crate::{
     client::{auth::AuthMechanism, options::ServerApi},
     cmap::Command,
     error::{Error, Result},
+    operation::{CommandErrorBody, CommandResponse},
 };
 
 /// Encapsulates the command building of a `saslStart` command.
@@ -98,12 +99,20 @@ fn validate_command_success(auth_mechanism: &str, response: &Document) -> Result
 
     match bson_util::get_int(ok) {
         Some(1) => Ok(()),
-        Some(_) => Err(Error::authentication_error(
-            auth_mechanism,
-            response
-                .get_str("errmsg")
-                .unwrap_or("Authentication failure"),
-        )),
+        Some(_) => {
+            let source = bson::from_bson::<CommandResponse<CommandErrorBody>>(Bson::Document(
+                response.clone(),
+            ))
+            .map(|cmd_resp| cmd_resp.body.into())
+            .ok();
+            Err(Error::authentication_error(
+                auth_mechanism,
+                response
+                    .get_str("errmsg")
+                    .unwrap_or("Authentication failure"),
+            )
+            .with_source(source))
+        }
         _ => Err(Error::invalid_authentication_response(auth_mechanism)),
     }
 }

diff --git a/src/client/executor.rs b/src/client/executor.rs
@@ -332,16 +332,24 @@ impl Client {
             Ok(c) => c,
             Err(mut err) => {
                 err.add_labels_and_update_pin(None, &mut session, None)?;
+                if err.is_read_retryable() && self.inner.options.retry_writes != Some(false) {
+                    err.add_label(RETRYABLE_WRITE_ERROR);
+                }
 
-                if err.is_pool_cleared() {
+                let op_retry = match self.get_op_retryability(&op, &session) {
+                    Retryability::Read => err.is_read_retryable(),
+                    Retryability::Write => err.is_write_retryable(),
+                    _ => false,
+                };
+                if err.is_pool_cleared() || op_retry {
                     return self.execute_retry(&mut op, &mut session, None, err).await;
                 } else {
                     return Err(err);
                 }
             }
         };
 
-        let retryability = self.get_retryability(&conn, &op, &session).await?;
+        let retryability = self.get_retryability(&conn, &op, &session)?;
 
         let txn_number = match session {
             Some(ref mut session) => {
@@ -433,7 +441,7 @@ impl Client {
             Err(_) => return Err(first_error),
         };
 
-        let retryability = self.get_retryability(&conn, op, session).await?;
+        let retryability = self.get_retryability(&conn, op, session)?;
         if retryability == Retryability::None {
             return Err(first_error);
         }
@@ -825,35 +833,49 @@ impl Client {
     }
 
     /// Returns the retryability level for the execution of this operation.
-    async fn get_retryability<T: Operation>(
+    fn get_op_retryability<T: Operation>(
         &self,
-        conn: &Connection,
         op: &T,
         session: &Option<&mut ClientSession>,
-    ) -> Result<Retryability> {
-        if !session
+    ) -> Retryability {
+        if session
             .as_ref()
             .map(|session| session.in_transaction())
             .unwrap_or(false)
         {
-            match op.retryability() {
-                Retryability::Read if self.inner.options.retry_reads != Some(false) => {
-                    return Ok(Retryability::Read);
-                }
-                Retryability::Write if conn.stream_description()?.supports_retryable_writes() => {
-                    // commitTransaction and abortTransaction should be retried regardless of the
-                    // value for retry_writes set on the Client
-                    if op.name() == CommitTransaction::NAME
-                        || op.name() == AbortTransaction::NAME
-                        || self.inner.options.retry_writes != Some(false)
-                    {
-                        return Ok(Retryability::Write);
-                    }
-                }
-                _ => {}
+            return Retryability::None;
+        }
+        match op.retryability() {
+            Retryability::Read if self.inner.options.retry_reads != Some(false) => {
+                Retryability::Read
+            }
+            // commitTransaction and abortTransaction should be retried regardless of the
+            // value for retry_writes set on the Client
+            Retryability::Write
+                if op.name() == CommitTransaction::NAME
+                    || op.name() == AbortTransaction::NAME
+                    || self.inner.options.retry_writes != Some(false) =>
+            {
+                Retryability::Write
+            }
+            _ => Retryability::None,
+        }
+    }
+
+    /// Returns the retryability level for the execution of this operation on this connection.
+    fn get_retryability<T: Operation>(
+        &self,
+        conn: &Connection,
+        op: &T,
+        session: &Option<&mut ClientSession>,
+    ) -> Result<Retryability> {
+        match self.get_op_retryability(op, session) {
+            Retryability::Read => Ok(Retryability::Read),
+            Retryability::Write if conn.stream_description()?.supports_retryable_writes() => {
+                Ok(Retryability::Write)
             }
+            _ => Ok(Retryability::None),
         }
-        Ok(Retryability::None)
     }
 
     async fn update_cluster_time(

diff --git a/src/error.rs b/src/error.rs
@@ -47,6 +47,8 @@ pub struct Error {
     pub kind: Box<ErrorKind>,
     labels: HashSet<String>,
     pub(crate) wire_version: Option<i32>,
+    #[source]
+    pub(crate) source: Option<Box<Error>>,
 }
 
 impl Error {
@@ -61,6 +63,7 @@ impl Error {
             kind: Box::new(kind),
             labels,
             wire_version: None,
+            source: None,
         }
     }
 
@@ -237,6 +240,7 @@ impl Error {
             ErrorKind::Write(WriteFailure::WriteConcernError(wc_error)) => Some(wc_error.code),
             _ => None,
         }
+        .or_else(|| self.source.as_ref().and_then(|s| s.code()))
     }
 
     /// Gets the message for this error, if applicable, for use in testing.
@@ -345,6 +349,11 @@ impl Error {
         }
         false
     }
+
+    pub(crate) fn with_source<E: Into<Option<Error>>>(mut self, source: E) -> Self {
+        self.source = source.into().map(Box::new);
+        self
+    }
 }
 
 impl<E> From<E> for Error

diff --git a/src/test/spec/json/retryable-reads/README.rst b/src/test/spec/json/retryable-reads/README.rst
@@ -9,8 +9,10 @@ Retryable Reads Tests
 Introduction
 ============
 
-The YAML and JSON files in this directory tree are platform-independent tests
-that drivers can use to prove their conformance to the Retryable Reads spec.
+The YAML and JSON files in the ``legacy`` and ``unified`` sub-directories are platform-independent tests
+that drivers can use to prove their conformance to the Retryable Reads spec. Tests in the
+``unified`` directory are written using the `Unified Test Format <../../unified-test-format/unified-test-format.rst>`_.
+Tests in the ``legacy`` directory are written using the format described below.
 
 Prose tests, which are not easily expressed in YAML, are also presented
 in this file. Those tests will need to be manually implemented by each driver.
@@ -93,20 +95,20 @@ Each YAML file has the following keys:
 
 - ``database_name`` and ``collection_name``: Optional. The database and
   collection to use for testing.
-  
+
 - ``bucket_name``: Optional. The GridFS bucket name to use for testing.
 
 - ``data``: The data that should exist in the collection(s) under test before
   each test run. This will typically be an array of documents to be inserted
   into the collection under test (i.e. ``collection_name``); however, this field
   may also be an object mapping collection names to arrays of documents to be
   inserted into the specified collection.
-    
+
 - ``tests``: An array of tests that are to be run independently of each other.
   Each test will have some or all of the following fields:
 
   - ``description``: The name of the test.
-    
+
   - ``clientOptions``: Optional, parameters to pass to MongoClient().
 
   - ``useMultipleMongoses`` (optional): If ``true``, and the topology type is
@@ -121,7 +123,7 @@ Each YAML file has the following keys:
     server.
 
     ``useMultipleMongoses`` only affects ``Sharded`` and ``LoadBalanced`` topologies.
-    
+
   - ``skipReason``: Optional, string describing why this test should be skipped.
 
   - ``failPoint``: Optional, a server fail point to enable, expressed as the
@@ -140,10 +142,10 @@ Each YAML file has the following keys:
     - ``result``: Optional. The return value from the operation, if any. This
       field may be a scalar (e.g. in the case of a count), a single document, or
       an array of documents in the case of a multi-document read.
-      
+
     - ``error``: Optional. If ``true``, the test should expect an error or
       exception.
-        
+
   - ``expectations``: Optional list of command-started events.
 
 GridFS Tests
@@ -167,7 +169,7 @@ data.
 
 
 .. _GridFSBucket spec: https://github.com/mongodb/specifications/blob/master/source/gridfs/gridfs-spec.rst#configurable-gridfsbucket-class
-    
+
 
 Speeding Up Tests
 -----------------
@@ -229,6 +231,8 @@ This test requires MongoDB 4.2.9+ for ``blockConnection`` support in the failpoi
 Changelog
 =========
 
+:2022-01-10: Create legacy and unified subdirectories for new unified tests
+
 :2021-08-27: Clarify behavior of ``useMultipleMongoses`` for ``LoadBalanced`` topologies.
 
 :2019-03-19: Add top-level ``runOn`` field to denote server version and/or

diff --git a/...json/retryable-reads/aggregate-merge.json → ...tryable-reads/legacy/aggregate-merge.json b/...json/retryable-reads/aggregate-merge.json → ...tryable-reads/legacy/aggregate-merge.json
diff --git a/.../json/retryable-reads/aggregate-merge.yml → ...etryable-reads/legacy/aggregate-merge.yml b/.../json/retryable-reads/aggregate-merge.yml → ...etryable-reads/legacy/aggregate-merge.yml
diff --git a/...tryable-reads/aggregate-serverErrors.json → ...-reads/legacy/aggregate-serverErrors.json b/...tryable-reads/aggregate-serverErrors.json → ...-reads/legacy/aggregate-serverErrors.json
diff --git a/...etryable-reads/aggregate-serverErrors.yml → ...e-reads/legacy/aggregate-serverErrors.yml b/...etryable-reads/aggregate-serverErrors.yml → ...e-reads/legacy/aggregate-serverErrors.yml
diff --git a/.../spec/json/retryable-reads/aggregate.json → ...son/retryable-reads/legacy/aggregate.json b/.../spec/json/retryable-reads/aggregate.json → ...son/retryable-reads/legacy/aggregate.json
diff --git a/...t/spec/json/retryable-reads/aggregate.yml → ...json/retryable-reads/legacy/aggregate.yml b/...t/spec/json/retryable-reads/aggregate.yml → ...json/retryable-reads/legacy/aggregate.yml
diff --git a/...ngeStreams-client.watch-serverErrors.json → ...ngeStreams-client.watch-serverErrors.json b/...ngeStreams-client.watch-serverErrors.json → ...ngeStreams-client.watch-serverErrors.json
diff --git a/...angeStreams-client.watch-serverErrors.yml → ...angeStreams-client.watch-serverErrors.yml b/...angeStreams-client.watch-serverErrors.yml → ...angeStreams-client.watch-serverErrors.yml
diff --git a/...ble-reads/changeStreams-client.watch.json → ...ds/legacy/changeStreams-client.watch.json b/...ble-reads/changeStreams-client.watch.json → ...ds/legacy/changeStreams-client.watch.json
diff --git a/...able-reads/changeStreams-client.watch.yml → ...ads/legacy/changeStreams-client.watch.yml b/...able-reads/changeStreams-client.watch.yml → ...ads/legacy/changeStreams-client.watch.yml
diff --git a/...geStreams-db.coll.watch-serverErrors.json → ...geStreams-db.coll.watch-serverErrors.json b/...geStreams-db.coll.watch-serverErrors.json → ...geStreams-db.coll.watch-serverErrors.json
diff --git a/...ngeStreams-db.coll.watch-serverErrors.yml → ...ngeStreams-db.coll.watch-serverErrors.yml b/...ngeStreams-db.coll.watch-serverErrors.yml → ...ngeStreams-db.coll.watch-serverErrors.yml
diff --git a/...le-reads/changeStreams-db.coll.watch.json → ...s/legacy/changeStreams-db.coll.watch.json b/...le-reads/changeStreams-db.coll.watch.json → ...s/legacy/changeStreams-db.coll.watch.json
diff --git a/...ble-reads/changeStreams-db.coll.watch.yml → ...ds/legacy/changeStreams-db.coll.watch.yml b/...ble-reads/changeStreams-db.coll.watch.yml → ...ds/legacy/changeStreams-db.coll.watch.yml
diff --git a/.../changeStreams-db.watch-serverErrors.json → .../changeStreams-db.watch-serverErrors.json b/.../changeStreams-db.watch-serverErrors.json → .../changeStreams-db.watch-serverErrors.json
diff --git a/...s/changeStreams-db.watch-serverErrors.yml → ...y/changeStreams-db.watch-serverErrors.yml b/...s/changeStreams-db.watch-serverErrors.yml → ...y/changeStreams-db.watch-serverErrors.yml
diff --git a/...tryable-reads/changeStreams-db.watch.json → ...-reads/legacy/changeStreams-db.watch.json b/...tryable-reads/changeStreams-db.watch.json → ...-reads/legacy/changeStreams-db.watch.json
diff --git a/...etryable-reads/changeStreams-db.watch.yml → ...e-reads/legacy/changeStreams-db.watch.yml b/...etryable-reads/changeStreams-db.watch.yml → ...e-reads/legacy/changeStreams-db.watch.yml
diff --git a/...n/retryable-reads/count-serverErrors.json → ...able-reads/legacy/count-serverErrors.json b/...n/retryable-reads/count-serverErrors.json → ...able-reads/legacy/count-serverErrors.json
diff --git a/...on/retryable-reads/count-serverErrors.yml → ...yable-reads/legacy/count-serverErrors.yml b/...on/retryable-reads/count-serverErrors.yml → ...yable-reads/legacy/count-serverErrors.yml
diff --git a/...test/spec/json/retryable-reads/count.json → ...ec/json/retryable-reads/legacy/count.json b/...test/spec/json/retryable-reads/count.json → ...ec/json/retryable-reads/legacy/count.json
diff --git a/src/test/spec/json/retryable-reads/count.yml → ...pec/json/retryable-reads/legacy/count.yml b/src/test/spec/json/retryable-reads/count.yml → ...pec/json/retryable-reads/legacy/count.yml
diff --git a/...le-reads/countDocuments-serverErrors.json → ...s/legacy/countDocuments-serverErrors.json b/...le-reads/countDocuments-serverErrors.json → ...s/legacy/countDocuments-serverErrors.json
diff --git a/...ble-reads/countDocuments-serverErrors.yml → ...ds/legacy/countDocuments-serverErrors.yml b/...ble-reads/countDocuments-serverErrors.yml → ...ds/legacy/countDocuments-serverErrors.yml
diff --git a/.../json/retryable-reads/countDocuments.json → ...etryable-reads/legacy/countDocuments.json b/.../json/retryable-reads/countDocuments.json → ...etryable-reads/legacy/countDocuments.json
diff --git a/...c/json/retryable-reads/countDocuments.yml → ...retryable-reads/legacy/countDocuments.yml b/...c/json/retryable-reads/countDocuments.yml → ...retryable-reads/legacy/countDocuments.yml
diff --git a/...etryable-reads/distinct-serverErrors.json → ...e-reads/legacy/distinct-serverErrors.json b/...etryable-reads/distinct-serverErrors.json → ...e-reads/legacy/distinct-serverErrors.json
diff --git a/...retryable-reads/distinct-serverErrors.yml → ...le-reads/legacy/distinct-serverErrors.yml b/...retryable-reads/distinct-serverErrors.yml → ...le-reads/legacy/distinct-serverErrors.yml
diff --git a/...t/spec/json/retryable-reads/distinct.json → ...json/retryable-reads/legacy/distinct.json b/...t/spec/json/retryable-reads/distinct.json → ...json/retryable-reads/legacy/distinct.json
diff --git a/...st/spec/json/retryable-reads/distinct.yml → .../json/retryable-reads/legacy/distinct.yml b/...st/spec/json/retryable-reads/distinct.yml → .../json/retryable-reads/legacy/distinct.yml
diff --git a/...ble-reads/estimatedDocumentCount-4.9.json → ...ds/legacy/estimatedDocumentCount-4.9.json b/...ble-reads/estimatedDocumentCount-4.9.json → ...ds/legacy/estimatedDocumentCount-4.9.json
diff --git a/...able-reads/estimatedDocumentCount-4.9.yml → ...ads/legacy/estimatedDocumentCount-4.9.yml b/...able-reads/estimatedDocumentCount-4.9.yml → ...ads/legacy/estimatedDocumentCount-4.9.yml
diff --git a/...-reads/estimatedDocumentCount-pre4.9.json → ...legacy/estimatedDocumentCount-pre4.9.json b/...-reads/estimatedDocumentCount-pre4.9.json → ...legacy/estimatedDocumentCount-pre4.9.json
diff --git a/...e-reads/estimatedDocumentCount-pre4.9.yml → .../legacy/estimatedDocumentCount-pre4.9.yml b/...e-reads/estimatedDocumentCount-pre4.9.yml → .../legacy/estimatedDocumentCount-pre4.9.yml
diff --git a/...imatedDocumentCount-serverErrors-4.9.json → ...imatedDocumentCount-serverErrors-4.9.json b/...imatedDocumentCount-serverErrors-4.9.json → ...imatedDocumentCount-serverErrors-4.9.json
@@ -158,7 +158,7 @@
       ]
     },
     {
-      "description": "EstimatedDocumentCount succeeds after NotMaster",
+      "description": "EstimatedDocumentCount succeeds after NotWritablePrimary",
       "failPoint": {
         "configureFailPoint": "failCommand",
         "mode": {
@@ -228,7 +228,7 @@
       ]
     },
     {
-      "description": "EstimatedDocumentCount succeeds after NotMasterNoSlaveOk",
+      "description": "EstimatedDocumentCount succeeds after NotPrimaryNoSecondaryOk",
       "failPoint": {
         "configureFailPoint": "failCommand",
         "mode": {
@@ -298,7 +298,7 @@
       ]
     },
     {
-      "description": "EstimatedDocumentCount succeeds after NotMasterOrSecondary",
+      "description": "EstimatedDocumentCount succeeds after NotPrimaryOrSecondary",
       "failPoint": {
         "configureFailPoint": "failCommand",
         "mode": {
@@ -788,7 +788,7 @@
       ]
     },
     {
-      "description": "EstimatedDocumentCount fails after two NotMaster errors",
+      "description": "EstimatedDocumentCount fails after two NotWritablePrimary errors",
       "failPoint": {
         "configureFailPoint": "failCommand",
         "mode": {
@@ -858,7 +858,7 @@
       ]
     },
     {
-      "description": "EstimatedDocumentCount fails after NotMaster when retryReads is false",
+      "description": "EstimatedDocumentCount fails after NotWritablePrimary when retryReads is false",
       "clientOptions": {
         "retryReads": false
       },

diff --git a/...timatedDocumentCount-serverErrors-4.9.yml → ...timatedDocumentCount-serverErrors-4.9.yml b/...timatedDocumentCount-serverErrors-4.9.yml → ...timatedDocumentCount-serverErrors-4.9.yml
@@ -41,7 +41,7 @@ tests:
              - *retryable_command_started_event
              - *retryable_command_started_event
     -
-        description: "EstimatedDocumentCount succeeds after NotMaster"
+        description: "EstimatedDocumentCount succeeds after NotWritablePrimary"
         failPoint:
             <<: *failCommand_failPoint
             data: { failCommands: [aggregate], errorCode: 10107 }
@@ -50,7 +50,7 @@ tests:
              - *retryable_command_started_event
              - *retryable_command_started_event
     -
-        description: "EstimatedDocumentCount succeeds after NotMasterNoSlaveOk"
+        description: "EstimatedDocumentCount succeeds after NotPrimaryNoSecondaryOk"
         failPoint:
             <<: *failCommand_failPoint
             data: { failCommands: [aggregate], errorCode: 13435 }
@@ -59,7 +59,7 @@ tests:
              - *retryable_command_started_event
              - *retryable_command_started_event
     -
-        description: "EstimatedDocumentCount succeeds after NotMasterOrSecondary"
+        description: "EstimatedDocumentCount succeeds after NotPrimaryOrSecondary"
         failPoint:
             <<: *failCommand_failPoint
             data: { failCommands: [aggregate], errorCode: 13436 }
@@ -122,7 +122,7 @@ tests:
              - *retryable_command_started_event
              - *retryable_command_started_event
     -
-        description: "EstimatedDocumentCount fails after two NotMaster errors"
+        description: "EstimatedDocumentCount fails after two NotWritablePrimary errors"
         failPoint:
             <<: *failCommand_failPoint
             mode: { times: 2 }
@@ -135,7 +135,7 @@ tests:
              - *retryable_command_started_event
              - *retryable_command_started_event
     -
-        description: "EstimatedDocumentCount fails after NotMaster when retryReads is false"
+        description: "EstimatedDocumentCount fails after NotWritablePrimary when retryReads is false"
         clientOptions:
             retryReads: false
         failPoint:

diff --git a/...tedDocumentCount-serverErrors-pre4.9.json → ...tedDocumentCount-serverErrors-pre4.9.json b/...tedDocumentCount-serverErrors-pre4.9.json → ...tedDocumentCount-serverErrors-pre4.9.json
@@ -110,7 +110,7 @@
       ]
     },
     {
-      "description": "EstimatedDocumentCount succeeds after NotMaster",
+      "description": "EstimatedDocumentCount succeeds after NotWritablePrimary",
       "failPoint": {
         "configureFailPoint": "failCommand",
         "mode": {
@@ -150,7 +150,7 @@
       ]
     },
     {
-      "description": "EstimatedDocumentCount succeeds after NotMasterNoSlaveOk",
+      "description": "EstimatedDocumentCount succeeds after NotPrimaryNoSecondaryOk",
       "failPoint": {
         "configureFailPoint": "failCommand",
         "mode": {
@@ -190,7 +190,7 @@
       ]
     },
     {
-      "description": "EstimatedDocumentCount succeeds after NotMasterOrSecondary",
+      "description": "EstimatedDocumentCount succeeds after NotPrimaryOrSecondary",
       "failPoint": {
         "configureFailPoint": "failCommand",
         "mode": {
@@ -470,7 +470,7 @@
       ]
     },
     {
-      "description": "EstimatedDocumentCount fails after two NotMaster errors",
+      "description": "EstimatedDocumentCount fails after two NotWritablePrimary errors",
       "failPoint": {
         "configureFailPoint": "failCommand",
         "mode": {
@@ -510,7 +510,7 @@
       ]
     },
     {
-      "description": "EstimatedDocumentCount fails after NotMaster when retryReads is false",
+      "description": "EstimatedDocumentCount fails after NotWritablePrimary when retryReads is false",
       "clientOptions": {
         "retryReads": false
       },