Log results of requests.utils.super_len() when DANDI_DEVEL_INSTRUMENT_REQUESTS_SUPERLEN is set

.github/workflows/test.yml

@@ -82,6 +82,7 @@ jobs:
         sudo mount -t nfs localhost:/tmp/nfsmount_ /tmp/nfsmount
         echo TMPDIR=/tmp/nfsmount/tmp >> "$GITHUB_ENV"
         echo HOME=/tmp/nfsmount/home >> "$GITHUB_ENV"
+        echo DANDI_DEVEL_INSTRUMENT_REQUESTS_SUPERLEN=1 >> "$GITHUB_ENV"
 
     - name: Run all tests
       if: matrix.mode != 'dandi-api'

DEVELOPMENT.md

@@ -78,6 +78,10 @@ development command line options.
   tests will not pull the latest needed Docker images at the start of a run if
   older versions of the images are already present.
 
+- `DANDI_DEVEL_INSTRUMENT_REQUESTS_SUPERLEN` -- When set, the `upload()`
+  function will patch `requests` to log the results of calls to
+  `requests.utils.super_len()`
+
 ## Sourcegraph
 
 The [Sourcegraph](https://sourcegraph.com) browser extension can be used to

dandi/upload.py

@@ -1,10 +1,12 @@
 from collections import defaultdict
+from contextlib import ExitStack
 from functools import reduce
 import os.path
 from pathlib import Path
 import re
 import time
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union
+from unittest.mock import patch
 
 import click
 from packaging.version import Version
@@ -63,13 +65,31 @@ def upload(
     assert instance.api is not None
     api_url = instance.api
 
-    # We need to use this as a context manager in order to ensure the session
-    # gets properly closed.  Otherwise, pytest sometimes complains under
-    # obscure conditions.
-    with DandiAPIClient(api_url) as client:
+    with ExitStack() as stack:
+        # We need to use the client as a context manager in order to ensure the
+        # session gets properly closed.  Otherwise, pytest sometimes complains
+        # under obscure conditions.
+        client = stack.enter_context(DandiAPIClient(api_url))
         client.check_schema_version()
         client.dandi_authenticate()
 
+        if os.environ.get("DANDI_DEVEL_INSTRUMENT_REQUESTS_SUPERLEN"):
+            from requests.utils import super_len
+
+            def new_super_len(o):
+                try:
+                    n = super_len(o)
+                except Exception:
+                    lgr.debug(
+                        "requests.utils.super_len() failed on %r:", o, exc_info=True
+                    )
+                    raise
+                else:
+                    lgr.debug("requests.utils.super_len() reported %d for %r", n, o)
+                    return n
+
+            stack.enter_context(patch("requests.models.super_len", new_super_len))
+
         dandiset = APIDandiset(dandiset_.path)  # "cast" to a new API based dandiset
 
         ds_identifier = dandiset.identifier