fix: push max_items down into paging

Required some rewriting of a couple cassettes as the requests changed tightly.
stac-utils · Apr 21, 2023 · 419f82f · 419f82f
1 parent f39773d
commit 419f82f
Show file tree

Hide file tree

Showing 9 changed files with 475 additions and 594 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Double encoding of `intersects` parameter in GET requests [#362](https://github.com/stac-utils/pystac-client/pull/362)
 - Fix geometry instantiation in item-search-intersects.ipynb [#484](https://github.com/stac-utils/pystac-client/pull/484)
 - Three tests that were false positives due to out-of-date cassettes [#491](https://github.com/stac-utils/pystac-client/pull/491)
+- Max items checks when paging [#492](https://github.com/stac-utils/pystac-client/pull/492)
 
 ## [v0.6.1] - 2023-03-14
 

diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py
@@ -690,16 +690,10 @@ def items_as_dicts(self) -> Iterator[Dict[str, Any]]:
         Yields:
             Item : each Item matching the search criteria
         """
-        nitems = 0
-        for page in self._stac_io.get_pages(
-            self.url, self.method, self.get_parameters()
-        ):
+        for page in self.pages_as_dicts():
             for item in page.get("features", []):
-                call_modifier(self.modifier, item)
+                # already signed in pages_as_dicts
                 yield item
-                nitems += 1
-                if self._max_items and nitems >= self._max_items:
-                    return
 
     # ------------------------------------------------------------------------
     # By Page
@@ -727,11 +721,22 @@ def pages_as_dicts(self) -> Iterator[Dict[str, Any]]:
             criteria as a feature-collection-like dictionary.
         """
         if isinstance(self._stac_io, StacApiIO):
+            num_items = 0
             for page in self._stac_io.get_pages(
                 self.url, self.method, self.get_parameters()
             ):
                 call_modifier(self.modifier, page)
-                yield page
+                features = page.get("features", [])
+                if features:
+                    num_items += len(features)
+                    if self._max_items and num_items > self._max_items:
+                        # Slice the features down to make sure we hit max_items
+                        page["features"] = features[0 : -(num_items - self._max_items)]
+                    yield page
+                    if self._max_items and num_items >= self._max_items:
+                        return
+                else:
+                    return
 
     # ------------------------------------------------------------------------
     # Everything
@@ -767,20 +772,10 @@ def item_collection_as_dict(self) -> Dict[str, Any]:
             Dict : A GeoJSON FeatureCollection
         """
         features = []
-        for page in self._stac_io.get_pages(
-            self.url, self.method, self.get_parameters()
-        ):
+        for page in self.pages_as_dicts():
             for feature in page["features"]:
                 features.append(feature)
-                if self._max_items and len(features) >= self._max_items:
-                    feature_collection = {
-                        "type": "FeatureCollection",
-                        "features": features,
-                    }
-                    call_modifier(self.modifier, feature_collection)
-                    return feature_collection
         feature_collection = {"type": "FeatureCollection", "features": features}
-        call_modifier(self.modifier, feature_collection)
         return feature_collection
 
     # Deprecated methods

diff --git a/...em_search/TestItemSearch.test_deprecations[get_all_items-item_collection-False-True].yaml b/...em_search/TestItemSearch.test_deprecations[get_all_items-item_collection-False-True].yaml
@@ -19,7 +19,7 @@ interactions:
   response:
     body:
       string: !!binary |
-        H4sIACRsQWQC/+2cW4/iNhSA/wpC6r6USXxJHIdqVGmn6qqV2q22+9QRQgEMZDckUZK5tdr/3mMn
+        H4sIAN1tQWQC/+2cW4/iNhSA/wpC6r6USXxJHIdqVGmn6qqV2q22+9QRQgEMZDckUZK5tdr/3mMn
         AQcyMCzDcAsaRuA49rnl8MWx/V87e4pFu9v+VXjZXSJuoiAQw8yPwnanPc7L0nb39r+2P4Ja91l/
         1rcs5FBm9dOHPuZ9xFCfIMwxwq784CJCbDh5MIge4cQrhxqYIt6xqOG6DLukI4uQzRhxO5ZlIMYo
         cnqdJUGghcAPv+Z9JyKAI0NdtqK2F8eBP/RkofklVUemiRjDkWmWxWnXNOPAC0XmJU/DaBbfZSIx
@@ -71,7 +71,7 @@ interactions:
       Content-Type:
       - application/geo+json
       Date:
-      - Thu, 20 Apr 2023 16:45:24 GMT
+      - Thu, 20 Apr 2023 16:52:45 GMT
       Strict-Transport-Security:
       - max-age=15724800; includeSubDomains
       X-Cache:
@@ -81,7 +81,7 @@ interactions:
       vary:
       - Accept-Encoding
       x-azure-ref:
-      - 20230420T164524Z-etrte6b79d6x39rbwaqfbyrsw800000004m00000000087yx
+      - 20230420T165245Z-ph7t3013zp2n93v2zenxd3r61000000004tg00000000kpu0
     status:
       code: 200
       message: OK
@@ -106,7 +106,7 @@ interactions:
   response:
     body:
       string: !!binary |
-        H4sIACRsQWQC/+2d62/iuBbA/xWEtPNlafAjcRyuqpW2VzvaK+3OanY+3VGFAhjILCRRkr7uav73
+        H4sIAN1tQWQC/+2d62/iuBbA/xWEtPNlafAjcRyuqpW2VzvaK+3OanY+3VGFAhjILCRRkr7uav73
         e+wkxASaQls2PFy1FTiOz8Mn5pdTn/TvbvYUi+6g+4vws7tE3ESLhRhnQRR2e91p3pZ2B1//7gYT
         6HWfDZdD20YuZXSYiiHmw/mQIMwQRzacMRpFj9D7yqUW9iihtGdTy/MYIW5PNWIPe6Rn2xZijLjk
         tleTD2MsgvCvXGQiFnBkrKtU9PbjeBGMfdnY/5aqI/NETOHIPMvidNDvxws/FJmfPI2jZXyXicRa
@@ -160,7 +160,7 @@ interactions:
       Content-Type:
       - application/geo+json
       Date:
-      - Thu, 20 Apr 2023 16:45:25 GMT
+      - Thu, 20 Apr 2023 16:52:47 GMT
       Strict-Transport-Security:
       - max-age=15724800; includeSubDomains
       X-Cache:
@@ -170,7 +170,7 @@ interactions:
       vary:
       - Accept-Encoding
       x-azure-ref:
-      - 20230420T164524Z-etrte6b79d6x39rbwaqfbyrsw800000004m000000000880e
+      - 20230420T165245Z-ph7t3013zp2n93v2zenxd3r61000000004tg00000000kpvk
     status:
       code: 200
       message: OK

diff --git a/...mSearch.test_deprecations[get_all_items_as_dict-item_collection_as_dict-False-False].yaml b/...mSearch.test_deprecations[get_all_items_as_dict-item_collection_as_dict-False-False].yaml
@@ -19,7 +19,7 @@ interactions:
   response:
     body:
       string: !!binary |
-        H4sIACZsQWQC/+2cW4/iNhSA/wpC6r6USXxJHIdqVGmn6qqV2q22+9QRQgEMZDckUZK5tdr/3mMn
+        H4sIAN9tQWQC/+2cW4/iNhSA/wpC6r6USXxJHIdqVGmn6qqV2q22+9QRQgEMZDckUZK5tdr/3mMn
         AQcyMCzDcAsaRuA49rnl8MWx/V87e4pFu9v+VXjZXSJuoiAQw8yPwnanPc7L0nb39r+2P4Ja91l/
         1rcs5FBm9dOHPuZ9xFCfIMwxwq784CJCbDh5MIge4cQrhxqYIt6xqOG6DLukI4uQzRhxO5ZlIMYo
         cnqdJUGghcAPv+Z9JyKAI0NdtqK2F8eBP/RkofklVUemiRjDkWmWxWnXNOPAC0XmJU/DaBbfZSIx
@@ -58,26 +58,30 @@ interactions:
         0w7ip749Rm+VYEPxmG2ELIjgaSQZ+6+Pf3/eDbpSCIXhVOJ4NFIXRQA/61m7i1GV0POHIW5O5vmN
         INwbarGqbKyiVV560VcRFtp0NwDcEXHs7t0U5gRc+B8mSv9AInIAAA==
     headers:
+      Accept-Ranges:
+      - bytes
       Access-Control-Allow-Credentials:
       - 'true'
       Access-Control-Allow-Origin:
       - '*'
-      Content-Encoding:
-      - gzip
+      Connection:
+      - keep-alive
       Content-Length:
       - '2149'
       Content-Type:
       - application/geo+json
       Date:
-      - Thu, 20 Apr 2023 16:45:25 GMT
+      - Thu, 20 Apr 2023 16:52:47 GMT
       Strict-Transport-Security:
       - max-age=15724800; includeSubDomains
-      Vary:
-      - Accept-Encoding
-      X-Azure-Ref:
-      - 0JWxBZAAAAADY4JLgin9pTLLdcMIHuK14REVOMzAxMDAwMTA5MDMzADkyN2FiZmE2LTE5ZjYtNGFmMS1hMDlkLWM5NTlkOWExZTY0NA==
       X-Cache:
       - CONFIG_NOCACHE
+      content-encoding:
+      - gzip
+      vary:
+      - Accept-Encoding
+      x-azure-ref:
+      - 20230420T165247Z-tftb77d56d3778hnqxff1fw2ag00000001bg00000000g4xa
     status:
       code: 200
       message: OK
@@ -102,7 +106,7 @@ interactions:
   response:
     body:
       string: !!binary |
-        H4sIACZsQWQC/+2d62/iuBbA/xWEtPNlafAjcRyuqpW2VzvaK+3OanY+3VGFAhjILCRRkr7uav73
+        H4sIAN9tQWQC/+2d62/iuBbA/xWEtPNlafAjcRyuqpW2VzvaK+3OanY+3VGFAhjILCRRkr7uav73
         e+wkxASaQls2PFy1FTiOz8Mn5pdTn/TvbvYUi+6g+4vws7tE3ESLhRhnQRR2e91p3pZ2B1//7gYT
         6HWfDZdD20YuZXSYiiHmw/mQIMwQRzacMRpFj9D7yqUW9iihtGdTy/MYIW5PNWIPe6Rn2xZijLjk
         tleTD2MsgvCvXGQiFnBkrKtU9PbjeBGMfdnY/5aqI/NETOHIPMvidNDvxws/FJmfPI2jZXyXicRa
@@ -143,26 +147,30 @@ interactions:
         9d3Q+d8wvJzF8ziWc6VhmvSxij95JUV/ibCwZrArs9WgLbpLz8cR0qLBC/8v74gY/u1iCncCGP0f
         XydYnp1yAAA=
     headers:
+      Accept-Ranges:
+      - bytes
       Access-Control-Allow-Credentials:
       - 'true'
       Access-Control-Allow-Origin:
       - '*'
-      Content-Encoding:
-      - gzip
+      Connection:
+      - keep-alive
       Content-Length:
       - '2231'
       Content-Type:
       - application/geo+json
       Date:
-      - Thu, 20 Apr 2023 16:45:25 GMT
+      - Thu, 20 Apr 2023 16:52:47 GMT
       Strict-Transport-Security:
       - max-age=15724800; includeSubDomains
-      Vary:
-      - Accept-Encoding
-      X-Azure-Ref:
-      - 0JmxBZAAAAADgDbukw/n2SpkTgoEMHMLcREVOMzAxMDAwMTA5MDMzADkyN2FiZmE2LTE5ZjYtNGFmMS1hMDlkLWM5NTlkOWExZTY0NA==
       X-Cache:
       - CONFIG_NOCACHE
+      content-encoding:
+      - gzip
+      vary:
+      - Accept-Encoding
+      x-azure-ref:
+      - 20230420T165247Z-tftb77d56d3778hnqxff1fw2ag00000001bg00000000g4yk
     status:
       code: 200
       message: OK