adjust checkpoint filters when you want to retrieve all events (#380)

* adjust checkpoint filters when you want to retrieve all events

* Bugfix: only returning 10,000 events on first run

* move checkpoint None -> empty-string check to _get_all_file_events()

* release prep 1.14.3

CHANGELOG.md

@@ -8,6 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 The intended audience of this file is for py42 consumers -- as such, changes that don't affect
 how a consumer would use the library (e.g. adding unit tests, updating documentation, etc) are not captured here.
 
+## 1.14.3 - 2022-07-06
+
+### Fixed
+
+- Bug where the `code42 security-data search` command using a checkpoint and only the `--include-non-exposure` filter resulted in invalid page tokens.
+- Bug where `code42 security-data search` would only return 10,000 events on the first search when using a new checkpoint.
+
 ## 1.14.2 - 2022-06-17
 
 ### Fixed

src/code42cli/__version__.py

@@ -1 +1 @@
-__version__ = "1.14.2"
+__version__ = "1.14.3"

src/code42cli/click_ext/groups.py

@@ -13,6 +13,7 @@
 from py42.exceptions import Py42ForbiddenError
 from py42.exceptions import Py42HTTPError
 from py42.exceptions import Py42InvalidEmailError
+from py42.exceptions import Py42InvalidPageTokenError
 from py42.exceptions import Py42InvalidPasswordError
 from py42.exceptions import Py42InvalidRuleOperationError
 from py42.exceptions import Py42InvalidUsernameError
@@ -84,6 +85,7 @@ def invoke(self, ctx):
             Py42UpdateClosedCaseError,
             Py42UsernameMustBeEmailError,
             Py42InvalidEmailError,
+            Py42InvalidPageTokenError,
             Py42InvalidPasswordError,
             Py42InvalidUsernameError,
             Py42ActiveLegalHoldError,

src/code42cli/cmds/securitydata.py

@@ -1,4 +1,3 @@
-import datetime
 from pprint import pformat
 
 import click
@@ -478,13 +477,9 @@ def _construct_query(state, begin, end, saved_search, advanced_query, or_query):
 
     if not state.search_filters:
         # if a checkpoint and _only_ --include-non-exposure is passed, the filter list will be empty, which isn't a
-        # valid query, so in that case we want to fallback to a 90 day (max event age) date range. The checkpoint will
+        # valid query, so in that case we want to fallback to retrieving all events. The checkpoint will
         # still cause the query results to only contain events after the checkpointed event.
-        _90_days = datetime.datetime.utcnow() - datetime.timedelta(days=90)
-        timestamp = convert_datetime_to_timestamp(_90_days)
-        state.search_filters.append(
-            create_time_range_filter(f.EventTimestamp, timestamp, None)
-        )
+        state.search_filters.append(RiskSeverity.exists())
     query = FileEventQuery(*state.search_filters)
     query.page_size = MAX_EVENT_PAGE_SIZE
     query.sort_direction = "asc"
@@ -493,6 +488,8 @@ def _construct_query(state, begin, end, saved_search, advanced_query, or_query):
 
 
 def _get_all_file_events(state, query, checkpoint=""):
+    if checkpoint is None:
+        checkpoint = ""
     try:
         response = state.sdk.securitydata.search_all_file_events(
             query, page_token=checkpoint

tests/cmds/test_securitydata.py

@@ -4,6 +4,7 @@
 import pandas
 import py42.sdk.queries.fileevents.filters as f
 import pytest
+from py42.exceptions import Py42InvalidPageTokenError
 from py42.sdk.queries.fileevents.file_event_query import FileEventQuery
 from py42.sdk.queries.fileevents.filters import RiskIndicator
 from py42.sdk.queries.fileevents.filters import RiskSeverity
@@ -327,6 +328,17 @@ def test_search_and_send_to_when_advanced_query_passed_non_existent_filename_rai
         assert "Could not open file: not_a_file" in result.stdout
 
 
+@search_and_send_to_test
+def test_search_and_send_to_when_given_invalid_page_token_raises_error(
+    runner, cli_state, custom_error, file_event_cursor_with_eventid_checkpoint, command
+):
+    cli_state.sdk.securitydata.search_all_file_events.side_effect = (
+        Py42InvalidPageTokenError(custom_error, TEST_FILE_EVENT_ID_2)
+    )
+    result = runner.invoke(cli, [*command, "--use-checkpoint", "test"], obj=cli_state)
+    assert f'Invalid page token: "{TEST_FILE_EVENT_ID_2}"' in result.output
+
+
 @advanced_query_incompat_test_params
 def test_search_with_advanced_query_and_incompatible_argument_errors(
     runner, arg, cli_state