diff --git a/CHANGELOG.md b/CHANGELOG.md index 31b9e1fa9..a06ac9cd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 The intended audience of this file is for py42 consumers -- as such, changes that don't affect how a consumer would use the library (e.g. adding unit tests, updating documentation, etc) are not captured here. +## 1.3.1 - 2021-02-25 + +### Changed + +- Command options for `profile update`: + - `-n` `--name` is not required, and if omitted will use the default profile. + - `-s` `--server` and `-u` `--username` are not required and can be updated independently now. + - Example: `code42 profile update -s 1.2.3.4:1234` + +## 1.3.0 - 2021-02-11 + +### Fixed + +- Issue where `code42 alert-rules bulk add` would show as successful when adding users to a non-existent alert rule. + +### Added + +- New choice `TLS-TCP` for `--protocol` option used by `send-to` commands: + - `code42 security-data send-to` + - `code42 alerts send-to` + - `code42 audit-logs send-to` + for more securely transporting data. Included are new flags: + - `--certs` + - `--ignore-cert-validation` + +### Changed + +- The error text in cases command when: + - `cases create` sets a name that already exists in the system. + - `cases create` sets a description that has more than 250 characters. + - `cases update` sets a description that has more than 250 characters. + - `cases file-events add` is performed on an already closed case. + - `cases file-events add` sets an event id that is already added to the case. + - `cases file-events remove` is performed on an already closed case. + ## 1.2.0 - 2021-01-25 ### Added @@ -71,6 +106,12 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta - Now, when adding a cloud alias to a detection list user, such as during `departing-employee add`, it will remove the existing cloud alias if one exists. - Before, it would error and the cloud alias would not get added. +### Added + +- `code42 devices list` option: + - `--include-legal-hold-membership` prints the legal hold matter name and ID for any active device on legal hold + - `--include-total-storage` prints the backup archive count and total storage + ## 1.0.0 - 2020-08-31 ### Fixed diff --git a/src/code42cli/cmds/devices.py b/src/code42cli/cmds/devices.py index 8b0a920aa..8935752b2 100644 --- a/src/code42cli/cmds/devices.py +++ b/src/code42cli/cmds/devices.py @@ -1,8 +1,11 @@ from datetime import date import click +import numpy as np from pandas import concat from pandas import DataFrame +from pandas import json_normalize +from pandas import Series from pandas import to_datetime from py42 import exceptions from py42.exceptions import Py42NotFoundError @@ -244,6 +247,22 @@ def _get_device_info(sdk, device_guid): is_flag=True, help="Include device settings in output.", ) +@click.option( + "--include-legal-hold-membership", + required=False, + type=bool, + default=False, + is_flag=True, + help="Include legal hold membership in output.", +) +@click.option( + "--include-total-storage", + required=False, + type=bool, + default=False, + is_flag=True, + help="Include backup archive count and total storage in output.", +) @click.option( "--exclude-most-recently-connected", type=int, @@ -285,6 +304,8 @@ def list_devices( include_backup_usage, include_usernames, include_settings, + include_legal_hold_membership, + include_total_storage, exclude_most_recently_connected, last_connected_after, last_connected_before, @@ -309,7 +330,11 @@ def list_devices( "userUid", ] df = _get_device_dataframe( - state.sdk, columns, active, org_uid, include_backup_usage + state.sdk, + columns, + active, + org_uid, + (include_backup_usage or include_total_storage), ) if last_connected_after: df = df.loc[to_datetime(df.lastConnected) > last_connected_after] @@ -326,10 +351,14 @@ def list_devices( .head(exclude_most_recently_connected) ) df = df.drop(most_recent.index) + if include_total_storage: + df = _add_storage_totals_to_dataframe(df, include_backup_usage) if include_settings: df = _add_settings_to_dataframe(state.sdk, df) if include_usernames: df = _add_usernames_to_device_dataframe(state.sdk, df) + if include_legal_hold_membership: + df = _add_legal_hold_membership_to_device_dataframe(state.sdk, df) if df.empty: click.echo("No results found.") else: @@ -337,6 +366,42 @@ def list_devices( formatter.echo_formatted_dataframe(df) +def _add_legal_hold_membership_to_device_dataframe(sdk, df): + columns = ["legalHold.legalHoldUid", "legalHold.name", "user.userUid"] + + legal_hold_member_dataframe = ( + json_normalize(list(_get_all_active_hold_memberships(sdk)))[columns] + .groupby(["user.userUid"]) + .agg(",".join) + .rename( + { + "legalHold.legalHoldUid": "legalHoldUid", + "legalHold.name": "legalHoldName", + }, + axis=1, + ) + ) + df = df.merge( + legal_hold_member_dataframe, + how="left", + left_on="userUid", + right_on="user.userUid", + ) + + df.loc[df["status"] == "Deactivated", ["legalHoldUid", "legalHoldName"]] = np.nan + + return df + + +def _get_all_active_hold_memberships(sdk): + for page in sdk.legalhold.get_all_matters(active=True): + for matter in page["legalHolds"]: + for _page in sdk.legalhold.get_all_matter_custodians( + legal_hold_uid=matter["legalHoldUid"], active=True + ): + yield from _page["legalHoldMemberships"] + + def _get_device_dataframe( sdk, columns, active=None, org_uid=None, include_backup_usage=False ): @@ -392,6 +457,26 @@ def _add_usernames_to_device_dataframe(sdk, device_dataframe): return device_dataframe.merge(users_dataframe, how="left", on="userUid") +def _add_storage_totals_to_dataframe(df, include_backup_usage): + df[["archiveCount", "totalStorageBytes"]] = df["backupUsage"].apply( + _break_backup_usage_into_total_storage + ) + + if not include_backup_usage: + df = df.drop("backupUsage", axis=1) + return df + + +def _break_backup_usage_into_total_storage(backup_usage): + total_storage = 0 + archive_count = 0 + for archive in backup_usage: + if archive["archiveFormat"] != "ARCHIVE_V2": + archive_count += 1 + total_storage += archive["archiveBytes"] + return Series([archive_count, total_storage]) + + @devices.command() @active_option @inactive_option diff --git a/tests/cmds/test_devices.py b/tests/cmds/test_devices.py index 6281db4c3..5dbf31a22 100644 --- a/tests/cmds/test_devices.py +++ b/tests/cmds/test_devices.py @@ -1,7 +1,12 @@ +import json from datetime import date +import numpy as np import pytest from pandas import DataFrame +from pandas import Series +from pandas._testing import assert_frame_equal +from pandas._testing import assert_series_equal from py42.exceptions import Py42BadRequestError from py42.exceptions import Py42ForbiddenError from py42.exceptions import Py42NotFoundError @@ -11,7 +16,9 @@ from code42cli import PRODUCT_NAME from code42cli.cmds.devices import _add_backup_set_settings_to_dataframe +from code42cli.cmds.devices import _add_legal_hold_membership_to_device_dataframe from code42cli.cmds.devices import _add_usernames_to_device_dataframe +from code42cli.cmds.devices import _break_backup_usage_into_total_storage from code42cli.cmds.devices import _get_device_dataframe from code42cli.main import cli @@ -77,7 +84,19 @@ "836476656572622471","serverName":"cif-sea","serverHostName":"https://cif-sea.crashplan.com", "isProvider":false,"archiveGuid":"843293524842941560","archiveFormat":"ARCHIVE_V1","activity": {"connected":false,"backingUp":false,"restoring":false,"timeRemainingInMs":0, -"remainingFiles":0,"remainingBytes":0}}]}}""" +"remainingFiles":0,"remainingBytes":0}},{"targetComputerParentId":null,"targetComputerParentGuid": +null,"targetComputerGuid":"43","targetComputerName":"PROe Cloud, US","targetComputerOsName":null, +"targetComputerType":"SERVER","selectedFiles":1599,"selectedBytes":1529420143,"todoFiles":0, +"todoBytes":0,"archiveBytes":56848550,"billableBytes":1529420143,"sendRateAverage":0, +"completionRateAverage":0,"lastBackup":"2019-12-02T09:34:28.364-06:00","lastCompletedBackup": +"2019-12-02T09:34:28.364-06:00","lastConnected":"2019-12-02T11:02:36.108-06:00","lastMaintenanceDate": +"2021-02-16T07:01:11.697-06:00","lastCompactDate":"2021-02-16T07:01:11.694-06:00","modificationDate": +"2021-02-17T04:57:27.222-06:00","creationDate":"2019-09-26T15:27:38.806-05:00","using":true, +"alertState":16,"alertStates":["CriticalBackupAlert"],"percentComplete":100.0,"storePointId":10989, +"storePointName":"fsa-iad-2","serverId":160024121,"serverGuid":"883282371081742804","serverName": +"fsa-iad","serverHostName":"https://web-fsa-iad.crashplan.com","isProvider":false,"archiveGuid": +"92077743916530001","archiveFormat":"ARCHIVE_V1","activity":{"connected":false,"backingUp":false, +"restoring":false,"timeRemainingInMs":0,"remainingFiles":0,"remainingBytes":0}}]}}""" TEST_EMPTY_BACKUPUSAGE_RESPONSE = """{"metadata":{"timestamp":"2020-10-13T12:51:28.410Z","params": {"incBackupUsage":"True","idType":"guid"}},"data":{"computerId":1767,"name":"SNWINTEST1", "osHostname":"UNKNOWN","guid":"843290890230648046","type":"COMPUTER","status":"Active", @@ -222,6 +241,75 @@ }, ], } +MATTER_RESPONSE = { + "legalHolds": [ + { + "legalHoldUid": "123456789", + "name": "Test legal hold matter", + "description": "", + "notes": None, + "holdExtRef": None, + "active": True, + "creationDate": "2020-08-05T10:49:58.353-05:00", + "lastModified": "2020-08-05T10:49:58.358-05:00", + "creator": { + "userUid": "12345", + "username": "user@code42.com", + "email": "user@code42.com", + "userExtRef": None, + }, + "holdPolicyUid": "966191295667423997", + }, + { + "legalHoldUid": "987654321", + "name": "Another Matter", + "description": "", + "notes": None, + "holdExtRef": None, + "active": True, + "creationDate": "2020-05-20T15:58:31.375-05:00", + "lastModified": "2020-05-28T13:49:16.098-05:00", + "creator": { + "userUid": "76543", + "username": "user2@code42.com", + "email": "user2@code42.com", + "userExtRef": None, + }, + "holdPolicyUid": "946178665645035826", + }, + ] +} +ALL_CUSTODIANS_RESPONSE = { + "legalHoldMemberships": [ + { + "legalHoldMembershipUid": "99999", + "active": True, + "creationDate": "2020-07-16T08:50:23.405Z", + "legalHold": { + "legalHoldUid": "123456789", + "name": "Test legal hold matter", + }, + "user": { + "userUid": "840103986007089121", + "username": "ttranda_deactivated@ttrantest.com", + "email": "ttranda_deactivated@ttrantest.com", + "userExtRef": None, + }, + }, + { + "legalHoldMembershipUid": "88888", + "active": True, + "creationDate": "2020-07-16T08:50:23.405Z", + "legalHold": {"legalHoldUid": "987654321", "name": "Another Matter"}, + "user": { + "userUid": "840103986007089121", + "username": "ttranda_deactivated@ttrantest.com", + "email": "ttranda_deactivated@ttrantest.com", + "userExtRef": None, + }, + }, + ] +} def _create_py42_response(mocker, text): @@ -275,6 +363,14 @@ def users_list_generator(): yield TEST_USERS_LIST_PAGE +def matter_list_generator(): + yield MATTER_RESPONSE + + +def custodian_list_generator(): + yield ALL_CUSTODIANS_RESPONSE + + @pytest.fixture def backupusage_response(mocker): return _create_py42_response(mocker, TEST_BACKUPUSAGE_RESPONSE) @@ -357,6 +453,18 @@ def get_all_users_success(cli_state): cli_state.sdk.users.get_all.return_value = users_list_generator() +@pytest.fixture +def get_all_matter_success(cli_state): + cli_state.sdk.legalhold.get_all_matters.return_value = matter_list_generator() + + +@pytest.fixture +def get_all_custodian_success(cli_state): + cli_state.sdk.legalhold.get_all_matter_custodians.return_value = ( + custodian_list_generator() + ) + + def test_deactivate_deactivates_device( runner, cli_state, deactivate_device_success, get_device_by_guid_success ): @@ -559,6 +667,79 @@ def test_add_usernames_to_device_dataframe_adds_usernames_to_dataframe( assert "username" in result.columns +def test_add_legal_hold_membership_to_device_dataframe_adds_legal_hold_columns_to_dataframe( + cli_state, get_all_matter_success, get_all_custodian_success +): + testdf = DataFrame.from_records( + [ + {"userUid": "840103986007089121", "status": "Active"}, + {"userUid": "836473273124890369", "status": "Active, Deauthorized"}, + ] + ) + result = _add_legal_hold_membership_to_device_dataframe(cli_state.sdk, testdf) + assert "legalHoldUid" in result.columns + assert "legalHoldName" in result.columns + + +def test_list_include_legal_hold_membership_pops_legal_hold_if_device_deactivated( + cli_state, get_all_matter_success, get_all_custodian_success +): + testdf = DataFrame.from_records( + [ + {"userUid": "840103986007089121", "status": "Deactivated"}, + {"userUid": "840103986007089121", "status": "Active"}, + ] + ) + + testdf_result = DataFrame.from_records( + [ + { + "userUid": "840103986007089121", + "status": "Deactivated", + "legalHoldUid": np.nan, + "legalHoldName": np.nan, + }, + { + "userUid": "840103986007089121", + "status": "Active", + "legalHoldUid": "123456789,987654321", + "legalHoldName": "Test legal hold matter,Another Matter", + }, + ] + ) + result = _add_legal_hold_membership_to_device_dataframe(cli_state.sdk, testdf) + + assert_frame_equal(result, testdf_result) + + +def test_list_include_legal_hold_membership_merges_in_and_concats_legal_hold_info( + runner, + cli_state, + get_all_devices_success, + get_all_custodian_success, + get_all_matter_success, +): + result = runner.invoke( + cli, ["devices", "list", "--include-legal-hold-membership"], obj=cli_state + ) + + assert "Test legal hold matter,Another Matter" in result.output + assert "123456789,987654321" in result.output + + +def test_break_backup_usage_into_total_storage_correctly_calculates_values(): + test_backupusage_cell = json.loads(TEST_BACKUPUSAGE_RESPONSE)["data"]["backupUsage"] + result = _break_backup_usage_into_total_storage(test_backupusage_cell) + + test_empty_backupusage_cell = json.loads(TEST_EMPTY_BACKUPUSAGE_RESPONSE)["data"][ + "backupUsage" + ] + empty_result = _break_backup_usage_into_total_storage(test_empty_backupusage_cell) + + assert_series_equal(result, Series([2, 56968051])) + assert_series_equal(empty_result, Series([0, 0])) + + def test_last_connected_after_filters_appropriate_results( cli_state, runner, get_all_devices_success ):