From 652dc2f20c9d1dcf79b0b2bfcb3932e0acf97667 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 11:26:06 -0800 Subject: [PATCH 1/9] Add logs attribute to Job class --- hyp3_sdk/jobs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 0902cdf..1a64271 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -24,6 +24,7 @@ def __init__( name: Optional[str] = None, job_parameters: Optional[dict] = None, files: Optional[List] = None, + logs: Optional[List] = None, browse_images: Optional[List] = None, thumbnail_images: Optional[List] = None, expiration_time: Optional[datetime] = None @@ -36,6 +37,7 @@ def __init__( self.name = name self.job_parameters = job_parameters self.files = files + self.logs = logs self.browse_images = browse_images self.thumbnail_images = thumbnail_images self.expiration_time = expiration_time @@ -61,6 +63,7 @@ def from_dict(input_dict: dict): name=input_dict.get('name'), job_parameters=input_dict.get('job_parameters'), files=input_dict.get('files'), + logs=input_dict.get('logs'), browse_images=input_dict.get('browse_images'), thumbnail_images=input_dict.get('thumbnail_images'), expiration_time=expiration_time @@ -77,7 +80,7 @@ def to_dict(self, for_resubmit: bool = False): job_dict[key] = value if not for_resubmit: - for key in ['files', 'browse_images', 'thumbnail_images', 'job_id', 'status_code', 'user_id', + for key in ['files', 'logs', 'browse_images', 'thumbnail_images', 'job_id', 'status_code', 'user_id', 'expiration_time', 'request_time']: value = self.__getattribute__(key) if value is not None: From 9bff4c431df2ba4ae34b30892a365cfc0c54c91f Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 11:57:43 -0800 Subject: [PATCH 2/9] Add test for job attributes --- hyp3_sdk/jobs.py | 6 ++++-- tests/test_jobs.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 1a64271..fd788be 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -14,6 +14,9 @@ # TODO: actually looks like a good candidate for a dataclass (python 3.7+) # https://docs.python.org/3/library/dataclasses.html class Job: + _finished_job_keys = {'job_id', 'logs', 'request_time', 'status_code', 'user_id'} + _successful_job_keys = _finished_job_keys.union({'browse_images', 'expiration_time', 'files', 'thumbnail_images'}) + def __init__( self, job_type: str, @@ -80,8 +83,7 @@ def to_dict(self, for_resubmit: bool = False): job_dict[key] = value if not for_resubmit: - for key in ['files', 'logs', 'browse_images', 'thumbnail_images', 'job_id', 'status_code', 'user_id', - 'expiration_time', 'request_time']: + for key in self._successful_job_keys: value = self.__getattribute__(key) if value is not None: if isinstance(value, datetime): diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 1f2a821..875a94f 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -12,6 +12,7 @@ "browse_images": ["https://PAIR_PROCESS.png"], "expiration_time": "2020-10-08T00:00:00+00:00", "files": [{"filename": "PAIR_PROCESS.nc", "size": 5949932, "url": "https://PAIR_PROCESS.nc"}], + "logs": ["https://d1c05104-b455-4f35-a95a-84155d63f855.log"], "job_id": "d1c05104-b455-4f35-a95a-84155d63f855", "job_parameters": {"granules": [ "S1A_IW_SLC__1SDH_20180511T204719_20180511T204746_021862_025C12_6F77", @@ -26,6 +27,7 @@ } FAILED_JOB = { + "logs": ["https://281b2087-9e7d-4d17-a9b3-aebeb2ad23c6.log"], "job_id": "281b2087-9e7d-4d17-a9b3-aebeb2ad23c6", "job_parameters": { "granules": [ @@ -41,6 +43,20 @@ } +def test_job_attributes(): + job = Job.from_dict(SUCCEEDED_JOB) + for key in Job._successful_job_keys: + assert job.__getattribute__(key) + + job = Job.from_dict(FAILED_JOB) + for key in Job._finished_job_keys: + assert job.__getattribute__(key) + + job = Job.from_dict(FAILED_JOB) + for key in Job._successful_job_keys.difference(Job._finished_job_keys): + assert job.__getattribute__(key) is None + + def test_job_dict_transforms(): job = Job.from_dict(SUCCEEDED_JOB) assert job.to_dict() == SUCCEEDED_JOB @@ -48,6 +64,18 @@ def test_job_dict_transforms(): job = Job.from_dict(FAILED_JOB) assert job.to_dict() == FAILED_JOB + job = Job.from_dict(SUCCEEDED_JOB) + retry = job.to_dict(for_resubmit=True) + for key in Job._successful_job_keys: + assert job.__getattribute__(key) + assert not retry.get(key, False) + + job = Job.from_dict(FAILED_JOB) + retry = job.to_dict(for_resubmit=True) + for key in Job._finished_job_keys: + assert job.__getattribute__(key) + assert not retry.get(key, False) + def test_job_complete_succeeded_failed_running(): job = Job.from_dict(SUCCEEDED_JOB) From 0deb1fd426b050f8e2be19219d34acd964f49f15 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 13:47:25 -0800 Subject: [PATCH 3/9] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index decdeb3..4e6b27d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [1.2.0](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.3...v1.2.0) ### Added +- `Job` class now has a `logs` attribute containing links to job log files - Added missing [container methods](https://docs.python.org/3/reference/datamodel.html#emulating-container-types) - batches are now subscriptable: `batch[0]` - jobs can be searched for in batches:`job in batch` From b05e53e4fe6229784822fabf6b08edd0debc6a33 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:00:43 -0800 Subject: [PATCH 4/9] refactor for resubmit --- CHANGELOG.md | 2 +- hyp3_sdk/jobs.py | 12 ++++-------- tests/test_jobs.py | 10 +++++----- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6b27d..89ffc73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed - [#92](https://github.com/ASFHyP3/hyp3-sdk/issues/92) -- `ImportError` being raised when showing a progress bar because `ipywidgets` may not always be - installed when running in a Jupyter kernel + installed when running in a Jupyter kernel ## [1.1.3](https://github.com/ASFHyP3/hyp3-sdk/compare/v1.1.2...v1.1.3) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index fd788be..83d84bf 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -14,8 +14,7 @@ # TODO: actually looks like a good candidate for a dataclass (python 3.7+) # https://docs.python.org/3/library/dataclasses.html class Job: - _finished_job_keys = {'job_id', 'logs', 'request_time', 'status_code', 'user_id'} - _successful_job_keys = _finished_job_keys.union({'browse_images', 'expiration_time', 'files', 'thumbnail_images'}) + _attributes_for_resubmit = {'name', 'job_parameters', 'job_type'} def __init__( self, @@ -73,17 +72,14 @@ def from_dict(input_dict: dict): ) def to_dict(self, for_resubmit: bool = False): - job_dict = { - 'job_type': self.job_type, - } - - for key in ['name', 'job_parameters']: + job_dict = {} + for key in Job._attributes_for_resubmit: value = self.__getattribute__(key) if value is not None: job_dict[key] = value if not for_resubmit: - for key in self._successful_job_keys: + for key in set(vars(self).keys()) - Job._attributes_for_resubmit: value = self.__getattribute__(key) if value is not None: if isinstance(value, datetime): diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 875a94f..285551f 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -45,15 +45,15 @@ def test_job_attributes(): job = Job.from_dict(SUCCEEDED_JOB) - for key in Job._successful_job_keys: + for key in SUCCEEDED_JOB.keys(): assert job.__getattribute__(key) job = Job.from_dict(FAILED_JOB) - for key in Job._finished_job_keys: + for key in FAILED_JOB.keys(): assert job.__getattribute__(key) job = Job.from_dict(FAILED_JOB) - for key in Job._successful_job_keys.difference(Job._finished_job_keys): + for key in set(vars(job).keys()) - set(FAILED_JOB.keys()): assert job.__getattribute__(key) is None @@ -66,13 +66,13 @@ def test_job_dict_transforms(): job = Job.from_dict(SUCCEEDED_JOB) retry = job.to_dict(for_resubmit=True) - for key in Job._successful_job_keys: + for key in set(SUCCEEDED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) assert not retry.get(key, False) job = Job.from_dict(FAILED_JOB) retry = job.to_dict(for_resubmit=True) - for key in Job._finished_job_keys: + for key in set(FAILED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) assert not retry.get(key, False) From e0b7abcb8336270ef88f8f49d9d28b9fe21c660a Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:08:04 -0800 Subject: [PATCH 5/9] more refactoring --- hyp3_sdk/jobs.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 83d84bf..6e3a579 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -71,21 +71,26 @@ def from_dict(input_dict: dict): expiration_time=expiration_time ) + def _attribute_value(self, attribute_name): + value = self.__getattribute__(attribute_name) + if isinstance(value, datetime): + return value.isoformat(timespec='seconds') + else: + return value + def to_dict(self, for_resubmit: bool = False): job_dict = {} - for key in Job._attributes_for_resubmit: - value = self.__getattribute__(key) - if value is not None: - job_dict[key] = value - - if not for_resubmit: - for key in set(vars(self).keys()) - Job._attributes_for_resubmit: - value = self.__getattribute__(key) + if for_resubmit: + for key in Job._attributes_for_resubmit: + value = self._attribute_value(key) if value is not None: - if isinstance(value, datetime): - job_dict[key] = value.isoformat(timespec='seconds') - else: - job_dict[key] = value + job_dict[key] = value + else: + for key in vars(self).keys(): + value = self._attribute_value(key) + if value is not None: + job_dict[key] = value + return job_dict def succeeded(self) -> bool: From 33ec09f1032c5a3c83ae7f35fba10d56991cbb15 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:11:02 -0800 Subject: [PATCH 6/9] Moar refactor! --- hyp3_sdk/jobs.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/hyp3_sdk/jobs.py b/hyp3_sdk/jobs.py index 6e3a579..fbe31e8 100644 --- a/hyp3_sdk/jobs.py +++ b/hyp3_sdk/jobs.py @@ -71,24 +71,19 @@ def from_dict(input_dict: dict): expiration_time=expiration_time ) - def _attribute_value(self, attribute_name): - value = self.__getattribute__(attribute_name) - if isinstance(value, datetime): - return value.isoformat(timespec='seconds') - else: - return value - def to_dict(self, for_resubmit: bool = False): job_dict = {} if for_resubmit: - for key in Job._attributes_for_resubmit: - value = self._attribute_value(key) - if value is not None: - job_dict[key] = value + keys_to_process = Job._attributes_for_resubmit else: - for key in vars(self).keys(): - value = self._attribute_value(key) - if value is not None: + keys_to_process = vars(self).keys() + + for key in keys_to_process: + value = self.__getattribute__(key) + if value is not None: + if isinstance(value, datetime): + job_dict[key] = value.isoformat(timespec='seconds') + else: job_dict[key] = value return job_dict From 9f09fd95d53f98580a67d37593fe18e347c79255 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:17:21 -0800 Subject: [PATCH 7/9] Better test names/flow --- tests/test_jobs.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 285551f..a9a7579 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -52,8 +52,8 @@ def test_job_attributes(): for key in FAILED_JOB.keys(): assert job.__getattribute__(key) - job = Job.from_dict(FAILED_JOB) - for key in set(vars(job).keys()) - set(FAILED_JOB.keys()): + unprovided_attributes = set(vars(job).keys()) - set(FAILED_JOB.keys()) + for key in unprovided_attributes: assert job.__getattribute__(key) is None @@ -61,16 +61,14 @@ def test_job_dict_transforms(): job = Job.from_dict(SUCCEEDED_JOB) assert job.to_dict() == SUCCEEDED_JOB - job = Job.from_dict(FAILED_JOB) - assert job.to_dict() == FAILED_JOB - - job = Job.from_dict(SUCCEEDED_JOB) retry = job.to_dict(for_resubmit=True) for key in set(SUCCEEDED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) assert not retry.get(key, False) job = Job.from_dict(FAILED_JOB) + assert job.to_dict() == FAILED_JOB + retry = job.to_dict(for_resubmit=True) for key in set(FAILED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) From 40d8980dbe701bd665c42b20e93d9ae13f1cceed Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:25:40 -0800 Subject: [PATCH 8/9] test resubmit keys in retry dict --- tests/test_jobs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index a9a7579..839dc79 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -62,6 +62,9 @@ def test_job_dict_transforms(): assert job.to_dict() == SUCCEEDED_JOB retry = job.to_dict(for_resubmit=True) + for key in Job._attributes_for_resubmit: + _ = retry[key] + for key in set(SUCCEEDED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) assert not retry.get(key, False) @@ -70,6 +73,9 @@ def test_job_dict_transforms(): assert job.to_dict() == FAILED_JOB retry = job.to_dict(for_resubmit=True) + for key in Job._attributes_for_resubmit: + _ = retry[key] + for key in set(FAILED_JOB.keys()) - Job._attributes_for_resubmit: assert job.__getattribute__(key) assert not retry.get(key, False) From 65b43e47b20bae0146d751e074e89c10214813b0 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 29 Jun 2021 15:28:54 -0800 Subject: [PATCH 9/9] simplify retry test --- tests/test_jobs.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 839dc79..35c409d 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -62,23 +62,13 @@ def test_job_dict_transforms(): assert job.to_dict() == SUCCEEDED_JOB retry = job.to_dict(for_resubmit=True) - for key in Job._attributes_for_resubmit: - _ = retry[key] - - for key in set(SUCCEEDED_JOB.keys()) - Job._attributes_for_resubmit: - assert job.__getattribute__(key) - assert not retry.get(key, False) + assert retry.keys() == Job._attributes_for_resubmit job = Job.from_dict(FAILED_JOB) assert job.to_dict() == FAILED_JOB retry = job.to_dict(for_resubmit=True) - for key in Job._attributes_for_resubmit: - _ = retry[key] - - for key in set(FAILED_JOB.keys()) - Job._attributes_for_resubmit: - assert job.__getattribute__(key) - assert not retry.get(key, False) + assert retry.keys() == Job._attributes_for_resubmit def test_job_complete_succeeded_failed_running():