From 475cd4c4be285cb7ea590ba3bb38f69f14d11561 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Thu, 13 Feb 2020 10:57:45 -0800 Subject: [PATCH 01/20] use nbconvert to write output as html --- xgboost_synthetic/testing/execute_notebook.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/xgboost_synthetic/testing/execute_notebook.py b/xgboost_synthetic/testing/execute_notebook.py index 3482c7715..50dcc70e7 100644 --- a/xgboost_synthetic/testing/execute_notebook.py +++ b/xgboost_synthetic/testing/execute_notebook.py @@ -8,6 +8,8 @@ def prepare_env(): subprocess.check_call(["pip3", "install", "-U", "papermill"]) + subprocess.check_call(["pip3", "install", "-U", "nbconvert"]) + subprocess.check_call(["pip3", "install", "-U", "nbformat"]) subprocess.check_call(["pip3", "install", "-r", "../requirements.txt"]) @@ -22,7 +24,17 @@ def execute_notebook(notebook_path, parameters=None): def run_notebook_test(notebook_path, expected_messages, parameters=None): output_path = execute_notebook(notebook_path, parameters=parameters) + + import nbformat #pylint: disable=import-error + import nbconvert #pylint: disable=import-error + actual_output = open(output_path, 'r').read() + + nb = nbformat.reads(actual_output, as_version=4) + html_exporter = nbconvert.HTMLExporter() + (html_output, _) = html_exporter.from_notebook_node(nb) + # Write to local/GCS. + for expected_message in expected_messages: if not expected_message in actual_output: logger.error(actual_output) From 10f9e06e39c31e4b6b8c6dc525902fdc7759d0f1 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Thu, 13 Feb 2020 11:11:28 -0800 Subject: [PATCH 02/20] write local file --- xgboost_synthetic/testing/execute_notebook.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xgboost_synthetic/testing/execute_notebook.py b/xgboost_synthetic/testing/execute_notebook.py index 50dcc70e7..7edb41fd0 100644 --- a/xgboost_synthetic/testing/execute_notebook.py +++ b/xgboost_synthetic/testing/execute_notebook.py @@ -33,7 +33,10 @@ def run_notebook_test(notebook_path, expected_messages, parameters=None): nb = nbformat.reads(actual_output, as_version=4) html_exporter = nbconvert.HTMLExporter() (html_output, _) = html_exporter.from_notebook_node(nb) - # Write to local/GCS. + logger.info(html_output[:10] + "...") + html_path = os.path.join(os.path.dirname(output_path), "out.html") + with open(html_path, "w") as f: + f.write(html_output) for expected_message in expected_messages: if not expected_message in actual_output: From 8fcbbc642279a9ba880399949920169142160d90 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Thu, 13 Feb 2020 14:22:01 -0800 Subject: [PATCH 03/20] change dir --- xgboost_synthetic/testing/execute_notebook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xgboost_synthetic/testing/execute_notebook.py b/xgboost_synthetic/testing/execute_notebook.py index 7edb41fd0..04ef64284 100644 --- a/xgboost_synthetic/testing/execute_notebook.py +++ b/xgboost_synthetic/testing/execute_notebook.py @@ -34,7 +34,7 @@ def run_notebook_test(notebook_path, expected_messages, parameters=None): html_exporter = nbconvert.HTMLExporter() (html_output, _) = html_exporter.from_notebook_node(nb) logger.info(html_output[:10] + "...") - html_path = os.path.join(os.path.dirname(output_path), "out.html") + html_path = os.path.join(os.path.dirname(__file__), "out.html") with open(html_path, "w") as f: f.write(html_output) From abc339f789c726f301186275d4cacee80c309f4e Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Thu, 13 Feb 2020 15:31:44 -0800 Subject: [PATCH 04/20] write to gcs --- xgboost_synthetic/testing/execute_notebook.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xgboost_synthetic/testing/execute_notebook.py b/xgboost_synthetic/testing/execute_notebook.py index 04ef64284..2228b3c44 100644 --- a/xgboost_synthetic/testing/execute_notebook.py +++ b/xgboost_synthetic/testing/execute_notebook.py @@ -3,6 +3,7 @@ import os import subprocess +from kubeflow.testing import util as kf_util logger = logging.getLogger(__name__) @@ -33,10 +34,11 @@ def run_notebook_test(notebook_path, expected_messages, parameters=None): nb = nbformat.reads(actual_output, as_version=4) html_exporter = nbconvert.HTMLExporter() (html_output, _) = html_exporter.from_notebook_node(nb) - logger.info(html_output[:10] + "...") - html_path = os.path.join(os.path.dirname(__file__), "out.html") - with open(html_path, "w") as f: - f.write(html_output) + gcs_path = "gs://kubeflow-ci-deployment/" + "/".join([ + "xgboost_synthetic_testing", + "temp.html" + ]) + kf_util.upload_to_gcs(html_output, gcs_path) for expected_message in expected_messages: if not expected_message in actual_output: From 4fba5f3eb2b9f9cb4777d2d797fe8efbc056da96 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Thu, 13 Feb 2020 15:46:52 -0800 Subject: [PATCH 05/20] add kubeflow/testing --- xgboost_synthetic/testing/job.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xgboost_synthetic/testing/job.yaml b/xgboost_synthetic/testing/job.yaml index 4267e4e82..f2aa9ce9f 100644 --- a/xgboost_synthetic/testing/job.yaml +++ b/xgboost_synthetic/testing/job.yaml @@ -23,7 +23,7 @@ spec: # This init container checks out the source code. - command: - /usr/local/bin/checkout_repos.sh - - --repos=kubeflow/examples@$(CHECK_TAG) + - --repos=kubeflow/examples@$(CHECK_TAG),kubeflow/testing@HEAD - --src_dir=/src name: checkout image: gcr.io/kubeflow-ci/test-worker:v20190802-c6f9140-e3b0c4 @@ -35,6 +35,9 @@ spec: image: execute-image command: ["python3", "execute_notebook.py"] workingDir: /src/kubeflow/examples/xgboost_synthetic/testing + env: + - name: PYTHONPATH + value: /src/kubeflow/testing/py volumeMounts: - mountPath: /src name: src From 64e3f076d4f677c2d89c8934ad647ead6c297bd3 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Thu, 13 Feb 2020 15:59:26 -0800 Subject: [PATCH 06/20] update to env and checkout_repos --- xgboost_synthetic/testing/job.yaml | 5 +---- xgboost_synthetic/testing/xgboost_test.py | 4 ++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/xgboost_synthetic/testing/job.yaml b/xgboost_synthetic/testing/job.yaml index f2aa9ce9f..4267e4e82 100644 --- a/xgboost_synthetic/testing/job.yaml +++ b/xgboost_synthetic/testing/job.yaml @@ -23,7 +23,7 @@ spec: # This init container checks out the source code. - command: - /usr/local/bin/checkout_repos.sh - - --repos=kubeflow/examples@$(CHECK_TAG),kubeflow/testing@HEAD + - --repos=kubeflow/examples@$(CHECK_TAG) - --src_dir=/src name: checkout image: gcr.io/kubeflow-ci/test-worker:v20190802-c6f9140-e3b0c4 @@ -35,9 +35,6 @@ spec: image: execute-image command: ["python3", "execute_notebook.py"] workingDir: /src/kubeflow/examples/xgboost_synthetic/testing - env: - - name: PYTHONPATH - value: /src/kubeflow/testing/py volumeMounts: - mountPath: /src name: src diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 2d0fb697f..5e5743a40 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -35,6 +35,7 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa if not repos: repos = argo_build_util.get_repo_from_prow_env() + repos += ",kubeflow/testing@HEAD" logging.info("Repos set to %s", repos) job["spec"]["template"]["spec"]["initContainers"][0]["command"] = [ "/usr/local/bin/checkout_repos.sh", @@ -42,6 +43,9 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa "--src_dir=/src", "--depth=all", ] + job["spec"]["template"]["spec"]["containers"][0]["env"] = [ + {"name": "PYTHONPATH", "value": "/src/kubeflow/testing/py"}, + ] job["spec"]["template"]["spec"]["containers"][0]["image"] = image util.load_kube_config(persist_config=False) From cc0e0460844aa96c8778af84ab68c03d79bc1dbd Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Thu, 13 Feb 2020 16:33:30 -0800 Subject: [PATCH 07/20] format gcs path --- xgboost_synthetic/testing/execute_notebook.py | 5 +---- xgboost_synthetic/testing/xgboost_test.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/xgboost_synthetic/testing/execute_notebook.py b/xgboost_synthetic/testing/execute_notebook.py index 2228b3c44..7e95d42ef 100644 --- a/xgboost_synthetic/testing/execute_notebook.py +++ b/xgboost_synthetic/testing/execute_notebook.py @@ -34,10 +34,7 @@ def run_notebook_test(notebook_path, expected_messages, parameters=None): nb = nbformat.reads(actual_output, as_version=4) html_exporter = nbconvert.HTMLExporter() (html_output, _) = html_exporter.from_notebook_node(nb) - gcs_path = "gs://kubeflow-ci-deployment/" + "/".join([ - "xgboost_synthetic_testing", - "temp.html" - ]) + gcs_path = os.getenv("OUTPUT_GCS") kf_util.upload_to_gcs(html_output, gcs_path) for expected_message in expected_messages: diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 5e5743a40..a7275d676 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -43,8 +43,18 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa "--src_dir=/src", "--depth=all", ] + + output_gcs = "/".join([ + "gs://kubeflow-ci-deployment/", + "xgboost_synthetic_testing", + os.getenv("JOB_TYPE"), + os.getenv("HOSTNAME"), + "notebook.html", + ]) + logging.info("Tested notebook will be outputed to: %s", output_gcs) job["spec"]["template"]["spec"]["containers"][0]["env"] = [ {"name": "PYTHONPATH", "value": "/src/kubeflow/testing/py"}, + {"name": "OUTPUT_GCS", "value": output_gcs, ] job["spec"]["template"]["spec"]["containers"][0]["image"] = image util.load_kube_config(persist_config=False) From e715a9eab46d6fd0dff41038f3dc24b8aed12333 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Thu, 13 Feb 2020 16:37:40 -0800 Subject: [PATCH 08/20] fix syntax --- xgboost_synthetic/testing/xgboost_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index a7275d676..08af6e200 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -54,7 +54,7 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa logging.info("Tested notebook will be outputed to: %s", output_gcs) job["spec"]["template"]["spec"]["containers"][0]["env"] = [ {"name": "PYTHONPATH", "value": "/src/kubeflow/testing/py"}, - {"name": "OUTPUT_GCS", "value": output_gcs, + {"name": "OUTPUT_GCS", "value": output_gcs}, ] job["spec"]["template"]["spec"]["containers"][0]["image"] = image util.load_kube_config(persist_config=False) From d1b4df6a785d21816c4d88cbb1232eb57991ea87 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Thu, 13 Feb 2020 16:42:47 -0800 Subject: [PATCH 09/20] fix --- xgboost_synthetic/testing/xgboost_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 08af6e200..05205fdd9 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -45,7 +45,7 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa ] output_gcs = "/".join([ - "gs://kubeflow-ci-deployment/", + "gs://kubeflow-ci-deployment", "xgboost_synthetic_testing", os.getenv("JOB_TYPE"), os.getenv("HOSTNAME"), From 25133bf1a065366fe4c43312c5f3d7829cd5b8e1 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 13:02:26 -0800 Subject: [PATCH 10/20] add option notebook_artifacts_dir --- py/kubeflow/examples/create_e2e_workflow.py | 1 + xgboost_synthetic/testing/conftest.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/py/kubeflow/examples/create_e2e_workflow.py b/py/kubeflow/examples/create_e2e_workflow.py index b17d0a52c..1b13bbd57 100644 --- a/py/kubeflow/examples/create_e2e_workflow.py +++ b/py/kubeflow/examples/create_e2e_workflow.py @@ -251,6 +251,7 @@ def _build_tests_dag_notebooks(self): # Test timeout in seconds. "--timeout=1800", "--junitxml=" + self.artifacts_dir + "/junit_xgboost-synthetic-test.xml", + "--notebook_artifacts_dir=" + self.artifacts_dir + "/xgboost-synthetic-test-notebooks", ] dependencies = [] diff --git a/xgboost_synthetic/testing/conftest.py b/xgboost_synthetic/testing/conftest.py index ba4dbdf3e..a7da35e52 100644 --- a/xgboost_synthetic/testing/conftest.py +++ b/xgboost_synthetic/testing/conftest.py @@ -16,6 +16,9 @@ def pytest_addoption(parser): parser.addoption( "--repos", help="The repos to checkout; leave blank to use defaults", type=str, default="") + parser.addoption( + "--notebook_artifacts_dir", help="Directory to store notebook artifacts", + type=str, default="") @pytest.fixture def name(request): @@ -32,3 +35,7 @@ def image(request): @pytest.fixture def repos(request): return request.config.getoption("--repos") + +@pytest.fixture +def notebook_artifacts_dir(request): + return request.config.getoption("--notebook_artifacts_dir") From f6b689313cdde171322efee3c9b9779c9d6852a3 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 13:20:24 -0800 Subject: [PATCH 11/20] download to artifacts --- xgboost_synthetic/testing/xgboost_test.py | 24 +++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 05205fdd9..3aab943ef 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -6,6 +6,7 @@ import pytest +from google.cloud import storage from kubernetes import client as k8s_client from kubeflow.testing import argo_build_util from kubeflow.testing import util @@ -17,7 +18,7 @@ # and we want signal in postsubmits and periodics @pytest.mark.xfail(os.getenv("JOB_TYPE") == "presubmit", reason="Flaky") def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disable=too-many-branches,too-many-statements - repos, image): + repos, image, notebook_artifacts_dir): '''Generate Job and summit.''' util.set_pytest_junit(record_xml_attribute, "test_xgboost_synthetic") @@ -44,13 +45,14 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa "--depth=all", ] - output_gcs = "/".join([ - "gs://kubeflow-ci-deployment", - "xgboost_synthetic_testing", - os.getenv("JOB_TYPE"), - os.getenv("HOSTNAME"), - "notebook.html", - ]) + nb_bucket = "kubeflow-ci-deployment" + nb_path = os.path.join( + "xgboost_synthetic_testing", + os.getenv("JOB_TYPE"), + os.getenv("HOSTNAME"), + "notebook.html", + ) + output_gcs = kf_util.to_gcs_uri(nb_bucket, nb_path) logging.info("Tested notebook will be outputed to: %s", output_gcs) job["spec"]["template"]["spec"]["containers"][0]["env"] = [ {"name": "PYTHONPATH", "value": "/src/kubeflow/testing/py"}, @@ -89,6 +91,12 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa last_condition = final_job.status.conditions[-1] + # Download notebook html to artifacts + with open(os.path.join(notebook_artifacts_dir, os.getenv("HOSTNAME"), + "notebook.html"), "w") as f: + storage_client = storage.Client() + storage_client.download_blob_to_file(output_gcs, f) + if last_condition.type not in ["Complete"]: logging.error("Job didn't complete successfully") raise RuntimeError("Job {0}.{1} failed".format(namespace, name)) From e63ea8a5e5ac3fb2a1de24b229d6b566da7e6b63 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 13:26:36 -0800 Subject: [PATCH 12/20] fix --- xgboost_synthetic/testing/xgboost_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 3aab943ef..228521054 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -50,7 +50,7 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa "xgboost_synthetic_testing", os.getenv("JOB_TYPE"), os.getenv("HOSTNAME"), - "notebook.html", + "notebook.html" ) output_gcs = kf_util.to_gcs_uri(nb_bucket, nb_path) logging.info("Tested notebook will be outputed to: %s", output_gcs) From c6dc0c1b83ec7c5f701a634dc70d29ea8848bb95 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 13:56:53 -0800 Subject: [PATCH 13/20] shorten name --- xgboost_synthetic/testing/xgboost_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 228521054..9d6795818 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -92,8 +92,7 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa last_condition = final_job.status.conditions[-1] # Download notebook html to artifacts - with open(os.path.join(notebook_artifacts_dir, os.getenv("HOSTNAME"), - "notebook.html"), "w") as f: + with open(os.path.join(notebook_artifacts_dir, "notebook.html"), "w") as f: storage_client = storage.Client() storage_client.download_blob_to_file(output_gcs, f) From 20a7d26d09ffacc0344124c3288ae8a89f86b11c Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 14:08:30 -0800 Subject: [PATCH 14/20] fix --- xgboost_synthetic/testing/xgboost_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 9d6795818..be67d58c7 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -52,7 +52,7 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa os.getenv("HOSTNAME"), "notebook.html" ) - output_gcs = kf_util.to_gcs_uri(nb_bucket, nb_path) + output_gcs = util.to_gcs_uri(nb_bucket, nb_path) logging.info("Tested notebook will be outputed to: %s", output_gcs) job["spec"]["template"]["spec"]["containers"][0]["env"] = [ {"name": "PYTHONPATH", "value": "/src/kubeflow/testing/py"}, From dcfb49382584397fd8b14da9c5d8e720c0b887b5 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 14:43:22 -0800 Subject: [PATCH 15/20] fix --- xgboost_synthetic/testing/xgboost_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index be67d58c7..c9936b909 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -93,6 +93,10 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa # Download notebook html to artifacts with open(os.path.join(notebook_artifacts_dir, "notebook.html"), "w") as f: + notebook_artifacts_path = os.path.join(notebook_artifacts_dir, "notebook.html") + logging.info("Writing notebook artifact to: %s", notebook_artifacts_path) + # Download notebook html to artifacts + with open(notebook_artifacts_path, "w") as f: storage_client = storage.Client() storage_client.download_blob_to_file(output_gcs, f) From 480fa8a684af3f9215295c1ac86a79a7654f25ef Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 14:54:12 -0800 Subject: [PATCH 16/20] mkdirs --- xgboost_synthetic/testing/xgboost_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index c9936b909..49602c75c 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -95,6 +95,7 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa with open(os.path.join(notebook_artifacts_dir, "notebook.html"), "w") as f: notebook_artifacts_path = os.path.join(notebook_artifacts_dir, "notebook.html") logging.info("Writing notebook artifact to: %s", notebook_artifacts_path) + os.makedirs(notebook_artifacts_dir, exist_ok=True) # Download notebook html to artifacts with open(notebook_artifacts_path, "w") as f: storage_client = storage.Client() From cf23322c2d13316b3cb6cbe608aab362477391eb Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 15:21:30 -0800 Subject: [PATCH 17/20] fix --- xgboost_synthetic/testing/xgboost_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 49602c75c..7becb519a 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -99,7 +99,9 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa # Download notebook html to artifacts with open(notebook_artifacts_path, "w") as f: storage_client = storage.Client() - storage_client.download_blob_to_file(output_gcs, f) + bucket = storage_client.get_bucket(nb_bucket) + blob = bucket.get_blob(nb_path) + blob.download_to_file(f) if last_condition.type not in ["Complete"]: logging.error("Job didn't complete successfully") From e94aa01d105a76e840ef990d99bc4e49414e775d Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 15:26:27 -0800 Subject: [PATCH 18/20] fix --- xgboost_synthetic/testing/xgboost_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 7becb519a..1cf8418c1 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -92,7 +92,6 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa last_condition = final_job.status.conditions[-1] # Download notebook html to artifacts - with open(os.path.join(notebook_artifacts_dir, "notebook.html"), "w") as f: notebook_artifacts_path = os.path.join(notebook_artifacts_dir, "notebook.html") logging.info("Writing notebook artifact to: %s", notebook_artifacts_path) os.makedirs(notebook_artifacts_dir, exist_ok=True) From 85ba65a23925a5614bd4545b11a1426a8372686e Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 15:37:48 -0800 Subject: [PATCH 19/20] log error --- xgboost_synthetic/testing/xgboost_test.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 1cf8418c1..973d784e9 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -97,10 +97,14 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa os.makedirs(notebook_artifacts_dir, exist_ok=True) # Download notebook html to artifacts with open(notebook_artifacts_path, "w") as f: - storage_client = storage.Client() - bucket = storage_client.get_bucket(nb_bucket) - blob = bucket.get_blob(nb_path) - blob.download_to_file(f) + try: + storage_client = storage.Client() + bucket = storage_client.get_bucket(nb_bucket) + blob = bucket.get_blob(nb_path) + blob.download_to_file(f) + except Exception as e: + logging.info("downloading notebook fail: %s", e) + raise e if last_condition.type not in ["Complete"]: logging.error("Job didn't complete successfully") From 09045fb4e93a92ff3c472d1a4d2274fed61835b1 Mon Sep 17 00:00:00 2001 From: Gabriel Wen Date: Fri, 14 Feb 2020 15:52:33 -0800 Subject: [PATCH 20/20] use notebook_artifacts_path --- xgboost_synthetic/testing/xgboost_test.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 973d784e9..f9ec960fb 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -95,16 +95,10 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa notebook_artifacts_path = os.path.join(notebook_artifacts_dir, "notebook.html") logging.info("Writing notebook artifact to: %s", notebook_artifacts_path) os.makedirs(notebook_artifacts_dir, exist_ok=True) - # Download notebook html to artifacts - with open(notebook_artifacts_path, "w") as f: - try: - storage_client = storage.Client() - bucket = storage_client.get_bucket(nb_bucket) - blob = bucket.get_blob(nb_path) - blob.download_to_file(f) - except Exception as e: - logging.info("downloading notebook fail: %s", e) - raise e + storage_client = storage.Client() + bucket = storage_client.get_bucket(nb_bucket) + blob = bucket.get_blob(nb_path) + blob.download_to_filename(notebook_artifacts_path) if last_condition.type not in ["Complete"]: logging.error("Job didn't complete successfully")