diff --git a/py/kubeflow/examples/create_e2e_workflow.py b/py/kubeflow/examples/create_e2e_workflow.py index b17d0a52c..1b13bbd57 100644 --- a/py/kubeflow/examples/create_e2e_workflow.py +++ b/py/kubeflow/examples/create_e2e_workflow.py @@ -251,6 +251,7 @@ def _build_tests_dag_notebooks(self): # Test timeout in seconds. "--timeout=1800", "--junitxml=" + self.artifacts_dir + "/junit_xgboost-synthetic-test.xml", + "--notebook_artifacts_dir=" + self.artifacts_dir + "/xgboost-synthetic-test-notebooks", ] dependencies = [] diff --git a/xgboost_synthetic/testing/conftest.py b/xgboost_synthetic/testing/conftest.py index ba4dbdf3e..a7da35e52 100644 --- a/xgboost_synthetic/testing/conftest.py +++ b/xgboost_synthetic/testing/conftest.py @@ -16,6 +16,9 @@ def pytest_addoption(parser): parser.addoption( "--repos", help="The repos to checkout; leave blank to use defaults", type=str, default="") + parser.addoption( + "--notebook_artifacts_dir", help="Directory to store notebook artifacts", + type=str, default="") @pytest.fixture def name(request): @@ -32,3 +35,7 @@ def image(request): @pytest.fixture def repos(request): return request.config.getoption("--repos") + +@pytest.fixture +def notebook_artifacts_dir(request): + return request.config.getoption("--notebook_artifacts_dir") diff --git a/xgboost_synthetic/testing/execute_notebook.py b/xgboost_synthetic/testing/execute_notebook.py index 3482c7715..7e95d42ef 100644 --- a/xgboost_synthetic/testing/execute_notebook.py +++ b/xgboost_synthetic/testing/execute_notebook.py @@ -3,11 +3,14 @@ import os import subprocess +from kubeflow.testing import util as kf_util logger = logging.getLogger(__name__) def prepare_env(): subprocess.check_call(["pip3", "install", "-U", "papermill"]) + subprocess.check_call(["pip3", "install", "-U", "nbconvert"]) + subprocess.check_call(["pip3", "install", "-U", "nbformat"]) subprocess.check_call(["pip3", "install", "-r", "../requirements.txt"]) @@ -22,7 +25,18 @@ def execute_notebook(notebook_path, parameters=None): def run_notebook_test(notebook_path, expected_messages, parameters=None): output_path = execute_notebook(notebook_path, parameters=parameters) + + import nbformat #pylint: disable=import-error + import nbconvert #pylint: disable=import-error + actual_output = open(output_path, 'r').read() + + nb = nbformat.reads(actual_output, as_version=4) + html_exporter = nbconvert.HTMLExporter() + (html_output, _) = html_exporter.from_notebook_node(nb) + gcs_path = os.getenv("OUTPUT_GCS") + kf_util.upload_to_gcs(html_output, gcs_path) + for expected_message in expected_messages: if not expected_message in actual_output: logger.error(actual_output) diff --git a/xgboost_synthetic/testing/xgboost_test.py b/xgboost_synthetic/testing/xgboost_test.py index 2d0fb697f..f9ec960fb 100644 --- a/xgboost_synthetic/testing/xgboost_test.py +++ b/xgboost_synthetic/testing/xgboost_test.py @@ -6,6 +6,7 @@ import pytest +from google.cloud import storage from kubernetes import client as k8s_client from kubeflow.testing import argo_build_util from kubeflow.testing import util @@ -17,7 +18,7 @@ # and we want signal in postsubmits and periodics @pytest.mark.xfail(os.getenv("JOB_TYPE") == "presubmit", reason="Flaky") def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disable=too-many-branches,too-many-statements - repos, image): + repos, image, notebook_artifacts_dir): '''Generate Job and summit.''' util.set_pytest_junit(record_xml_attribute, "test_xgboost_synthetic") @@ -35,6 +36,7 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa if not repos: repos = argo_build_util.get_repo_from_prow_env() + repos += ",kubeflow/testing@HEAD" logging.info("Repos set to %s", repos) job["spec"]["template"]["spec"]["initContainers"][0]["command"] = [ "/usr/local/bin/checkout_repos.sh", @@ -42,6 +44,20 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa "--src_dir=/src", "--depth=all", ] + + nb_bucket = "kubeflow-ci-deployment" + nb_path = os.path.join( + "xgboost_synthetic_testing", + os.getenv("JOB_TYPE"), + os.getenv("HOSTNAME"), + "notebook.html" + ) + output_gcs = util.to_gcs_uri(nb_bucket, nb_path) + logging.info("Tested notebook will be outputed to: %s", output_gcs) + job["spec"]["template"]["spec"]["containers"][0]["env"] = [ + {"name": "PYTHONPATH", "value": "/src/kubeflow/testing/py"}, + {"name": "OUTPUT_GCS", "value": output_gcs}, + ] job["spec"]["template"]["spec"]["containers"][0]["image"] = image util.load_kube_config(persist_config=False) @@ -75,6 +91,15 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa last_condition = final_job.status.conditions[-1] + # Download notebook html to artifacts + notebook_artifacts_path = os.path.join(notebook_artifacts_dir, "notebook.html") + logging.info("Writing notebook artifact to: %s", notebook_artifacts_path) + os.makedirs(notebook_artifacts_dir, exist_ok=True) + storage_client = storage.Client() + bucket = storage_client.get_bucket(nb_bucket) + blob = bucket.get_blob(nb_path) + blob.download_to_filename(notebook_artifacts_path) + if last_condition.type not in ["Complete"]: logging.error("Job didn't complete successfully") raise RuntimeError("Job {0}.{1} failed".format(namespace, name))