From e7eb02a4d419bfb83a82ffd16d2e65755eb76afa Mon Sep 17 00:00:00 2001 From: Jash Parekh Date: Sat, 18 Feb 2023 18:15:50 +0530 Subject: [PATCH] Fail pipeline on first exception (#379) Co-authored-by: Jash Parekh --- .flake8 | 4 ++-- CHANGELOG.md | 6 ++++++ README.md | 14 +++++++++++--- hooks/command | 5 +++++ plugin.yml | 4 +++- plugin_scripts/deploy.py | 10 +++++++++- 6 files changed, 36 insertions(+), 7 deletions(-) diff --git a/.flake8 b/.flake8 index fc79732..2b79e42 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] max-line-length = 88 max-complexity = 8 -ignore = W503,E501 +ignore = W503,E501,C901 builtins = unicode tee = True -exclude = venv,env,.venv,scratch \ No newline at end of file +exclude = venv,env,.venv,scratch diff --git a/CHANGELOG.md b/CHANGELOG.md index ba5e212..e3984da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.2.0] - 2023-02-18 + +### Added + +- Option to specify `fail_pipeline_on_first_exception`, default to `true` + ## [1.1.0] - 2023-02-02 ### Added diff --git a/README.md b/README.md index e290f90..5aeacc5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![Actions Status](https://github.com/wayfair-incubator/bigquery-buildkite-plugin/workflows/Lint/badge.svg?branch=main)](https://github.com/wayfair-incubator/bigquery-buildkite-plugin/actions) [![Actions Status](https://github.com/wayfair-incubator/bigquery-buildkite-plugin/workflows/Unit%20Tests/badge.svg?branch=main)](https://github.com/wayfair-incubator/bigquery-buildkite-plugin/actions) -![Version](https://img.shields.io/static/v1.svg?label=Version&message=0.5.0&color=lightgrey&?link=http://left&link=https://github.com/wayfair-incubator/bigquery-buildkite-plugin/tree/v1.0.0) +![Version](https://img.shields.io/static/v1.svg?label=Version&message=1.2.0&color=lightgrey&?link=http://left&link=https://github.com/wayfair-incubator/bigquery-buildkite-plugin/tree/v1.2.0) ![Plugin Status](https://img.shields.io/static/v1.svg?label=&message=Buildkite%20Plugin&color=blue&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAMAAADVRocKAAAAk1BMVEX///+DrRik1Cb+/vyErhin1yeAqhfJ5n+dzCO50X2VwiC2zna020vg8LSo1i+q1zTN3qL7/fav2UDT4q33++yXuj3z+eLO6IqMuByJsSPM54XX5bWcvUbH5Xrg6sWiwVHB1ovk7cymxFnq8deOtC2xzG7c6L6/1YfU7JbI2pj2+e+jzzPd763K3J3j8ryRtDbY7aOqCe3pAAACTElEQVRoge3Z11KDQBSAYUpItSRqVGwxEVOwxPd/OndhCbA5BTaLM87sueb83+yMStHz3Lhx48bN/5iw+2VjImy0fDvzQiNCLKVrdjn0nq++QwNCLMy+9uOzc2Y59AZBwF4F55NefxxxyxK4aEvI/C7x/QxglrMTBNxVej6V+QNALh+AhkRY5isAsVwBxFWfM/rnLsu/qnwNQIkawBBy/abMawBCaEAQXGFElt/Evo8CIHEEIESWH9XyAAAQACCIH40A8yBwRICARiB5BNAIBKgQ8tLbCZBHgRqBAgWB5wmgQhCAIt7ekTwJ5AQHSGKC1TkgiM7WIs8A0bBvCETR8H7CA4EhIPP9fgPA7AR53u91BBR53+8EKPPdACLvq3wXQC1vH9DytoGjvF0gGo71vE0gCoC8PQDJ2wJEvgfmbQFo3g5A5HNA3K+eL0yBePdO5AtAEAOUoIB4c+ONiHwBZHddjMCB5DUV6yOqfzgBQWCAzMu9ZoAiHgACBpJdmj3SNAdQAgKSXfFQ1gZQz28PlxxQ5tsCIKED+6/qU2tbQBF3lxgwn9YfitsDR0QVmE9D7bHeBNCIEphf63lToEYUAJQ3BxSxlUQGPD3Cr5DmwIGQJ8DypwHqlXX7gec5oMcAXv5OT31OYU6weuM/9tDv/iSwWnJxfghA5s0+QzUCFi828iiwWNvJI4C9PAg8WcwDAPVLYwGwndeAufV8DZB/bm3nK0A3+RyI81tdF3l1gn1neQlM4qnpp+9ms0xP/P8AP/8778aNGzdu/mh+AQp1NCB/JInXAAAAAElFTkSuQmCC) @@ -19,7 +19,7 @@ If the version number is not provided then the most recent version of the plugin ```yaml steps: - plugins: - - wayfair-incubator/bigquery#v1.0.0: + - wayfair-incubator/bigquery#v1.2.0: gcp_project: gcp-us-project dataset_schema_directory: schemas/gcp-us-project/dataset ``` @@ -48,7 +48,7 @@ Example: `main` Default: `master` -### `execute_only_changed_files` (optional, string) +### `execute_only_changed_files` (optional, boolean) Whether to execute just the files that changed in a directory or not. @@ -56,6 +56,14 @@ Example: `true` Default: `true` +### `fail_pipeline_on_first_exception` (optional, boolean) + +Whether to fail pipeline as soon as there is one failure. + +Example: `true` + +Default: `true` + ## Secret This plugin expects `GCP_SERVICE_ACCOUNT` is placed as environment variable. Make sure to store it [securely](https://buildkite.com/docs/pipelines/secrets)! diff --git a/hooks/command b/hooks/command index 74e3b2c..344fa80 100755 --- a/hooks/command +++ b/hooks/command @@ -79,6 +79,9 @@ else changed_files=$(git diff-tree --diff-filter=ABMRT --no-commit-id --name-only -r origin..HEAD | xargs | sed -e 's/ /,/g') fi +default_fail_pipeline_on_first_exception="true" +fail_pipeline_on_first_exception="${BUILDKITE_PLUGIN_BIGQUERY_DEFAULT_FAIL_PIPELINE_ON_FIRST_EXCEPTION:-$default_fail_pipeline_on_first_exception}" + echo "$gcp_service_account" >"$PIPELINE_FILE" if [[ ${debug_mode} == "true" ]]; then @@ -86,6 +89,7 @@ if [[ ${debug_mode} == "true" ]]; then echo "GCP Project: ${gcp_project}" echo "Dataset Schema Directory: ${dataset_schema_directory}" echo "Execute Only Changed Files: ${execute_only_changed_files}" + echo "Fail Pipeline on First Exception: ${fail_pipeline_on_first_exception}" echo "Changed Files: ${changed_files}" echo "Is Default Branch: ${is_default_branch}" echo "Docker Image: ${image}" @@ -120,6 +124,7 @@ args+=( "--env" "dataset_schema_directory=$dataset_schema_directory" "--env" "gcp_project=$gcp_project" "--env" "execute_only_changed_files=$execute_only_changed_files" + "--env" "fail_pipeline_on_first_exception=$fail_pipeline_on_first_exception" "--env" "credentials=$( bool: deploy_failed = False for root, dirs, files in os.walk(dataset_schema_directory): dataset = root.split("/").pop() @@ -91,6 +96,9 @@ def _deploy_from_directory( print(f"Failed to deploy to Bigquery: {str(e)}") deploy_failed = True + if fail_pipeline_on_first_exception: + return deploy_failed + return deploy_failed