From 456d07ea66f55fbc347dd5580f618823f96dba7f Mon Sep 17 00:00:00 2001
From: Lars Kellogg-Stedman <lars@redhat.com>
Date: Fri, 24 May 2024 11:07:18 -0400
Subject: [PATCH] Add command and workflow to validate rates.yaml

We want to ensure that new changes do not introduce errors in rates.yaml.
This commit adds a `validate-rates-file` command and an associated Github
workflow that will use the new command to validate `rates.yaml` on pull
requests.

The validate-rates-file command knows how to generate github annotations [1]
for pydantic validation errors and yaml parse errors.

[1]: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
---
 .github/workflows/validate-rates.yaml     | 29 ++++++++++++++
 pyproject.toml                            |  4 ++
 src/nerc_rates/cmd/__init__.py            |  0
 src/nerc_rates/cmd/validate_rates_file.py | 46 +++++++++++++++++++++++
 src/nerc_rates/rates.py                   | 22 +++++++----
 src/nerc_rates/tests/test_rates.py        |  2 +-
 6 files changed, 94 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/validate-rates.yaml
 create mode 100644 src/nerc_rates/cmd/__init__.py
 create mode 100644 src/nerc_rates/cmd/validate_rates_file.py

diff --git a/.github/workflows/validate-rates.yaml b/.github/workflows/validate-rates.yaml
new file mode 100644
index 0000000..235a302
--- /dev/null
+++ b/.github/workflows/validate-rates.yaml
@@ -0,0 +1,29 @@
+name: Validate rates file
+on:
+  push:
+    paths:
+      - rates.yaml
+  pull_request:
+    paths:
+      - rates.yaml
+
+jobs:
+  validate-rates-file:
+    name: Validate rates file
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          pip install -e .
+
+      - name: Validate rates file
+        run: |
+          validate-rates-file -g rates.yaml
diff --git a/pyproject.toml b/pyproject.toml
index bd7d591..66636a5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,6 +15,10 @@ dependencies = [
   "requests",
 ]
 
+[project.entry-points."console_scripts"]
+validate-rates-file = "nerc_rates.cmd.validate_rates_file:main"
+
+
 [build-system]
 requires = [
   "setuptools>=42",
diff --git a/src/nerc_rates/cmd/__init__.py b/src/nerc_rates/cmd/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/nerc_rates/cmd/validate_rates_file.py b/src/nerc_rates/cmd/validate_rates_file.py
new file mode 100644
index 0000000..e1f1316
--- /dev/null
+++ b/src/nerc_rates/cmd/validate_rates_file.py
@@ -0,0 +1,46 @@
+import sys
+import argparse
+import pydantic
+import yaml
+
+from .. import rates
+
+
+def pydantic_to_github(err, rates_file):
+    """Produce a github error annotation from a pydantic ValidationError"""
+    for error in err.errors():
+        print(f"::error file={rates_file},title=Validation error::{error['msg']}")
+
+
+def yaml_to_github(err, rates_file):
+    """Produce a github error annotation from a YAML ParserError"""
+    line = err.problem_mark.line
+    print(
+        f"::error file={rates_file},line={line},title=Parser error::{err.context}: {err.problem}"
+    )
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument(
+        "-g", "--github", action="store_true", help="Emit github workflow annotations"
+    )
+    p.add_argument("-u", "--url", action="store_true", help="Rate file is a url")
+    p.add_argument("rates_file", default="rates.yaml", nargs="?")
+    args = p.parse_args()
+
+    try:
+        if args.url:
+            r = rates.load_from_url(args.rates_file)
+        else:
+            r = rates.load_from_file(args.rates_file)
+
+        print(f"OK [{len(r.root)} entries]")
+    except pydantic.ValidationError as err:
+        if args.github:
+            pydantic_to_github(err, args.rates_file)
+        sys.exit(err)
+    except yaml.parser.ParserError as err:
+        if args.github:
+            yaml_to_github(err, args.rates_file)
+        sys.exit(err)
diff --git a/src/nerc_rates/rates.py b/src/nerc_rates/rates.py
index a4f6a90..d2d7493 100644
--- a/src/nerc_rates/rates.py
+++ b/src/nerc_rates/rates.py
@@ -3,20 +3,26 @@
 
 from .models import Rates
 
-DEFAULT_URL = "https://raw.githubusercontent.com/knikolla/nerc-rates/main/rates.yaml"
+DEFAULT_RATES_FILE = "rates.yaml"
+DEFAULT_RATES_URL = (
+    "https://raw.githubusercontent.com/CCI-MOC/nerc-rates/main/rates.yaml"
+)
 
 
-def load_from_url(url=DEFAULT_URL) -> Rates:
+def load_from_url(url: str | None = None) -> Rates:
+    if url is None:
+        url = DEFAULT_RATES_URL
+
     r = requests.get(url, allow_redirects=True)
-    # Using the BaseLoader prevents conversion of numeric
-    # values to floats and loads them as strings.
+    r.raise_for_status()
     config = yaml.safe_load(r.content.decode("utf-8"))
     return Rates.model_validate(config)
 
 
-def load_from_file() -> Rates:
-    with open("rates.yaml", "r") as f:
-        # Using the BaseLoader prevents conversion of numeric
-        # values to floats and loads them as strings.
+def load_from_file(path: str | None = None) -> Rates:
+    if path is None:
+        path = DEFAULT_RATES_FILE
+
+    with open(path, "r") as f:
         config = yaml.safe_load(f)
     return Rates.model_validate(config)
diff --git a/src/nerc_rates/tests/test_rates.py b/src/nerc_rates/tests/test_rates.py
index e765ca8..54d9362 100644
--- a/src/nerc_rates/tests/test_rates.py
+++ b/src/nerc_rates/tests/test_rates.py
@@ -12,7 +12,7 @@ def test_load_from_url():
           from: 2023-06
     """
     with requests_mock.Mocker() as m:
-        m.get(rates.DEFAULT_URL, text=mock_response_text)
+        m.get(rates.DEFAULT_RATES_URL, text=mock_response_text)
         r = load_from_url()
         assert r.get_value_at("CPU SU Rate", "2023-06") == "0.013"