Add fallback to starters pull on kedro new

kedro/framework/cli/starters.py

@@ -5,6 +5,7 @@
 """
 from __future__ import annotations
 
+import logging
 import os
 import re
 import shutil
@@ -17,6 +18,7 @@
 from typing import Any, Callable
 
 import click
+import requests
 import yaml
 from attrs import define, field
 from importlib_metadata import EntryPoints
@@ -95,7 +97,44 @@ class KedroStarterSpec:
 KEDRO_PATH = Path(kedro.__file__).parent
 TEMPLATE_PATH = KEDRO_PATH / "templates" / "project"
 
-_STARTERS_REPO = "git+https://github.com/kedro-org/kedro-starters.git"
+
+def _get_latest_starters_version() -> str:
+    if "KEDRO_STARTERS_VERSION" not in os.environ:
+        GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
+        headers = {}
+        if GITHUB_TOKEN:
+            headers["Authorization"] = f"token {GITHUB_TOKEN}"
+
+        try:
+            response = requests.get(
+                "https://api.github.com/repos/kedro-org/kedro-starters/releases/latest",
+                headers=headers,
+                timeout=10,
+            )
+            response.raise_for_status()  # Raise an HTTPError for bad status codes
+            latest_release = response.json()
+        except requests.exceptions.RequestException as e:
+            logging.error(f"Error fetching kedro-starters latest release version: {e}")
+            return ""
+
+        os.environ["KEDRO_STARTERS_VERSION"] = latest_release["tag_name"]
+        return str(latest_release["tag_name"])
+    else:
+        return str(os.getenv("KEDRO_STARTERS_VERSION"))
+
+
+def _kedro_and_starters_version_identical() -> bool:
+    starters_version = _get_latest_starters_version()
+    return True if version == starters_version else False
+
+
+_STARTERS_REPO = (
+    "git+https://github.com/kedro-org/kedro-starters.git"
+    if _kedro_and_starters_version_identical()
+    else "https://github.com/kedro-org/kedro-starters.git@main"
+)
+
+
 _OFFICIAL_STARTER_SPECS = [
     KedroStarterSpec("astro-airflow-iris", _STARTERS_REPO, "astro-airflow-iris"),
     KedroStarterSpec("spaceflights-pandas", _STARTERS_REPO, "spaceflights-pandas"),
@@ -766,35 +805,39 @@ def _make_cookiecutter_args_and_fetch_template(
         "extra_context": config,
     }
 
-    if checkout:
-        cookiecutter_args["checkout"] = checkout
+    kedro_version_match_starters = _kedro_and_starters_version_identical()
+
     if directory:
         cookiecutter_args["directory"] = directory
 
     tools = config["tools"]
     example_pipeline = config["example_pipeline"]
     starter_path = "git+https://github.com/kedro-org/kedro-starters.git"
+    checkout_version = version if kedro_version_match_starters else "main"
 
     if "PySpark" in tools and "Kedro Viz" in tools:
         # Use the spaceflights-pyspark-viz starter if both PySpark and Kedro Viz are chosen.
         cookiecutter_args["directory"] = "spaceflights-pyspark-viz"
         # Ensures we use the same tag version of kedro for kedro-starters
-        cookiecutter_args["checkout"] = version
+        cookiecutter_args["checkout"] = checkout_version
     elif "PySpark" in tools:
         # Use the spaceflights-pyspark starter if only PySpark is chosen.
         cookiecutter_args["directory"] = "spaceflights-pyspark"
-        cookiecutter_args["checkout"] = version
+        cookiecutter_args["checkout"] = checkout_version
     elif "Kedro Viz" in tools:
         # Use the spaceflights-pandas-viz starter if only Kedro Viz is chosen.
         cookiecutter_args["directory"] = "spaceflights-pandas-viz"
+        cookiecutter_args["checkout"] = checkout_version
     elif example_pipeline == "True":
         # Use spaceflights-pandas starter if example was selected, but PySpark or Viz wasn't
         cookiecutter_args["directory"] = "spaceflights-pandas"
-        cookiecutter_args["checkout"] = version
+        cookiecutter_args["checkout"] = checkout_version
     else:
         # Use the default template path for non PySpark, Viz or example options:
         starter_path = template_path
-
+        cookiecutter_args["checkout"] = (
+            checkout if checkout and kedro_version_match_starters else "main"
+        )
     return cookiecutter_args, starter_path
 
 

pyproject.toml

@@ -71,12 +71,14 @@ test = [
     "pytest-mock>=1.7.1, <4.0",
     "pytest-xdist[psutil]~=2.2.1",
     "pytest>=7.2,<9.0",
-    "s3fs>=2021.4",
+    "s3fs>=2021.4, <2024.6",  # Upper bound set arbitrarily, to be reassessed in early 2024
+    "requests_mock",
     "trufflehog~=2.1",
     # mypy related dependencies
     "pandas-stubs",
     "types-PyYAML",
     "types-cachetools",
+    "types-requests",
     "types-toml"
 ]
 docs = [

tests/framework/cli/test_starters.py

@@ -2,10 +2,13 @@
 """
 from __future__ import annotations
 
+import logging
+import os
 import shutil
 from pathlib import Path
 
 import pytest
+import requests
 import toml
 import yaml
 from click.testing import CliRunner
@@ -18,6 +21,7 @@
     KedroStarterSpec,
     _convert_tool_short_names_to_numbers,
     _fetch_validate_parse_config_from_user_prompts,
+    _get_latest_starters_version,
     _make_cookiecutter_args_and_fetch_template,
     _parse_tools_input,
     _parse_yes_no_to_bool,
@@ -1687,3 +1691,174 @@ def test_flag_is_absent(self, fake_kedro_cli):
 
         telemetry_file_path = Path(repo_name + "/.telemetry")
         assert not telemetry_file_path.exists()
+
+
+@pytest.fixture
+def mock_env_vars(mocker):
+    """Fixture to mock environment variables"""
+    mocker.patch.dict(os.environ, {"GITHUB_TOKEN": "fake_token"}, clear=True)
+
+
+class TestGetLatestStartersVersion:
+    def test_get_latest_starters_version(self, mock_env_vars, requests_mock):
+        """Test _get_latest_starters_version when KEDRO_STARTERS_VERSION is not set"""
+        latest_version = "1.2.3"
+
+        # Mock the GitHub API response
+        requests_mock.get(
+            "https://api.github.com/repos/kedro-org/kedro-starters/releases/latest",
+            json={"tag_name": latest_version},
+            status_code=200,
+        )
+
+        result = _get_latest_starters_version()
+
+        assert result == latest_version
+        assert os.getenv("KEDRO_STARTERS_VERSION") == latest_version
+
+    def test_get_latest_starters_version_with_env_set(self, mocker):
+        """Test _get_latest_starters_version when KEDRO_STARTERS_VERSION is already set"""
+        expected_version = "1.2.3"
+        mocker.patch.dict(os.environ, {"KEDRO_STARTERS_VERSION": expected_version})
+
+        result = _get_latest_starters_version()
+
+        assert result == expected_version
+
+    def test_get_latest_starters_version_error(
+        self, mock_env_vars, requests_mock, caplog
+    ):
+        """Test _get_latest_starters_version when the request raises an exception"""
+        with caplog.at_level(logging.ERROR):
+            # Mock the request to raise a RequestException
+            requests_mock.get(
+                "https://api.github.com/repos/kedro-org/kedro-starters/releases/latest",
+                exc=requests.exceptions.RequestException("Request failed"),
+            )
+
+            result = _get_latest_starters_version()
+
+            assert result == ""
+            assert (
+                "Error fetching kedro-starters latest release version: Request failed"
+                in caplog.text
+            )
+
+
+class TestFetchCookiecutterArgsWhenKedroVersionDifferentFromStarters:
+    @pytest.fixture
+    def config(self):
+        return {
+            "tools": [],
+            "example_pipeline": "False",
+            "output_dir": "/my/output/dir",
+        }
+
+    def test_kedro_version_match_starters_true(self, config, mocker):
+        mocker.patch(
+            "kedro.framework.cli.starters._kedro_and_starters_version_identical",
+            return_value=True,
+        )
+
+        config["tools"] = ["PySpark", "Kedro Viz"]
+        checkout = ""
+        directory = "my-directory"
+        template_path = "my/template/path"
+
+        expected_args = {
+            "output_dir": "/my/output/dir",
+            "no_input": True,
+            "extra_context": config,
+            "checkout": version,
+            "directory": "spaceflights-pyspark-viz",
+        }
+        expected_path = "git+https://github.com/kedro-org/kedro-starters.git"
+
+        result_args, result_path = _make_cookiecutter_args_and_fetch_template(
+            config, checkout, directory, template_path
+        )
+
+        assert result_args == expected_args
+        assert result_path == expected_path
+
+    def test_kedro_version_match_starters_false(self, config, mocker):
+        mocker.patch(
+            "kedro.framework.cli.starters._kedro_and_starters_version_identical",
+            return_value=False,
+        )
+
+        config["tools"] = ["PySpark", "Kedro Viz"]
+        checkout = "my-checkout"
+        directory = "my-directory"
+        template_path = "my/template/path"
+
+        expected_args = {
+            "output_dir": "/my/output/dir",
+            "no_input": True,
+            "extra_context": config,
+            "checkout": "main",
+            "directory": "spaceflights-pyspark-viz",
+        }
+        expected_path = "git+https://github.com/kedro-org/kedro-starters.git"
+
+        result_args, result_path = _make_cookiecutter_args_and_fetch_template(
+            config, checkout, directory, template_path
+        )
+
+        assert result_args == expected_args
+        assert result_path == expected_path
+
+    def test_no_tools_and_example_pipeline(self, config, mocker):
+        mocker.patch(
+            "kedro.framework.cli.starters._kedro_and_starters_version_identical",
+            return_value=False,
+        )
+
+        config["tools"] = []
+        config["example_pipeline"] = "True"
+        checkout = ""
+        directory = ""
+        template_path = "my/template/path"
+
+        expected_args = {
+            "output_dir": "/my/output/dir",
+            "no_input": True,
+            "extra_context": config,
+            "checkout": "main",
+            "directory": "spaceflights-pandas",
+        }
+        expected_path = "git+https://github.com/kedro-org/kedro-starters.git"
+
+        result_args, result_path = _make_cookiecutter_args_and_fetch_template(
+            config, checkout, directory, template_path
+        )
+
+        assert result_args == expected_args
+        assert result_path == expected_path
+
+    def test_no_tools_no_example_pipeline(self, config, mocker):
+        mocker.patch(
+            "kedro.framework.cli.starters._kedro_and_starters_version_identical",
+            return_value=False,
+        )
+
+        config["tools"] = []
+        config["example_pipeline"] = "False"
+        checkout = ""
+        directory = ""
+        template_path = "my/template/path"
+
+        expected_args = {
+            "output_dir": "/my/output/dir",
+            "no_input": True,
+            "extra_context": config,
+            "checkout": "main",
+        }
+        expected_path = "my/template/path"
+
+        result_args, result_path = _make_cookiecutter_args_and_fetch_template(
+            config, checkout, directory, template_path
+        )
+
+        assert result_args == expected_args
+        assert result_path == expected_path