Skip to content

Commit

Permalink
Migrate default project template to static project metadata
Browse files Browse the repository at this point in the history
Uses `requirements.txt` for dev requirements in project template.

Fix gh-2280.
Fix gh-2519.

Signed-off-by: Juan Luis Cano Rodríguez <juan_luis_cano@mckinsey.com>
  • Loading branch information
astrojuanlu committed May 26, 2023
1 parent ced5ba9 commit 5500c6f
Show file tree
Hide file tree
Showing 11 changed files with 76 additions and 84 deletions.
13 changes: 9 additions & 4 deletions features/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,13 +116,18 @@ def _setup_minimal_env(context):

def _install_project_requirements(context):
install_reqs = (
Path(
"kedro/templates/project/{{ cookiecutter.repo_name }}/src/requirements.txt"
)
Path("kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt")
.read_text(encoding="utf-8")
.splitlines()
)
install_reqs = [req for req in install_reqs if "{" not in req]
install_reqs = [
req
for req in install_reqs
if (req.strip())
and ("{" not in req)
and (not req.startswith("-e"))
and (not req.startswith("#"))
]
install_reqs.append(".[pandas.CSVDataSet]")
call([context.pip, "install", *install_reqs], env=context.env)
return context
10 changes: 5 additions & 5 deletions features/steps/cli_steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ def create_config_file(context):
@given("I have installed the project dependencies")
def pip_install_dependencies(context):
"""Install project dependencies using pip."""
reqs_path = "src/requirements.txt"
reqs_path = "requirements.txt"
res = run(
[context.pip, "install", "-r", reqs_path],
env=context.env,
Expand Down Expand Up @@ -410,7 +410,7 @@ def update_kedro_req(context: behave.runner.Context):
"""Replace kedro as a standalone requirement with a line
that includes all of kedro's dependencies (-r kedro/requirements.txt)
"""
reqs_path = context.root_project_dir / "src" / "requirements.txt"
reqs_path = context.root_project_dir / "requirements.txt"
kedro_reqs = f"-r {context.requirements_path.as_posix()}"

if reqs_path.is_file():
Expand All @@ -428,7 +428,7 @@ def update_kedro_req(context: behave.runner.Context):

@when("I add {dependency} to the requirements")
def add_req(context: behave.runner.Context, dependency: str):
reqs_path = context.root_project_dir / "src" / "requirements.txt"
reqs_path = context.root_project_dir / "requirements.txt"
if reqs_path.is_file():
reqs_path.write_text(reqs_path.read_text() + "\n" + str(dependency) + "\n")

Expand Down Expand Up @@ -610,14 +610,14 @@ def check_docs_generated(context: behave.runner.Context):
@then("requirements should be generated")
def check_reqs_generated(context: behave.runner.Context):
"""Check that new project requirements are generated."""
reqs_path = context.root_project_dir / "src" / "requirements.lock"
reqs_path = context.root_project_dir / "requirements.lock"
assert reqs_path.is_file()
assert "This file is autogenerated by pip-compile" in reqs_path.read_text()


@then("{dependency} should be in the requirements")
def check_dependency_in_reqs(context: behave.runner.Context, dependency: str):
reqs_path = context.root_project_dir / "src" / "requirements.txt"
reqs_path = context.root_project_dir / "requirements.txt"
assert dependency in reqs_path.read_text()


Expand Down
4 changes: 2 additions & 2 deletions kedro/framework/cli/micropkg.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ def _pull_package(
package_reqs = re.findall(reqs_element_pattern, list_reqs[0])

if package_reqs:
requirements_txt = metadata.source_dir / "requirements.txt"
requirements_txt = metadata.project_path / "requirements.txt"
_append_package_reqs(requirements_txt, package_reqs, package_name)

_clean_pycache(temp_dir_path)
Expand Down Expand Up @@ -741,7 +741,7 @@ def _generate_sdist_file(
# Build a setup.py on the fly
try:
install_requires = _make_install_requires(
package_source / "requirements.txt" # type: ignore
metadata.project_path / "requirements.txt" # type: ignore
)
except Exception as exc:
click.secho("FAILED", fg="red")
Expand Down
19 changes: 11 additions & 8 deletions kedro/framework/cli/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,9 @@ def test(metadata: ProjectMetadata, args, **kwargs): # pylint: disable=unused-a
try:
_check_module_importable("pytest")
except KedroCliError as exc:
source_path = metadata.source_dir
project_path = metadata.project_path
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(source_path))
NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(project_path))
) from exc
python_call("pytest", args)

Expand All @@ -110,20 +110,22 @@ def lint(
click.secho(deprecation_message, fg="red")

source_path = metadata.source_dir
project_path = metadata.project_path
package_name = metadata.package_name
files = files or (str(source_path / "tests"), str(source_path / package_name))
files = files or (str(project_path / "tests"), str(source_path / package_name))

if "PYTHONPATH" not in os.environ:
# isort needs the source path to be in the 'PYTHONPATH' environment
# variable to treat it as a first-party import location
# NOTE: Actually, `pip install [-e] .` achieves the same
os.environ["PYTHONPATH"] = str(source_path) # pragma: no cover

for module_name in ("flake8", "isort", "black"):
try:
_check_module_importable(module_name)
except KedroCliError as exc:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(source_path))
NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(project_path))
) from exc

python_call("black", ("--check",) + files if check_only else files)
Expand Down Expand Up @@ -199,10 +201,11 @@ def build_docs(metadata: ProjectMetadata, open_docs):
click.secho(deprecation_message, fg="red")

source_path = metadata.source_dir
project_path = metadata.project_path
package_name = metadata.package_name

python_call("pip", ["install", str(source_path / "[docs]")])
python_call("pip", ["install", "-r", str(source_path / "requirements.txt")])
python_call("pip", ["install", str(project_path / "[docs]")])
python_call("pip", ["install", "-r", str(project_path / "requirements.txt")])
python_call("ipykernel", ["install", "--user", f"--name={package_name}"])
shutil.rmtree("docs/build", ignore_errors=True)
call(
Expand Down Expand Up @@ -291,7 +294,7 @@ def activate_nbstripout(
)
click.secho(deprecation_message, fg="red")

source_path = metadata.source_dir
project_path = metadata.source_dir
click.secho(
(
"Notebook output cells will be automatically cleared before committing"
Expand All @@ -304,7 +307,7 @@ def activate_nbstripout(
_check_module_importable("nbstripout")
except KedroCliError as exc:
raise KedroCliError(
NO_DEPENDENCY_MESSAGE.format(module="nbstripout", src=str(source_path))
NO_DEPENDENCY_MESSAGE.format(module="nbstripout", src=str(project_path))
) from exc

try:
Expand Down
12 changes: 0 additions & 12 deletions kedro/templates/project/{{ cookiecutter.repo_name }}/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,25 +64,13 @@ After this, if you'd like to update your project requirements, please update `re
> Jupyter, JupyterLab, and IPython are already included in the project requirements by default, so once you have run `pip install -r requirements.txt` you will not need to take any extra steps before you use them.
### Jupyter
To use Jupyter notebooks in your Kedro project, you need to install Jupyter:

```
pip install jupyter
```

After installing Jupyter, you can start a local notebook server:

```
kedro jupyter notebook
```

### JupyterLab
To use JupyterLab, you need to install it:

```
pip install jupyterlab
```

You can also start JupyterLab:

```
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,38 @@
[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"

[project]
name = "{{ cookiecutter.python_package }}"
dependencies = [
"kedro~={{ cookiecutter.kedro_version }}",
]
dynamic = ["version"]

[project.scripts]
{{ cookiecutter.repo_name }} = "{{ cookiecutter.python_package }}.__main__:main"

[project.optional-dependencies]
docs = [
"docutils<0.18.0",
"sphinx~=3.4.3",
"sphinx_rtd_theme==0.5.1",
"nbsphinx==0.8.1",
"nbstripout~=0.4",
"sphinx-autodoc-typehints==1.11.1",
"sphinx_copybutton==0.3.1",
"ipykernel>=5.3, <7.0",
"Jinja2<3.1.0",
"myst-parser~=0.17.2",
]

[tool.setuptools.dynamic]
version = {attr = "{{ cookiecutter.python_package }}.__version__"}

[tool.setuptools.packages.find]
where = ["src"]
namespaces = false

[tool.kedro]
package_name = "{{ cookiecutter.python_package }}"
project_name = "{{ cookiecutter.project_name }}"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# Install library code
-e file:.

# Development dependencies
black~=22.0
flake8>=3.7.9, <5.0
ipython>=7.31.1, <8.0; python_version < '3.8'
Expand All @@ -6,8 +10,6 @@ isort~=5.0
jupyter~=1.0
jupyterlab_server>=2.11.1, <2.16.0
jupyterlab~=3.0, <3.6.0
kedro~={{ cookiecutter.kedro_version }}
kedro-telemetry~=0.2.0
nbstripout~=0.4
pytest-cov~=3.0
pytest-mock>=1.7.1, <2.0
Expand Down
39 changes: 0 additions & 39 deletions kedro/templates/project/{{ cookiecutter.repo_name }}/setup.py

This file was deleted.

6 changes: 3 additions & 3 deletions tests/framework/cli/micropkg/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def cleanup_micropackages(fake_repo_path, fake_package_path):
if each.is_file():
each.unlink()

tests = fake_repo_path / "src" / "tests" / micropackage
tests = fake_repo_path / "tests" / micropackage
if tests.is_dir():
shutil.rmtree(str(tests))

Expand All @@ -35,7 +35,7 @@ def cleanup_micropackages(fake_repo_path, fake_package_path):
def cleanup_pipelines(fake_repo_path, fake_package_path):
pipes_path = fake_package_path / "pipelines"
old_pipelines = {p.name for p in pipes_path.iterdir() if p.is_dir()}
requirements_txt = fake_repo_path / "src" / "requirements.txt"
requirements_txt = fake_repo_path / "requirements.txt"
requirements = requirements_txt.read_text()
yield

Expand All @@ -53,7 +53,7 @@ def cleanup_pipelines(fake_repo_path, fake_package_path):
if each.is_file():
each.unlink()

tests = fake_repo_path / "src" / "tests" / "pipelines" / pipeline
tests = fake_repo_path / "tests" / "pipelines" / pipeline
if tests.is_dir():
shutil.rmtree(str(tests))

Expand Down
10 changes: 5 additions & 5 deletions tests/framework/cli/test_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def test_pytest_not_installed(
fake_project_cli, ["test", "--random-arg", "value"], obj=fake_metadata
)
expected_message = NO_DEPENDENCY_MESSAGE.format(
module="pytest", src=str(fake_repo_path / "src")
module="pytest", src=str(fake_repo_path)
)

assert result.exit_code
Expand All @@ -148,7 +148,7 @@ def test_lint(
assert not result.exit_code, result.stdout

expected_files = files or (
str(fake_repo_path / "src/tests"),
str(fake_repo_path / "tests"),
str(fake_repo_path / "src/dummy_package"),
)
expected_calls = [
Expand Down Expand Up @@ -185,7 +185,7 @@ def test_lint_check_only(
assert not result.exit_code, result.stdout

expected_files = files or (
str(fake_repo_path / "src/tests"),
str(fake_repo_path / "tests"),
str(fake_repo_path / "src/dummy_package"),
)
expected_calls = [
Expand Down Expand Up @@ -217,7 +217,7 @@ def test_import_not_installed(

result = CliRunner().invoke(fake_project_cli, ["lint"], obj=fake_metadata)
expected_message = NO_DEPENDENCY_MESSAGE.format(
module=module_name, src=str(fake_repo_path / "src")
module=module_name, src=str(fake_repo_path)
)

assert result.exit_code, result.stdout
Expand Down Expand Up @@ -354,7 +354,7 @@ def test_happy_path(
mocker.call("pip", ["install", str(fake_repo_path / "src/[docs]")]),
mocker.call(
"pip",
["install", "-r", str(fake_repo_path / "src/requirements.txt")],
["install", "-r", str(fake_repo_path / "requirements.txt")],
),
mocker.call("ipykernel", ["install", "--user", "--name=dummy_package"]),
]
Expand Down
6 changes: 2 additions & 4 deletions tests/framework/cli/test_starters.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
KedroStarterSpec,
)

FILES_IN_TEMPLATE = 31
FILES_IN_TEMPLATE = 30


@pytest.fixture
Expand Down Expand Up @@ -70,9 +70,7 @@ def _assert_template_ok(
assert (full_path / ".gitignore").is_file()
assert project_name in (full_path / "README.md").read_text(encoding="utf-8")
assert "KEDRO" in (full_path / ".gitignore").read_text(encoding="utf-8")
assert kedro_version in (full_path / "src" / "requirements.txt").read_text(
encoding="utf-8"
)
assert kedro_version in (full_path / "pyproject.toml").read_text(encoding="utf-8")
assert (full_path / "src" / python_package / "__init__.py").is_file()


Expand Down

0 comments on commit 5500c6f

Please sign in to comment.