diff --git a/quetz/main.py b/quetz/main.py index 631b24f9..3a076f3f 100644 --- a/quetz/main.py +++ b/quetz/main.py @@ -1090,6 +1090,7 @@ def delete_package_version( filename: str, channel_name: str, package_name: str, + background_tasks: BackgroundTasks, dao: Dao = Depends(get_dao), db=Depends(get_db), auth: authorization.Rules = Depends(get_rules), @@ -1115,6 +1116,10 @@ def delete_package_version( dao.update_channel_size(channel_name) + wrapped_bg_task = background_task_wrapper(indexing.update_indexes, logger) + # Background task to update indexes + background_tasks.add_task(wrapped_bg_task, dao, pkgstore, channel_name, [platform]) + @api_router.get( "/packages/search/", response_model=List[rest_models.PackageSearch], tags=["search"] @@ -1300,6 +1305,10 @@ def post_file_to_package( handle_package_files(package.channel, files, dao, auth, force, package=package) dao.update_channel_size(package.channel_name) + wrapped_bg_task = background_task_wrapper(indexing.update_indexes, logger) + # Background task to update indexes + background_tasks.add_task(wrapped_bg_task, dao, pkgstore, package.channel_name) + @api_router.post( "/channels/{channel_name}/upload/{filename}", status_code=201, tags=["upload"] diff --git a/quetz/tests/api/__init__.py b/quetz/tests/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/quetz/tests/api/conftest.py b/quetz/tests/api/conftest.py index d7abe02a..8f9784aa 100644 --- a/quetz/tests/api/conftest.py +++ b/quetz/tests/api/conftest.py @@ -3,22 +3,10 @@ import pytest -from quetz.config import Config -from quetz.dao import Dao -from quetz.db_models import Identity, PackageVersion, Profile, User +from quetz.db_models import Identity, Profile, User from quetz.rest_models import Channel, Package -@pytest.fixture -def package_name(): - return "my-package" - - -@pytest.fixture -def channel_name(): - return "my-channel" - - @pytest.fixture def private_channel(dao, other_user): @@ -43,7 +31,9 @@ def private_package(dao, other_user, private_channel): @pytest.fixture -def private_package_version(dao, private_channel, private_package, other_user, config): +def private_package_version( + dao, private_channel, private_package, other_user, config, package_name +): package_format = "tarbz2" package_info = "{}" channel_name = private_channel.name @@ -77,68 +67,6 @@ def private_package_version(dao, private_channel, private_package, other_user, c return version -@pytest.fixture -def make_package_version( - db, - user, - public_channel, - channel_name, - package_name, - public_package, - dao: Dao, - config: Config, -): - - pkgstore = config.get_package_store() - - versions = [] - - def _make_package_version(filename, version_number, platform="linux-64"): - filename = Path(filename) - with open(filename, "rb") as fid: - pkgstore.add_file(fid.read(), channel_name, platform / filename) - package_format = "tarbz2" - package_info = "{}" - version = dao.create_version( - channel_name, - package_name, - package_format, - platform, - version_number, - 0, - "", - str(filename), - package_info, - user.id, - size=11, - ) - - dao.update_package_channeldata( - channel_name, - package_name, - {'name': package_name, 'subdirs': [platform]}, - ) - - dao.update_channel_size(channel_name) - - versions.append(version) - - return version - - yield _make_package_version - - for version in versions: - db.query(PackageVersion).filter(PackageVersion.id == version.id).delete() - db.commit() - - -@pytest.fixture -def package_version(db, make_package_version): - version = make_package_version("test-package-0.1-0.tar.bz2", "0.1") - - return version - - @pytest.fixture() def other_user_without_profile(db): user = User(id=uuid.uuid4().bytes, username="other") @@ -163,37 +91,6 @@ def other_user(other_user_without_profile, db): yield other_user_without_profile -@pytest.fixture -def channel_role(): - return "owner" - - -@pytest.fixture -def package_role(): - return "owner" - - -@pytest.fixture -def public_channel(dao: Dao, user, channel_role, channel_name): - - channel_data = Channel(name=channel_name, private=False) - channel = dao.create_channel(channel_data, user.id, channel_role) - - return channel - - -@pytest.fixture -def public_package(db, user, public_channel, dao, package_role, package_name): - - package_data = Package(name=package_name) - - package = dao.create_package( - public_channel.name, package_data, user.id, package_role - ) - - return package - - @pytest.fixture def pkgstore(config): return config.get_package_store() diff --git a/quetz/tests/api/test_main_packages.py b/quetz/tests/api/test_main_packages.py index 45f37a97..5c2f47d1 100644 --- a/quetz/tests/api/test_main_packages.py +++ b/quetz/tests/api/test_main_packages.py @@ -1,4 +1,6 @@ +import json import os +import time from pathlib import Path from typing import BinaryIO @@ -10,7 +12,6 @@ from quetz.config import Config from quetz.db_models import ChannelMember, Package, PackageMember, PackageVersion from quetz.errors import ValidationError -from quetz.pkgstores import PackageStore from quetz.tasks.indexing import update_indexes @@ -72,10 +73,22 @@ def test_delete_package_versions_with_package( update_indexes(dao, pkgstore, public_channel.name) + # Get package files package_filenames = [ os.path.join(version.platform, version.filename) for version in public_package.package_versions # type: ignore ] + + # Get repodata content + package_dir = Path(pkgstore.channels_dir) / public_channel.name / 'linux-64' + with open(package_dir / 'repodata.json', 'r') as fd: + repodata = json.load(fd) + + # Check that all packages are initially in repodata + for filename in package_filenames: + assert os.path.basename(filename) in repodata["packages"].keys() + + # Get channel files init_files = sorted(pkgstore.list_files(public_channel.name)) response = auth_client.delete( @@ -95,9 +108,19 @@ def test_delete_package_versions_with_package( assert len(versions) == 0 + # Check that repodata content has been updated + with open(package_dir / 'repodata.json', 'r') as fd: + repodata = json.load(fd) + + assert repodata["info"] == repodata["info"] + + # Remove package files from files list + # Check that packages have been removed from repodata for filename in package_filenames: init_files.remove(filename) + assert os.path.basename(filename) not in repodata["packages"] + # Check that the package tree files is the same except for package files files = sorted(pkgstore.list_files(public_channel.name)) assert files == init_files @@ -312,10 +335,6 @@ def test_upload_package_version_wrong_filename( files=files, ) - with open(package_filename, "rb") as fid: - condainfo = CondaInfo(fid, package_filename) - condainfo._parse_conda() - package_dir = Path(pkgstore.channels_dir) / public_channel.name / 'linux-64' assert response.status_code == 400 @@ -326,6 +345,78 @@ def test_upload_package_version_wrong_filename( assert not os.path.exists(package_dir) +@pytest.mark.parametrize("package_name", ["test-package"]) +def test_upload_duplicate_package_version( + auth_client, + public_channel, + public_package, + package_name, + db, + config, + remove_package_versions, +): + pkgstore = config.get_package_store() + + package_filename = "test-package-0.1-0.tar.bz2" + package_filename_copy = "test-package-0.1-0_copy.tar.bz2" + + with open(package_filename, "rb") as fid: + files = {"files": (package_filename, fid)} + response = auth_client.post( + f"/api/channels/{public_channel.name}/packages/" + f"{public_package.name}/files/", + files=files, + ) + + # Get repodata content + package_dir = Path(pkgstore.channels_dir) / public_channel.name / 'linux-64' + with open(package_dir / 'repodata.json', 'r') as fd: + repodata_init = json.load(fd) + + # Try submitting the same package without 'force' flag + with open(package_filename, "rb") as fid: + files = {"files": (package_filename, fid)} + response = auth_client.post( + f"/api/channels/{public_channel.name}/packages/" + f"{public_package.name}/files/", + files=files, + ) + assert response.status_code == 409 + detail = response.json()['detail'] + assert "Duplicate" in detail + + # Change the archive to test force update + os.remove(package_filename) + os.rename(package_filename_copy, package_filename) + + # Ensure the 'time_modified' value change in repodata.json + time.sleep(1) + + # Submit the same package with 'force' flag + with open(package_filename, "rb") as fid: + files = {"files": (package_filename, fid)} + response = auth_client.post( + f"/api/channels/{public_channel.name}/packages/" + f"{public_package.name}/files/", + files=files, + data={"force": True}, + ) + + assert response.status_code == 201 + + # Check that repodata content has been updated + with open(package_dir / 'repodata.json', 'r') as fd: + repodata = json.load(fd) + + assert repodata_init["info"] == repodata["info"] + assert repodata_init["packages"].keys() == repodata["packages"].keys() + repodata_init_pkg = repodata_init["packages"][package_filename] + repodata_pkg = repodata["packages"][package_filename] + assert repodata_init_pkg["time_modified"] != repodata_pkg["time_modified"] + assert repodata_init_pkg["md5"] != repodata_pkg["md5"] + assert repodata_init_pkg["sha256"] != repodata_pkg["sha256"] + + @pytest.mark.parametrize("package_name", ["test-package"]) def test_check_channel_size_limits( auth_client, public_channel, public_package, db, config @@ -353,13 +444,22 @@ def test_check_channel_size_limits( def test_delete_package_version( - auth_client, public_channel, package_version, dao, pkgstore: PackageStore, db + auth_client, public_channel, package_version, dao, pkgstore, db ): assert public_channel.size > 0 assert public_channel.size == package_version.size filename = "test-package-0.1-0.tar.bz2" platform = "linux-64" + + update_indexes(dao, pkgstore, public_channel.name) + + # Get repodata content and check that package is inside + package_dir = Path(pkgstore.channels_dir) / public_channel.name / 'linux-64' + with open(package_dir / 'repodata.json', 'r') as fd: + repodata = json.load(fd) + assert filename in repodata["packages"].keys() + response = auth_client.delete( f"/api/channels/{public_channel.name}/" f"packages/{package_version.package_name}/versions/{platform}/{filename}" @@ -381,6 +481,11 @@ def test_delete_package_version( db.refresh(public_channel) assert public_channel.size == 0 + # Check that repodata content has been updated + with open(package_dir / 'repodata.json', 'r') as fd: + repodata = json.load(fd) + assert filename not in repodata["packages"].keys() + def test_package_name_length_limit(auth_client, public_channel, db): diff --git a/quetz/tests/conftest.py b/quetz/tests/conftest.py index d7e15bf5..3047c9eb 100644 --- a/quetz/tests/conftest.py +++ b/quetz/tests/conftest.py @@ -10,10 +10,14 @@ import uuid +from pathlib import Path from pytest import fixture -from quetz.db_models import Profile, User +from quetz.config import Config +from quetz.dao import Dao +from quetz.db_models import PackageVersion, Profile, User +from quetz.rest_models import Channel, Package pytest_plugins = "quetz.testing.fixtures" @@ -53,3 +57,106 @@ def user(db, user_without_profile): ).delete() db.commit() + + +@fixture +def make_package_version( + db, + user, + public_channel, + channel_name, + package_name, + public_package, + dao: Dao, + config: Config, +): + + pkgstore = config.get_package_store() + + versions = [] + + def _make_package_version(filename, version_number, platform="linux-64"): + filename = Path(filename) + with open(filename, "rb") as fid: + pkgstore.add_file(fid.read(), channel_name, platform / filename) + package_format = "tarbz2" + package_info = "{}" + version = dao.create_version( + channel_name, + package_name, + package_format, + platform, + version_number, + 0, + "", + str(filename), + package_info, + user.id, + size=11, + ) + + dao.update_package_channeldata( + channel_name, + package_name, + {'name': package_name, 'subdirs': [platform]}, + ) + + dao.update_channel_size(channel_name) + + versions.append(version) + + return version + + yield _make_package_version + + for version in versions: + db.query(PackageVersion).filter(PackageVersion.id == version.id).delete() + db.commit() + + +@fixture +def package_version(db, make_package_version): + version = make_package_version("test-package-0.1-0.tar.bz2", "0.1") + + return version + + +@fixture +def public_channel(dao: Dao, user, channel_role, channel_name): + + channel_data = Channel(name=channel_name, private=False) + channel = dao.create_channel(channel_data, user.id, channel_role) + + return channel + + +@fixture +def public_package(db, user, public_channel, dao, package_role, package_name): + + package_data = Package(name=package_name) + + package = dao.create_package( + public_channel.name, package_data, user.id, package_role + ) + + return package + + +@fixture +def channel_name(): + return "my-channel" + + +@fixture +def package_name(): + return "my-package" + + +@fixture +def channel_role(): + return "owner" + + +@fixture +def package_role(): + return "owner" diff --git a/quetz/tests/data/test-package-0.1-0_copy.tar.bz2 b/quetz/tests/data/test-package-0.1-0_copy.tar.bz2 new file mode 100644 index 00000000..52b7318f Binary files /dev/null and b/quetz/tests/data/test-package-0.1-0_copy.tar.bz2 differ diff --git a/quetz/tests/test_dao.py b/quetz/tests/test_dao.py index 180a0699..bd95106d 100644 --- a/quetz/tests/test_dao.py +++ b/quetz/tests/test_dao.py @@ -12,16 +12,6 @@ from quetz.metrics.db_models import IntervalType, PackageVersionMetric, round_timestamp -@pytest.fixture -def package_name(): - return "my-package" - - -@pytest.fixture -def channel_name(): - return "my-channel" - - @pytest.fixture def channel(dao, db, user, channel_name): diff --git a/quetz/tests/test_indexing.py b/quetz/tests/test_indexing.py index 4393a241..dce32551 100644 --- a/quetz/tests/test_indexing.py +++ b/quetz/tests/test_indexing.py @@ -1,29 +1,51 @@ +import json +from pathlib import Path + import pytest -from quetz.config import Config -from quetz.db_models import Channel +from quetz import channel_data from quetz.tasks.indexing import update_indexes @pytest.fixture -def local_channel(db): +def empty_channeldata(dao): + return channel_data.export(dao, "") + + +def test_update_indexes_empty_channel(config, public_channel, dao, empty_channeldata): + pkgstore = config.get_package_store() + + update_indexes(dao, pkgstore, public_channel.name) + + files = pkgstore.list_files(public_channel.name) + + base_files = [ + 'channeldata.json', + 'index.html', + 'noarch/index.html', + 'noarch/repodata.json', + ] - channel = Channel(name="test-local-channel") - db.add(channel) - db.commit() + expected_files = base_files.copy() - yield channel + for suffix in ['.bz2', '.gz']: + expected_files.extend(s + suffix for s in base_files) - db.delete(channel) - db.commit() + assert sorted(files) == sorted(expected_files) + channel_dir = Path(pkgstore.channels_dir) / public_channel.name + with open(channel_dir / 'channeldata.json', 'r') as fd: + assert json.load(fd) == empty_channeldata -def test_update_indexes(config: Config, local_channel, dao): + +def test_update_indexes_empty_package( + config, public_channel, public_package, dao, empty_channeldata +): pkgstore = config.get_package_store() - update_indexes(dao, pkgstore, local_channel.name) + update_indexes(dao, pkgstore, public_channel.name) - files = pkgstore.list_files(local_channel.name) + files = pkgstore.list_files(public_channel.name) base_files = [ 'channeldata.json', @@ -38,3 +60,46 @@ def test_update_indexes(config: Config, local_channel, dao): expected_files.extend(s + suffix for s in base_files) assert sorted(files) == sorted(expected_files) + + channel_dir = Path(pkgstore.channels_dir) / public_channel.name + with open(channel_dir / 'channeldata.json', 'r') as fd: + channeldata = json.load(fd) + + assert public_package.name in channeldata["packages"].keys() + + assert channeldata["packages"].pop(public_package.name) == {} + assert channeldata == empty_channeldata + + +def test_update_indexes_with_package_version( + config, public_channel, public_package, package_version, dao +): + pkgstore = config.get_package_store() + + update_indexes(dao, pkgstore, public_channel.name) + + files = pkgstore.list_files(public_channel.name) + + base_files = [ + 'channeldata.json', + 'index.html', + 'linux-64/index.html', + 'linux-64/repodata.json', + 'noarch/index.html', + 'noarch/repodata.json', + ] + + expected_files = base_files.copy() + + for suffix in ['.bz2', '.gz']: + expected_files.extend(s + suffix for s in base_files) + + expected_files.append(f"linux-64/{package_version.filename}") + + assert sorted(files) == sorted(expected_files) + + channel_dir = Path(pkgstore.channels_dir) / public_channel.name + with open(channel_dir / 'channeldata.json', 'r') as fd: + channeldata = json.load(fd) + + assert public_package.name in channeldata["packages"].keys() diff --git a/quetz/tests/test_jobs.py b/quetz/tests/test_jobs.py index 6c7d7124..6873781b 100644 --- a/quetz/tests/test_jobs.py +++ b/quetz/tests/test_jobs.py @@ -31,16 +31,6 @@ def sqlite_in_memory(): return False -@pytest.fixture -def package_name(): - return "my-package" - - -@pytest.fixture -def channel_name(): - return "my-channel" - - def add_package_version( filename, package_version, channel_name, user, dao, package_name=None ): @@ -98,16 +88,6 @@ def package_version( db.commit() -@pytest.fixture -def channel_role(): - return "owner" - - -@pytest.fixture -def package_role(): - return "owner" - - @pytest.fixture def public_channel(dao: Dao, user, channel_role, channel_name, db): diff --git a/quetz/tests/test_tasks.py b/quetz/tests/test_tasks.py index 102fc118..93c36b4e 100644 --- a/quetz/tests/test_tasks.py +++ b/quetz/tests/test_tasks.py @@ -9,11 +9,6 @@ from quetz.tasks.reindexing import reindex_packages_from_store -@pytest.fixture -def channel_name(): - return "my-channel" - - @pytest.fixture def pkgstore(config): pkgstore = config.get_package_store() diff --git a/quetz/tests/test_versionorder.py b/quetz/tests/test_versionorder.py index 961d11db..cf82a690 100644 --- a/quetz/tests/test_versionorder.py +++ b/quetz/tests/test_versionorder.py @@ -19,16 +19,6 @@ from quetz.versionorder import VersionOrder -@pytest.fixture -def package_name(): - return "my-package" - - -@pytest.fixture -def channel_name(): - return "my-channel" - - def test_versionorder(): versions = [ ("0.4", [[0], [0], [4]]),