From 5f2832c0b39ef56c44c17a0460bc876ae350fae8 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Mon, 14 Mar 2022 18:41:36 +0100 Subject: [PATCH] fix sqlalchemy and pytest warnings (#502) --- plugins/quetz_harvester/tests/conftest.py | 4 ++-- quetz/authentication/registry.py | 2 +- quetz/dao.py | 3 +++ quetz/db_models.py | 21 ++++++++++++--------- quetz/jobs/models.py | 6 ++++-- quetz/jobs/runner.py | 2 ++ quetz/tasks/workers.py | 2 +- quetz/testing/mockups.py | 2 +- quetz/tests/api/conftest.py | 4 ++-- quetz/tests/api/test_channels.py | 4 ++-- quetz/tests/conftest.py | 7 ++++++- quetz/tests/test_jobs.py | 4 ++-- quetz/tests/test_mirror.py | 4 ++-- quetz/utils.py | 3 ++- 14 files changed, 42 insertions(+), 26 deletions(-) diff --git a/plugins/quetz_harvester/tests/conftest.py b/plugins/quetz_harvester/tests/conftest.py index ddc78ba2..f4e84582 100644 --- a/plugins/quetz_harvester/tests/conftest.py +++ b/plugins/quetz_harvester/tests/conftest.py @@ -10,7 +10,7 @@ from quetz.db_models import ApiKey, Profile, User from quetz.jobs.runner import Supervisor from quetz.rest_models import Channel, Package -from quetz.testing.mockups import TestWorker +from quetz.testing.mockups import MockWorker pytest_plugins = "quetz.testing.fixtures" @@ -51,7 +51,7 @@ def auth_client(client, user): @pytest.fixture def supervisor(config, db, dao): - manager = TestWorker(config, db, dao) + manager = MockWorker(config, db, dao) supervisor = Supervisor(db, manager) return supervisor diff --git a/quetz/authentication/registry.py b/quetz/authentication/registry.py index b296f376..2a657efa 100644 --- a/quetz/authentication/registry.py +++ b/quetz/authentication/registry.py @@ -39,7 +39,7 @@ def register(self, auth: BaseAuthenticator): ) if len(self.enabled_authenticators) > 1: - logger.warn( + logger.warning( "You have registered multiple authentication providers." "Please note that this is currently discouraged in production setups!" ) diff --git a/quetz/dao.py b/quetz/dao.py index 951b188d..a28e3243 100644 --- a/quetz/dao.py +++ b/quetz/dao.py @@ -99,6 +99,8 @@ class Upsert(Insert): incr: increment """ + inherit_cache = False + def __init__(self, table, values, index_elements, column, incr=1): self.values = values self.index_elements = index_elements @@ -802,6 +804,7 @@ def create_version( .with_for_update() .filter(Package.channel_name == channel_name) .filter(Package.name == package_name) + .join(PackageVersion) .filter(PackageVersion.package_format == package_format) .filter(PackageVersion.platform == platform) ).first() diff --git a/quetz/db_models.py b/quetz/db_models.py index 893611f2..56429c0a 100644 --- a/quetz/db_models.py +++ b/quetz/db_models.py @@ -124,9 +124,8 @@ class ChannelMember(Base): user_id = Column(UUID, ForeignKey('users.id'), primary_key=True, index=True) role = Column(String) - channel = relationship( - 'Channel', backref=backref("channel_members", cascade="all,delete-orphan") - ) + channel = relationship('Channel', back_populates="members") + user = relationship( 'User', backref=backref("channel_members", cascade="all,delete-orphan") ) @@ -210,9 +209,9 @@ class Channel(Base): mirrors = relationship("ChannelMirror", cascade="all, delete", uselist=True) members_count = column_property( - select([func.count(ChannelMember.user_id)]).where( - ChannelMember.channel_name == name - ), + select([func.count(ChannelMember.user_id)]) + .where(ChannelMember.channel_name == name) + .scalar_subquery(), # type: ignore deferred=True, ) @@ -224,7 +223,9 @@ def load_channel_metadata(self): return {} packages_count = column_property( - select([func.count(Package.name)]).where(Package.channel_name == name), + select([func.count(Package.name)]) + .where(Package.channel_name == name) + .scalar_subquery(), # type: ignore deferred=True, ) @@ -263,9 +264,11 @@ class PackageMember(Base): 'Package', backref=backref("members", cascade="all,delete"), primaryjoin="and_(Package.name == foreign(PackageMember.package_name)," - "Package.channel_name == Channel.name)", + "Package.channel_name == PackageMember.channel_name)", + ) + channel = relationship( + 'Channel', backref=backref("package_members", cascade="all,delete") ) - channel = relationship('Channel') user = relationship('User', backref=backref("packages", cascade="all,delete")) def __repr__(self): diff --git a/quetz/jobs/models.py b/quetz/jobs/models.py index 141530c3..4a35ba9b 100644 --- a/quetz/jobs/models.py +++ b/quetz/jobs/models.py @@ -60,7 +60,9 @@ class Job(Base): default=JobStatus.pending, ) - tasks = sa.orm.relationship('Task', cascade="all,delete-orphan", uselist=True) + tasks = sa.orm.relationship( + 'Task', back_populates='job', cascade="all,delete-orphan", uselist=True + ) extra_args = sa.Column(sa.String(), nullable=True) @@ -83,7 +85,7 @@ class Task(Base): job_id = sa.Column( sa.Integer, sa.ForeignKey("jobs.id", ondelete="cascade"), nullable=False ) - job = sa.orm.relationship("Job") + job = sa.orm.relationship("Job", back_populates='tasks') package_version_id = sa.Column( UUID, sa.ForeignKey("package_versions.id", ondelete="cascade"), nullable=True ) diff --git a/quetz/jobs/runner.py b/quetz/jobs/runner.py index fcd5eb0a..59104eca 100644 --- a/quetz/jobs/runner.py +++ b/quetz/jobs/runner.py @@ -22,6 +22,7 @@ class any_true(FunctionElement): + inherit_cache = True name = "anytrue" type = Boolean() @@ -37,6 +38,7 @@ def pg_any(element, compiler, **kw): class all_true(FunctionElement): + inherit_cache = True name = "alltrue" type = Boolean() diff --git a/quetz/tasks/workers.py b/quetz/tasks/workers.py index 7994a724..50c1e2c8 100644 --- a/quetz/tasks/workers.py +++ b/quetz/tasks/workers.py @@ -260,7 +260,7 @@ def done(self): async def wait(self, waittime=0.1): while not self.done: - asyncio.sleep(waittime) + await asyncio.sleep(waittime) if self.status == "failed": raise self._future.exception() diff --git a/quetz/testing/mockups.py b/quetz/testing/mockups.py index 73f82104..e2a835c9 100644 --- a/quetz/testing/mockups.py +++ b/quetz/testing/mockups.py @@ -7,7 +7,7 @@ from quetz.tasks.workers import job_wrapper -class TestWorker: +class MockWorker: "synchronous worker for testing" def __init__( diff --git a/quetz/tests/api/conftest.py b/quetz/tests/api/conftest.py index baa9326f..d7abe02a 100644 --- a/quetz/tests/api/conftest.py +++ b/quetz/tests/api/conftest.py @@ -5,7 +5,7 @@ from quetz.config import Config from quetz.dao import Dao -from quetz.db_models import Identity, Profile, User +from quetz.db_models import Identity, PackageVersion, Profile, User from quetz.rest_models import Channel, Package @@ -128,7 +128,7 @@ def _make_package_version(filename, version_number, platform="linux-64"): yield _make_package_version for version in versions: - db.delete(version) + db.query(PackageVersion).filter(PackageVersion.id == version.id).delete() db.commit() diff --git a/quetz/tests/api/test_channels.py b/quetz/tests/api/test_channels.py index 692dbe0d..64cdf68d 100644 --- a/quetz/tests/api/test_channels.py +++ b/quetz/tests/api/test_channels.py @@ -18,7 +18,7 @@ from quetz.config import Config from quetz.jobs.models import Job from quetz.jobs.runner import Supervisor -from quetz.testing.mockups import TestWorker +from quetz.testing.mockups import MockWorker @pytest.fixture @@ -135,7 +135,7 @@ def test_permissions_channel_endpoints( @pytest.fixture def sync_supervisor(db, dao, config): "supervisor with synchronous test worker" - manager = TestWorker(config, db, dao) + manager = MockWorker(config, db, dao) supervisor = Supervisor(db, manager) return supervisor diff --git a/quetz/tests/conftest.py b/quetz/tests/conftest.py index 4d5f0535..7d3986e8 100644 --- a/quetz/tests/conftest.py +++ b/quetz/tests/conftest.py @@ -46,7 +46,12 @@ def user(db, user_without_profile): yield user_without_profile - db.delete(profile) + db.query(Profile).filter( + Profile.name == profile.name, + Profile.avatar_url == profile.avatar_url, + Profile.user_id == user_without_profile.id, + ).delete() + db.commit() diff --git a/quetz/tests/test_jobs.py b/quetz/tests/test_jobs.py index 4e106d71..6c7d7124 100644 --- a/quetz/tests/test_jobs.py +++ b/quetz/tests/test_jobs.py @@ -16,7 +16,7 @@ from quetz.jobs.runner import Supervisor, mk_sql_expr, parse_conda_spec from quetz.rest_models import Channel, Package from quetz.tasks.workers import SubprocessWorker -from quetz.testing.mockups import TestWorker +from quetz.testing.mockups import MockWorker pytest_plugins = ("pytest_asyncio",) @@ -891,7 +891,7 @@ def package_version_job(db, user, package_version): @pytest.fixture def sync_supervisor(db, dao, config): "supervisor with synchronous test worker" - manager = TestWorker(config, db, dao) + manager = MockWorker(config, db, dao) supervisor = Supervisor(db, manager) return supervisor diff --git a/quetz/tests/test_mirror.py b/quetz/tests/test_mirror.py index 29bef286..8301006b 100644 --- a/quetz/tests/test_mirror.py +++ b/quetz/tests/test_mirror.py @@ -24,12 +24,12 @@ handle_repodata_package, initial_sync_mirror, ) -from quetz.testing.mockups import TestWorker +from quetz.testing.mockups import MockWorker @pytest.fixture def job_supervisor(db, config, dao, dummy_remote_session_object): - manager = TestWorker(config, db, dao, dummy_remote_session_object) + manager = MockWorker(config, db, dao, dummy_remote_session_object) supervisor = Supervisor(db, manager) return supervisor diff --git a/quetz/utils.py b/quetz/utils.py index ec371ff7..f6bb2d3a 100644 --- a/quetz/utils.py +++ b/quetz/utils.py @@ -162,7 +162,8 @@ def apply_custom_query(search_type, db, keywords, filters): f'%{each_keyword}%' ) keyword_conditions.append(each_keyword_condition) - query = db.filter(and_(*keyword_conditions)) + + query = db.filter(and_(True, *keyword_conditions)) for each_filter in filters: key, values = each_filter