diff --git a/backend/api/comments/resources.py b/backend/api/comments/resources.py index 00eedf26c7..7cb8b38435 100644 --- a/backend/api/comments/resources.py +++ b/backend/api/comments/resources.py @@ -3,7 +3,7 @@ from fastapi.responses import JSONResponse from loguru import logger -from backend.db import get_db, get_session +from backend.db import get_db from backend.models.dtos.mapping_dto import TaskCommentDTO from backend.models.dtos.message_dto import ChatMessageDTO from backend.models.dtos.user_dto import AuthUserDTO @@ -14,9 +14,6 @@ from backend.services.users.authentication_service import login_required from backend.services.users.user_service import UserService -session = get_session() - - router = APIRouter( prefix="/projects", tags=["projects"], diff --git a/backend/api/tasks/statistics.py b/backend/api/tasks/statistics.py index eae8025d6a..3476323d37 100644 --- a/backend/api/tasks/statistics.py +++ b/backend/api/tasks/statistics.py @@ -1,21 +1,28 @@ from datetime import date, timedelta -from backend.services.stats_service import StatsService -from backend.api.utils import validate_date_input + +from databases import Database from fastapi import APIRouter, Depends, Request -from backend.db import get_session -from starlette.authentication import requires + +from backend.api.utils import validate_date_input +from backend.db import get_db +from backend.models.dtos.user_dto import AuthUserDTO +from backend.services.stats_service import StatsService +from backend.services.users.authentication_service import login_required router = APIRouter( prefix="/tasks", tags=["tasks"], - dependencies=[Depends(get_session)], responses={404: {"description": "Not found"}}, ) @router.get("/statistics/") -@requires("authenticated") -async def get(request: Request): +async def get( + request: Request, + organisation_id: int, + db: Database = Depends(get_db), + user: AuthUserDTO = Depends(login_required), +): """ Get Task Stats --- diff --git a/backend/api/users/actions.py b/backend/api/users/actions.py index 924a7e6cbf..b46905f78a 100644 --- a/backend/api/users/actions.py +++ b/backend/api/users/actions.py @@ -1,21 +1,18 @@ -from fastapi import APIRouter, Depends, Request, Body +from databases import Database +from fastapi import APIRouter, Body, Depends, Request from fastapi.responses import JSONResponse from loguru import logger -from backend.models.dtos.user_dto import UserDTO, UserRegisterEmailDTO -from databases import Database from backend.db import get_db -from backend.services.users.authentication_service import login_required -from backend.models.dtos.user_dto import AuthUserDTO +from backend.models.dtos.user_dto import AuthUserDTO, UserDTO, UserRegisterEmailDTO +from backend.services.interests_service import InterestService from backend.services.messaging.message_service import MessageService +from backend.services.users.authentication_service import login_required from backend.services.users.user_service import UserService, UserServiceError -from backend.services.interests_service import InterestService -from backend.db import get_session router = APIRouter( prefix="/users", tags=["users"], - dependencies=[Depends(get_session)], responses={404: {"description": "Not found"}}, ) diff --git a/backend/api/users/openstreetmap.py b/backend/api/users/openstreetmap.py index 2da87450e2..9c448ddbc3 100644 --- a/backend/api/users/openstreetmap.py +++ b/backend/api/users/openstreetmap.py @@ -1,23 +1,26 @@ -from backend.services.users.user_service import UserService, OSMServiceError +from databases import Database from fastapi import APIRouter, Depends, Request -from backend.db import get_session -from starlette.authentication import requires +from fastapi.responses import JSONResponse + from backend.db import get_db -from databases import Database +from backend.models.dtos.user_dto import AuthUserDTO +from backend.services.users.authentication_service import login_required +from backend.services.users.user_service import OSMServiceError, UserService router = APIRouter( prefix="/users", tags=["users"], - dependencies=[Depends(get_session)], responses={404: {"description": "Not found"}}, ) -# class UsersOpenStreetMapAPI(Resource): -# @token_auth.login_required @router.get("/{username}/openstreetmap/") -@requires("authenticated") -async def get(request: Request, db: Database = Depends(get_db), username: str = None): +async def get( + request: Request, + db: Database = Depends(get_db), + user: AuthUserDTO = Depends(login_required), + username: str = None, +): """ Get details from OpenStreetMap for a specified username --- @@ -54,4 +57,4 @@ async def get(request: Request, db: Database = Depends(get_db), username: str = osm_dto = await UserService.get_osm_details_for_user(username, db) return osm_dto.model_dump(by_alias=True) except OSMServiceError as e: - return {"Error": str(e)}, 502 + return JSONResponse(content={"Error": str(e)}, status_code=502) diff --git a/backend/api/users/resources.py b/backend/api/users/resources.py index 4fe7661b9c..59dccdc8db 100644 --- a/backend/api/users/resources.py +++ b/backend/api/users/resources.py @@ -6,7 +6,7 @@ from loguru import logger -from backend.db import get_db, get_session +from backend.db import get_db from backend.models.dtos.user_dto import AuthUserDTO, UserSearchQuery from backend.services.project_service import ProjectService from backend.services.users.authentication_service import login_required @@ -16,7 +16,6 @@ router = APIRouter( prefix="/users", tags=["users"], - dependencies=[Depends(get_session)], responses={404: {"description": "Not found"}}, ) diff --git a/backend/api/users/tasks.py b/backend/api/users/tasks.py index edb2ef4e5e..20e506cec7 100644 --- a/backend/api/users/tasks.py +++ b/backend/api/users/tasks.py @@ -118,7 +118,7 @@ async def get( ) sort_by = request.query_params.get("sort_by", "-action_date") - tasks = UserService.get_tasks_dto( + tasks = await UserService.get_tasks_dto( user.id, project_id=project_id, project_status=project_status, diff --git a/backend/db.py b/backend/db.py index b4973bcce6..5d7a3026cc 100644 --- a/backend/db.py +++ b/backend/db.py @@ -43,12 +43,6 @@ def create_db_session(self): db_connection = DatabaseConnection() # Create a single instance -# remove -def get_session(): - """Yield a new database session.""" - return db_connection.create_db_session() - - async def get_db(): """Get the database connection from the pool.""" async with db_connection.database.connection() as connection: diff --git a/backend/models/dtos/interests_dto.py b/backend/models/dtos/interests_dto.py index b0d900b953..d43c1f32f9 100644 --- a/backend/models/dtos/interests_dto.py +++ b/backend/models/dtos/interests_dto.py @@ -1,17 +1,6 @@ from pydantic import BaseModel, Field from typing import Optional, List -# class InterestDTO(Model): -# """DTO for a interest.""" - -# id = IntType() -# name = StringType(required=True, min_length=1) -# user_selected = BooleanType( -# serialized_name="userSelected", serialize_when_none=False -# ) -# count_projects = IntType(serialize_when_none=False, serialized_name="countProjects") -# count_users = IntType(serialize_when_none=False, serialized_name="countUsers") - class InterestDTO(BaseModel): id: Optional[int] = None diff --git a/backend/models/dtos/project_dto.py b/backend/models/dtos/project_dto.py index 3916b261d0..a9f823ea50 100644 --- a/backend/models/dtos/project_dto.py +++ b/backend/models/dtos/project_dto.py @@ -1,7 +1,5 @@ -# from schematics import Model -# from schematics.exceptions import ValidationError from datetime import date, datetime -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Union, Any from fastapi import HTTPException from pydantic import BaseModel, Field, root_validator @@ -452,7 +450,7 @@ class Config: class ProjectSearchResultsDTO(BaseModel): """Contains all results for the search criteria""" - map_results: Optional[List] = Field(default_factory=list, alias="mapResults") + map_results: Optional[Any] = Field(default_factory=list, alias="mapResults") results: Optional[List["ListSearchResultDTO"]] = Field(default_factory=list) pagination: Optional["Pagination"] = Field(default_factory=dict) diff --git a/backend/models/dtos/user_dto.py b/backend/models/dtos/user_dto.py index 87386b80cf..f15bf41ad0 100644 --- a/backend/models/dtos/user_dto.py +++ b/backend/models/dtos/user_dto.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Optional +from typing import List, Optional, Dict from pydantic import BaseModel, Field from pydantic.functional_validators import field_validator @@ -154,7 +154,7 @@ class MappedProject(BaseModel): tasks_mapped: Optional[int] = Field(None, alias="tasksMapped") tasks_validated: Optional[int] = Field(None, alias="tasksValidated") status: Optional[str] = None - centroid: Optional[str] = None + centroid: Optional[Dict] = None class Config: populate_by_name = True diff --git a/backend/models/postgis/application.py b/backend/models/postgis/application.py index 9deec88224..6922354bd1 100644 --- a/backend/models/postgis/application.py +++ b/backend/models/postgis/application.py @@ -10,13 +10,11 @@ select, ) -from backend.db import Base, get_session +from backend.db import Base from backend.models.dtos.application_dto import ApplicationDTO, ApplicationsDTO from backend.models.postgis.utils import timestamp from backend.services.users.authentication_service import AuthenticationService -session = get_session() - class Application(Base): """Describes an application that is authorized to access the TM""" @@ -45,9 +43,6 @@ async def create(self, user_id, db: Database): await db.execute(query) return application - def save(self): - session.commit() - async def delete(self, db: Database): query = delete(Application).where(Application.id == self.id) await db.execute(query) diff --git a/backend/models/postgis/banner.py b/backend/models/postgis/banner.py index d9c341e5a9..16190eb0c8 100644 --- a/backend/models/postgis/banner.py +++ b/backend/models/postgis/banner.py @@ -3,11 +3,9 @@ from markdown import markdown from sqlalchemy import Boolean, Column, Integer, String, insert, update -from backend.db import Base, get_session +from backend.db import Base from backend.models.dtos.banner_dto import BannerDTO -session = get_session() - class Banner(Base): """Model for Banners""" @@ -26,10 +24,6 @@ async def create(self, db: Database): ) await db.execute(query) - def update(self): - """Updates the current model in the DB""" - session.commit() - async def update_from_dto(self, db: Database, dto: BannerDTO): """Updates the current model in the DB""" self.message = dto.message diff --git a/backend/models/postgis/campaign.py b/backend/models/postgis/campaign.py index 70dcce24a6..6d6c706fd8 100644 --- a/backend/models/postgis/campaign.py +++ b/backend/models/postgis/campaign.py @@ -1,9 +1,8 @@ from sqlalchemy import Column, ForeignKey, Integer, String, Table, UniqueConstraint -from backend.db import Base, get_session +from backend.db import Base from backend.models.dtos.campaign_dto import CampaignDTO, CampaignListDTO -session = get_session() campaign_projects = Table( "campaign_projects", @@ -34,27 +33,6 @@ class Campaign(Base): url = Column(String) description = Column(String) - def create(self): - """Creates and saves the current model to the DB""" - session.add(self) - session.commit() - - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - - def save(self): - session.commit() - - def update(self, dto: CampaignDTO): - """Update the user details""" - self.name = dto.name if dto.name else self.name - self.logo = dto.logo if dto.logo else self.logo - self.url = dto.url if dto.url else self.url - self.description = dto.description if dto.description else self.description - session.commit() - @classmethod def from_dto(cls, dto: CampaignDTO): """Creates new message from DTO""" diff --git a/backend/models/postgis/custom_editors.py b/backend/models/postgis/custom_editors.py index 4521171644..6fb1031f47 100644 --- a/backend/models/postgis/custom_editors.py +++ b/backend/models/postgis/custom_editors.py @@ -1,11 +1,9 @@ from databases import Database from sqlalchemy import Column, ForeignKey, Integer, String, delete, update -from backend.db import Base, get_session +from backend.db import Base from backend.models.dtos.project_dto import CustomEditorDTO -session = get_session() - class CustomEditor(Base): """Model for user defined editors for a project""" @@ -16,20 +14,6 @@ class CustomEditor(Base): description = Column(String) url = Column(String, nullable=False) - def create(self): - """Creates and saves the current model to the DB""" - session.add(self) - session.commit() - - def save(self): - """Save changes to db""" - session.commit() - - @staticmethod - def get_by_project_id(project_id: int): - """Get custom editor by it's project id""" - return session.get(CustomEditor, project_id) - @classmethod async def create_from_dto(cls, project_id: int, dto: CustomEditorDTO, db: Database): """Creates a new CustomEditor from dto, used in project edit""" diff --git a/backend/models/postgis/interests.py b/backend/models/postgis/interests.py index 57bb98c3ec..3a7a31e7b4 100644 --- a/backend/models/postgis/interests.py +++ b/backend/models/postgis/interests.py @@ -1,11 +1,9 @@ from databases import Database from sqlalchemy import BigInteger, Column, ForeignKey, Integer, String, Table, select -from backend.db import Base, get_session -from backend.exceptions import NotFound -from backend.models.dtos.interests_dto import InterestDTO, InterestsListDTO +from backend.db import Base +from backend.models.dtos.interests_dto import InterestDTO -session = get_session() # Secondary table defining many-to-many join for interests of a user. user_interests = Table( @@ -43,34 +41,6 @@ async def get_by_id(interest_id: int, db: Database): return Interest(**result) return None - @staticmethod - def get_by_name(name: str): - """Get interest by name""" - interest = session.query(Interest).filter(Interest.name == name).first() - if interest is None: - raise NotFound(sub_code="INTEREST_NOT_FOUND", interest_name=name) - - return interest - - def update(self, dto): - """Update existing interest""" - self.name = dto.name - session.commit() - - def create(self): - """Creates and saves the current model to the DB""" - session.add(self) - session.commit() - - def save(self): - """Save changes to db""" - session.commit() - - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - def as_dto(self) -> InterestDTO: """Get the interest from the DB""" dto = InterestDTO() @@ -78,12 +48,3 @@ def as_dto(self) -> InterestDTO: dto.name = self.name return dto - - @staticmethod - def get_all_interests(): - """Get all interests""" - query = session.query(Interest).all() - interest_list_dto = InterestsListDTO() - interest_list_dto.interests = [interest.as_dto() for interest in query] - - return interest_list_dto diff --git a/backend/models/postgis/licenses.py b/backend/models/postgis/licenses.py index 983bdaa23f..11cbfbdee2 100644 --- a/backend/models/postgis/licenses.py +++ b/backend/models/postgis/licenses.py @@ -2,11 +2,10 @@ from sqlalchemy import BigInteger, Column, ForeignKey, Integer, String, Table from sqlalchemy.orm import relationship -from backend.db import Base, get_session +from backend.db import Base from backend.exceptions import NotFound -from backend.models.dtos.licenses_dto import LicenseDTO, LicenseListDTO +from backend.models.dtos.licenses_dto import LicenseDTO -session = get_session() # Secondary table defining the many-to-many join user_licenses_table = Table( @@ -64,34 +63,6 @@ async def create_from_dto(license_dto: LicenseDTO, db: Database) -> int: new_license_id = await db.execute(query, values) return new_license_id - def update_license(self, dto: LicenseDTO): - """Update existing license""" - self.name = dto.name - self.description = dto.description - self.plain_text = dto.plain_text - session.commit() - - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - - @staticmethod - def get_all() -> LicenseListDTO: - """Gets all licenses currently stored""" - results = session.query(License).all() - - dto = LicenseListDTO() - for result in results: - imagery_license = LicenseDTO() - imagery_license.license_id = result.id - imagery_license.name = result.name - imagery_license.description = result.description - imagery_license.plain_text = result.plain_text - dto.licenses.append(imagery_license) - - return dto - def as_dto(self) -> LicenseDTO: """Get the license from the DB""" dto = LicenseDTO() diff --git a/backend/models/postgis/message.py b/backend/models/postgis/message.py index 59e68956d6..95447c8ecc 100644 --- a/backend/models/postgis/message.py +++ b/backend/models/postgis/message.py @@ -15,7 +15,7 @@ from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import false -from backend.db import Base, get_session +from backend.db import Base from backend.exceptions import NotFound from backend.models.dtos.message_dto import MessageDTO, MessagesDTO from backend.models.postgis.project import Project @@ -23,8 +23,6 @@ from backend.models.postgis.user import User from backend.models.postgis.utils import timestamp -session = get_session() - class MessageType(Enum): """Describes the various kinds of messages a user might receive""" @@ -181,11 +179,6 @@ async def get_all_tasks_contributors(project_id: int, task_id: int, db: Database return [contributor["username"] for contributor in contributors] - def mark_as_read(self): - """Mark the message in scope as Read""" - self.read = True - session.commit() - @staticmethod def get_unread_message_count(user_id: int): """Get count of unread messages for user""" @@ -240,11 +233,6 @@ async def delete_all_messages( params["message_type_filters"] = message_type_filters await db.execute(delete_query, params) - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - @staticmethod async def mark_multiple_messages_read( message_ids: list, user_id: int, db: Database diff --git a/backend/models/postgis/notification.py b/backend/models/postgis/notification.py index e8f606b3b9..0ffa718022 100644 --- a/backend/models/postgis/notification.py +++ b/backend/models/postgis/notification.py @@ -11,13 +11,11 @@ ) from sqlalchemy.orm import relationship -from backend.db import Base, get_session +from backend.db import Base from backend.models.dtos.notification_dto import NotificationDTO from backend.models.postgis.user import User from backend.models.postgis.utils import timestamp -session = get_session() - class Notification(Base): """Describes a Notification for a user""" @@ -43,14 +41,6 @@ def as_dto(self) -> NotificationDTO: return dto - def save(self): - session.add(self) - session.commit() - - def update(self): - self.date = timestamp() - session.commit() - @staticmethod async def get_unread_message_count(user_id: int, db: Database) -> int: """Get count of unread messages for user""" diff --git a/backend/models/postgis/organisation.py b/backend/models/postgis/organisation.py index a2ea5512c6..abc3c431d8 100644 --- a/backend/models/postgis/organisation.py +++ b/backend/models/postgis/organisation.py @@ -12,7 +12,7 @@ ) from sqlalchemy.orm import backref, relationship -from backend.db import Base, get_session +from backend.db import Base from backend.exceptions import NotFound from backend.models.dtos.organisation_dto import ( NewOrganisationDTO, @@ -24,9 +24,6 @@ from backend.models.postgis.statuses import OrganisationType from backend.models.postgis.user import User -session = get_session() - - # Secondary table defining many-to-many relationship between organisations and managers organisation_managers = Table( "organisation_managers", @@ -65,14 +62,6 @@ class Organisation(Base): Campaign, secondary=campaign_organisations, backref="organisation" ) - def create(self, session): - """Creates and saves the current model to the DB""" - session.add(self) - session.commit() - - def save(self): - session.commit() - async def create_from_dto(new_organisation_dto: NewOrganisationDTO, db: Database): """Creates a new organisation from a DTO and associates managers""" slug = new_organisation_dto.slug or slugify(new_organisation_dto.name) @@ -165,11 +154,6 @@ async def update(organisation_dto: UpdateOrganisationDTO, db: Database): except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - async def can_be_deleted(organisation_id: int, db) -> bool: # Check if the organization has any projects projects_query = """ diff --git a/backend/models/postgis/partner.py b/backend/models/postgis/partner.py index 35d7162390..56b865d8e0 100644 --- a/backend/models/postgis/partner.py +++ b/backend/models/postgis/partner.py @@ -3,7 +3,6 @@ from databases import Database from sqlalchemy import Column, Integer, String -from backend import db from backend.db import Base from backend.exceptions import NotFound from backend.models.dtos.partner_dto import PartnerDTO @@ -28,20 +27,6 @@ class Partner(Base): website_links = Column(String, nullable=True) mapswipe_group_id = Column(String, nullable=True) - def create(self): - """Creates and saves the current model to the DB""" - db.session.add(self) - db.session.commit() - - def save(self): - """Save changes to DB""" - db.session.commit() - - def delete(self): - """Deletes from the DB""" - db.session.delete(self) - db.session.commit() - @staticmethod async def get_all_partners(db: Database): """ diff --git a/backend/models/postgis/priority_area.py b/backend/models/postgis/priority_area.py index 0e0243bff5..9afc62210c 100644 --- a/backend/models/postgis/priority_area.py +++ b/backend/models/postgis/priority_area.py @@ -5,11 +5,9 @@ from geoalchemy2 import Geometry from sqlalchemy import Column, ForeignKey, Integer, Table -from backend.db import Base, get_session +from backend.db import Base from backend.models.postgis.utils import InvalidGeoJson -session = get_session() - # Priority areas aren't shared, however, this arch was taken from TM2 to ease data migration project_priority_areas = Table( "project_priority_areas", @@ -76,9 +74,3 @@ async def from_dict(cls, area_poly: dict, db: Database): return pa else: raise Exception("Failed to insert Priority Area") - - def get_as_geojson(self): - """Helper to translate geometry back to a GEOJson Poly""" - with db.engine.connect() as conn: - pa_geojson = conn.execute(self.geometry.ST_AsGeoJSON()).scalar() - return geojson.loads(pa_geojson) diff --git a/backend/models/postgis/project.py b/backend/models/postgis/project.py index d935aa6324..b4c9759227 100644 --- a/backend/models/postgis/project.py +++ b/backend/models/postgis/project.py @@ -114,15 +114,6 @@ async def create(self, db: Database): ) ) - def save(self): - """Save changes to db""" - session.commit() - - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - # cache mapper counts for 30 seconds active_mappers_cache = TTLCache(maxsize=1024, ttl=30) diff --git a/backend/models/postgis/project_chat.py b/backend/models/postgis/project_chat.py index 015b03c658..c6e150390b 100644 --- a/backend/models/postgis/project_chat.py +++ b/backend/models/postgis/project_chat.py @@ -5,7 +5,7 @@ from sqlalchemy import BigInteger, Column, DateTime, ForeignKey, Integer, String from sqlalchemy.orm import relationship -from backend.db import Base, get_session +from backend.db import Base from backend.models.dtos.message_dto import ( ChatMessageDTO, ListChatMessageDTO, @@ -15,8 +15,6 @@ from backend.models.postgis.user import User from backend.models.postgis.utils import timestamp -session = get_session() - class ProjectChat(Base): """Contains all project info localized into supported languages""" diff --git a/backend/models/postgis/project_info.py b/backend/models/postgis/project_info.py index 2891878389..42ef511337 100644 --- a/backend/models/postgis/project_info.py +++ b/backend/models/postgis/project_info.py @@ -1,4 +1,3 @@ -# # from flask import current_app from typing import List from databases import Database @@ -14,11 +13,9 @@ ) from sqlalchemy.dialects.postgresql import TSVECTOR -from backend.db import Base, get_session +from backend.db import Base from backend.models.dtos.project_dto import ProjectInfoDTO -session = get_session() - class ProjectInfo(Base): """Contains all project info localized into supported languages""" diff --git a/backend/models/postgis/project_partner.py b/backend/models/postgis/project_partner.py index a77ecfa70c..5b99008b6b 100644 --- a/backend/models/postgis/project_partner.py +++ b/backend/models/postgis/project_partner.py @@ -3,7 +3,6 @@ from databases import Database from sqlalchemy import Column, DateTime, ForeignKey, Integer -from backend import db from backend.db import Base from backend.models.dtos.project_partner_dto import ( ProjectPartnerAction, @@ -105,15 +104,6 @@ async def create(self, db: Database) -> int: result = await db.fetch_one(query, values=values) return result["id"] - def save(self): - """Save changes to db""" - db.session.commit() - - def delete(self): - """Deletes the current model from the DB""" - db.session.delete(self) - db.session.commit() - class ProjectPartnership(Base): """Describes the relationship between a Project and a Partner""" diff --git a/backend/models/postgis/release_version.py b/backend/models/postgis/release_version.py index a075c40dd9..26a34493af 100644 --- a/backend/models/postgis/release_version.py +++ b/backend/models/postgis/release_version.py @@ -1,9 +1,7 @@ from databases import Database from sqlalchemy import Column, String, DateTime, insert -from backend.db import Base, get_session - -session = get_session() +from backend.db import Base class ReleaseVersion(Base): @@ -13,9 +11,6 @@ class ReleaseVersion(Base): tag_name = Column(String(64), nullable=False, primary_key=True) published_at = Column(DateTime, nullable=False) - def update(self): - session.commit() - async def save(self, db: Database): query = insert(ReleaseVersion.__table__).values( tag_name=self.tag_name, published_at=self.published_at diff --git a/backend/models/postgis/tags.py b/backend/models/postgis/tags.py index c50aa30af8..740c126062 100644 --- a/backend/models/postgis/tags.py +++ b/backend/models/postgis/tags.py @@ -1,8 +1,6 @@ from sqlalchemy import Column, String, Integer from backend.models.dtos.tags_dto import TagsDTO -from backend.db import Base, get_session - -session = get_session() +from backend.db import Base class Tags(Base): diff --git a/backend/models/postgis/task.py b/backend/models/postgis/task.py index 83b29abe98..befecf7396 100644 --- a/backend/models/postgis/task.py +++ b/backend/models/postgis/task.py @@ -2,10 +2,11 @@ import json from datetime import timezone from enum import Enum -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import bleach import geojson +from databases import Database from geoalchemy2 import Geometry from shapely.geometry import shape @@ -24,11 +25,13 @@ desc, distinct, func, + select, ) from sqlalchemy.orm import relationship from sqlalchemy.orm.exc import MultipleResultsFound -from backend.db import Base, get_session +from backend.config import settings +from backend.db import Base from backend.exceptions import NotFound from backend.models.dtos.mapping_dto import TaskDTO, TaskHistoryDTO from backend.models.dtos.mapping_issues_dto import TaskMappingIssueDTO @@ -49,14 +52,6 @@ timestamp, ) -session = get_session() -from typing import Optional - -from databases import Database -from sqlalchemy import select - -from backend.config import settings - class TaskAction(Enum): """Describes the possible actions that can happen to to a task, that we'll record history for""" @@ -109,11 +104,6 @@ def __init__(self, project_id, task_id): self.task_id = task_id self.is_closed = False - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - @staticmethod async def get_open_for_task(project_id: int, task_id: int, db: Database): """ @@ -298,11 +288,6 @@ def __init__(self, issue, count, mapping_issue_category_id, task_history_id=None self.count = count self.mapping_issue_category_id = mapping_issue_category_id - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - def as_dto(self): issue_dto = TaskMappingIssueDTO() issue_dto.category_id = self.mapping_issue_category_id @@ -378,11 +363,6 @@ def set_state_change_action(new_state: TaskStatus) -> str: def set_auto_unlock_action(task_action: TaskAction) -> str: return task_action.name, None - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - async def update_task_locked_with_duration( task_id: int, project_id: int, @@ -729,20 +709,6 @@ class Task(Base): lock_holder = relationship(User, foreign_keys=[locked_by]) mapper = relationship(User, foreign_keys=[mapped_by]) - def create(self): - """Creates and saves the current model to the DB""" - session.add(self) - session.commit() - - def update(self): - """Updates the DB with the current state of the Task""" - session.commit() - - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - @classmethod def from_geojson_feature(cls, task_id, task_feature): """ diff --git a/backend/models/postgis/task_annotation.py b/backend/models/postgis/task_annotation.py index f813de9a69..522fab5f8b 100644 --- a/backend/models/postgis/task_annotation.py +++ b/backend/models/postgis/task_annotation.py @@ -9,13 +9,11 @@ String, ) -from backend.db import Base, get_session +from backend.db import Base from backend.models.dtos.project_dto import ProjectTaskAnnotationsDTO from backend.models.dtos.task_annotation_dto import TaskAnnotationDTO from backend.models.postgis.utils import timestamp -session = get_session() - class TaskAnnotation(Base): """Describes Task annotaions like derived ML attributes""" @@ -57,20 +55,6 @@ def __init__( self.annotation_markdown = annotation_markdown self.properties = properties - def create(self): - """Creates and saves the current model to the DB""" - session.add(self) - session.commit() - - def update(self): - """Updates the DB with the current state of the Task Annotations""" - session.commit() - - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - @staticmethod def get_task_annotation(task_id, project_id, annotation_type): """Get annotations for a task with supplied type""" diff --git a/backend/models/postgis/team.py b/backend/models/postgis/team.py index fb26f1de2c..a440942747 100644 --- a/backend/models/postgis/team.py +++ b/backend/models/postgis/team.py @@ -11,7 +11,7 @@ ) from sqlalchemy.orm import backref, relationship -from backend.db import Base, get_session +from backend.db import Base from backend.exceptions import NotFound from backend.models.dtos.organisation_dto import OrganisationTeamsDTO from backend.models.dtos.team_dto import ( @@ -29,8 +29,6 @@ ) from backend.models.postgis.user import User -session = get_session() - class TeamMembers(Base): __tablename__ = "team_members" @@ -63,11 +61,6 @@ async def create(self, db: Database): ) return team_member - def delete(self): - """Deletes the current model from the DB""" - session.delete(self) - session.commit() - async def update(self, db: Database): """Updates the current model in the DB""" await db.execute( diff --git a/backend/models/postgis/user.py b/backend/models/postgis/user.py index e51560a870..731ed1e024 100644 --- a/backend/models/postgis/user.py +++ b/backend/models/postgis/user.py @@ -1,4 +1,5 @@ import geojson +from databases import Database from sqlalchemy import ( ARRAY, BigInteger, @@ -13,7 +14,7 @@ ) from sqlalchemy.orm import relationship -from backend.db import Base, get_session +from backend.db import Base from backend.exceptions import NotFound from backend.models.dtos.user_dto import ( ListedUser, @@ -37,9 +38,6 @@ ) from backend.models.postgis.utils import timestamp -session = get_session() -from databases import Database - class User(Base): """Describes the history associated with a task""" @@ -86,14 +84,6 @@ class User(Base): ) interests = relationship(Interest, secondary=user_interests, backref="users") - def create(self): - """Creates and saves the current model to the DB""" - session.add(self) - session.commit() - - def save(self): - session.commit() - @staticmethod async def get_by_id(user_id: int, db: Database): """ @@ -434,11 +424,6 @@ def has_user_accepted_licence(self, license_id: int): return False - def delete(self): - """Delete the user in scope from DB""" - session.delete(self) - session.commit() - def as_dto(self, logged_in_username: str) -> UserDTO: """Create DTO object from user in scope""" user_dto = UserDTO() @@ -482,12 +467,6 @@ def as_dto(self, logged_in_username: str) -> UserDTO: user_dto.self_description_gender = self.self_description_gender return user_dto - def create_or_update_interests(self, interests_ids): - self.interests = [] - objs = [Interest.get_by_id(i) for i in interests_ids] - self.interests.extend(objs) - session.commit() - class UserEmail(Base): __tablename__ = "users_with_email" @@ -500,9 +479,6 @@ async def create(self, db: Database): user = await db.execute(insert(UserEmail.__table__).values(email=self.email)) return user - def save(self): - session.commit() - async def delete(self, db: Database): """Deletes the current model from the DB""" await db.execute(delete(UserEmail.__table__).where(UserEmail.id == self.id)) diff --git a/backend/services/campaign_service.py b/backend/services/campaign_service.py index 2be6251203..796f19c0f1 100644 --- a/backend/services/campaign_service.py +++ b/backend/services/campaign_service.py @@ -1,10 +1,7 @@ -# from flask import current_app -# from psycopg2.errors import UniqueViolation, NotNullViolation from databases import Database from fastapi import HTTPException from sqlalchemy.exc import IntegrityError -from backend.db import get_session from backend.exceptions import NotFound from backend.models.dtos.campaign_dto import ( CampaignDTO, @@ -16,8 +13,6 @@ from backend.services.organisation_service import OrganisationService from backend.services.project_service import ProjectService -session = get_session() - class CampaignService: @staticmethod diff --git a/backend/services/interests_service.py b/backend/services/interests_service.py index c8dc831ec4..392e75f7c0 100644 --- a/backend/services/interests_service.py +++ b/backend/services/interests_service.py @@ -28,11 +28,6 @@ def get_by_id(interest_id): interest = Interest.get_by_id(interest_id) return interest - @staticmethod - def get_by_name(name): - interest = Interest.get_by_name(name) - return interest - @staticmethod async def create(interest_name: str, db: Database) -> InterestDTO: query = """ @@ -80,8 +75,7 @@ async def get_all_interests(db: Database) -> InterestsListDTO: interest_list_dto = InterestsListDTO() for record in results: interest_dto = InterestDTO(**record) - interest_dict = interest_dto.dict(exclude_unset=True) - interest_list_dto.interests.append(interest_dict) + interest_list_dto.interests.append(interest_dto) return interest_list_dto @staticmethod diff --git a/backend/services/organisation_service.py b/backend/services/organisation_service.py index 2270afc328..57d04c753d 100644 --- a/backend/services/organisation_service.py +++ b/backend/services/organisation_service.py @@ -4,11 +4,8 @@ from databases import Database from fastapi import HTTPException from loguru import logger - -# from flask import current_app from sqlalchemy.exc import IntegrityError -from backend.db import get_session from backend.exceptions import NotFound from backend.models.dtos.organisation_dto import ( ListOrganisationsDTO, @@ -34,8 +31,6 @@ from backend.models.postgis.team import TeamVisibility from backend.services.users.user_service import UserService -session = get_session() - class OrganisationServiceError(Exception): """Custom Exception to notify callers an error occurred when handling organisations""" diff --git a/backend/services/project_search_service.py b/backend/services/project_search_service.py index 27425c8024..3eeaa8b3a1 100644 --- a/backend/services/project_search_service.py +++ b/backend/services/project_search_service.py @@ -12,7 +12,6 @@ from shapely.geometry import Polygon, box from backend.api.utils import validate_date_input -from backend.db import get_session from backend.exceptions import NotFound from backend.models.dtos.project_dto import ( ListSearchResultDTO, @@ -37,8 +36,6 @@ ) from backend.services.users.user_service import UserService -session = get_session() - search_cache = TTLCache(maxsize=128, ttl=300) csv_download_cache = TTLCache(maxsize=16, ttl=600) diff --git a/backend/services/recommendation_service.py b/backend/services/recommendation_service.py index 12bec64213..2ffc90adc5 100644 --- a/backend/services/recommendation_service.py +++ b/backend/services/recommendation_service.py @@ -1,18 +1,15 @@ import pandas as pd +from cachetools import TTLCache +from databases import Database from sklearn.metrics.pairwise import cosine_similarity from sklearn.preprocessing import MultiLabelBinarizer -from cachetools import TTLCache from backend.exceptions import NotFound +from backend.models.dtos.project_dto import ProjectSearchResultsDTO from backend.models.postgis.project import Project from backend.models.postgis.statuses import ProjectStatus -from backend.models.dtos.project_dto import ProjectSearchResultsDTO from backend.services.project_search_service import ProjectSearchService from backend.services.users.user_service import UserService -from backend.db import get_session -from databases import Database - -session = get_session() similar_projects_cache = TTLCache(maxsize=1000, ttl=60 * 60 * 24) # 24 hours diff --git a/backend/services/stats_service.py b/backend/services/stats_service.py index f4bdce41b0..6687eb03e1 100644 --- a/backend/services/stats_service.py +++ b/backend/services/stats_service.py @@ -5,7 +5,6 @@ from databases import Database from sqlalchemy import func, or_, select -from backend.db import get_session from backend.exceptions import NotFound from backend.models.dtos.project_dto import ProjectSearchResultsDTO from backend.models.dtos.stats_dto import ( @@ -36,8 +35,6 @@ from backend.services.project_service import ProjectService from backend.services.users.user_service import UserService -session = get_session() - homepage_stats_cache = TTLCache(maxsize=4, ttl=30) diff --git a/backend/services/users/user_service.py b/backend/services/users/user_service.py index 9ba2e30103..e73be5f151 100644 --- a/backend/services/users/user_service.py +++ b/backend/services/users/user_service.py @@ -7,7 +7,6 @@ from sqlalchemy.sql import outerjoin from backend.config import Settings -from backend.db import get_session from backend.exceptions import NotFound from backend.models.dtos.interests_dto import InterestDTO, InterestsListDTO from backend.models.dtos.project_dto import ProjectFavoritesDTO, ProjectSearchResultsDTO @@ -40,7 +39,6 @@ from backend.services.users.osm_service import OSMService, OSMServiceError settings = Settings() -session = get_session() user_filter_cache = TTLCache(maxsize=1024, ttl=600) @@ -514,8 +512,7 @@ async def get_detailed_stats(username: str, db: Database) -> UserStatsDTO: if result and result["total_time"]: total_validation_time = result["total_time"] - # TODO Handle typecasting. - stats_dto.time_spent_validating = round(float(total_validation_time), 1) + stats_dto.time_spent_validating = int(total_validation_time) stats_dto.total_time_spent += stats_dto.time_spent_validating # Total mapping time @@ -533,9 +530,7 @@ async def get_detailed_stats(username: str, db: Database) -> UserStatsDTO: total_mapping_time = await db.fetch_one(total_mapping_time_query) if total_mapping_time and total_mapping_time[0]: - stats_dto.time_spent_mapping = round( - total_mapping_time[0].total_seconds(), 1 - ) + stats_dto.time_spent_mapping = int(total_mapping_time[0].total_seconds()) stats_dto.total_time_spent += stats_dto.time_spent_mapping stats_dto.contributions_interest = await UserService.get_interests_stats( @@ -703,90 +698,106 @@ async def get_mapped_projects(user_name: str, preferred_locale: str, db: Databas @staticmethod async def get_recommended_projects( user_name: str, preferred_locale: str, db: Database - ): - """Gets all projects a user has mapped or validated on""" + ) -> ProjectSearchResultsDTO: from backend.services.project_search_service import ProjectSearchService + """Gets all projects a user has mapped or validated on""" limit = 20 - # 1. Retrieve the user information - query = select(User.id, User.mapping_level).where(User.username == user_name) - user = await db.fetch_one(query) - if user is None: - raise NotFound(sub_code="USER_NOT_FOUND", username=user_name) - user_id = user["id"] - user_mapping_level = user["mapping_level"] + # Get user details + user_query = """ + SELECT id, mapping_level + FROM users + WHERE username = :user_name + """ + user = await db.fetch_one(user_query, {"user_name": user_name}) + if not user: + raise NotFound(sub_code="USER_NOT_FOUND", username=user_name) - # 2. Get all project IDs the user has contributed to - sq = ( - select(distinct(TaskHistory.project_id)) - .where(TaskHistory.user_id == user_id) - .alias("contributed_projects") + # Get all projects the user has contributed to + contributed_projects_query = """ + SELECT DISTINCT project_id + FROM task_history + WHERE user_id = :user_id + """ + contributed_projects = await db.fetch_all( + contributed_projects_query, {"user_id": user["id"]} ) - - # 3. Get all campaigns for the contributed projects or authored by the user - campaign_tags_query = select(distinct(Project.campaign).label("tag")).where( - or_(Project.author_id == user_id, Project.id.in_(sq)) + contributed_project_ids = [row["project_id"] for row in contributed_projects] + + # Fetch campaign tags for contributed or authored projects + campaign_tags_query = """ + SELECT DISTINCT c.name AS tag + FROM campaigns c + JOIN campaign_projects cp ON c.id = cp.campaign_id + WHERE cp.project_id = ANY(:project_ids) OR :user_id IN ( + SELECT p.author_id + FROM projects p + WHERE p.id = cp.project_id + ) + """ + campaign_tags = await db.fetch_all( + query=campaign_tags_query, + values={"user_id": user["id"], "project_ids": contributed_project_ids}, ) - campaign_tags = await db.fetch_all(campaign_tags_query) - campaign_tags_list = [tag["tag"] for tag in campaign_tags] - # 4. Get projects that match these campaign tags but exclude those already contributed - query, params = await ProjectSearchService.create_search_query(db) + campaign_tags_set = {row["tag"] for row in campaign_tags} + # Get projects with matching campaign tags but exclude user contributions + recommended_projects_query = """ + SELECT DISTINCT + p.*, + o.name AS organisation_name, + o.logo AS organisation_logo + FROM projects p + LEFT JOIN organisations o ON p.organisation_id = o.id + JOIN campaign_projects cp ON p.id = cp.project_id + JOIN campaigns c ON cp.campaign_id = c.id + WHERE c.name = ANY(:campaign_tags) + AND p.author_id != :user_id + LIMIT :limit + """ + recommended_projects = await db.fetch_all( + query=recommended_projects_query, + values={ + "campaign_tags": list(campaign_tags_set), + "user_id": user["id"], + "limit": limit, + }, + ) - # Prepare the campaign tags condition - if campaign_tags_list: - campaign_tags_placeholder = ", ".join( - [f":tag{i}" for i in range(len(campaign_tags_list))] - ) - campaign_tags_condition = ( - f" AND p.campaign IN ({campaign_tags_placeholder})" - ) - else: - campaign_tags_condition = "" # No condition if list is empty - - # Modify the query to include the campaign tags condition and limit - final_query = f"{query} {campaign_tags_condition} LIMIT :limit" - - campagin_params = params.copy() - # Update params to include campaign tags - for i, tag in enumerate(campaign_tags_list): - campagin_params[f"tag{i}"] = tag - campagin_params["limit"] = limit - - # Execute the final query with parameters - projs = await db.fetch_all(final_query, campagin_params) - project_ids = [proj["id"] for proj in projs] - - # 5. Get projects filtered by user's mapping level if fewer than the limit - if len(projs) < limit: - remaining_projs_query = f""" - {query} - AND p.difficulty = :difficulty + # Get only projects matching the user's mapping level if needed + len_projs = len(recommended_projects) + if len_projs < limit: + remaining_projects_query = """ + SELECT DISTINCT p.*, o.name AS organisation_name, o.logo AS organisation_logo + FROM projects p + LEFT JOIN organisations o ON p.organisation_id = o.id + WHERE difficulty = :mapping_level LIMIT :remaining_limit """ + remaining_projects = await db.fetch_all( + remaining_projects_query, + { + "mapping_level": user["mapping_level"], + "remaining_limit": limit - len_projs, + }, + ) + recommended_projects.extend(remaining_projects) - params["difficulty"] = user_mapping_level - params["remaining_limit"] = limit - len(projs) - remaining_projs = await db.fetch_all(remaining_projs_query, params) - remaining_projs_ids = [proj["id"] for proj in remaining_projs] - project_ids.extend(remaining_projs_ids) - projs.extend(remaining_projs) - - # 6. Create DTO for the results dto = ProjectSearchResultsDTO() - # Get all total contributions for each project + project_ids = [project["id"] for project in recommended_projects] contrib_counts = await ProjectSearchService.get_total_contributions( project_ids, db ) - # Combine projects and their contribution counts - zip_items = zip(projs, contrib_counts) dto.results = [ - await ProjectSearchService.create_result_dto(p, preferred_locale, t, db) - for p, t in zip_items + await ProjectSearchService.create_result_dto( + project, preferred_locale, contrib_count, db + ) + for project, contrib_count in zip(recommended_projects, contrib_counts) ] + dto.pagination = None return dto diff --git a/manage.py b/manage.py index acd5cae448..8a2cb4b27d 100644 --- a/manage.py +++ b/manage.py @@ -1,23 +1,22 @@ -import os -import warnings +import atexit import base64 import csv import datetime +import os +import warnings + import click -from flask_migrate import Migrate +from apscheduler.schedulers.background import BackgroundScheduler from dotenv import load_dotenv +from flask_migrate import Migrate +from sqlalchemy import func -from backend import create_app, initialise_counters, db +from backend import create_app, db, initialise_counters +from backend.models.postgis.task import Task, TaskHistory +from backend.services.interests_service import InterestService +from backend.services.stats_service import StatsService from backend.services.users.authentication_service import AuthenticationService from backend.services.users.user_service import UserService -from backend.services.stats_service import StatsService -from backend.services.interests_service import InterestService -from backend.models.postgis.task import Task, TaskHistory - -from sqlalchemy import func -import atexit -from apscheduler.schedulers.background import BackgroundScheduler - # Load configuration from file into environment load_dotenv(os.path.join(os.path.dirname(__file__), "tasking-manager.env")) @@ -137,8 +136,9 @@ def update_project_categories(filename): # This is compatibility code with previous releases # People should generally prefer `flask `. from sys import argv - from flask.cli import FlaskGroup + from click import Command + from flask.cli import FlaskGroup cli = FlaskGroup(create_app=lambda: application) cli.add_command(