Skip to content

Commit

Permalink
misc: Migrate MobyGamesHandler to async
Browse files Browse the repository at this point in the history 
Convert `MobyGamesHandler` methods to be asynchronous, and use an
`httpx` async client, instead of `requests` sync client.
  • Loading branch information
@adamantike
adamantike committed Jul 22, 2024
1 parent e1cec57 commit 7a45282
Show file tree
Hide file tree
Showing 6 changed files with 249 additions and 188 deletions.
16 changes: 10 additions & 6 deletions backend/endpoints/rom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,6 @@
from typing import Annotated
from urllib.parse import quote

from config import (
DISABLE_DOWNLOAD_ENDPOINT_AUTH,
LIBRARY_BASE_PATH,
RESOURCES_BASE_PATH,
)
from decorators.auth import protected_route
from endpoints.responses import MessageResponse
from endpoints.responses.rom import (
Expand All @@ -31,6 +26,12 @@
from logger.logger import log
from stream_zip import ZIP_AUTO, stream_zip # type: ignore[import]

from config import (
DISABLE_DOWNLOAD_ENDPOINT_AUTH,
LIBRARY_BASE_PATH,
RESOURCES_BASE_PATH,
)

router = APIRouter()


Expand Down Expand Up @@ -305,7 +306,10 @@ async def update_rom(
cleaned_data.get("moby_id", "")
and int(cleaned_data.get("moby_id", "")) != rom.moby_id
):
moby_rom = meta_moby_handler.get_rom_by_id(cleaned_data["moby_id"])
moby_rom = await meta_moby_handler.get_rom_by_id(
requests_client=request.app.requests_client,
moby_id=cleaned_data["moby_id"],
)
cleaned_data.update(moby_rom)
path_screenshots = fs_resource_handler.get_rom_screenshots(
rom=rom,
Expand Down
10 changes: 6 additions & 4 deletions backend/endpoints/search.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ async def search_rom(
igdb_matched_roms = meta_igdb_handler.get_matched_roms_by_id(
int(search_term)
)
moby_matched_roms = meta_moby_handler.get_matched_roms_by_id(
int(search_term)
moby_matched_roms = await meta_moby_handler.get_matched_roms_by_id(
requests_client=request.app.requests_client, moby_id=int(search_term)
)
except ValueError as exc:
log.error(f"Search error: invalid ID '{search_term}'")
Expand All @@ -74,8 +74,10 @@ async def search_rom(
igdb_matched_roms = meta_igdb_handler.get_matched_roms_by_name(
search_term, _get_main_platform_igdb_id(rom.platform)
)
moby_matched_roms = meta_moby_handler.get_matched_roms_by_name(
search_term, rom.platform.moby_id
moby_matched_roms = await meta_moby_handler.get_matched_roms_by_name(
requests_client=request.app.requests_client,
search_term=search_term,
platform_moby_id=rom.platform.moby_id,
)

merged_dict = {
Expand Down
288 changes: 148 additions & 140 deletions backend/endpoints/sockets/scan.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from typing import Final

Check failure on line 1 in backend/endpoints/sockets/scan.py

View check run for this annotation

Trunk.io / Trunk Check

isort 

Incorrect formatting, autoformat by running 'trunk fmt'

Check failure on line 1 in backend/endpoints/sockets/scan.py

View workflow job for this annotation

GitHub Actions / Trunk Check

isort 

Incorrect formatting, autoformat by running 'trunk fmt'

import emoji
import httpx
import socketio # type: ignore
from config import SCAN_TIMEOUT
from endpoints.responses.platform import PlatformSchema
from endpoints.responses.rom import RomSchema
from exceptions.fs_exceptions import (
Expand All @@ -28,6 +28,8 @@
from rq.job import Job
from sqlalchemy.inspection import inspect

from config import SCAN_TIMEOUT

STOP_SCAN_FLAG: Final = "scan:stop"


Expand Down Expand Up @@ -132,172 +134,178 @@ async def stop_scan():
else:
log.info(f"Found {len(platform_list)} platforms in file system ")

for platform_slug in platform_list:
# Stop the scan if the flag is set
if redis_client.get(STOP_SCAN_FLAG):
await stop_scan()
break

platform = db_platform_handler.get_platform_by_fs_slug(platform_slug)
if platform and scan_type == ScanType.NEW_PLATFORMS:
continue

scanned_platform = scan_platform(
platform_slug, fs_platforms, metadata_sources=metadata_sources
)
if platform:
scanned_platform.id = platform.id
# Keep the existing ids if they exist on the platform
scanned_platform.igdb_id = scanned_platform.igdb_id or platform.igdb_id
scanned_platform.moby_id = scanned_platform.moby_id or platform.moby_id

scan_stats.scanned_platforms += 1
scan_stats.added_platforms += 1 if not platform else 0
scan_stats.metadata_platforms += (
1 if scanned_platform.igdb_id or scanned_platform.moby_id else 0
)

platform = db_platform_handler.add_platform(scanned_platform)

await sm.emit(
"scan:scanning_platform",
PlatformSchema.model_validate(platform).model_dump(
include={"id", "name", "slug"}
),
)
await sm.emit("", None)

# Scanning firmware
try:
fs_firmware = fs_firmware_handler.get_firmware(platform)
except FirmwareNotFoundException:
fs_firmware = []

if len(fs_firmware) == 0:
log.warning(
" ⚠️ No firmware found, skipping firmware scan for this platform"
)
else:
log.info(f" {len(fs_firmware)} firmware files found")

for fs_fw in fs_firmware:
# Break early if the flag is set
async with httpx.AsyncClient() as requests_client:
for platform_slug in platform_list:
# Stop the scan if the flag is set
if redis_client.get(STOP_SCAN_FLAG):
await stop_scan()
break

firmware = db_firmware_handler.get_firmware_by_filename(
platform.id, fs_fw
platform = db_platform_handler.get_platform_by_fs_slug(platform_slug)
if platform and scan_type == ScanType.NEW_PLATFORMS:
continue

scanned_platform = scan_platform(
platform_slug, fs_platforms, metadata_sources=metadata_sources
)
if platform:
scanned_platform.id = platform.id
# Keep the existing ids if they exist on the platform
scanned_platform.igdb_id = (
scanned_platform.igdb_id or platform.igdb_id
)
scanned_platform.moby_id = (
scanned_platform.moby_id or platform.moby_id
)

scanned_firmware = scan_firmware(
platform=platform,
file_name=fs_fw,
firmware=firmware,
scan_stats.scanned_platforms += 1
scan_stats.added_platforms += 1 if not platform else 0
scan_stats.metadata_platforms += (
1 if scanned_platform.igdb_id or scanned_platform.moby_id else 0
)

scan_stats.scanned_firmware += 1
scan_stats.added_firmware += 1 if not firmware else 0
platform = db_platform_handler.add_platform(scanned_platform)

_added_firmware = db_firmware_handler.add_firmware(scanned_firmware)
firmware = db_firmware_handler.get_firmware(_added_firmware.id)
await sm.emit(
"scan:scanning_platform",
PlatformSchema.model_validate(platform).model_dump(
include={"id", "name", "slug"}
),
)
await sm.emit("", None)

# Scanning roms
try:
fs_roms = fs_rom_handler.get_roms(platform)
except RomsNotFoundException as e:
log.error(e)
continue
# Scanning firmware
try:
fs_firmware = fs_firmware_handler.get_firmware(platform)
except FirmwareNotFoundException:
fs_firmware = []

if len(fs_roms) == 0:
log.warning(
" ⚠️ No roms found, verify that the folder structure is correct"
)
else:
log.info(f" {len(fs_roms)} roms found")
if len(fs_firmware) == 0:
log.warning(
" ⚠️ No firmware found, skipping firmware scan for this platform"
)
else:
log.info(f" {len(fs_firmware)} firmware files found")

for fs_rom in fs_roms:
# Break early if the flag is set
if redis_client.get(STOP_SCAN_FLAG):
break
for fs_fw in fs_firmware:
# Break early if the flag is set
if redis_client.get(STOP_SCAN_FLAG):
break

rom = db_rom_handler.get_rom_by_filename(
platform.id, fs_rom["file_name"]
)
firmware = db_firmware_handler.get_firmware_by_filename(
platform.id, fs_fw
)

if _should_scan_rom(
scan_type=scan_type, rom=rom, selected_roms=selected_roms
):
scanned_rom = await scan_rom(
scanned_firmware = scan_firmware(
platform=platform,
rom_attrs=fs_rom,
scan_type=scan_type,
rom=rom,
metadata_sources=metadata_sources,
file_name=fs_fw,
firmware=firmware,
)

scan_stats.scanned_roms += 1
scan_stats.added_roms += 1 if not rom else 0
scan_stats.metadata_roms += (
1 if scanned_rom.igdb_id or scanned_rom.moby_id else 0
)
scan_stats.scanned_firmware += 1
scan_stats.added_firmware += 1 if not firmware else 0

_added_rom = db_rom_handler.add_rom(scanned_rom)
_added_firmware = db_firmware_handler.add_firmware(scanned_firmware)
firmware = db_firmware_handler.get_firmware(_added_firmware.id)

path_cover_s, path_cover_l = fs_resource_handler.get_cover(
overwrite=True,
entity=_added_rom,
url_cover=_added_rom.url_cover,
)
# Scanning roms
try:
fs_roms = fs_rom_handler.get_roms(platform)
except RomsNotFoundException as e:
log.error(e)
continue

path_screenshots = fs_resource_handler.get_rom_screenshots(
rom=_added_rom,
url_screenshots=_added_rom.url_screenshots,
if len(fs_roms) == 0:
log.warning(
" ⚠️ No roms found, verify that the folder structure is correct"
)
else:
log.info(f" {len(fs_roms)} roms found")

_added_rom.path_cover_s = path_cover_s
_added_rom.path_cover_l = path_cover_l
_added_rom.path_screenshots = path_screenshots
# Update the scanned rom with the cover and screenshots paths and update database
db_rom_handler.update_rom(
_added_rom.id,
{
c: getattr(_added_rom, c)
for c in inspect(_added_rom).mapper.column_attrs.keys()
},
for fs_rom in fs_roms:
# Break early if the flag is set
if redis_client.get(STOP_SCAN_FLAG):
break

rom = db_rom_handler.get_rom_by_filename(
platform.id, fs_rom["file_name"]
)

await sm.emit(
"scan:scanning_rom",
{
"platform_name": platform.name,
"platform_slug": platform.slug,
**RomSchema.model_validate(_added_rom).model_dump(
exclude={"created_at", "updated_at", "rom_user"}
),
},
if _should_scan_rom(
scan_type=scan_type, rom=rom, selected_roms=selected_roms
):
scanned_rom = await scan_rom(
requests_client=requests_client,
platform=platform,
rom_attrs=fs_rom,
scan_type=scan_type,
rom=rom,
metadata_sources=metadata_sources,
)

scan_stats.scanned_roms += 1
scan_stats.added_roms += 1 if not rom else 0
scan_stats.metadata_roms += (
1 if scanned_rom.igdb_id or scanned_rom.moby_id else 0
)

_added_rom = db_rom_handler.add_rom(scanned_rom)

path_cover_s, path_cover_l = fs_resource_handler.get_cover(
overwrite=True,
entity=_added_rom,
url_cover=_added_rom.url_cover,
)

path_screenshots = fs_resource_handler.get_rom_screenshots(
rom=_added_rom,
url_screenshots=_added_rom.url_screenshots,
)

_added_rom.path_cover_s = path_cover_s
_added_rom.path_cover_l = path_cover_l
_added_rom.path_screenshots = path_screenshots
# Update the scanned rom with the cover and screenshots paths and update database
db_rom_handler.update_rom(
_added_rom.id,
{
c: getattr(_added_rom, c)
for c in inspect(_added_rom).mapper.column_attrs.keys()
},
)

await sm.emit(
"scan:scanning_rom",
{
"platform_name": platform.name,
"platform_slug": platform.slug,
**RomSchema.model_validate(_added_rom).model_dump(
exclude={"created_at", "updated_at", "rom_user"}
),
},
)
await sm.emit("", None)

# Only purge entries if there are some file remaining in the library
# This protects against accidental deletion of entries when
# the folder structure is not correct or the drive is not mounted
if len(fs_roms) > 0:
db_rom_handler.purge_roms(
platform.id, [rom["file_name"] for rom in fs_roms]
)
await sm.emit("", None)

# Only purge entries if there are some file remaining in the library
# This protects against accidental deletion of entries when
# the folder structure is not correct or the drive is not mounted
if len(fs_roms) > 0:
db_rom_handler.purge_roms(
platform.id, [rom["file_name"] for rom in fs_roms]
)

# Same protection for firmware
if len(fs_firmware) > 0:
db_firmware_handler.purge_firmware(
platform.id, [fw for fw in fs_firmware]
)
# Same protection for firmware
if len(fs_firmware) > 0:
db_firmware_handler.purge_firmware(
platform.id, [fw for fw in fs_firmware]
)

# Same protection for platforms
if len(fs_platforms) > 0:
db_platform_handler.purge_platforms(fs_platforms)
# Same protection for platforms
if len(fs_platforms) > 0:
db_platform_handler.purge_platforms(fs_platforms)

log.info(emoji.emojize(":check_mark: Scan completed "))
await sm.emit("scan:done", scan_stats.__dict__)
log.info(emoji.emojize(":check_mark: Scan completed "))
await sm.emit("scan:done", scan_stats.__dict__)
except Exception as e:
log.error(e)
# Catch all exceptions and emit error to the client
Expand Down
Loading

0 comments on commit 7a45282

Please sign in to comment.