Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

misc: Migrate MobyGamesHandler to async #1011

Merged
merged 2 commits into from
Jul 23, 2024
Merged

misc: Migrate MobyGamesHandler to async #1011

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6d8a871
misc: Migrate MobyGamesHandler to async
adamantike Jul 22, 2024
7c6dfa8
feat: Use ContextVar to share httpx AsyncClient instance between requ…
adamantike Jul 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion backend/endpoints/rom.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,10 @@ async def update_rom(
cleaned_data.get("moby_id", "")
and int(cleaned_data.get("moby_id", "")) != rom.moby_id
):
moby_rom = meta_moby_handler.get_rom_by_id(cleaned_data["moby_id"])
moby_rom = await meta_moby_handler.get_rom_by_id(
requests_client=request.app.requests_client,
moby_id=cleaned_data["moby_id"],
)
cleaned_data.update(moby_rom)
path_screenshots = fs_resource_handler.get_rom_screenshots(
rom=rom,
Expand Down
10 changes: 6 additions & 4 deletions backend/endpoints/search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ async def search_rom(
igdb_matched_roms = meta_igdb_handler.get_matched_roms_by_id(
int(search_term)
)
moby_matched_roms = meta_moby_handler.get_matched_roms_by_id(
int(search_term)
moby_matched_roms = await meta_moby_handler.get_matched_roms_by_id(
requests_client=request.app.requests_client, moby_id=int(search_term)
)
except ValueError as exc:
log.error(f"Search error: invalid ID '{search_term}'")
Expand All @@ -74,8 +74,10 @@ async def search_rom(
igdb_matched_roms = meta_igdb_handler.get_matched_roms_by_name(
search_term, _get_main_platform_igdb_id(rom.platform)
)
moby_matched_roms = meta_moby_handler.get_matched_roms_by_name(
search_term, rom.platform.moby_id
moby_matched_roms = await meta_moby_handler.get_matched_roms_by_name(
requests_client=request.app.requests_client,
search_term=search_term,
platform_moby_id=rom.platform.moby_id,
)

merged_dict = {
Expand Down
285 changes: 146 additions & 139 deletions backend/endpoints/sockets/scan.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from typing import Final

import emoji
import httpx
import socketio # type: ignore
from config import SCAN_TIMEOUT
from endpoints.responses.platform import PlatformSchema
Expand Down Expand Up @@ -132,172 +133,178 @@ async def stop_scan():
else:
log.info(f"Found {len(platform_list)} platforms in file system ")

for platform_slug in platform_list:
# Stop the scan if the flag is set
if redis_client.get(STOP_SCAN_FLAG):
await stop_scan()
break

platform = db_platform_handler.get_platform_by_fs_slug(platform_slug)
if platform and scan_type == ScanType.NEW_PLATFORMS:
continue

scanned_platform = scan_platform(
platform_slug, fs_platforms, metadata_sources=metadata_sources
)
if platform:
scanned_platform.id = platform.id
# Keep the existing ids if they exist on the platform
scanned_platform.igdb_id = scanned_platform.igdb_id or platform.igdb_id
scanned_platform.moby_id = scanned_platform.moby_id or platform.moby_id

scan_stats.scanned_platforms += 1
scan_stats.added_platforms += 1 if not platform else 0
scan_stats.metadata_platforms += (
1 if scanned_platform.igdb_id or scanned_platform.moby_id else 0
)

platform = db_platform_handler.add_platform(scanned_platform)

await sm.emit(
"scan:scanning_platform",
PlatformSchema.model_validate(platform).model_dump(
include={"id", "name", "slug"}
),
)
await sm.emit("", None)

# Scanning firmware
try:
fs_firmware = fs_firmware_handler.get_firmware(platform)
except FirmwareNotFoundException:
fs_firmware = []

if len(fs_firmware) == 0:
log.warning(
" ⚠️ No firmware found, skipping firmware scan for this platform"
)
else:
log.info(f" {len(fs_firmware)} firmware files found")

for fs_fw in fs_firmware:
# Break early if the flag is set
async with httpx.AsyncClient() as requests_client:
for platform_slug in platform_list:
# Stop the scan if the flag is set
if redis_client.get(STOP_SCAN_FLAG):
await stop_scan()
break

firmware = db_firmware_handler.get_firmware_by_filename(
platform.id, fs_fw
platform = db_platform_handler.get_platform_by_fs_slug(platform_slug)
if platform and scan_type == ScanType.NEW_PLATFORMS:
continue

scanned_platform = scan_platform(
platform_slug, fs_platforms, metadata_sources=metadata_sources
)
if platform:
scanned_platform.id = platform.id
# Keep the existing ids if they exist on the platform
scanned_platform.igdb_id = (
scanned_platform.igdb_id or platform.igdb_id
)
scanned_platform.moby_id = (
scanned_platform.moby_id or platform.moby_id
)

scanned_firmware = scan_firmware(
platform=platform,
file_name=fs_fw,
firmware=firmware,
scan_stats.scanned_platforms += 1
scan_stats.added_platforms += 1 if not platform else 0
scan_stats.metadata_platforms += (
1 if scanned_platform.igdb_id or scanned_platform.moby_id else 0
)

scan_stats.scanned_firmware += 1
scan_stats.added_firmware += 1 if not firmware else 0
platform = db_platform_handler.add_platform(scanned_platform)

_added_firmware = db_firmware_handler.add_firmware(scanned_firmware)
firmware = db_firmware_handler.get_firmware(_added_firmware.id)
await sm.emit(
"scan:scanning_platform",
PlatformSchema.model_validate(platform).model_dump(
include={"id", "name", "slug"}
),
)
await sm.emit("", None)

# Scanning roms
try:
fs_roms = fs_rom_handler.get_roms(platform)
except RomsNotFoundException as e:
log.error(e)
continue
# Scanning firmware
try:
fs_firmware = fs_firmware_handler.get_firmware(platform)
except FirmwareNotFoundException:
fs_firmware = []

if len(fs_roms) == 0:
log.warning(
" ⚠️ No roms found, verify that the folder structure is correct"
)
else:
log.info(f" {len(fs_roms)} roms found")
if len(fs_firmware) == 0:
log.warning(
" ⚠️ No firmware found, skipping firmware scan for this platform"
)
else:
log.info(f" {len(fs_firmware)} firmware files found")

for fs_rom in fs_roms:
# Break early if the flag is set
if redis_client.get(STOP_SCAN_FLAG):
break
for fs_fw in fs_firmware:
# Break early if the flag is set
if redis_client.get(STOP_SCAN_FLAG):
break

rom = db_rom_handler.get_rom_by_filename(
platform.id, fs_rom["file_name"]
)
firmware = db_firmware_handler.get_firmware_by_filename(
platform.id, fs_fw
)

if _should_scan_rom(
scan_type=scan_type, rom=rom, selected_roms=selected_roms
):
scanned_rom = await scan_rom(
scanned_firmware = scan_firmware(
platform=platform,
rom_attrs=fs_rom,
scan_type=scan_type,
rom=rom,
metadata_sources=metadata_sources,
file_name=fs_fw,
firmware=firmware,
)

scan_stats.scanned_roms += 1
scan_stats.added_roms += 1 if not rom else 0
scan_stats.metadata_roms += (
1 if scanned_rom.igdb_id or scanned_rom.moby_id else 0
)
scan_stats.scanned_firmware += 1
scan_stats.added_firmware += 1 if not firmware else 0

_added_rom = db_rom_handler.add_rom(scanned_rom)
_added_firmware = db_firmware_handler.add_firmware(scanned_firmware)
firmware = db_firmware_handler.get_firmware(_added_firmware.id)

path_cover_s, path_cover_l = fs_resource_handler.get_cover(
overwrite=True,
entity=_added_rom,
url_cover=_added_rom.url_cover,
)
# Scanning roms
try:
fs_roms = fs_rom_handler.get_roms(platform)
except RomsNotFoundException as e:
log.error(e)
continue

path_screenshots = fs_resource_handler.get_rom_screenshots(
rom=_added_rom,
url_screenshots=_added_rom.url_screenshots,
if len(fs_roms) == 0:
log.warning(
" ⚠️ No roms found, verify that the folder structure is correct"
)
else:
log.info(f" {len(fs_roms)} roms found")

_added_rom.path_cover_s = path_cover_s
_added_rom.path_cover_l = path_cover_l
_added_rom.path_screenshots = path_screenshots
# Update the scanned rom with the cover and screenshots paths and update database
db_rom_handler.update_rom(
_added_rom.id,
{
c: getattr(_added_rom, c)
for c in inspect(_added_rom).mapper.column_attrs.keys()
},
for fs_rom in fs_roms:
# Break early if the flag is set
if redis_client.get(STOP_SCAN_FLAG):
break

rom = db_rom_handler.get_rom_by_filename(
platform.id, fs_rom["file_name"]
)

await sm.emit(
"scan:scanning_rom",
{
"platform_name": platform.name,
"platform_slug": platform.slug,
**RomSchema.model_validate(_added_rom).model_dump(
exclude={"created_at", "updated_at", "rom_user"}
),
},
if _should_scan_rom(
scan_type=scan_type, rom=rom, selected_roms=selected_roms
):
scanned_rom = await scan_rom(
requests_client=requests_client,
platform=platform,
rom_attrs=fs_rom,
scan_type=scan_type,
rom=rom,
metadata_sources=metadata_sources,
)

scan_stats.scanned_roms += 1
scan_stats.added_roms += 1 if not rom else 0
scan_stats.metadata_roms += (
1 if scanned_rom.igdb_id or scanned_rom.moby_id else 0
)

_added_rom = db_rom_handler.add_rom(scanned_rom)

path_cover_s, path_cover_l = fs_resource_handler.get_cover(
overwrite=True,
entity=_added_rom,
url_cover=_added_rom.url_cover,
)

path_screenshots = fs_resource_handler.get_rom_screenshots(
rom=_added_rom,
url_screenshots=_added_rom.url_screenshots,
)

_added_rom.path_cover_s = path_cover_s
_added_rom.path_cover_l = path_cover_l
_added_rom.path_screenshots = path_screenshots
# Update the scanned rom with the cover and screenshots paths and update database
db_rom_handler.update_rom(
_added_rom.id,
{
c: getattr(_added_rom, c)
for c in inspect(_added_rom).mapper.column_attrs.keys()
},
)

await sm.emit(
"scan:scanning_rom",
{
"platform_name": platform.name,
"platform_slug": platform.slug,
**RomSchema.model_validate(_added_rom).model_dump(
exclude={"created_at", "updated_at", "rom_user"}
),
},
)
await sm.emit("", None)

# Only purge entries if there are some file remaining in the library
# This protects against accidental deletion of entries when
# the folder structure is not correct or the drive is not mounted
if len(fs_roms) > 0:
db_rom_handler.purge_roms(
platform.id, [rom["file_name"] for rom in fs_roms]
)
await sm.emit("", None)

# Only purge entries if there are some file remaining in the library
# This protects against accidental deletion of entries when
# the folder structure is not correct or the drive is not mounted
if len(fs_roms) > 0:
db_rom_handler.purge_roms(
platform.id, [rom["file_name"] for rom in fs_roms]
)

# Same protection for firmware
if len(fs_firmware) > 0:
db_firmware_handler.purge_firmware(
platform.id, [fw for fw in fs_firmware]
)
# Same protection for firmware
if len(fs_firmware) > 0:
db_firmware_handler.purge_firmware(
platform.id, [fw for fw in fs_firmware]
)

# Same protection for platforms
if len(fs_platforms) > 0:
db_platform_handler.purge_platforms(fs_platforms)
# Same protection for platforms
if len(fs_platforms) > 0:
db_platform_handler.purge_platforms(fs_platforms)

log.info(emoji.emojize(":check_mark: Scan completed "))
await sm.emit("scan:done", scan_stats.__dict__)
log.info(emoji.emojize(":check_mark: Scan completed "))
await sm.emit("scan:done", scan_stats.__dict__)
except Exception as e:
log.error(e)
# Catch all exceptions and emit error to the client
Expand Down
Loading
Loading