add concurrency to LocalLibrary load tracks/playlists functions (#41)

docs/release-history.rst

@@ -32,6 +32,16 @@ The format is based on `Keep a Changelog <https://keepachangelog.com/en>`_,
 and this project adheres to `Semantic Versioning <https://semver.org/spec/v2.0.0.html>`_
 
 
+0.9.0
+=====
+
+Changed
+-------
+
+* :py:meth:`.LocalLibrary.load_tracks` and :py:meth:`.LocalLibrary.load_playlists` now run concurrently.
+* Made :py:func:`.load_tracks` and :py:func:`.load_playlists` utility functions more DRY
+* Move :py:meth:`.TagReader.load` from :py:class:`.LocalTrack` to super class :py:class:`.TagReader`
+
 0.8.1
 =====
 

musify/local/file.py

@@ -17,7 +17,7 @@
 
 from PIL import Image, UnidentifiedImageError
 
-from musify.local.exception import InvalidFileType, ImageLoadError
+from musify.local.exception import InvalidFileType, ImageLoadError, FileDoesNotExistError
 from musify.shared.core.misc import PrettyPrinter
 
 
@@ -94,7 +94,7 @@ def date_modified(self) -> datetime | None:
         return datetime.fromtimestamp(getmtime(self.path)) if exists(self.path) else None
 
     def _validate_type(self, path: str) -> None:
-        """Raises exception if the path's extension is not accepted"""
+        """Raises an exception if the ``path`` extension is not accepted"""
         ext = splitext(path)[1].casefold()
         if ext not in self.valid_extensions:
             raise InvalidFileType(
@@ -103,6 +103,12 @@ def _validate_type(self, path: str) -> None:
                 f"Use only: {', '.join(self.valid_extensions)}"
             )
 
+    @staticmethod
+    def _validate_existence(path: str):
+        """Raises an exception if there is no file at the given ``path``"""
+        if not path or not exists(path):
+            raise FileDoesNotExistError(f"File not found | {path}")
+
     @classmethod
     def get_filepaths(cls, folder: str) -> set[str]:
         """Get all files in a given folder that match this File object's valid filetypes recursively."""

musify/local/library/library.py

@@ -2,11 +2,13 @@
 The core, basic library implementation which is just a simple set of folders.
 """
 import itertools
-from collections.abc import Collection, Mapping, Iterable
+from collections.abc import Collection, Mapping
 from functools import reduce
 from os.path import splitext, join, exists, basename, dirname
 from typing import Any
 
+from concurrent.futures import ThreadPoolExecutor
+
 from musify.local.collection import LocalCollection, LocalFolder, LocalAlbum, LocalArtist, LocalGenres
 from musify.local.file import PathMapper, PathStemMapper
 from musify.local.playlist import LocalPlaylist, load_playlist, PLAYLIST_CLASSES
@@ -274,27 +276,37 @@ def _log_errors(self, message: str = "Could not load") -> None:
     ###########################################################################
     ## Tracks
     ###########################################################################
+    def load_track(self, path: str) -> LocalTrack | None:
+        """
+        Wrapper for :py:func:`load_track` which automatically loads the track at the given ``path``
+        and assigns optional arguments using this library's attributes.
+
+        Handles exceptions by logging paths which produce errors to internal list of ``errors``.
+        """
+        try:
+            return load_track(path=path, remote_wrangler=self.remote_wrangler)
+        except MusifyError as ex:
+            self.logger.debug(f"Load error for track: {path} - {ex}")
+            self.errors.append(path)
+
     def load_tracks(self) -> None:
         """Load all tracks from all the valid paths in this library, replacing currently loaded tracks."""
+        if not self._track_paths:
+            return
+
         self.logger.debug(f"Load {self.name} tracks: START")
         self.logger.info(
             f"\33[1;95m  >\33[1;97m Extracting metadata and properties for {len(self._track_paths)} tracks \33[0m"
         )
 
-        tracks: list[LocalTrack] = []
-        bar: Iterable[str] = self.logger.get_progress_bar(
-            iterable=self._track_paths, desc="Loading tracks", unit="tracks"
-        )
-        for path in bar:
-            try:
-                tracks.append(load_track(path=path, remote_wrangler=self.remote_wrangler))
-            except MusifyError as ex:
-                self.logger.debug(f"Load error: {path} - {ex}")
-                self.errors.append(path)
-                continue
+        with ThreadPoolExecutor(thread_name_prefix="track-loader") as executor:
+            tasks = executor.map(self.load_track, self._track_paths)
+            bar = self.logger.get_progress_bar(
+                tasks, desc="Loading tracks", unit="tracks", total=len(self._track_paths)
+            )
+            self._tracks = list(bar)
 
-        self._tracks = tracks
-        self._log_errors()
+        self._log_errors("Could not load the following tracks")
         self.logger.debug(f"Load {self.name} tracks: DONE\n")
 
     def log_tracks(self) -> None:
@@ -310,6 +322,21 @@ def log_tracks(self) -> None:
     ###########################################################################
     ## Playlists
     ###########################################################################
+    def load_playlist(self, path: str) -> LocalPlaylist:
+        """
+        Wrapper for :py:func:`load_playlist` which automatically loads the playlist at the given ``path``
+        and assigns optional arguments using this library's attributes.
+
+        Handles exceptions by logging paths which produce errors to internal list of ``errors``.
+        """
+        try:
+            return load_playlist(
+                path=path, tracks=self.tracks, path_mapper=self.path_mapper, remote_wrangler=self.remote_wrangler,
+            )
+        except MusifyError as ex:
+            self.logger.debug(f"Load error for playlist: {path} - {ex}")
+            self.errors.append(path)
+
     def load_playlists(self) -> None:
         """
         Load all playlists found in this library's ``playlist_folder``,
@@ -326,15 +353,14 @@ def load_playlists(self) -> None:
             f"\33[1;95m  >\33[1;97m Loading playlist data for {len(self._playlist_paths)} playlists \33[0m"
         )
 
-        iterable = self._playlist_paths.items()
-        bar = self.logger.get_progress_bar(iterable=iterable, desc="Loading playlists", unit="playlists")
-        playlists: list[LocalPlaylist] = [
-            load_playlist(
-                path=path, tracks=self.tracks, path_mapper=self.path_mapper, remote_wrangler=self.remote_wrangler,
-            ) for name, path in bar
-        ]
+        with ThreadPoolExecutor(thread_name_prefix="playlist-loader") as executor:
+            tasks = executor.map(self.load_playlist, self._playlist_paths.values())
+            bar = self.logger.get_progress_bar(
+                tasks, desc="Loading playlists", unit="playlists", total=len(self._playlist_paths)
+            )
+            self._playlists = {pl.name: pl for pl in sorted(bar, key=lambda x: x.name.casefold())}
 
-        self._playlists = {pl.name: pl for pl in sorted(playlists, key=lambda x: x.name.casefold())}
+        self._log_errors("Could not load the following playlists")
         self.logger.debug(f"Load {self.name} playlists: DONE\n")
 
     def log_playlists(self) -> None:

musify/local/playlist/utils.py

@@ -45,10 +45,8 @@ def load_playlist(
     :raise InvalidFileType: If the file type is not supported.
     """
     ext = splitext(path)[1].casefold()
+    if ext not in PLAYLIST_FILETYPES:
+        raise InvalidFileType(ext, f"Not an accepted extension. Use only: {', '.join(PLAYLIST_FILETYPES)}")
 
-    if ext in M3U.valid_extensions:
-        return M3U(path=path, tracks=tracks, path_mapper=path_mapper, remote_wrangler=remote_wrangler)
-    elif ext in XAutoPF.valid_extensions:
-        return XAutoPF(path=path, tracks=tracks, path_mapper=path_mapper)
-
-    raise InvalidFileType(ext, f"Not an accepted extension. Use only: {', '.join(PLAYLIST_FILETYPES)}")
+    cls = next(cls for cls in PLAYLIST_CLASSES if ext in cls.valid_extensions)
+    return cls(path=path, tracks=tracks, path_mapper=path_mapper, remote_wrangler=remote_wrangler)

musify/local/playlist/xautopf.py

@@ -87,7 +87,9 @@ def description(self, value: str | None):
     def image_links(self):
         return {}
 
-    def __init__(self, path: str, tracks: Collection[LocalTrack] = (), path_mapper: PathMapper = PathMapper()):
+    def __init__(
+            self, path: str, tracks: Collection[LocalTrack] = (), path_mapper: PathMapper = PathMapper(), *_, **__
+    ):
         self._validate_type(path)
         if not exists(path):
             # TODO: implement creation of auto-playlist from scratch (very low priority)

musify/local/track/base/reader.py

@@ -1,7 +1,6 @@
 """
 Implements all functionality pertaining to reading metadata/tags/properties of a :py:class:`LocalTrack`.
 """
-
 import datetime
 import re
 from abc import ABCMeta, abstractmethod
@@ -22,7 +21,7 @@
 from musify.shared.utils import to_collection
 
 
-class TagReader(LocalItem, Track, metaclass=ABCMeta):
+class TagReader[T: mutagen.FileType](LocalItem, Track, metaclass=ABCMeta):
     """
     Functionality for reading tags/metadata/properties from a loaded audio file.
 
@@ -383,6 +382,21 @@ def __init__(self, remote_wrangler: RemoteDataWrangler = None):
 
         self._loaded = False
 
+    def load(self, path: str | None = None) -> T:
+        """
+        Load local file using mutagen from the given path or the path stored in the object's ``file``.
+        Re-formats to case-sensitive system path if applicable.
+
+        :param path: The path to the file. If not given, use the stored ``file`` path.
+        :return: Mutagen file object or None if load error.
+        :raise FileDoesNotExistError: If the file cannot be found.
+        :raise InvalidFileType: If the file type is not supported.
+        """
+        path = path or self.path
+        self._validate_type(path)
+        self._validate_existence(path)
+        return mutagen.File(path)
+
     def load_metadata(self) -> None:
         """Driver for extracting all supported metadata from a loaded file"""
 

musify/local/track/base/track.py

@@ -5,12 +5,11 @@
 import os
 from abc import ABCMeta
 from copy import deepcopy
-from os.path import join, exists, dirname
+from os.path import join, dirname
 from typing import Any, Self
 
 import mutagen
 
-from musify.local.exception import FileDoesNotExistError
 from musify.local.track.base.reader import TagReader
 from musify.local.track.base.writer import TagWriter
 from musify.shared.core.base import Item
@@ -45,24 +44,6 @@ def __init__(self, file: str | T, remote_wrangler: RemoteDataWrangler = None):
         self._file: T = self.load(file) if isinstance(file, str) else file
         self.load_metadata()
 
-    def load(self, path: str | None = None) -> T:
-        """
-        Load local file using mutagen from the given path or the path stored in the object's ``file``.
-        Re-formats to case-sensitive system path if applicable.
-
-        :param path: The path to the file. If not given, use the stored ``file`` path.
-        :return: Mutagen file object or None if load error.
-        :raise FileDoesNotExistError: If the file cannot be found.
-        :raise InvalidFileType: If the file type is not supported.
-        """
-        path = path or self.path
-        self._validate_type(path)
-
-        if not path or not exists(path):
-            raise FileDoesNotExistError(f"File not found | {path}")
-
-        return mutagen.File(path)
-
     def merge(self, track: Track, tags: UnitIterable[TrackField] = TrackField.ALL) -> None:
         """Set the tags of this track equal to the given ``track``. Give a list of ``tags`` to limit which are set"""
         tag_names = TrackField.__tags__ if tags == TrackField.ALL else set(TrackField.to_tags(tags))
@@ -103,12 +84,12 @@ def __eq__(self, item: Item):
     def __copy__(self):
         """Copy object by reloading from the file object in memory"""
         if not self.file.tags:  # file is not a real file, used in testing
-            obj = self.__class__.__new__(self.__class__)
+            new = self.__class__.__new__(self.__class__)
             for key in TagReader.__slots__:
-                setattr(obj, key, getattr(self, key))
+                setattr(new, key, getattr(self, key))
             for key in self.__slots__:
-                setattr(obj, key, getattr(self, key))
-            return obj
+                setattr(new, key, getattr(self, key))
+            return new
         return self.__class__(file=self.file, remote_wrangler=self.remote_wrangler)
 
     def __deepcopy__(self, _: dict = None):

musify/local/track/base/writer.py

@@ -26,7 +26,7 @@ class SyncResultTrack(Result):
     updated: Mapping[Tags, int]
 
 
-class TagWriter(TagReader, metaclass=ABCMeta):
+class TagWriter[T: mutagen.FileType](TagReader[T], metaclass=ABCMeta):
     """Functionality for updating and removing tags/metadata/properties from a loaded audio file."""
 
     #: The date format to use when saving string representations of dates to tag values

musify/local/track/utils.py

@@ -32,16 +32,8 @@ def load_track(path: str, remote_wrangler: RemoteDataWrangler = None) -> LocalTr
     :raise InvalidFileType: If the file type is not supported.
     """
     ext = splitext(path)[1].casefold()
+    if ext not in TRACK_FILETYPES:
+        raise InvalidFileType(ext, f"Not an accepted extension. Use only: {', '.join(TRACK_FILETYPES)}")
 
-    if ext in FLAC.valid_extensions:
-        return FLAC(file=path, remote_wrangler=remote_wrangler)
-    elif ext in MP3.valid_extensions:
-        return MP3(file=path, remote_wrangler=remote_wrangler)
-    elif ext in M4A.valid_extensions:
-        return M4A(file=path, remote_wrangler=remote_wrangler)
-    elif ext in WMA.valid_extensions:
-        return WMA(file=path, remote_wrangler=remote_wrangler)
-
-    raise InvalidFileType(
-        ext, f"Not an accepted extension. Use only: {', '.join(TRACK_FILETYPES)}"
-    )
+    cls = next(cls for cls in TRACK_CLASSES if ext in cls.valid_extensions)
+    return cls(file=path, remote_wrangler=remote_wrangler)

musify/spotify/object.py

@@ -221,22 +221,22 @@ def load(
             *_,
             **__
     ) -> Self:
-        obj = cls.__new__(cls)
-        obj.api = api
+        self = cls.__new__(cls)
+        self.api = api
 
         # set a mock response with URL to load from
         id_ = cls.extract_ids(value)[0]
-        obj._response = {
+        self._response = {
             "href": cls.convert(id_, kind=RemoteObjectType.TRACK, type_in=RemoteIDType.ID, type_out=RemoteIDType.URL)
         }
-        obj.reload(
+        self.reload(
             features=features,
             analysis=analysis,
             extend_album=extend_album,
             extend_artists=extend_artists,
             use_cache=use_cache
         )
-        return obj
+        return self
 
     def reload(
             self,
@@ -295,11 +295,11 @@ def load(
             if kind == RemoteObjectType.PLAYLIST:
                 value = api.get_playlist_url(value)
 
-            obj = cls.__new__(cls)
-            obj.api = api
-            obj._response = {"href": value}
-            obj.reload(*args, **kwargs, extend_tracks=extend_tracks, use_cache=use_cache)
-            return obj
+            self = cls.__new__(cls)
+            self.api = api
+            self._response = {"href": value}
+            self.reload(*args, **kwargs, extend_tracks=extend_tracks, use_cache=use_cache)
+            return self
 
         # get response
         if isinstance(value, MutableMapping) and cls.get_item_type(value) == kind:
@@ -692,16 +692,16 @@ def load(
             *_,
             **__
     ) -> Self:
-        obj = cls.__new__(cls)
-        obj.api = api
+        self = cls.__new__(cls)
+        self.api = api
 
         # set a mock response with URL to load from
         id_ = cls.extract_ids(value)[0]
-        obj._response = {
+        self._response = {
             "href": cls.convert(id_, kind=RemoteObjectType.ARTIST, type_in=RemoteIDType.ID, type_out=RemoteIDType.URL)
         }
-        obj.reload(extend_albums=extend_albums, extend_tracks=extend_tracks, use_cache=use_cache)
-        return obj
+        self.reload(extend_albums=extend_albums, extend_tracks=extend_tracks, use_cache=use_cache)
+        return self
 
     def reload(
             self, extend_albums: bool = False, extend_tracks: bool = False, use_cache: bool = True, *_, **__

tests/processors/test_filter.py

@@ -100,7 +100,7 @@ def test_filter_nested(self, comparers: list[Comparer], tracks: list[LocalTrack]
         for i, track in enumerate(tracks):
             track.track_number = i + 1
             track.year = 2020 if i >= 8 else 1990
-            track.bpm = randrange(80, 100) if 10 < i < 15 else 120
+            track.bpm = randrange(85, 95) if 10 < i < 15 else 120
 
         comparer = FilterComparers(
             comparers={comparers[0]: (False, sub_filter_1), comparers[1]: (False, sub_filter_2)},