From 604c7c0f417d24da918ac135cd25514050cdf5e8 Mon Sep 17 00:00:00 2001 From: blast Date: Mon, 7 Oct 2024 13:07:03 +0200 Subject: [PATCH] implementing log and tasks --- requirements.txt | 3 +- spotisub/__init__.py | 9 ++--- spotisub/database.py | 21 +++++++++++ spotisub/generator.py | 37 +++++++++++++------ spotisub/helpers/subsonic_helper.py | 57 +++++++++++++++++------------ spotisub/routes.py | 24 ++++++------ spotisub/static/css/light.css | 22 +++++++++++ spotisub/static/js/logs.js | 2 +- spotisub/templates/base.html | 2 +- spotisub/templates/logs.html | 4 +- spotisub/templates/tasks.html | 43 +++++++++++++++++++++- spotisub/utils.py | 13 +++++++ 12 files changed, 180 insertions(+), 57 deletions(-) diff --git a/requirements.txt b/requirements.txt index 475d207..f413121 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,4 +30,5 @@ Mako==1.2.3 MarkupSafe==2.1.1 visitor==0.1.3 WTForms==3.0.1 -zipp==3.10.0 \ No newline at end of file +zipp==3.10.0 +pygtail==0.14.0 \ No newline at end of file diff --git a/spotisub/__init__.py b/spotisub/__init__.py index ab954c2..6e51351 100644 --- a/spotisub/__init__.py +++ b/spotisub/__init__.py @@ -1,7 +1,7 @@ """Spotisub init module""" import logging import os - +from logging.handlers import RotatingFileHandler from flask import Flask from flask_bootstrap import Bootstrap from flask_sqlalchemy import SQLAlchemy @@ -15,11 +15,10 @@ logging.basicConfig( handlers=[ - logging.FileHandler( - os.path.abspath( + RotatingFileHandler(os.path.abspath( os.curdir) + - "/cache/spotisub.log", - mode="w"), + "/cache/spotisub.log", maxBytes=1048576, + backupCount=5), logging.StreamHandler()], format='%(asctime)s %(levelname)-8s %(message)s', level=int( diff --git a/spotisub/database.py b/spotisub/database.py index 3d101b2..ba72f8a 100644 --- a/spotisub/database.py +++ b/spotisub/database.py @@ -19,6 +19,7 @@ from sqlalchemy import text from sqlalchemy import desc from sqlalchemy import or_ +from sqlalchemy import distinct from sqlalchemy import collate VERSION = "0.3.3-alpha5" @@ -420,6 +421,25 @@ def select_playlist_info_by_name(name): return value +def select_distinct_type_name(): + """select spotify artists by uuid""" + values = [] + with dbms.db_engine.connect() as conn: + value = None + stmt = select( + distinct(dbms.playlist_info.c.type)) + stmt.compile() + cursor = conn.execute(stmt) + records = cursor.fetchall() + + for row in records: + values.append(row.type) + cursor.close() + conn.close() + + return values + + def select_playlist_info_by_name_with_conn(conn, name): """select spotify artists by uuid""" value = None @@ -1264,6 +1284,7 @@ def limit_and_order_stmt(stmt, page=None, limit=None, order=None, asc=None): if page is not None and limit is not None: stmt = stmt.limit(limit).offset(page * limit) order_by = [] + order = "case when "+order+" is null then 1 else 0 end, "+order if order is not None: if asc: stmt = stmt.order_by(collate(text(order), 'NOCASE')) diff --git a/spotisub/generator.py b/spotisub/generator.py index 0599f44..7bef031 100644 --- a/spotisub/generator.py +++ b/spotisub/generator.py @@ -4,6 +4,8 @@ import random import time import re +import string +import math import threading from datetime import datetime from datetime import timedelta @@ -11,6 +13,7 @@ from spotisub import spotisub from spotisub import constants from spotisub import database +from spotisub import utils from spotisub.helpers import spotipy_helper from spotisub.helpers import subsonic_helper from spotisub.threading.spotisub_thread import thread_with_trace @@ -264,7 +267,7 @@ def artist_top_tracks(uuid): def show_recommendations_for_artist(uuid): """get user saved tracks""" - if not check_thread_running_by_name("reimport_all"): + if not utils.check_thread_running_by_name("reimport_all"): thread = thread_with_trace( target=lambda: show_recommendations_for_artist_run(uuid), name=constants.JOB_AR_ID + "_" + uuid) @@ -325,7 +328,7 @@ def show_recommendations_for_artist_run(uuid): def my_recommendations(uuid): """get user saved tracks""" - if not check_thread_running_by_name("reimport_all"): + if not utils.check_thread_running_by_name("reimport_all"): thread = thread_with_trace( target=lambda: my_recommendations_run(uuid), name=constants.JOB_MR_ID + "_" + uuid) @@ -403,7 +406,7 @@ def my_recommendations_run(uuid): def get_user_saved_tracks(uuid): """get user saved tracks""" - if not check_thread_running_by_name("reimport_all"): + if not utils.check_thread_running_by_name("reimport_all"): thread = thread_with_trace( target=lambda: get_user_saved_tracks_run(uuid), name=constants.JOB_ST_ID + "_" + uuid) @@ -435,7 +438,7 @@ def get_user_saved_tracks_run(uuid): def get_user_playlists(uuid): """get user saved tracks""" - if not check_thread_running_by_name("reimport_all"): + if not utils.check_thread_running_by_name("reimport_all"): thread = thread_with_trace( target=lambda: get_user_playlists_run(uuid), name=constants.JOB_UP_ID + "_" + uuid) @@ -649,6 +652,23 @@ def reimport(uuid): constants.SAVED_GEN_SCHED, constants.SAVED_GEN_SCHED_DEFAULT_VALUE) +def get_tasks(): + tasks = [] + types = database.select_distinct_type_name() + for type in types: + job = scheduler.get_job(type) + if job is not None: + task = {} + task["type"] = type + task["type_desc"] = string.capwords(type.replace("_", " ")) + task["next_execution"] = job.next_run_time.strftime("%H:%M:%S") + task["interval"] = str( math.trunc(job.trigger.interval_length / 60 / 60) ) + " hour(s)" + if len(job.args) > 0: + task["args"] = str( job.args[0] ) + else: + task["args"] = "" + tasks.append(task) + return tasks def poll_playlist(uuid): for thread in threading.enumerate(): @@ -696,17 +716,11 @@ def scan_library(): def reimport_all(): """Used to reimport everything""" - if not check_thread_running_by_name("reimport_all"): + if not utils.check_thread_running_by_name("reimport_all"): thread = thread_with_trace( target=lambda: reimport_all_thread(), name="reimport_all").start() -def check_thread_running_by_name(name): - for thread in threading.enumerate(): - if thread.name == name and thread.is_alive(): - return True - return False - def reimport_all_thread(): """Used to reimport everything""" import_all_user_saved_tracks() @@ -791,6 +805,7 @@ def import_all_user_playlists(): order='playlist_info.subsonic_playlist_name', asc=True) except BaseException: + utils.write_exception() pass scheduler.modify_job(id="init_jobs", next_run_time=datetime.now()) diff --git a/spotisub/helpers/subsonic_helper.py b/spotisub/helpers/subsonic_helper.py index 8ef4eb0..4e85341 100644 --- a/spotisub/helpers/subsonic_helper.py +++ b/spotisub/helpers/subsonic_helper.py @@ -7,6 +7,7 @@ import threading import libsonic import string +from concurrent.futures import ThreadPoolExecutor from expiringdict import ExpiringDict from libsonic.errors import DataNotFoundError from spotipy.exceptions import SpotifyException @@ -20,6 +21,7 @@ from spotisub.classes import ComparisonHelper from spotisub.helpers import musicbrainz_helper +cache_executor = ThreadPoolExecutor(max_workers=2) if os.environ.get(constants.SPOTDL_ENABLED, constants.SPOTDL_ENABLED_DEFAULT_VALUE) == "1": @@ -64,10 +66,13 @@ def load_spotify_cache_from_file(): object = ExpiringDict(max_len=10000, max_age_seconds=43200) path = os.path.abspath(os.curdir) + '/cache/spotify_object_cache.pkl' if os.path.exists(path): - with open(path, 'rb') as f: - old_cache_obj = pickle.load(f) - for key, value in old_cache_obj.items(): - object[key] = value + if os.stat(path).st_size == 0: + os.remove(path) + else: + with open(path, 'rb') as f: + old_cache_obj = pickle.load(f) + for key, value in old_cache_obj.items(): + object[key] = value return object @@ -78,28 +83,34 @@ def save_spotify_cache_to_file(object): def get_spotify_object_from_cache(sp, spotify_uri): - spotify_object = None - if spotify_uri not in spotify_cache: - try: - spotify_object = None - if "track" in spotify_uri: - spotify_object = sp.track(spotify_uri) - elif "album" in spotify_uri: - spotify_object = sp.album(spotify_uri) - elif "artist" in spotify_uri: - spotify_object = sp.artist(spotify_uri) - elif "playlist" in spotify_uri: - spotify_object = sp.playlist(spotify_uri) - if spotify_object is not None: - spotify_cache[spotify_uri] = spotify_object - save_spotify_cache_to_file(spotify_cache) - except SpotifyException: - pass + if spotify_uri in spotify_cache: + return spotify_cache[spotify_uri] else: - spotify_object = spotify_cache[spotify_uri] - return spotify_object + cache_executor.submit(load_spotify_object_to_cache, sp, spotify_uri) + return None +def load_spotify_object_to_cache(sp, spotify_uri): + try: + global spotify_cache + if spotify_uri in spotify_cache: + return + spotify_object = None + if "track" in spotify_uri: + spotify_object = sp.track(spotify_uri) + elif "album" in spotify_uri: + spotify_object = sp.album(spotify_uri) + elif "artist" in spotify_uri: + spotify_object = sp.artist(spotify_uri) + elif "playlist" in spotify_uri: + spotify_object = sp.playlist(spotify_uri) + if spotify_object is not None: + spotify_cache[spotify_uri] = spotify_object + save_spotify_cache_to_file(spotify_cache) + except SpotifyException: + utils.write_exception() + pass + def check_pysonic_connection(): """Return SubsonicOfflineException if pysonic is offline""" if pysonic.ping(): diff --git a/spotisub/routes.py b/spotisub/routes.py index 0e85d14..8c97c53 100644 --- a/spotisub/routes.py +++ b/spotisub/routes.py @@ -23,6 +23,7 @@ from flask_login import login_user from flask_login import logout_user from flask_login import login_required +from pygtail import Pygtail from spotipy.exceptions import SpotifyException from spotisub import spotisub from spotisub import configuration_db @@ -94,7 +95,7 @@ def get_json_message(message, is_ok): @login_required def overview( page=1, - limit=25, + limit=100, order='playlist_info.subsonic_playlist_name', asc=1): title = 'Overview' @@ -131,7 +132,7 @@ def overview( @login_required def overview_content( page=1, - limit=25, + limit=100, order='playlist_info.subsonic_playlist_name', asc=1): spotipy_helper.get_secrets() @@ -351,7 +352,8 @@ def artist(uuid=None, page=1, limit=25, order='spotify_song.title', asc=1): def tasks(): title = 'Tasks' return render_template('tasks.html', - title=title) + title=title, + tasks=generator.get_tasks()) @spotisub.route('/logs') @@ -366,15 +368,11 @@ def logs(): @login_required def streamlog(): def generate(): - with open(os.path.abspath(os.curdir) + '/cache/spotisub.log') as f: - while True: - yield f.read() - sleep(1) - - r = Response(response=generate(), status=200, mimetype="text/plain") - r.headers["Content-Type"] = "text/plain; charset=utf-8" - r.headers["Accept"] = "text/plain" - return r + path = os.path.abspath(os.curdir) + '/cache/spotisub.log' + for line in Pygtail(path, every_n=1): + yield "data:" + str(line) + "\n" + sleep(0.1) + return Response(response=generate(), status=200, mimetype= 'text/event-stream') @spotisub.route('/poll_playlist//') @@ -390,7 +388,7 @@ def poll_playlist(uuid=None): @spotisub.route('/poll_overview/') @login_required def poll_overview(): - if generator.check_thread_running_by_name("reimport_all"): + if utils.check_thread_running_by_name("reimport_all"): return get_response_json(get_json_message( "Job reimport_all is running", True), 200) else: diff --git a/spotisub/static/css/light.css b/spotisub/static/css/light.css index 9453b73..191a697 100644 --- a/spotisub/static/css/light.css +++ b/spotisub/static/css/light.css @@ -791,4 +791,26 @@ progress[value]::-webkit-progress-value { background-color: #ff0000; box-shadow: 0 5px #666; transform: translateY(4px); + } + + .output-log-div{ + width: 100%; + height: 90%; + padding: 0px; + overflow-y: scroll; + display: inline-block; + margin: 0px; + padding: 0px; + line-height: 1em; + } + .output-log-pre{ + width: 100%; + height: 100%; + overflow-y: hidden; + overflow-x: hidden; + padding: 0px; + display: inline-block; + margin: 0px; + padding: 0px; + line-height: 1em; } \ No newline at end of file diff --git a/spotisub/static/js/logs.js b/spotisub/static/js/logs.js index eb739f1..da7d2c9 100644 --- a/spotisub/static/js/logs.js +++ b/spotisub/static/js/logs.js @@ -5,7 +5,7 @@ function pollLogsJob(url){ xhr.onreadystatechange = function () { var output = document.getElementById('output-log'); - if (this.readyState == 4 && this.status == 200) { + if (this.readyState >= 3 && this.status == 200) { output.textContent = xhr.responseText; } } diff --git a/spotisub/templates/base.html b/spotisub/templates/base.html index aae29cf..c14ad22 100644 --- a/spotisub/templates/base.html +++ b/spotisub/templates/base.html @@ -43,6 +43,7 @@ @@ -61,7 +62,6 @@