implementing log and tasks

requirements.txt

@@ -30,4 +30,5 @@ Mako==1.2.3
 MarkupSafe==2.1.1
 visitor==0.1.3
 WTForms==3.0.1
-zipp==3.10.0
+zipp==3.10.0
+pygtail==0.14.0

spotisub/__init__.py

@@ -1,7 +1,7 @@
 """Spotisub init module"""
 import logging
 import os
-
+from logging.handlers import RotatingFileHandler
 from flask import Flask
 from flask_bootstrap import Bootstrap
 from flask_sqlalchemy import SQLAlchemy
@@ -15,11 +15,10 @@
 
 logging.basicConfig(
     handlers=[
-        logging.FileHandler(
-            os.path.abspath(
+        RotatingFileHandler(os.path.abspath(
                 os.curdir) +
-            "/cache/spotisub.log",
-            mode="w"),
+            "/cache/spotisub.log", maxBytes=1048576,
+                                  backupCount=5),
         logging.StreamHandler()],
     format='%(asctime)s %(levelname)-8s %(message)s',
     level=int(

spotisub/database.py

@@ -19,6 +19,7 @@
 from sqlalchemy import text
 from sqlalchemy import desc
 from sqlalchemy import or_
+from sqlalchemy import distinct
 from sqlalchemy import collate
 
 VERSION = "0.3.3-alpha5"
@@ -420,6 +421,25 @@ def select_playlist_info_by_name(name):
     return value
 
 
+def select_distinct_type_name():
+    """select spotify artists by uuid"""
+    values = []
+    with dbms.db_engine.connect() as conn:
+        value = None
+        stmt = select(
+            distinct(dbms.playlist_info.c.type))
+        stmt.compile()
+        cursor = conn.execute(stmt)
+        records = cursor.fetchall()
+
+        for row in records:
+            values.append(row.type)
+        cursor.close()
+        conn.close()
+
+    return values
+
+
 def select_playlist_info_by_name_with_conn(conn, name):
     """select spotify artists by uuid"""
     value = None
@@ -1264,6 +1284,7 @@ def limit_and_order_stmt(stmt, page=None, limit=None, order=None, asc=None):
     if page is not None and limit is not None:
         stmt = stmt.limit(limit).offset(page * limit)
     order_by = []
+    order = "case when "+order+" is null then 1 else 0 end, "+order
     if order is not None:
         if asc:
             stmt = stmt.order_by(collate(text(order), 'NOCASE'))

spotisub/generator.py

@@ -4,13 +4,16 @@
 import random
 import time
 import re
+import string
+import math
 import threading
 from datetime import datetime
 from datetime import timedelta
 from flask_apscheduler import APScheduler
 from spotisub import spotisub
 from spotisub import constants
 from spotisub import database
+from spotisub import utils
 from spotisub.helpers import spotipy_helper
 from spotisub.helpers import subsonic_helper
 from spotisub.threading.spotisub_thread import thread_with_trace
@@ -264,7 +267,7 @@ def artist_top_tracks(uuid):
 
 def show_recommendations_for_artist(uuid):
     """get user saved tracks"""
-    if not check_thread_running_by_name("reimport_all"):
+    if not utils.check_thread_running_by_name("reimport_all"):
         thread = thread_with_trace(
             target=lambda: show_recommendations_for_artist_run(uuid),
             name=constants.JOB_AR_ID + "_" + uuid)
@@ -325,7 +328,7 @@ def show_recommendations_for_artist_run(uuid):
 
 def my_recommendations(uuid):
     """get user saved tracks"""
-    if not check_thread_running_by_name("reimport_all"):
+    if not utils.check_thread_running_by_name("reimport_all"):
         thread = thread_with_trace(
             target=lambda: my_recommendations_run(uuid),
             name=constants.JOB_MR_ID + "_" + uuid)
@@ -403,7 +406,7 @@ def my_recommendations_run(uuid):
 
 def get_user_saved_tracks(uuid):
     """get user saved tracks"""
-    if not check_thread_running_by_name("reimport_all"):
+    if not utils.check_thread_running_by_name("reimport_all"):
         thread = thread_with_trace(
             target=lambda: get_user_saved_tracks_run(uuid),
             name=constants.JOB_ST_ID + "_" + uuid)
@@ -435,7 +438,7 @@ def get_user_saved_tracks_run(uuid):
 
 def get_user_playlists(uuid):
     """get user saved tracks"""
-    if not check_thread_running_by_name("reimport_all"):
+    if not utils.check_thread_running_by_name("reimport_all"):
         thread = thread_with_trace(
             target=lambda: get_user_playlists_run(uuid),
             name=constants.JOB_UP_ID + "_" + uuid)
@@ -649,6 +652,23 @@ def reimport(uuid):
                 constants.SAVED_GEN_SCHED,
                 constants.SAVED_GEN_SCHED_DEFAULT_VALUE)
 
+def get_tasks():
+    tasks = []
+    types = database.select_distinct_type_name()
+    for type in types:
+        job = scheduler.get_job(type)
+        if job is not None:
+            task = {}
+            task["type"] = type
+            task["type_desc"] = string.capwords(type.replace("_", " "))
+            task["next_execution"] = job.next_run_time.strftime("%H:%M:%S")
+            task["interval"] = str( math.trunc(job.trigger.interval_length / 60 / 60) ) + " hour(s)"
+            if len(job.args) > 0:
+                task["args"] = str( job.args[0] )
+            else:
+                task["args"] = ""
+            tasks.append(task)
+    return tasks
 
 def poll_playlist(uuid):
     for thread in threading.enumerate():
@@ -696,17 +716,11 @@ def scan_library():
 
 def reimport_all():
     """Used to reimport everything"""
-    if not check_thread_running_by_name("reimport_all"):
+    if not utils.check_thread_running_by_name("reimport_all"):
         thread = thread_with_trace(
             target=lambda: reimport_all_thread(),
             name="reimport_all").start()
 
-def check_thread_running_by_name(name):
-    for thread in threading.enumerate():
-        if thread.name == name and thread.is_alive():
-            return True
-    return False
-
 def reimport_all_thread():
     """Used to reimport everything"""
     import_all_user_saved_tracks()
@@ -791,6 +805,7 @@ def import_all_user_playlists():
         order='playlist_info.subsonic_playlist_name',
         asc=True)
 except BaseException:
+    utils.write_exception()
     pass
 
 scheduler.modify_job(id="init_jobs", next_run_time=datetime.now())

spotisub/helpers/subsonic_helper.py

@@ -7,6 +7,7 @@
 import threading
 import libsonic
 import string
+from concurrent.futures import ThreadPoolExecutor
 from expiringdict import ExpiringDict
 from libsonic.errors import DataNotFoundError
 from spotipy.exceptions import SpotifyException
@@ -20,6 +21,7 @@
 from spotisub.classes import ComparisonHelper
 from spotisub.helpers import musicbrainz_helper
 
+cache_executor = ThreadPoolExecutor(max_workers=2)
 
 if os.environ.get(constants.SPOTDL_ENABLED,
                   constants.SPOTDL_ENABLED_DEFAULT_VALUE) == "1":
@@ -64,10 +66,13 @@ def load_spotify_cache_from_file():
     object = ExpiringDict(max_len=10000, max_age_seconds=43200)
     path = os.path.abspath(os.curdir) + '/cache/spotify_object_cache.pkl'
     if os.path.exists(path):
-        with open(path, 'rb') as f:
-            old_cache_obj = pickle.load(f)
-            for key, value in old_cache_obj.items():
-                object[key] = value
+        if os.stat(path).st_size == 0:
+            os.remove(path)
+        else:
+            with open(path, 'rb') as f:
+                old_cache_obj = pickle.load(f)
+                for key, value in old_cache_obj.items():
+                    object[key] = value
     return object
 
 
@@ -78,28 +83,34 @@ def save_spotify_cache_to_file(object):
 
 
 def get_spotify_object_from_cache(sp, spotify_uri):
-    spotify_object = None
-    if spotify_uri not in spotify_cache:
-        try:
-            spotify_object = None
-            if "track" in spotify_uri:
-                spotify_object = sp.track(spotify_uri)
-            elif "album" in spotify_uri:
-                spotify_object = sp.album(spotify_uri)
-            elif "artist" in spotify_uri:
-                spotify_object = sp.artist(spotify_uri)
-            elif "playlist" in spotify_uri:
-                spotify_object = sp.playlist(spotify_uri)
-            if spotify_object is not None:
-                spotify_cache[spotify_uri] = spotify_object
-                save_spotify_cache_to_file(spotify_cache)
-        except SpotifyException:
-            pass
+    if spotify_uri in spotify_cache:
+        return spotify_cache[spotify_uri]
     else:
-        spotify_object = spotify_cache[spotify_uri]
-    return spotify_object
+        cache_executor.submit(load_spotify_object_to_cache, sp, spotify_uri)
+    return None
 
 
+def load_spotify_object_to_cache(sp, spotify_uri):
+    try:
+        global spotify_cache
+        if spotify_uri in spotify_cache:
+            return
+        spotify_object = None
+        if "track" in spotify_uri:
+            spotify_object = sp.track(spotify_uri)
+        elif "album" in spotify_uri:
+            spotify_object = sp.album(spotify_uri)
+        elif "artist" in spotify_uri:
+            spotify_object = sp.artist(spotify_uri)
+        elif "playlist" in spotify_uri:
+            spotify_object = sp.playlist(spotify_uri)
+        if spotify_object is not None:
+            spotify_cache[spotify_uri] = spotify_object
+            save_spotify_cache_to_file(spotify_cache)
+    except SpotifyException:
+        utils.write_exception()
+        pass
+
 def check_pysonic_connection():
     """Return SubsonicOfflineException if pysonic is offline"""
     if pysonic.ping():

spotisub/routes.py

@@ -23,6 +23,7 @@
 from flask_login import login_user
 from flask_login import logout_user
 from flask_login import login_required
+from pygtail import Pygtail
 from spotipy.exceptions import SpotifyException
 from spotisub import spotisub
 from spotisub import configuration_db
@@ -94,7 +95,7 @@ def get_json_message(message, is_ok):
 @login_required
 def overview(
         page=1,
-        limit=25,
+        limit=100,
         order='playlist_info.subsonic_playlist_name',
         asc=1):
     title = 'Overview'
@@ -131,7 +132,7 @@ def overview(
 @login_required
 def overview_content(
         page=1,
-        limit=25,
+        limit=100,
         order='playlist_info.subsonic_playlist_name',
         asc=1):
     spotipy_helper.get_secrets()
@@ -351,7 +352,8 @@ def artist(uuid=None, page=1, limit=25, order='spotify_song.title', asc=1):
 def tasks():
     title = 'Tasks'
     return render_template('tasks.html',
-                           title=title)
+                           title=title,
+                           tasks=generator.get_tasks())
 
 
 @spotisub.route('/logs')
@@ -366,15 +368,11 @@ def logs():
 @login_required
 def streamlog():
     def generate():
-        with open(os.path.abspath(os.curdir) + '/cache/spotisub.log') as f:
-            while True:
-                yield f.read()
-                sleep(1)
-
-    r = Response(response=generate(), status=200, mimetype="text/plain")
-    r.headers["Content-Type"] = "text/plain; charset=utf-8"
-    r.headers["Accept"] = "text/plain"
-    return r
+        path = os.path.abspath(os.curdir) + '/cache/spotisub.log'
+        for line in Pygtail(path, every_n=1):
+            yield "data:" + str(line) + "\n"
+            sleep(0.1)
+    return Response(response=generate(), status=200, mimetype= 'text/event-stream')
 
 
 @spotisub.route('/poll_playlist/<string:uuid>/')
@@ -390,7 +388,7 @@ def poll_playlist(uuid=None):
 @spotisub.route('/poll_overview/')
 @login_required
 def poll_overview():
-    if generator.check_thread_running_by_name("reimport_all"):
+    if utils.check_thread_running_by_name("reimport_all"):
         return get_response_json(get_json_message(
             "Job reimport_all is running", True), 200)
     else:

spotisub/static/css/light.css

@@ -791,4 +791,26 @@ progress[value]::-webkit-progress-value {
       background-color: #ff0000;
       box-shadow: 0 5px #666;
       transform: translateY(4px);
+    }
+
+    .output-log-div{
+        width: 100%;
+        height: 90%;
+        padding: 0px;
+        overflow-y: scroll;
+        display: inline-block;
+        margin: 0px;
+        padding: 0px;
+        line-height: 1em;
+    }
+    .output-log-pre{
+        width: 100%;
+        height: 100%;
+        overflow-y: hidden;
+        overflow-x: hidden;
+        padding: 0px;
+        display: inline-block;
+        margin: 0px;
+        padding: 0px;
+        line-height: 1em;
     }

spotisub/static/js/logs.js

@@ -5,7 +5,7 @@ function pollLogsJob(url){
 
     xhr.onreadystatechange = function () {
         var output = document.getElementById('output-log');
-        if (this.readyState == 4 && this.status == 200) {
+        if (this.readyState >= 3 && this.status == 200) {
             output.textContent = xhr.responseText;
         }
     }

spotisub/templates/base.html

@@ -43,6 +43,7 @@
         <script>
             $( document ).ready(function() {
                 pollOverviewJob('{{url_for("poll_overview")}}')
+                setInterval(function(){pollOverviewJob('{{url_for("poll_overview")}}')}, 5000);
                 filterOverview()
             });
         </script>
@@ -61,7 +62,6 @@
         <script>
             $( document ).ready(function() {
                 pollLogsJob('{{url_for("streamlog")}}')
-                setInterval(function(){pollLogsJob('{{url_for("streamlog")}}')}, 1000);
             });
         </script>
         <script>

spotisub/templates/logs.html

@@ -5,5 +5,7 @@
     {% include '_flash_message.html' %}
     <!-- End of flash message -->
         {% include 'header.html' %}
-        <pre id="output-log"></pre>
+        <div class="output-log-div">
+            <pre id="output-log" class="output-log-pre"></pre>
+        </div>
 {% endblock %}