From 6f19e80bef1dce7f412bee96f32a2692b3c5b0b2 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 12:57:38 -0500 Subject: [PATCH 01/25] Create test.txt --- test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..e69de29 From 7e6fd9113e1b6af2a9c1731a207f81bd48c13aa6 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 12:59:30 -0500 Subject: [PATCH 02/25] Delete test.txt --- test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test.txt diff --git a/test.txt b/test.txt deleted file mode 100644 index e69de29..0000000 From 4a9f1d3763ff5daded8d7358fbfc4ff2d0c5007e Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 16:02:33 -0500 Subject: [PATCH 03/25] Add test folder and being testing loguru --- autobookstesting/AutoBooks.py | 331 +++++++++++++++++++++++++++ autobookstesting/AutoBooksDiscord.py | 104 +++++++++ autobookstesting/__init__.py | 0 autobookstesting/main.py | 3 + autobookstesting/utils.py | 34 +++ 5 files changed, 472 insertions(+) create mode 100644 autobookstesting/AutoBooks.py create mode 100644 autobookstesting/AutoBooksDiscord.py create mode 100644 autobookstesting/__init__.py create mode 100644 autobookstesting/main.py create mode 100644 autobookstesting/utils.py diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py new file mode 100644 index 0000000..9275097 --- /dev/null +++ b/autobookstesting/AutoBooks.py @@ -0,0 +1,331 @@ +from selenium import webdriver +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.common.by import By +from selenium.webdriver.chrome.options import Options +from configparser import ConfigParser +from unittest.mock import patch +from datetime import datetime +from time import sleep +from pathlib import Path +import odmpy.odm as odmpy +from loguru import logger +from utils import InterceptHandler, RedactingFormatter +import pandas as pd +import selenium +import cronitor +import glob +import sys +import shutil +import logging +import os +import requests + +# Set Vars +scriptver = "0.2.1" # Version number of script +error_count = 0 +good_odm_list = [] +bad_odm_list = [] +log_list = [] +library_list = [] +book_id_list = [] +book_title_list = [] +book_odm_list = [] +scriptdir = os.path.join(Path.home(), "AutoBooks") +csv_path = os.path.join(scriptdir, 'web_known_files.csv') + +# Check paths, and if not found do first time setup +if os.path.exists(scriptdir): + os.chdir(scriptdir) +else: + os.mkdir(scriptdir) + main_conf = requests.get('https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') + odmpy_conf = requests.get("https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") + folders = ['log', 'web_downloads', 'chrome_profile', 'sourcefiles'] + for folder in folders: + os.mkdir(os.path.join(scriptdir, folder)) + with open(os.path.join(scriptdir, "autobooks.conf"), mode='wb') as localfile: + localfile.write(main_conf.content) + with open(os.path.join(scriptdir, "odmpydl.conf"), mode='wb') as localfile: + localfile.write(odmpy_conf.content) + print("Finished setup please configure settings in file: ", os.path.join(scriptdir, "autobooks.conf")) + sys.exit(1) + +# Logging Config +LOG_FILENAME = os.path.join(scriptdir, 'log', 'AutoBooks-{:%H-%M-$S_%m-%d-%Y}-Main.log'.format(datetime.now())) +console_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {message}\n{exception}" +file_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {extra[scrubbed]}\n{exception}" +redacting_formatter = RedactingFormatter(patterns=["", "", "", ""], source_fmt=file_log_format) +logger.configure(handlers=[ + {'sink': sys.stderr, "format": console_log_format}, + {'sink': LOG_FILENAME, "format": redacting_formatter.format} + ]) +odmpy.logger.handlers.clear() +odmpy.logger.addHandler(InterceptHandler()) + +# Read config file +parser = ConfigParser() +parser.read(os.path.join(scriptdir, "autobooks.conf")) +odmdir = parser.get("DEFAULT", + "odm_folder") # Folder that contains the .odm files to process. For windows use / slashes +outdir = parser.get("DEFAULT", + "out_folder") # Folder where the finished audiobooks will be moved to. For windows use / slashes + +# Cronitor Setup +cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") # Cronitor API key https://cronitor.io/ +monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) # Set Cronitor monitor name +web_monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_web")) # Set Cronitor monitor name + + +# Function to process the books. +def process_books(odm_list): + global error_count + logger.info('Begin processing booklist: {}', " ".join(odm_list)) + for x in odm_list: + if parser.get('DEFAULT', "test_args") == "true": + odmpy_args = ["odmpy", "dl", x] + else: + odmpy_args = ["odmpy", "dl", "@" + os.path.join(scriptdir, "odmpydl.conf"), x] + with patch.object(sys, 'argv', odmpy_args): + try: + odmpy.run() + except FileNotFoundError: + logger.error("Could not find odm file {}", x) + except FileExistsError: + logger.error("FileAlreadyExists, likely from m4b creation attempt") + except SystemExit as e: + bad_odm_list.append(x) + try: + os.remove("cover.jpg") + except FileNotFoundError: + logger.debug("Could not remove cover.jpg, moving on") + else: + logger.debug("Removed cover.jpg to prep for next attempt") + error_count += 1 + else: + good_odm_list.append(x) + logger.info("Book Processing Finished") + + +# Function to cleanup in and out files. +def cleanup(m4bs, odms, odmfolder): + global error_count + # Move m4b files to outdir + for x in m4bs: + exists = os.path.isfile(os.path.join(outdir + x)) + if exists: + logger.error("Book {} already exists in outdir skipped", x) + error_count += 1 + else: + shutil.move(os.path.join(odmfolder, x), os.path.join(outdir, x)) + logger.info("Moved book {} to outdir", x) + # Backup source files + sourcefiles = odms + glob.glob("*.license") + for x in sourcefiles: + if os.path.isfile(os.path.join(scriptdir, "sourcefiles", x)): + logger.error("File {} already exists in sourcefiles dir skipped", x) + error_count += 1 + else: + shutil.move(x, os.path.join(scriptdir, "sourcefiles", x)) + logger.info("Moved file {} to sourcefiles", x) + + +# Function for login +def web_login(driver, name, cardno, pin, select): + global error_count + logger.info("web_login: Logging into library: {}", name) + # Attempt selecting library from dropdown + if select != "false": + select_box = driver.find_element(By.XPATH, '//input[@id="signin-options"]') + webdriver.ActionChains(driver).move_to_element(select_box).click().send_keys(select).perform() + sleep(1) + webdriver.ActionChains(driver).send_keys(Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform() + # Attempt sending card number + try: + driver.find_element(By.ID, "username").send_keys(cardno) + except selenium.common.exceptions.NoSuchElementException: + logger.critical("web_login: Can't find card number field skipped library {}", ) + error_count += 1 + # Attempt sending pin Note:Some pages don't have pin input + if pin != "false": + driver.find_element(By.ID, "password").send_keys(pin) + driver.find_element(By.CSS_SELECTOR, "button.signin-button.button.secondary").click() + sleep(5) + + +# Function to download loans from OverDrive page +def web_dl(driver, df, name): + global error_count + #Gather all book title elements and check if any found + books = driver.find_elements(By.XPATH, '//a[@tabindex="0"][@role="link"]') + if len(books) == 0: + logger.warning("Can't find books skipped library: {}", name) + error_count += 1 + return () + else: + logger.info("web_dl: Begin DL from library: {} ", name) + bookcount = 0 + for i in books: + #Fetch info about the book + book_url = i.get_attribute('href') + book_info = i.get_attribute('aria-label') + book_info_split = book_info.split(". Audiobook. Expires in") + book_dl_url = book_url.replace('/media/', '/media/download/audiobook-mp3/') + book_id = int(''.join(filter(str.isdigit, book_url))) + book_title = book_info_split[0] + if "Audiobook." in book_info: + if str(book_id) in df['book_id'].to_string(): + logger.info('web_dl: Skipped {} found in known books', book_title) + else: + # Download book + driver.get(book_dl_url) + logger.info("web_dl: Downloaded book: {}", book_title) + book_odm = max(glob.glob("*.odm"), key=os.path.getmtime) + bookcount += 1 + + #Add book data to vars + library_list.append(name) + book_id_list.append(book_id) + book_title_list.append(book_title) + book_odm_list.append(book_odm) + sleep(1) + + + sleep(1) + logger.info("web_dl: Finished downloading {} books from library {}", bookcount, name) + return () + +def main_run(): + # AutoBooks + logger.info("Started AutoBooks V.{} By:IvyB", scriptver) + # Try to change to ODM folder + try: + os.chdir(odmdir) + except FileNotFoundError: + logger.critical("The provided .odm dir was not found, exiting") + sys.exit(1) + else: + odm_list = glob.glob("*.odm") + monitor.ping(state='run', + message='AutoBooks by IvyB Version:' + scriptver + '\n odmdir:' + odmdir + '\n outdir:' + outdir + '\n logfile:' + LOG_FILENAME + '\n Found the following books \n' + " ".join( + odm_list)) + + # Check if any .odm files exist in odmdir + if len(odm_list) == 0: + monitor.ping(state='fail', message='Error: No .odm files found, exiting', + metrics={'error_count': error_count}) + logger.critical("No .odm files found, exiting") + sys.exit(1) + else: + process_books(odm_list) + # Cleanup input and output files + m4blist = glob.glob("*.m4b") + cleanup(m4blist, good_odm_list, odmdir) + # Process log file for Cronitor + with open(LOG_FILENAME) as logs: + lines = logs.readlines() + log_list = [] + for line in lines: + if any(term in line for term in ("Downloading", "expired", "generating", "merged")): + log_list.append(line) + # Send complete event and log to Cronitor + monitor.ping(state='complete', message="".join(log_list), + metrics={'count': len(odm_list), 'error_count': error_count}) + + +# AutoBooks Web Code +def web_run(): + if len(parser.sections()) == 0: + logger.critical("No libraries configured!") + sys.exit(1) + else: + logger.info("Started AutoBooks Web V.{} By:IvyB", scriptver) + monitor.ping(state='run', + message='AutoBooks Web by IvyB Version:' + scriptver + '\n logfile:' + LOG_FILENAME + '\n LibraryCount: ' + str( + len(parser.sections()))) + # Configure WebDriver options + options = Options() + prefs = { + "download.default_directory": os.path.join(scriptdir, "web_downloads"), + "download.prompt_for_download": False, + "download.directory_upgrade": True + } + options.add_argument('user-data-dir=' + os.path.join(scriptdir, "chrome_profile")) + # Headless mode check + if parser.get('DEFAULT', "web_headless") == "true": + options.add_argument('--headless') + options.add_argument('--disable-gpu') + options.add_experimental_option('prefs', prefs) + driver = webdriver.Chrome(options=options) + + # Attempt to read known files csv for checking books + if os.path.exists(csv_path): + df = pd.read_csv(csv_path, sep=",") + else: + df = pd.DataFrame({ + 'book_id': book_id_list, + }) + os.chdir(os.path.join(scriptdir,"web_downloads")) + # For every library, open site, attempt sign in, and attempt download. + for i in range(0, len(parser.sections())): + library_index = 'library_' + str(i) + library_subdomain = parser.get(library_index, "library_subdomain") + library_name = parser.get(library_index, "library_name") + logger.info("Started library {}", library_name) + url = "https://" + library_subdomain + ".overdrive.com/" + driver.get(url + "account/loans") + sleep(3) + # Check signed in status and either sign in or move on + if "/account/ozone/sign-in" in driver.current_url: + web_login(driver, library_name, parser.get(library_index, "card_number"), + parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) + web_dl(driver, df, library_name) + sleep(2) + # Output book data to csv + df_out = pd.DataFrame({ + 'library_name': library_list, + 'book_id': book_id_list, + 'book_title': book_title_list, + 'book_odm': book_odm_list + }) + if os.path.exists(csv_path): + df_out.to_csv(csv_path, mode='a', index=False, header=False) + else: + df_out.to_csv(csv_path, mode='w', index=False, header=True) + driver.close() + logger.info("AutoBooksWeb Complete") + odmlist = glob.glob("*.odm") + + # Process log file for Cronitor. + with open(LOG_FILENAME) as logs: + lines = logs.readlines() + log_list = [] + for line in lines: + if "AutoBooks.web" in line: + log_list.append(line) + monitor.ping(state='complete', message="".join(odmlist), + metrics={'count': len(odmlist), 'error_count': error_count}) + + # Call Minimum DL functions + if len(odmlist) != 0: + logger.info("Started AutoBooks V.{} By:IvyB", scriptver) + monitor.ping(state='run', + message='AutoBooks by IvyB Started from web Version:' + scriptver + '\n outdir:' + outdir + '\n logfile:' + LOG_FILENAME + '\n Found the following books \n' + " ".join( + odmlist)) + process_books(odmlist) + m4blist = glob.glob("*.m4b") + cleanup(m4blist, good_odm_list, os.path.join(scriptdir, "web_downloads")) + # Process log file for Cronitor + with open(LOG_FILENAME) as logs: + lines = logs.readlines() + log_list = [] + for line in lines: + if any(term in line for term in ("Downloading", "expired", "generating", "merged")): + log_list.append(line) + # Send complete event and log to Cronitor + monitor.ping(state='complete', message="".join(log_list), + metrics={'count': len(odmlist), 'error_count': error_count}) + #return["\n".join(title_list), error_count] + + +if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "true": + web_run() diff --git a/autobookstesting/AutoBooksDiscord.py b/autobookstesting/AutoBooksDiscord.py new file mode 100644 index 0000000..9439a34 --- /dev/null +++ b/autobookstesting/AutoBooksDiscord.py @@ -0,0 +1,104 @@ +from configparser import ConfigParser +from unittest.mock import patch +from datetime import datetime +from discord.ext import commands +from pathlib import Path +import os +import discord +import logging +import glob +import sys +import shutil +import platform +from AutoBooks import web_run, main_run, scriptver, scriptdir, parser, csv_path, fh, discord_logger, LOG_FILENAME +import pandas as pd + +# Log Settings +# DISCORD_LOGFILE = os.path.join(AutoBooks.scriptdir,'log','AutoBooks-{:%H-%M-%S_%m-%d-%Y}-Discord.log'.format(datetime.now())) +# formatter = logging.Formatter('%(asctime)s [%(name)s] %(levelname)s: %(message)s', datefmt='%I:%M:%S %p',) +# discord_fh = logging.FileHandler(DISCORD_LOGFILE) +# discord_fh.setFormatter(formatter) +# AutoBooks.discord_logger.removeHandler(AutoBooks.fh) +# AutoBooks.discord_logger.addHandler(discord_fh) +logger = logging.getLogger('discord') +logger.removeHandler(fh) +# logger.addHandler(discord_fh) +# Bot Settings +try: + token = parser.get("DEFAULT", "discord_bot_token") +except KeyError: + discord_logger.critical("Bot token not found in config file, exiting.") +bot = commands.Bot(command_prefix='?') + + +@bot.event +async def on_ready(): + discord_logger.info(f'{bot.user} has connected to Discord!') + + +@bot.command(name='web') +async def hello(ctx): + #Starting embed and running web + embed_start = discord.Embed(title="Running AutoBooks Web. This may take awhile....", description="Version: "+scriptver+" \nLogfile: "+LOG_FILENAME, color=0xFFAFCC) + embed_start.set_image(url="https://raw.githubusercontent.com/ivybowman/AutoBooks/main/img/logo/small_pink.png") + embed_start.set_footer(text="OS: "+ platform.platform()+" Host: "+platform.node()) + await ctx.channel.send(embed=embed_start) + web_info = web_run() + + #Ending Embed + embed_end = discord.Embed(title="AutoBooks Web Finished", description="See log info below for details. ErrorCount: "+str(web_info[1]), color=0xFFAFCC) + embed_end.set_thumbnail(url="https://raw.githubusercontent.com/ivybowman/AutoBooks/main/img/icon_pink.png") + if web_info[0] != "": + embed_end.add_field(name="Book List", value=str(web_info[0]), inline=False) + await ctx.channel.send(embed=embed_end) + #Logfile fetching + files = glob.glob(os.path.join(scriptdir, "log", "*-Main.log")) + files2 = sorted(files, key=os.path.getmtime, reverse=True) + print(files2[0]) + await ctx.channel.send(file=discord.File(files2[0])) + + +@bot.command(name='main') +async def hello(ctx): + embedVar = discord.Embed(title="Title", description="Desc", color=0xFFAFCC) + embedVar.add_field(name="Field1", value="hi", inline=False) + embedVar.add_field(name="Field2", value="hi2", inline=False) + + await ctx.channel.send(embed=embedVar) + main_run() + + +@bot.command(name='log') +async def hello(ctx): + files = glob.glob(os.path.join(scriptdir, "log", "*-Main.log")) + max_file = max(files, key=os.path.getmtime) + print(max_file) + await ctx.channel.send("Fetched latest AutoBooks logfile: \n" + max_file) + await ctx.channel.send(file=discord.File(max_file)) + + +@bot.command(name='csv') +async def hello(ctx): + try: + df = pd.read_csv(csv_path, sep=",") + embedVar = discord.Embed(title="Autobooks Known Books", description=df['audiobook_title'].to_string(index=False), color=0xFFAFCC) + embedVar.set_footer(text="OS: "+ platform.platform()+" Host: "+os.uname()) + await ctx.channel.send(embed=embedVar) + except FileNotFoundError: + await ctx.channel.send("Known Books CSV not found.") + + # await ctx.channel.send(file=discord.File(max_file)) + + +def discord_run(): + if token == "": + discord_logger.critical("Bot token not found in config file, exiting.") + else: + bot.run(token) + + +if __name__ == "__main__": + try: + discord_run() + except KeyboardInterrupt: + sys.exit(1) diff --git a/autobookstesting/__init__.py b/autobookstesting/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/autobookstesting/main.py b/autobookstesting/main.py new file mode 100644 index 0000000..4a4b60e --- /dev/null +++ b/autobookstesting/main.py @@ -0,0 +1,3 @@ +from AutoBooks import web_run +if __name__ == "__main__": + web_run() \ No newline at end of file diff --git a/autobookstesting/utils.py b/autobookstesting/utils.py new file mode 100644 index 0000000..0dc5969 --- /dev/null +++ b/autobookstesting/utils.py @@ -0,0 +1,34 @@ +import logging +from loguru import logger +import sys +from datetime import datetime +#Formatter to remove patterns from log output +class RedactingFormatter: + def __init__(self, patterns=None, source_fmt=None): + super().__init__() + self.patterns = patterns + self.fmt = source_fmt + + def format(self, record): + scrubbed = record["message"] + for pattern in self.patterns: + scrubbed = scrubbed.replace(pattern, "") + record["extra"]["scrubbed"] = scrubbed + return self.fmt + +#Handler to intercept logging messages for loguru +class InterceptHandler(logging.Handler): + def emit(self, record): + try: + level = logger.level(record.levelname).name + except ValueError: + level = record.levelno + + frame, depth = logging.currentframe(), 2 + while frame.f_code.co_filename == logging.__file__: + frame = frame.f_back + depth += 1 + + logger.opt(depth=depth, exception=record.exc_info).log( + level, record.getMessage() + ) \ No newline at end of file From 3ab44cea2ff65aef4620442217812dfe36261ac0 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 16:08:41 -0500 Subject: [PATCH 04/25] Update AutoBooks.py --- autobookstesting/AutoBooks.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index 9275097..e6f1e27 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -51,7 +51,7 @@ sys.exit(1) # Logging Config -LOG_FILENAME = os.path.join(scriptdir, 'log', 'AutoBooks-{:%H-%M-$S_%m-%d-%Y}-Main.log'.format(datetime.now())) +LOG_FILENAME = os.path.join(scriptdir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}-Main.log'.format(datetime.now())) console_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {message}\n{exception}" file_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {extra[scrubbed]}\n{exception}" redacting_formatter = RedactingFormatter(patterns=["", "", "", ""], source_fmt=file_log_format) @@ -132,7 +132,7 @@ def cleanup(m4bs, odms, odmfolder): # Function for login def web_login(driver, name, cardno, pin, select): global error_count - logger.info("web_login: Logging into library: {}", name) + logger.info("Logging into library: {}", name) # Attempt selecting library from dropdown if select != "false": select_box = driver.find_element(By.XPATH, '//input[@id="signin-options"]') @@ -143,7 +143,7 @@ def web_login(driver, name, cardno, pin, select): try: driver.find_element(By.ID, "username").send_keys(cardno) except selenium.common.exceptions.NoSuchElementException: - logger.critical("web_login: Can't find card number field skipped library {}", ) + logger.critical("Can't find card number field skipped library {}", ) error_count += 1 # Attempt sending pin Note:Some pages don't have pin input if pin != "false": @@ -162,7 +162,7 @@ def web_dl(driver, df, name): error_count += 1 return () else: - logger.info("web_dl: Begin DL from library: {} ", name) + logger.info("Begin DL from library: {} ", name) bookcount = 0 for i in books: #Fetch info about the book @@ -174,11 +174,11 @@ def web_dl(driver, df, name): book_title = book_info_split[0] if "Audiobook." in book_info: if str(book_id) in df['book_id'].to_string(): - logger.info('web_dl: Skipped {} found in known books', book_title) + logger.info('Skipped {} found in known books', book_title) else: # Download book driver.get(book_dl_url) - logger.info("web_dl: Downloaded book: {}", book_title) + logger.info("Downloaded book: {}", book_title) book_odm = max(glob.glob("*.odm"), key=os.path.getmtime) bookcount += 1 @@ -191,7 +191,7 @@ def web_dl(driver, df, name): sleep(1) - logger.info("web_dl: Finished downloading {} books from library {}", bookcount, name) + logger.info("Finished downloading {} books from library {}", bookcount, name) return () def main_run(): From 2631b9c31e87b8ed3120fd4f242f58dcafc438a0 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 16:12:07 -0500 Subject: [PATCH 05/25] Rename discord bot --- .../{AutoBooksDiscord.py => DiscordBot.py} | 19 +++++-------------- autobookstesting/main.py | 1 + 2 files changed, 6 insertions(+), 14 deletions(-) rename autobookstesting/{AutoBooksDiscord.py => DiscordBot.py} (78%) diff --git a/autobookstesting/AutoBooksDiscord.py b/autobookstesting/DiscordBot.py similarity index 78% rename from autobookstesting/AutoBooksDiscord.py rename to autobookstesting/DiscordBot.py index 9439a34..e7fa851 100644 --- a/autobookstesting/AutoBooksDiscord.py +++ b/autobookstesting/DiscordBot.py @@ -10,30 +10,21 @@ import sys import shutil import platform -from AutoBooks import web_run, main_run, scriptver, scriptdir, parser, csv_path, fh, discord_logger, LOG_FILENAME +from AutoBooks import web_run, main_run, scriptver, scriptdir, parser, csv_path, LOG_FILENAME, logger import pandas as pd -# Log Settings -# DISCORD_LOGFILE = os.path.join(AutoBooks.scriptdir,'log','AutoBooks-{:%H-%M-%S_%m-%d-%Y}-Discord.log'.format(datetime.now())) -# formatter = logging.Formatter('%(asctime)s [%(name)s] %(levelname)s: %(message)s', datefmt='%I:%M:%S %p',) -# discord_fh = logging.FileHandler(DISCORD_LOGFILE) -# discord_fh.setFormatter(formatter) -# AutoBooks.discord_logger.removeHandler(AutoBooks.fh) -# AutoBooks.discord_logger.addHandler(discord_fh) -logger = logging.getLogger('discord') -logger.removeHandler(fh) -# logger.addHandler(discord_fh) + # Bot Settings try: token = parser.get("DEFAULT", "discord_bot_token") except KeyError: - discord_logger.critical("Bot token not found in config file, exiting.") + logger.critical("Bot token not found in config file, exiting.") bot = commands.Bot(command_prefix='?') @bot.event async def on_ready(): - discord_logger.info(f'{bot.user} has connected to Discord!') + logger.info(f'{bot.user} has connected to Discord!') @bot.command(name='web') @@ -92,7 +83,7 @@ async def hello(ctx): def discord_run(): if token == "": - discord_logger.critical("Bot token not found in config file, exiting.") + logger.critical("Bot token not found in config file, exiting.") else: bot.run(token) diff --git a/autobookstesting/main.py b/autobookstesting/main.py index 4a4b60e..6e5eb5d 100644 --- a/autobookstesting/main.py +++ b/autobookstesting/main.py @@ -1,3 +1,4 @@ from AutoBooks import web_run +import argparse if __name__ == "__main__": web_run() \ No newline at end of file From bea99be7273f65c5408fb3e12406bf3e0896f7d3 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 16:45:35 -0500 Subject: [PATCH 06/25] More progress in loguru testing and code cleanup --- autobookstesting/AutoBooks.py | 44 ++++++++++++---------------------- autobookstesting/DiscordBot.py | 3 ++- autobookstesting/main.py | 6 +++-- autobookstesting/utils.py | 13 +++++++++- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index e6f1e27..251fa1b 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -9,27 +9,20 @@ from pathlib import Path import odmpy.odm as odmpy from loguru import logger -from utils import InterceptHandler, RedactingFormatter +from utils import InterceptHandler, RedactingFormatter, process_logfile import pandas as pd import selenium import cronitor import glob import sys import shutil -import logging import os import requests # Set Vars scriptver = "0.2.1" # Version number of script error_count = 0 -good_odm_list = [] -bad_odm_list = [] -log_list = [] -library_list = [] -book_id_list = [] -book_title_list = [] -book_odm_list = [] +good_odm_list, bad_odm_list, log_list, library_list, book_id_list, book_title_list, book_odm_list = ([] for i in range(7)) scriptdir = os.path.join(Path.home(), "AutoBooks") csv_path = os.path.join(scriptdir, 'web_known_files.csv') @@ -66,14 +59,13 @@ parser = ConfigParser() parser.read(os.path.join(scriptdir, "autobooks.conf")) odmdir = parser.get("DEFAULT", - "odm_folder") # Folder that contains the .odm files to process. For windows use / slashes -outdir = parser.get("DEFAULT", - "out_folder") # Folder where the finished audiobooks will be moved to. For windows use / slashes + "odm_folder") +outdir = parser.get("DEFAULT", "out_folder") -# Cronitor Setup -cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") # Cronitor API key https://cronitor.io/ -monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) # Set Cronitor monitor name -web_monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_web")) # Set Cronitor monitor name +# Cronitor Setup https://cronitor.io/ +cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") +monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) +web_monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_web")) # Function to process the books. @@ -111,8 +103,7 @@ def cleanup(m4bs, odms, odmfolder): global error_count # Move m4b files to outdir for x in m4bs: - exists = os.path.isfile(os.path.join(outdir + x)) - if exists: + if os.path.isfile(os.path.join(outdir + x)): logger.error("Book {} already exists in outdir skipped", x) error_count += 1 else: @@ -220,16 +211,11 @@ def main_run(): # Cleanup input and output files m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, odmdir) - # Process log file for Cronitor - with open(LOG_FILENAME) as logs: - lines = logs.readlines() - log_list = [] - for line in lines: - if any(term in line for term in ("Downloading", "expired", "generating", "merged")): - log_list.append(line) - # Send complete event and log to Cronitor - monitor.ping(state='complete', message="".join(log_list), - metrics={'count': len(odm_list), 'error_count': error_count}) + #Fetch log messages + process_logfile(LOG_FILENAME, terms=("Downloading", "expired", "generating", "merged", "saved")) + # Send complete event and log to Cronitor + monitor.ping(state='complete', message="".join(log_list), + metrics={'count': len(odm_list), 'error_count': error_count}) # AutoBooks Web Code @@ -328,4 +314,4 @@ def web_run(): if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "true": - web_run() + main_run() diff --git a/autobookstesting/DiscordBot.py b/autobookstesting/DiscordBot.py index e7fa851..80e22bc 100644 --- a/autobookstesting/DiscordBot.py +++ b/autobookstesting/DiscordBot.py @@ -34,7 +34,8 @@ async def hello(ctx): embed_start.set_image(url="https://raw.githubusercontent.com/ivybowman/AutoBooks/main/img/logo/small_pink.png") embed_start.set_footer(text="OS: "+ platform.platform()+" Host: "+platform.node()) await ctx.channel.send(embed=embed_start) - web_info = web_run() + web_info = ["test-data", "test-data2"] + web_run() #Ending Embed embed_end = discord.Embed(title="AutoBooks Web Finished", description="See log info below for details. ErrorCount: "+str(web_info[1]), color=0xFFAFCC) diff --git a/autobookstesting/main.py b/autobookstesting/main.py index 6e5eb5d..55f3a1c 100644 --- a/autobookstesting/main.py +++ b/autobookstesting/main.py @@ -1,4 +1,6 @@ -from AutoBooks import web_run +from AutoBooks import web_run, main_run +from DiscordBot import discord_run import argparse + if __name__ == "__main__": - web_run() \ No newline at end of file + discord_run() \ No newline at end of file diff --git a/autobookstesting/utils.py b/autobookstesting/utils.py index 0dc5969..a81cbc7 100644 --- a/autobookstesting/utils.py +++ b/autobookstesting/utils.py @@ -2,6 +2,7 @@ from loguru import logger import sys from datetime import datetime + #Formatter to remove patterns from log output class RedactingFormatter: def __init__(self, patterns=None, source_fmt=None): @@ -31,4 +32,14 @@ def emit(self, record): logger.opt(depth=depth, exception=record.exc_info).log( level, record.getMessage() - ) \ No newline at end of file + ) +# Process log file for Cronitor +def process_logfile(LOG_FILENAME, terms=None): + + with open(LOG_FILENAME) as logs: + lines = logs.readlines() + log_list = [] + for line in lines: + if any(term in line for term in terms): + log_list.append(line) + return "".join(log_list) \ No newline at end of file From 150cb55a636bb263fa34ec4b59aad9c97a8c2b7a Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 17:08:47 -0500 Subject: [PATCH 07/25] Update config and use f strings for cronitor pings --- autobookstesting/AutoBooks.py | 43 +++++++++++++--------------------- autobookstesting/DiscordBot.py | 1 - 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index 251fa1b..417181f 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -61,7 +61,7 @@ odmdir = parser.get("DEFAULT", "odm_folder") outdir = parser.get("DEFAULT", "out_folder") - +library_count = len(parser.sections()) # Cronitor Setup https://cronitor.io/ cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) @@ -179,8 +179,6 @@ def web_dl(driver, df, name): book_title_list.append(book_title) book_odm_list.append(book_odm) sleep(1) - - sleep(1) logger.info("Finished downloading {} books from library {}", bookcount, name) return () @@ -212,9 +210,9 @@ def main_run(): m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, odmdir) #Fetch log messages - process_logfile(LOG_FILENAME, terms=("Downloading", "expired", "generating", "merged", "saved")) + logstr = process_logfile(LOG_FILENAME, terms=("Downloading", "expired", "generating", "merged", "saved")) # Send complete event and log to Cronitor - monitor.ping(state='complete', message="".join(log_list), + monitor.ping(state='complete', message=logstr, metrics={'count': len(odm_list), 'error_count': error_count}) @@ -225,9 +223,11 @@ def web_run(): sys.exit(1) else: logger.info("Started AutoBooks Web V.{} By:IvyB", scriptver) + #monitor.ping(state='run', + #message='AutoBooks Web by IvyB Version:' + scriptver + '\n logfile:' + LOG_FILENAME + '\n LibraryCount: ' + str( + # len(parser.sections()))) monitor.ping(state='run', - message='AutoBooks Web by IvyB Version:' + scriptver + '\n logfile:' + LOG_FILENAME + '\n LibraryCount: ' + str( - len(parser.sections()))) + message=f'AutoBooks Web by IvyB Version: {scriptver} \n logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}') # Configure WebDriver options options = Options() prefs = { @@ -282,34 +282,23 @@ def web_run(): odmlist = glob.glob("*.odm") # Process log file for Cronitor. - with open(LOG_FILENAME) as logs: - lines = logs.readlines() - log_list = [] - for line in lines: - if "AutoBooks.web" in line: - log_list.append(line) - monitor.ping(state='complete', message="".join(odmlist), - metrics={'count': len(odmlist), 'error_count': error_count}) - + process_logfile(LOG_FILENAME, terms=("web")) + monitor.ping(state='complete', message="".join(odmlist), + metrics={'count': len(odmlist), 'error_count': error_count}) + # Call Minimum DL functions if len(odmlist) != 0: logger.info("Started AutoBooks V.{} By:IvyB", scriptver) monitor.ping(state='run', - message='AutoBooks by IvyB Started from web Version:' + scriptver + '\n outdir:' + outdir + '\n logfile:' + LOG_FILENAME + '\n Found the following books \n' + " ".join( - odmlist)) + message=f'AutoBooks by IvyB Started from web V.{scriptver} \n outdir:{outdir}\n logfile:{LOG_FILENAME}\n odmlist: \n{" ".join(odmlist)}') process_books(odmlist) m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, os.path.join(scriptdir, "web_downloads")) # Process log file for Cronitor - with open(LOG_FILENAME) as logs: - lines = logs.readlines() - log_list = [] - for line in lines: - if any(term in line for term in ("Downloading", "expired", "generating", "merged")): - log_list.append(line) - # Send complete event and log to Cronitor - monitor.ping(state='complete', message="".join(log_list), - metrics={'count': len(odmlist), 'error_count': error_count}) + logstr = process_logfile(LOG_FILENAME, terms=("Downloading", "expired", "generating", "merged")) + # Send complete event and log to Cronitor + monitor.ping(state='complete', message=logstr, + metrics={'count': len(odmlist), 'error_count': error_count}) #return["\n".join(title_list), error_count] diff --git a/autobookstesting/DiscordBot.py b/autobookstesting/DiscordBot.py index 80e22bc..435ed4b 100644 --- a/autobookstesting/DiscordBot.py +++ b/autobookstesting/DiscordBot.py @@ -78,7 +78,6 @@ async def hello(ctx): await ctx.channel.send(embed=embedVar) except FileNotFoundError: await ctx.channel.send("Known Books CSV not found.") - # await ctx.channel.send(file=discord.File(max_file)) From a7d8cfe6aa606041193765c0d22cf9b1ed578ad0 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 17:26:53 -0500 Subject: [PATCH 08/25] Update AutoBooks.py --- autobookstesting/AutoBooks.py | 117 +++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 50 deletions(-) diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index 417181f..76daaad 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -22,7 +22,8 @@ # Set Vars scriptver = "0.2.1" # Version number of script error_count = 0 -good_odm_list, bad_odm_list, log_list, library_list, book_id_list, book_title_list, book_odm_list = ([] for i in range(7)) +good_odm_list, bad_odm_list, log_list, library_list, book_id_list, book_title_list, book_odm_list = ([ +] for i in range(7)) scriptdir = os.path.join(Path.home(), "AutoBooks") csv_path = os.path.join(scriptdir, 'web_known_files.csv') @@ -31,8 +32,10 @@ os.chdir(scriptdir) else: os.mkdir(scriptdir) - main_conf = requests.get('https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') - odmpy_conf = requests.get("https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") + main_conf = requests.get( + 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') + odmpy_conf = requests.get( + "https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") folders = ['log', 'web_downloads', 'chrome_profile', 'sourcefiles'] for folder in folders: os.mkdir(os.path.join(scriptdir, folder)) @@ -40,26 +43,28 @@ localfile.write(main_conf.content) with open(os.path.join(scriptdir, "odmpydl.conf"), mode='wb') as localfile: localfile.write(odmpy_conf.content) - print("Finished setup please configure settings in file: ", os.path.join(scriptdir, "autobooks.conf")) + print("Finished setup please configure settings in file: ", + os.path.join(scriptdir, "autobooks.conf")) sys.exit(1) # Logging Config -LOG_FILENAME = os.path.join(scriptdir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}-Main.log'.format(datetime.now())) +LOG_FILENAME = os.path.join( + scriptdir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}-Main.log'.format(datetime.now())) console_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {message}\n{exception}" file_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {extra[scrubbed]}\n{exception}" -redacting_formatter = RedactingFormatter(patterns=["", "", "", ""], source_fmt=file_log_format) +redacting_formatter = RedactingFormatter(patterns=[ + "", "", "", ""], source_fmt=file_log_format) logger.configure(handlers=[ - {'sink': sys.stderr, "format": console_log_format}, - {'sink': LOG_FILENAME, "format": redacting_formatter.format} - ]) + {'sink': sys.stderr, "format": console_log_format}, + {'sink': LOG_FILENAME, "format": redacting_formatter.format} +]) odmpy.logger.handlers.clear() odmpy.logger.addHandler(InterceptHandler()) # Read config file parser = ConfigParser() parser.read(os.path.join(scriptdir, "autobooks.conf")) -odmdir = parser.get("DEFAULT", - "odm_folder") +odmdir = parser.get("DEFAULT", "odm_folder") outdir = parser.get("DEFAULT", "out_folder") library_count = len(parser.sections()) # Cronitor Setup https://cronitor.io/ @@ -76,14 +81,16 @@ def process_books(odm_list): if parser.get('DEFAULT', "test_args") == "true": odmpy_args = ["odmpy", "dl", x] else: - odmpy_args = ["odmpy", "dl", "@" + os.path.join(scriptdir, "odmpydl.conf"), x] + odmpy_args = ["odmpy", "dl", "@" + + os.path.join(scriptdir, "odmpydl.conf"), x] with patch.object(sys, 'argv', odmpy_args): try: odmpy.run() except FileNotFoundError: logger.error("Could not find odm file {}", x) except FileExistsError: - logger.error("FileAlreadyExists, likely from m4b creation attempt") + logger.error( + "FileAlreadyExists, likely from m4b creation attempt") except SystemExit as e: bad_odm_list.append(x) try: @@ -113,11 +120,14 @@ def cleanup(m4bs, odms, odmfolder): sourcefiles = odms + glob.glob("*.license") for x in sourcefiles: if os.path.isfile(os.path.join(scriptdir, "sourcefiles", x)): - logger.error("File {} already exists in sourcefiles dir skipped", x) + logger.error( + "File {} already exists in sourcefiles dir skipped", x) error_count += 1 else: + #license_file = x.replace(".odm",".license") shutil.move(x, os.path.join(scriptdir, "sourcefiles", x)) - logger.info("Moved file {} to sourcefiles", x) + #shutil.move(license_file, os.path.join(scriptdir, "sourcefiles", license_file)) + logger.info("Moved file pair {} to sourcefiles", x) # Function for login @@ -126,10 +136,13 @@ def web_login(driver, name, cardno, pin, select): logger.info("Logging into library: {}", name) # Attempt selecting library from dropdown if select != "false": - select_box = driver.find_element(By.XPATH, '//input[@id="signin-options"]') - webdriver.ActionChains(driver).move_to_element(select_box).click().send_keys(select).perform() + select_box = driver.find_element( + By.XPATH, '//input[@id="signin-options"]') + webdriver.ActionChains(driver).move_to_element( + select_box).click().send_keys(select).perform() sleep(1) - webdriver.ActionChains(driver).send_keys(Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform() + webdriver.ActionChains(driver).send_keys( + Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform() # Attempt sending card number try: driver.find_element(By.ID, "username").send_keys(cardno) @@ -139,14 +152,15 @@ def web_login(driver, name, cardno, pin, select): # Attempt sending pin Note:Some pages don't have pin input if pin != "false": driver.find_element(By.ID, "password").send_keys(pin) - driver.find_element(By.CSS_SELECTOR, "button.signin-button.button.secondary").click() + driver.find_element( + By.CSS_SELECTOR, "button.signin-button.button.secondary").click() sleep(5) # Function to download loans from OverDrive page def web_dl(driver, df, name): global error_count - #Gather all book title elements and check if any found + # Gather all book title elements and check if any found books = driver.find_elements(By.XPATH, '//a[@tabindex="0"][@role="link"]') if len(books) == 0: logger.warning("Can't find books skipped library: {}", name) @@ -156,11 +170,12 @@ def web_dl(driver, df, name): logger.info("Begin DL from library: {} ", name) bookcount = 0 for i in books: - #Fetch info about the book + # Fetch info about the book book_url = i.get_attribute('href') book_info = i.get_attribute('aria-label') book_info_split = book_info.split(". Audiobook. Expires in") - book_dl_url = book_url.replace('/media/', '/media/download/audiobook-mp3/') + book_dl_url = book_url.replace( + '/media/', '/media/download/audiobook-mp3/') book_id = int(''.join(filter(str.isdigit, book_url))) book_title = book_info_split[0] if "Audiobook." in book_info: @@ -173,16 +188,18 @@ def web_dl(driver, df, name): book_odm = max(glob.glob("*.odm"), key=os.path.getmtime) bookcount += 1 - #Add book data to vars + # Add book data to vars library_list.append(name) book_id_list.append(book_id) book_title_list.append(book_title) book_odm_list.append(book_odm) sleep(1) sleep(1) - logger.info("Finished downloading {} books from library {}", bookcount, name) + logger.info("Finished downloading {} books from library {}", + bookcount, name) return () + def main_run(): # AutoBooks logger.info("Started AutoBooks V.{} By:IvyB", scriptver) @@ -195,8 +212,7 @@ def main_run(): else: odm_list = glob.glob("*.odm") monitor.ping(state='run', - message='AutoBooks by IvyB Version:' + scriptver + '\n odmdir:' + odmdir + '\n outdir:' + outdir + '\n logfile:' + LOG_FILENAME + '\n Found the following books \n' + " ".join( - odm_list)) + message=f'AutoBooks by IvyB Started from web V.{scriptver} \n odmdir: {odmdir} \n outdir:{outdir}\n logfile:{LOG_FILENAME}\n odmlist: \n{" ".join(odm_list)}') # Check if any .odm files exist in odmdir if len(odm_list) == 0: @@ -209,11 +225,12 @@ def main_run(): # Cleanup input and output files m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, odmdir) - #Fetch log messages - logstr = process_logfile(LOG_FILENAME, terms=("Downloading", "expired", "generating", "merged", "saved")) + # Fetch log messages + logstr = process_logfile(LOG_FILENAME, terms=( + "Downloading", "expired", "generating", "merged", "saved")) # Send complete event and log to Cronitor monitor.ping(state='complete', message=logstr, - metrics={'count': len(odm_list), 'error_count': error_count}) + metrics={'count': len(odm_list), 'error_count': error_count}) # AutoBooks Web Code @@ -223,11 +240,8 @@ def web_run(): sys.exit(1) else: logger.info("Started AutoBooks Web V.{} By:IvyB", scriptver) - #monitor.ping(state='run', - #message='AutoBooks Web by IvyB Version:' + scriptver + '\n logfile:' + LOG_FILENAME + '\n LibraryCount: ' + str( - # len(parser.sections()))) monitor.ping(state='run', - message=f'AutoBooks Web by IvyB Version: {scriptver} \n logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}') + message=f'AutoBooks Web by IvyB Version: {scriptver} \n logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}') # Configure WebDriver options options = Options() prefs = { @@ -235,7 +249,8 @@ def web_run(): "download.prompt_for_download": False, "download.directory_upgrade": True } - options.add_argument('user-data-dir=' + os.path.join(scriptdir, "chrome_profile")) + options.add_argument('user-data-dir=' + + os.path.join(scriptdir, "chrome_profile")) # Headless mode check if parser.get('DEFAULT', "web_headless") == "true": options.add_argument('--headless') @@ -247,10 +262,10 @@ def web_run(): if os.path.exists(csv_path): df = pd.read_csv(csv_path, sep=",") else: - df = pd.DataFrame({ - 'book_id': book_id_list, + df = pd.DataFrame({ + 'book_id': book_id_list, }) - os.chdir(os.path.join(scriptdir,"web_downloads")) + os.chdir(os.path.join(scriptdir, "web_downloads")) # For every library, open site, attempt sign in, and attempt download. for i in range(0, len(parser.sections())): library_index = 'library_' + str(i) @@ -263,16 +278,16 @@ def web_run(): # Check signed in status and either sign in or move on if "/account/ozone/sign-in" in driver.current_url: web_login(driver, library_name, parser.get(library_index, "card_number"), - parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) + parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) web_dl(driver, df, library_name) sleep(2) - # Output book data to csv + # Output book data to csv df_out = pd.DataFrame({ - 'library_name': library_list, - 'book_id': book_id_list, - 'book_title': book_title_list, - 'book_odm': book_odm_list - }) + 'library_name': library_list, + 'book_id': book_id_list, + 'book_title': book_title_list, + 'book_odm': book_odm_list + }) if os.path.exists(csv_path): df_out.to_csv(csv_path, mode='a', index=False, header=False) else: @@ -284,8 +299,8 @@ def web_run(): # Process log file for Cronitor. process_logfile(LOG_FILENAME, terms=("web")) monitor.ping(state='complete', message="".join(odmlist), - metrics={'count': len(odmlist), 'error_count': error_count}) - + metrics={'count': len(odmlist), 'error_count': error_count}) + # Call Minimum DL functions if len(odmlist) != 0: logger.info("Started AutoBooks V.{} By:IvyB", scriptver) @@ -293,13 +308,15 @@ def web_run(): message=f'AutoBooks by IvyB Started from web V.{scriptver} \n outdir:{outdir}\n logfile:{LOG_FILENAME}\n odmlist: \n{" ".join(odmlist)}') process_books(odmlist) m4blist = glob.glob("*.m4b") - cleanup(m4blist, good_odm_list, os.path.join(scriptdir, "web_downloads")) + cleanup(m4blist, good_odm_list, os.path.join( + scriptdir, "web_downloads")) # Process log file for Cronitor - logstr = process_logfile(LOG_FILENAME, terms=("Downloading", "expired", "generating", "merged")) + logstr = process_logfile(LOG_FILENAME, terms=( + "Downloading", "expired", "generating", "merged")) # Send complete event and log to Cronitor monitor.ping(state='complete', message=logstr, - metrics={'count': len(odmlist), 'error_count': error_count}) - #return["\n".join(title_list), error_count] + metrics={'count': len(odmlist), 'error_count': error_count}) + # return["\n".join(title_list), error_count] if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "true": From 31e31f954febb8ed14a9b4f291ac4dcba3e76f6d Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 17:33:21 -0500 Subject: [PATCH 09/25] Update DiscordBot.py --- autobookstesting/DiscordBot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/autobookstesting/DiscordBot.py b/autobookstesting/DiscordBot.py index 435ed4b..7038052 100644 --- a/autobookstesting/DiscordBot.py +++ b/autobookstesting/DiscordBot.py @@ -80,7 +80,6 @@ async def hello(ctx): await ctx.channel.send("Known Books CSV not found.") # await ctx.channel.send(file=discord.File(max_file)) - def discord_run(): if token == "": logger.critical("Bot token not found in config file, exiting.") From 08e24d21428d7f8db1fe7c0dcb1d8dd4d0c50a34 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 17:34:46 -0500 Subject: [PATCH 10/25] Update DiscordBot.py --- autobookstesting/DiscordBot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobookstesting/DiscordBot.py b/autobookstesting/DiscordBot.py index 7038052..aa76e18 100644 --- a/autobookstesting/DiscordBot.py +++ b/autobookstesting/DiscordBot.py @@ -29,7 +29,7 @@ async def on_ready(): @bot.command(name='web') async def hello(ctx): - #Starting embed and running web + #Send starting embed and running web embed_start = discord.Embed(title="Running AutoBooks Web. This may take awhile....", description="Version: "+scriptver+" \nLogfile: "+LOG_FILENAME, color=0xFFAFCC) embed_start.set_image(url="https://raw.githubusercontent.com/ivybowman/AutoBooks/main/img/logo/small_pink.png") embed_start.set_footer(text="OS: "+ platform.platform()+" Host: "+platform.node()) From 013b7cee6f359704851dd27d19ab549a14f80662 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 17:50:35 -0500 Subject: [PATCH 11/25] Fix comment labels and adjust discord --- autobookstesting/AutoBooks.py | 8 +++++--- autobookstesting/DiscordBot.py | 3 +-- autobookstesting/utils.py | 1 - 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index 76daaad..0168693 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -49,14 +49,16 @@ # Logging Config LOG_FILENAME = os.path.join( - scriptdir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}-Main.log'.format(datetime.now())) + scriptdir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}.log'.format(datetime.now())) console_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {message}\n{exception}" +cronitor_log_format = "[{name}:{function}] {level}: {message}\n{exception}" file_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {extra[scrubbed]}\n{exception}" redacting_formatter = RedactingFormatter(patterns=[ "", "", "", ""], source_fmt=file_log_format) logger.configure(handlers=[ {'sink': sys.stderr, "format": console_log_format}, - {'sink': LOG_FILENAME, "format": redacting_formatter.format} + {'sink': LOG_FILENAME, "format": redacting_formatter.format}, + #{'sink': process_logfile(), "format": cronitor_log_format.format} ]) odmpy.logger.handlers.clear() odmpy.logger.addHandler(InterceptHandler()) @@ -301,7 +303,7 @@ def web_run(): monitor.ping(state='complete', message="".join(odmlist), metrics={'count': len(odmlist), 'error_count': error_count}) - # Call Minimum DL functions + # Call DL to process files from web if len(odmlist) != 0: logger.info("Started AutoBooks V.{} By:IvyB", scriptver) monitor.ping(state='run', diff --git a/autobookstesting/DiscordBot.py b/autobookstesting/DiscordBot.py index aa76e18..5ea094b 100644 --- a/autobookstesting/DiscordBot.py +++ b/autobookstesting/DiscordBot.py @@ -5,7 +5,6 @@ from pathlib import Path import os import discord -import logging import glob import sys import shutil @@ -62,7 +61,7 @@ async def hello(ctx): @bot.command(name='log') async def hello(ctx): - files = glob.glob(os.path.join(scriptdir, "log", "*-Main.log")) + files = glob.glob(os.path.join(scriptdir, "log", "*.log")) max_file = max(files, key=os.path.getmtime) print(max_file) await ctx.channel.send("Fetched latest AutoBooks logfile: \n" + max_file) diff --git a/autobookstesting/utils.py b/autobookstesting/utils.py index a81cbc7..9dae618 100644 --- a/autobookstesting/utils.py +++ b/autobookstesting/utils.py @@ -35,7 +35,6 @@ def emit(self, record): ) # Process log file for Cronitor def process_logfile(LOG_FILENAME, terms=None): - with open(LOG_FILENAME) as logs: lines = logs.readlines() log_list = [] From 67b8cc06766ea76d4ecddd0bb0b6575f438cb269 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 18:02:25 -0500 Subject: [PATCH 12/25] Update DiscordBot.py --- autobookstesting/DiscordBot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobookstesting/DiscordBot.py b/autobookstesting/DiscordBot.py index 5ea094b..5cafd33 100644 --- a/autobookstesting/DiscordBot.py +++ b/autobookstesting/DiscordBot.py @@ -17,7 +17,7 @@ try: token = parser.get("DEFAULT", "discord_bot_token") except KeyError: - logger.critical("Bot token not found in config file, exiting.") + logger.critical("Bot token field not found in config file, exiting.") bot = commands.Bot(command_prefix='?') From 0120d432bc1c73ac3f131ab2454117ba3fb9f995 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 4 Mar 2022 18:19:34 -0500 Subject: [PATCH 13/25] Update AutoBooks.py --- autobookstesting/AutoBooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index 0168693..4738415 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -22,8 +22,8 @@ # Set Vars scriptver = "0.2.1" # Version number of script error_count = 0 -good_odm_list, bad_odm_list, log_list, library_list, book_id_list, book_title_list, book_odm_list = ([ -] for i in range(7)) +good_odm_list, bad_odm_list, library_list, book_id_list, book_title_list, book_odm_list = ([ +] for i in range(6)) scriptdir = os.path.join(Path.home(), "AutoBooks") csv_path = os.path.join(scriptdir, 'web_known_files.csv') From 13e329ca3ec4c756b84415f765e89bef64f973f5 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 11:26:06 -0500 Subject: [PATCH 14/25] Fixes --- .gitignore | 1 + autobookstesting/AutoBooks.py | 70 +++++++++++++++++----------------- autobookstesting/DiscordBot.py | 8 ++-- setup.py | 48 ++++++++++++----------- 4 files changed, 65 insertions(+), 62 deletions(-) diff --git a/.gitignore b/.gitignore index e46a929..2518e9f 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ MANIFEST autobooks.conf web_chromeprofile/ web_edgeprofile/ +venv/ diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index 4738415..89898fc 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -20,54 +20,54 @@ import requests # Set Vars -scriptver = "0.2.1" # Version number of script +version = "0.2.1" # Version number of script error_count = 0 good_odm_list, bad_odm_list, library_list, book_id_list, book_title_list, book_odm_list = ([ ] for i in range(6)) -scriptdir = os.path.join(Path.home(), "AutoBooks") -csv_path = os.path.join(scriptdir, 'web_known_files.csv') +script_dir = os.path.join(Path.home(), "AutoBooks") +csv_path = os.path.join(script_dir, 'web_known_files.csv') # Check paths, and if not found do first time setup -if os.path.exists(scriptdir): - os.chdir(scriptdir) +if os.path.exists(script_dir): + os.chdir(script_dir) else: - os.mkdir(scriptdir) + os.mkdir(script_dir) main_conf = requests.get( 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') odmpy_conf = requests.get( "https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") folders = ['log', 'web_downloads', 'chrome_profile', 'sourcefiles'] for folder in folders: - os.mkdir(os.path.join(scriptdir, folder)) - with open(os.path.join(scriptdir, "autobooks.conf"), mode='wb') as localfile: + os.mkdir(os.path.join(script_dir, folder)) + with open(os.path.join(script_dir, "autobooks.conf"), mode='wb') as localfile: localfile.write(main_conf.content) - with open(os.path.join(scriptdir, "odmpydl.conf"), mode='wb') as localfile: + with open(os.path.join(script_dir, "odmpydl.conf"), mode='wb') as localfile: localfile.write(odmpy_conf.content) print("Finished setup please configure settings in file: ", - os.path.join(scriptdir, "autobooks.conf")) + os.path.join(script_dir, "autobooks.conf")) sys.exit(1) # Logging Config LOG_FILENAME = os.path.join( - scriptdir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}.log'.format(datetime.now())) + script_dir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}.log'.format(datetime.now())) +patterns = ['', '', '', ''] console_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {message}\n{exception}" cronitor_log_format = "[{name}:{function}] {level}: {message}\n{exception}" file_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {extra[scrubbed]}\n{exception}" -redacting_formatter = RedactingFormatter(patterns=[ - "", "", "", ""], source_fmt=file_log_format) +redacting_formatter = RedactingFormatter(patterns=patterns, source_fmt=file_log_format) logger.configure(handlers=[ {'sink': sys.stderr, "format": console_log_format}, {'sink': LOG_FILENAME, "format": redacting_formatter.format}, - #{'sink': process_logfile(), "format": cronitor_log_format.format} + # {'sink': process_logfile(), "format": cronitor_log_format.format} ]) odmpy.logger.handlers.clear() odmpy.logger.addHandler(InterceptHandler()) # Read config file parser = ConfigParser() -parser.read(os.path.join(scriptdir, "autobooks.conf")) -odmdir = parser.get("DEFAULT", "odm_folder") -outdir = parser.get("DEFAULT", "out_folder") +parser.read(os.path.join(script_dir, "autobooks.conf")) +odm_dir = parser.get("DEFAULT", "odm_folder") +out_dir = parser.get("DEFAULT", "out_folder") library_count = len(parser.sections()) # Cronitor Setup https://cronitor.io/ cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") @@ -78,13 +78,13 @@ # Function to process the books. def process_books(odm_list): global error_count - logger.info('Begin processing booklist: {}', " ".join(odm_list)) + logger.info('Begin processing book list: {}', " ".join(odm_list)) for x in odm_list: if parser.get('DEFAULT', "test_args") == "true": odmpy_args = ["odmpy", "dl", x] else: odmpy_args = ["odmpy", "dl", "@" + - os.path.join(scriptdir, "odmpydl.conf"), x] + os.path.join(script_dir, "odmpydl.conf"), x] with patch.object(sys, 'argv', odmpy_args): try: odmpy.run() @@ -112,22 +112,22 @@ def cleanup(m4bs, odms, odmfolder): global error_count # Move m4b files to outdir for x in m4bs: - if os.path.isfile(os.path.join(outdir + x)): + if os.path.isfile(os.path.join(out_dir + x)): logger.error("Book {} already exists in outdir skipped", x) error_count += 1 else: - shutil.move(os.path.join(odmfolder, x), os.path.join(outdir, x)) + shutil.move(os.path.join(odmfolder, x), os.path.join(out_dir, x)) logger.info("Moved book {} to outdir", x) # Backup source files sourcefiles = odms + glob.glob("*.license") for x in sourcefiles: - if os.path.isfile(os.path.join(scriptdir, "sourcefiles", x)): + if os.path.isfile(os.path.join(script_dir, "sourcefiles", x)): logger.error( "File {} already exists in sourcefiles dir skipped", x) error_count += 1 else: #license_file = x.replace(".odm",".license") - shutil.move(x, os.path.join(scriptdir, "sourcefiles", x)) + shutil.move(x, os.path.join(script_dir, "sourcefiles", x)) #shutil.move(license_file, os.path.join(scriptdir, "sourcefiles", license_file)) logger.info("Moved file pair {} to sourcefiles", x) @@ -204,17 +204,17 @@ def web_dl(driver, df, name): def main_run(): # AutoBooks - logger.info("Started AutoBooks V.{} By:IvyB", scriptver) + logger.info("Started AutoBooks V.{} By:IvyB", version) # Try to change to ODM folder try: - os.chdir(odmdir) + os.chdir(odm_dir) except FileNotFoundError: logger.critical("The provided .odm dir was not found, exiting") sys.exit(1) else: odm_list = glob.glob("*.odm") monitor.ping(state='run', - message=f'AutoBooks by IvyB Started from web V.{scriptver} \n odmdir: {odmdir} \n outdir:{outdir}\n logfile:{LOG_FILENAME}\n odmlist: \n{" ".join(odm_list)}') + message=f'AutoBooks by IvyB Started from web V.{version} \n odmdir: {odm_dir} \n outdir:{out_dir}\n logfile:{LOG_FILENAME}\n odmlist: \n{" ".join(odm_list)}') # Check if any .odm files exist in odmdir if len(odm_list) == 0: @@ -226,7 +226,7 @@ def main_run(): process_books(odm_list) # Cleanup input and output files m4blist = glob.glob("*.m4b") - cleanup(m4blist, good_odm_list, odmdir) + cleanup(m4blist, good_odm_list, odm_dir) # Fetch log messages logstr = process_logfile(LOG_FILENAME, terms=( "Downloading", "expired", "generating", "merged", "saved")) @@ -241,18 +241,18 @@ def web_run(): logger.critical("No libraries configured!") sys.exit(1) else: - logger.info("Started AutoBooks Web V.{} By:IvyB", scriptver) + logger.info("Started AutoBooks Web V.{} By:IvyB", version) monitor.ping(state='run', - message=f'AutoBooks Web by IvyB Version: {scriptver} \n logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}') + message=f'AutoBooks Web by IvyB Version: {version} \n logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}') # Configure WebDriver options options = Options() prefs = { - "download.default_directory": os.path.join(scriptdir, "web_downloads"), + "download.default_directory": os.path.join(script_dir, "web_downloads"), "download.prompt_for_download": False, "download.directory_upgrade": True } options.add_argument('user-data-dir=' + - os.path.join(scriptdir, "chrome_profile")) + os.path.join(script_dir, "chrome_profile")) # Headless mode check if parser.get('DEFAULT', "web_headless") == "true": options.add_argument('--headless') @@ -267,7 +267,7 @@ def web_run(): df = pd.DataFrame({ 'book_id': book_id_list, }) - os.chdir(os.path.join(scriptdir, "web_downloads")) + os.chdir(os.path.join(script_dir, "web_downloads")) # For every library, open site, attempt sign in, and attempt download. for i in range(0, len(parser.sections())): library_index = 'library_' + str(i) @@ -305,13 +305,13 @@ def web_run(): # Call DL to process files from web if len(odmlist) != 0: - logger.info("Started AutoBooks V.{} By:IvyB", scriptver) + logger.info("Started AutoBooks V.{} By:IvyB", version) monitor.ping(state='run', - message=f'AutoBooks by IvyB Started from web V.{scriptver} \n outdir:{outdir}\n logfile:{LOG_FILENAME}\n odmlist: \n{" ".join(odmlist)}') + message=f'AutoBooks by IvyB Started from web V.{version} \n outdir:{out_dir}\n logfile:{LOG_FILENAME}\n odmlist: \n{" ".join(odmlist)}') process_books(odmlist) m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, os.path.join( - scriptdir, "web_downloads")) + script_dir, "web_downloads")) # Process log file for Cronitor logstr = process_logfile(LOG_FILENAME, terms=( "Downloading", "expired", "generating", "merged")) diff --git a/autobookstesting/DiscordBot.py b/autobookstesting/DiscordBot.py index 5cafd33..ba9b318 100644 --- a/autobookstesting/DiscordBot.py +++ b/autobookstesting/DiscordBot.py @@ -9,7 +9,7 @@ import sys import shutil import platform -from AutoBooks import web_run, main_run, scriptver, scriptdir, parser, csv_path, LOG_FILENAME, logger +from AutoBooks import web_run, main_run, version, script_dir, parser, csv_path, LOG_FILENAME, logger import pandas as pd @@ -29,7 +29,7 @@ async def on_ready(): @bot.command(name='web') async def hello(ctx): #Send starting embed and running web - embed_start = discord.Embed(title="Running AutoBooks Web. This may take awhile....", description="Version: "+scriptver+" \nLogfile: "+LOG_FILENAME, color=0xFFAFCC) + embed_start = discord.Embed(title="Running AutoBooks Web. This may take awhile....", description="Version: " + version + " \nLogfile: " + LOG_FILENAME, color=0xFFAFCC) embed_start.set_image(url="https://raw.githubusercontent.com/ivybowman/AutoBooks/main/img/logo/small_pink.png") embed_start.set_footer(text="OS: "+ platform.platform()+" Host: "+platform.node()) await ctx.channel.send(embed=embed_start) @@ -43,7 +43,7 @@ async def hello(ctx): embed_end.add_field(name="Book List", value=str(web_info[0]), inline=False) await ctx.channel.send(embed=embed_end) #Logfile fetching - files = glob.glob(os.path.join(scriptdir, "log", "*-Main.log")) + files = glob.glob(os.path.join(script_dir, "log", "*-Main.log")) files2 = sorted(files, key=os.path.getmtime, reverse=True) print(files2[0]) await ctx.channel.send(file=discord.File(files2[0])) @@ -61,7 +61,7 @@ async def hello(ctx): @bot.command(name='log') async def hello(ctx): - files = glob.glob(os.path.join(scriptdir, "log", "*.log")) + files = glob.glob(os.path.join(script_dir, "log", "*.log")) max_file = max(files, key=os.path.getmtime) print(max_file) await ctx.channel.send("Fetched latest AutoBooks logfile: \n" + max_file) diff --git a/setup.py b/setup.py index beb25b5..00c0b98 100644 --- a/setup.py +++ b/setup.py @@ -1,36 +1,38 @@ from setuptools import setup -VERSION = '0.2.1' +VERSION = '0.2.1' DESCRIPTION = 'Python tool to automate processing a batch of OverDrive audiobooks.' LONG_DESCRIPTION = 'Python tool to automate processing a batch of OverDrive audiobooks.' # Setting up setup( - name="AutoBooks", - version=VERSION, - author="Ivy Bowman", - license="GPL", - url="https://github.com/ivybowman/AutoBooks", - description=DESCRIPTION, - long_description=LONG_DESCRIPTION, - packages=["autobooks"], - entry_points={ + name="AutoBooks", + version=VERSION, + author="Ivy Bowman", + license="GPL", + url="https://github.com/ivybowman/AutoBooks", + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + packages=["autobooks", "autobookstesting"], + entry_points={ "console_scripts": [ "autobooks = autobooks.AutoBooks:main_run", + "autobooks-web-test = autobookstesting.AutoBooks:web_run", "autobooks-web = autobooks.AutoBooks:web_run", "autobooks-discord = autobooks.AutoBooksDiscord:run" ] }, - install_requires=['odmpy @ git+https://git@github.com/ping/odmpy.git', "cronitor", "pandas", "discord.py", "selenium", "requests"], - include_package_data=True, - platforms="any", - keywords=['python', 'AutoBooks'], - classifiers= [ - "Development Status :: 4 - Beta", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Environment :: Console", - "Intended Audience :: End Users/Desktop", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - ] -) \ No newline at end of file + install_requires=['odmpy @ git+https://git@github.com/ping/odmpy.git', "cronitor", "pandas", "discord.py", + "selenium", "requests", "loguru"], + include_package_data=True, + platforms="any", + keywords=['python', 'AutoBooks'], + classifiers=[ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Environment :: Console", + "Intended Audience :: End Users/Desktop", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + ] +) From 04b2bd7779e7366b0499dae3d475f6d830e8952a Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 12:23:22 -0500 Subject: [PATCH 15/25] Lot's of formatting fixes --- autobookstesting/AutoBooks.py | 120 ++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 57 deletions(-) diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index 89898fc..b4721c6 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -1,23 +1,25 @@ -from selenium import webdriver -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options +import glob +import os +import shutil +import sys from configparser import ConfigParser -from unittest.mock import patch from datetime import datetime -from time import sleep from pathlib import Path +from time import sleep +from unittest.mock import patch + +import cronitor import odmpy.odm as odmpy -from loguru import logger -from utils import InterceptHandler, RedactingFormatter, process_logfile import pandas as pd -import selenium -import cronitor -import glob -import sys -import shutil -import os import requests +import selenium +from loguru import logger +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys + +from utils import InterceptHandler, RedactingFormatter, process_logfile # Set Vars version = "0.2.1" # Version number of script @@ -36,13 +38,13 @@ 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') odmpy_conf = requests.get( "https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") - folders = ['log', 'web_downloads', 'chrome_profile', 'sourcefiles'] + folders = ['log', 'web_downloads', 'web_profile', 'source_files'] for folder in folders: os.mkdir(os.path.join(script_dir, folder)) - with open(os.path.join(script_dir, "autobooks.conf"), mode='wb') as localfile: - localfile.write(main_conf.content) - with open(os.path.join(script_dir, "odmpydl.conf"), mode='wb') as localfile: - localfile.write(odmpy_conf.content) + with open(os.path.join(script_dir, "autobooks.conf"), mode='wb') as local_file: + local_file.write(main_conf.content) + with open(os.path.join(script_dir, "odmpydl.conf"), mode='wb') as local_file: + local_file.write(odmpy_conf.content) print("Finished setup please configure settings in file: ", os.path.join(script_dir, "autobooks.conf")) sys.exit(1) @@ -107,29 +109,29 @@ def process_books(odm_list): logger.info("Book Processing Finished") -# Function to cleanup in and out files. -def cleanup(m4bs, odms, odmfolder): +# Function to clean up in and out files. +def cleanup(m4b_list, odm_list, odm_folder): global error_count - # Move m4b files to outdir - for x in m4bs: + # Move m4b files to out_dir + for x in m4b_list: if os.path.isfile(os.path.join(out_dir + x)): - logger.error("Book {} already exists in outdir skipped", x) + logger.error("Book {} already exists in out dir skipped", x) error_count += 1 else: - shutil.move(os.path.join(odmfolder, x), os.path.join(out_dir, x)) - logger.info("Moved book {} to outdir", x) + shutil.move(os.path.join(odm_folder, x), os.path.join(out_dir, x)) + logger.info("Moved book {} to out_dir", x) # Backup source files - sourcefiles = odms + glob.glob("*.license") - for x in sourcefiles: - if os.path.isfile(os.path.join(script_dir, "sourcefiles", x)): + source_files = odm_list + glob.glob("*.license") + for x in source_files: + if os.path.isfile(os.path.join(script_dir, "source_files", x)): logger.error( - "File {} already exists in sourcefiles dir skipped", x) + "File {} already exists in source files dir skipped", x) error_count += 1 else: - #license_file = x.replace(".odm",".license") - shutil.move(x, os.path.join(script_dir, "sourcefiles", x)) - #shutil.move(license_file, os.path.join(scriptdir, "sourcefiles", license_file)) - logger.info("Moved file pair {} to sourcefiles", x) + # license_file = x.replace(".odm",".license") + shutil.move(x, os.path.join(script_dir, "source_files", x)) + # shutil.move(license_file, os.path.join(script_dir, "source_files", license_file)) + logger.info("Moved file pair {} to source files", x) # Function for login @@ -149,7 +151,7 @@ def web_login(driver, name, cardno, pin, select): try: driver.find_element(By.ID, "username").send_keys(cardno) except selenium.common.exceptions.NoSuchElementException: - logger.critical("Can't find card number field skipped library {}", ) + logger.critical("Can't find card number field skipped library {}", name) error_count += 1 # Attempt sending pin Note:Some pages don't have pin input if pin != "false": @@ -170,7 +172,7 @@ def web_dl(driver, df, name): return () else: logger.info("Begin DL from library: {} ", name) - bookcount = 0 + book_count = 0 for i in books: # Fetch info about the book book_url = i.get_attribute('href') @@ -188,7 +190,7 @@ def web_dl(driver, df, name): driver.get(book_dl_url) logger.info("Downloaded book: {}", book_title) book_odm = max(glob.glob("*.odm"), key=os.path.getmtime) - bookcount += 1 + book_count += 1 # Add book data to vars library_list.append(name) @@ -198,7 +200,7 @@ def web_dl(driver, df, name): sleep(1) sleep(1) logger.info("Finished downloading {} books from library {}", - bookcount, name) + book_count, name) return () @@ -214,9 +216,11 @@ def main_run(): else: odm_list = glob.glob("*.odm") monitor.ping(state='run', - message=f'AutoBooks by IvyB Started from web V.{version} \n odmdir: {odm_dir} \n outdir:{out_dir}\n logfile:{LOG_FILENAME}\n odmlist: \n{" ".join(odm_list)}') + message=f'AutoBooks by IvyB Started from web V.{version} \n' + + f'odm_dir: {odm_dir} \n out_dir:{out_dir}\n logfile:{LOG_FILENAME}\n' + + f'odm_list: \n{" ".join(odm_list)}') - # Check if any .odm files exist in odmdir + # Check if any .odm files exist in odm_dir if len(odm_list) == 0: monitor.ping(state='fail', message='Error: No .odm files found, exiting', metrics={'error_count': error_count}) @@ -227,11 +231,10 @@ def main_run(): # Cleanup input and output files m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, odm_dir) - # Fetch log messages - logstr = process_logfile(LOG_FILENAME, terms=( - "Downloading", "expired", "generating", "merged", "saved")) # Send complete event and log to Cronitor - monitor.ping(state='complete', message=logstr, + log_str = process_logfile(LOG_FILENAME, terms=( + "Downloading", "expired", "generating", "merged", "saved")) + monitor.ping(state='complete', message=log_str, metrics={'count': len(odm_list), 'error_count': error_count}) @@ -243,7 +246,8 @@ def web_run(): else: logger.info("Started AutoBooks Web V.{} By:IvyB", version) monitor.ping(state='run', - message=f'AutoBooks Web by IvyB Version: {version} \n logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}') + message=f'AutoBooks Web by IvyB V.{version} \n' + + f'logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}') # Configure WebDriver options options = Options() prefs = { @@ -252,7 +256,7 @@ def web_run(): "download.directory_upgrade": True } options.add_argument('user-data-dir=' + - os.path.join(script_dir, "chrome_profile")) + os.path.join(script_dir, "web_profile")) # Headless mode check if parser.get('DEFAULT', "web_headless") == "true": options.add_argument('--headless') @@ -268,6 +272,7 @@ def web_run(): 'book_id': book_id_list, }) os.chdir(os.path.join(script_dir, "web_downloads")) + # For every library, open site, attempt sign in, and attempt download. for i in range(0, len(parser.sections())): library_index = 'library_' + str(i) @@ -296,30 +301,31 @@ def web_run(): df_out.to_csv(csv_path, mode='w', index=False, header=True) driver.close() logger.info("AutoBooksWeb Complete") - odmlist = glob.glob("*.odm") + web_odm_list = glob.glob("*.odm") # Process log file for Cronitor. - process_logfile(LOG_FILENAME, terms=("web")) - monitor.ping(state='complete', message="".join(odmlist), - metrics={'count': len(odmlist), 'error_count': error_count}) + process_logfile(LOG_FILENAME, terms=("web", "ERROR")) + monitor.ping(state='complete', message="".join(web_odm_list), + metrics={'count': len(web_odm_list), 'error_count': error_count}) # Call DL to process files from web - if len(odmlist) != 0: + if len(web_odm_list) != 0: logger.info("Started AutoBooks V.{} By:IvyB", version) monitor.ping(state='run', - message=f'AutoBooks by IvyB Started from web V.{version} \n outdir:{out_dir}\n logfile:{LOG_FILENAME}\n odmlist: \n{" ".join(odmlist)}') - process_books(odmlist) + message=f'AutoBooks by IvyB Started from web V.{version} \n' + + f'out_dir:{out_dir}\n logfile:{LOG_FILENAME}\n odm_list: \n{" ".join(web_odm_list)}') + process_books(web_odm_list) m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, os.path.join( script_dir, "web_downloads")) # Process log file for Cronitor - logstr = process_logfile(LOG_FILENAME, terms=( + log_str = process_logfile(LOG_FILENAME, terms=( "Downloading", "expired", "generating", "merged")) # Send complete event and log to Cronitor - monitor.ping(state='complete', message=logstr, - metrics={'count': len(odmlist), 'error_count': error_count}) + monitor.ping(state='complete', message=log_str, + metrics={'count': len(web_odm_list), 'error_count': error_count}) # return["\n".join(title_list), error_count] if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "true": - main_run() + web_run() From 1a0425ddc34e95292f2dea46995629d736127b1b Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 12:39:11 -0500 Subject: [PATCH 16/25] Update AutoBooks.py --- autobookstesting/AutoBooks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index b4721c6..9b5b068 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -135,7 +135,7 @@ def cleanup(m4b_list, odm_list, odm_folder): # Function for login -def web_login(driver, name, cardno, pin, select): +def web_login(driver, name, card_num, pin, select): global error_count logger.info("Logging into library: {}", name) # Attempt selecting library from dropdown @@ -149,7 +149,7 @@ def web_login(driver, name, cardno, pin, select): Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform() # Attempt sending card number try: - driver.find_element(By.ID, "username").send_keys(cardno) + driver.find_element(By.ID, "username").send_keys(card_num) except selenium.common.exceptions.NoSuchElementException: logger.critical("Can't find card number field skipped library {}", name) error_count += 1 @@ -182,6 +182,8 @@ def web_dl(driver, df, name): '/media/', '/media/download/audiobook-mp3/') book_id = int(''.join(filter(str.isdigit, book_url))) book_title = book_info_split[0] + + # Check if found book is an audiobook then download or skip. if "Audiobook." in book_info: if str(book_id) in df['book_id'].to_string(): logger.info('Skipped {} found in known books', book_title) @@ -268,6 +270,7 @@ def web_run(): if os.path.exists(csv_path): df = pd.read_csv(csv_path, sep=",") else: + # Failing above create an empty df for checking df = pd.DataFrame({ 'book_id': book_id_list, }) From 2a3addf4f0cd77a4c00816de0b9232c81c48dba5 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 13:39:28 -0500 Subject: [PATCH 17/25] Change folder names --- autobookstesting/AutoBooks.py | 48 ++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index 9b5b068..be02278 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -38,7 +38,7 @@ 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') odmpy_conf = requests.get( "https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") - folders = ['log', 'web_downloads', 'web_profile', 'source_files'] + folders = ['log', 'downloads', 'profile', 'source_backup'] for folder in folders: os.mkdir(os.path.join(script_dir, folder)) with open(os.path.join(script_dir, "autobooks.conf"), mode='wb') as local_file: @@ -78,23 +78,21 @@ # Function to process the books. -def process_books(odm_list): +def process(odm_list): global error_count logger.info('Begin processing book list: {}', " ".join(odm_list)) for x in odm_list: if parser.get('DEFAULT', "test_args") == "true": odmpy_args = ["odmpy", "dl", x] else: - odmpy_args = ["odmpy", "dl", "@" + - os.path.join(script_dir, "odmpydl.conf"), x] + odmpy_args = ["odmpy", "dl", "-c", "-m", "--mergeformat", "m4b", "--nobookfolder", x] with patch.object(sys, 'argv', odmpy_args): try: odmpy.run() except FileNotFoundError: logger.error("Could not find odm file {}", x) except FileExistsError: - logger.error( - "FileAlreadyExists, likely from m4b creation attempt") + logger.error("FileAlreadyExists, likely from m4b creation attempt") except SystemExit as e: bad_odm_list.append(x) try: @@ -109,6 +107,21 @@ def process_books(odm_list): logger.info("Book Processing Finished") +# Function to merge the books. +def merge(odm_list): + global error_count + logger.info('Begin merging book list: {}', " ".join(odm_list)) + for x in odm_list: + odmpy_args = ["odmpy", "dl", "-c", "-m", "--mergeformat", "m4b", "--nobookfolder", x] + with patch.object(sys, 'argv', odmpy_args): + try: + odmpy.run() + except FileNotFoundError: + logger.error("Could not find odm file {}", x) + + logger.info("Book merging Finished") + + # Function to clean up in and out files. def cleanup(m4b_list, odm_list, odm_folder): global error_count @@ -121,16 +134,15 @@ def cleanup(m4b_list, odm_list, odm_folder): shutil.move(os.path.join(odm_folder, x), os.path.join(out_dir, x)) logger.info("Moved book {} to out_dir", x) # Backup source files - source_files = odm_list + glob.glob("*.license") - for x in source_files: + for x in odm_list: if os.path.isfile(os.path.join(script_dir, "source_files", x)): logger.error( "File {} already exists in source files dir skipped", x) error_count += 1 else: - # license_file = x.replace(".odm",".license") + license_file = x.replace(".odm", ".license") shutil.move(x, os.path.join(script_dir, "source_files", x)) - # shutil.move(license_file, os.path.join(script_dir, "source_files", license_file)) + shutil.move(license_file, os.path.join(script_dir, "source_files", license_file)) logger.info("Moved file pair {} to source files", x) @@ -182,7 +194,7 @@ def web_dl(driver, df, name): '/media/', '/media/download/audiobook-mp3/') book_id = int(''.join(filter(str.isdigit, book_url))) book_title = book_info_split[0] - + # Check if found book is an audiobook then download or skip. if "Audiobook." in book_info: if str(book_id) in df['book_id'].to_string(): @@ -229,8 +241,8 @@ def main_run(): logger.critical("No .odm files found, exiting") sys.exit(1) else: - process_books(odm_list) - # Cleanup input and output files + process(odm_list) + # Cleanup files m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, odm_dir) # Send complete event and log to Cronitor @@ -253,12 +265,12 @@ def web_run(): # Configure WebDriver options options = Options() prefs = { - "download.default_directory": os.path.join(script_dir, "web_downloads"), + "download.default_directory": os.path.join(script_dir, "downloads"), "download.prompt_for_download": False, "download.directory_upgrade": True } options.add_argument('user-data-dir=' + - os.path.join(script_dir, "web_profile")) + os.path.join(script_dir, "profile")) # Headless mode check if parser.get('DEFAULT', "web_headless") == "true": options.add_argument('--headless') @@ -274,7 +286,7 @@ def web_run(): df = pd.DataFrame({ 'book_id': book_id_list, }) - os.chdir(os.path.join(script_dir, "web_downloads")) + os.chdir(os.path.join(script_dir, "downloads")) # For every library, open site, attempt sign in, and attempt download. for i in range(0, len(parser.sections())): @@ -317,10 +329,10 @@ def web_run(): monitor.ping(state='run', message=f'AutoBooks by IvyB Started from web V.{version} \n' + f'out_dir:{out_dir}\n logfile:{LOG_FILENAME}\n odm_list: \n{" ".join(web_odm_list)}') - process_books(web_odm_list) + process(web_odm_list) m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, os.path.join( - script_dir, "web_downloads")) + script_dir, "downloads")) # Process log file for Cronitor log_str = process_logfile(LOG_FILENAME, terms=( "Downloading", "expired", "generating", "merged")) From 91b5ca46e93a3e3a50e19d29505d2b123c21f3c5 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 13:52:19 -0500 Subject: [PATCH 18/25] Cleaned up process function --- autobookstesting/AutoBooks.py | 31 +++++---------------- autobookstesting/DiscordBot.py | 49 +++++++++++++++++----------------- autobookstesting/main.py | 2 +- autobookstesting/utils.py | 17 +++++++----- 4 files changed, 42 insertions(+), 57 deletions(-) diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py index be02278..a808ece 100644 --- a/autobookstesting/AutoBooks.py +++ b/autobookstesting/AutoBooks.py @@ -36,15 +36,15 @@ os.mkdir(script_dir) main_conf = requests.get( 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') - odmpy_conf = requests.get( - "https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") + # odmpy_conf = requests.get( + # "https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") folders = ['log', 'downloads', 'profile', 'source_backup'] for folder in folders: os.mkdir(os.path.join(script_dir, folder)) with open(os.path.join(script_dir, "autobooks.conf"), mode='wb') as local_file: local_file.write(main_conf.content) - with open(os.path.join(script_dir, "odmpydl.conf"), mode='wb') as local_file: - local_file.write(odmpy_conf.content) + # with open(os.path.join(script_dir, "odmpydl.conf"), mode='wb') as local_file: + # local_file.write(odmpy_conf.content) print("Finished setup please configure settings in file: ", os.path.join(script_dir, "autobooks.conf")) sys.exit(1) @@ -95,33 +95,14 @@ def process(odm_list): logger.error("FileAlreadyExists, likely from m4b creation attempt") except SystemExit as e: bad_odm_list.append(x) - try: + if os.path.isfile("cover.jpg"): os.remove("cover.jpg") - except FileNotFoundError: - logger.debug("Could not remove cover.jpg, moving on") - else: - logger.debug("Removed cover.jpg to prep for next attempt") error_count += 1 else: good_odm_list.append(x) logger.info("Book Processing Finished") -# Function to merge the books. -def merge(odm_list): - global error_count - logger.info('Begin merging book list: {}', " ".join(odm_list)) - for x in odm_list: - odmpy_args = ["odmpy", "dl", "-c", "-m", "--mergeformat", "m4b", "--nobookfolder", x] - with patch.object(sys, 'argv', odmpy_args): - try: - odmpy.run() - except FileNotFoundError: - logger.error("Could not find odm file {}", x) - - logger.info("Book merging Finished") - - # Function to clean up in and out files. def cleanup(m4b_list, odm_list, odm_folder): global error_count @@ -195,7 +176,7 @@ def web_dl(driver, df, name): book_id = int(''.join(filter(str.isdigit, book_url))) book_title = book_info_split[0] - # Check if found book is an audiobook then download or skip. + # Check if found book is a not known audiobook if "Audiobook." in book_info: if str(book_id) in df['book_id'].to_string(): logger.info('Skipped {} found in known books', book_title) diff --git a/autobookstesting/DiscordBot.py b/autobookstesting/DiscordBot.py index ba9b318..7de3707 100644 --- a/autobookstesting/DiscordBot.py +++ b/autobookstesting/DiscordBot.py @@ -1,17 +1,13 @@ -from configparser import ConfigParser -from unittest.mock import patch -from datetime import datetime -from discord.ext import commands -from pathlib import Path -import os -import discord import glob -import sys -import shutil +import os import platform -from AutoBooks import web_run, main_run, version, script_dir, parser, csv_path, LOG_FILENAME, logger +import sys + +import discord import pandas as pd +from discord.ext import commands +from AutoBooks import web_run, main_run, version, script_dir, parser, csv_path, LOG_FILENAME, logger # Bot Settings try: @@ -28,21 +24,24 @@ async def on_ready(): @bot.command(name='web') async def hello(ctx): - #Send starting embed and running web - embed_start = discord.Embed(title="Running AutoBooks Web. This may take awhile....", description="Version: " + version + " \nLogfile: " + LOG_FILENAME, color=0xFFAFCC) + # Send starting embed and running web + embed_start = discord.Embed(title="Running AutoBooks Web. This may take awhile....", + description="Version: " + version + " \nLogfile: " + LOG_FILENAME, color=0xFFAFCC) embed_start.set_image(url="https://raw.githubusercontent.com/ivybowman/AutoBooks/main/img/logo/small_pink.png") - embed_start.set_footer(text="OS: "+ platform.platform()+" Host: "+platform.node()) + embed_start.set_footer(text="OS: " + platform.platform() + " Host: " + platform.node()) await ctx.channel.send(embed=embed_start) web_info = ["test-data", "test-data2"] web_run() - #Ending Embed - embed_end = discord.Embed(title="AutoBooks Web Finished", description="See log info below for details. ErrorCount: "+str(web_info[1]), color=0xFFAFCC) + # Ending Embed + embed_end = discord.Embed(title="AutoBooks Web Finished", + description="See log info below for details. ErrorCount: " + str(web_info[1]), + color=0xFFAFCC) embed_end.set_thumbnail(url="https://raw.githubusercontent.com/ivybowman/AutoBooks/main/img/icon_pink.png") if web_info[0] != "": embed_end.add_field(name="Book List", value=str(web_info[0]), inline=False) await ctx.channel.send(embed=embed_end) - #Logfile fetching + # Logfile fetching files = glob.glob(os.path.join(script_dir, "log", "*-Main.log")) files2 = sorted(files, key=os.path.getmtime, reverse=True) print(files2[0]) @@ -51,11 +50,11 @@ async def hello(ctx): @bot.command(name='main') async def hello(ctx): - embedVar = discord.Embed(title="Title", description="Desc", color=0xFFAFCC) - embedVar.add_field(name="Field1", value="hi", inline=False) - embedVar.add_field(name="Field2", value="hi2", inline=False) - - await ctx.channel.send(embed=embedVar) + embed_var = discord.Embed(title="Title", description="Desc", color=0xFFAFCC) + embed_var.add_field(name="Field1", value="hi", inline=False) + embed_var.add_field(name="Field2", value="hi2", inline=False) + + await ctx.channel.send(embed=embed_var) main_run() @@ -72,13 +71,15 @@ async def hello(ctx): async def hello(ctx): try: df = pd.read_csv(csv_path, sep=",") - embedVar = discord.Embed(title="Autobooks Known Books", description=df['audiobook_title'].to_string(index=False), color=0xFFAFCC) - embedVar.set_footer(text="OS: "+ platform.platform()+" Host: "+os.uname()) - await ctx.channel.send(embed=embedVar) + embed_var = discord.Embed(title="Autobooks Known Books", + description=df['audiobook_title'].to_string(index=False), color=0xFFAFCC) + embed_var.set_footer(text="OS: " + platform.platform() + " Host: " + os.uname()) + await ctx.channel.send(embed=embed_var) except FileNotFoundError: await ctx.channel.send("Known Books CSV not found.") # await ctx.channel.send(file=discord.File(max_file)) + def discord_run(): if token == "": logger.critical("Bot token not found in config file, exiting.") diff --git a/autobookstesting/main.py b/autobookstesting/main.py index 55f3a1c..d552299 100644 --- a/autobookstesting/main.py +++ b/autobookstesting/main.py @@ -1,4 +1,4 @@ -from AutoBooks import web_run, main_run +# from AutoBooks import web_run, main_run from DiscordBot import discord_run import argparse diff --git a/autobookstesting/utils.py b/autobookstesting/utils.py index 9dae618..43c98f8 100644 --- a/autobookstesting/utils.py +++ b/autobookstesting/utils.py @@ -1,9 +1,9 @@ import logging + from loguru import logger -import sys -from datetime import datetime -#Formatter to remove patterns from log output + +# Formatter to remove patterns from log output class RedactingFormatter: def __init__(self, patterns=None, source_fmt=None): super().__init__() @@ -17,7 +17,8 @@ def format(self, record): record["extra"]["scrubbed"] = scrubbed return self.fmt -#Handler to intercept logging messages for loguru + +# Handler to intercept logging messages for loguru class InterceptHandler(logging.Handler): def emit(self, record): try: @@ -33,12 +34,14 @@ def emit(self, record): logger.opt(depth=depth, exception=record.exc_info).log( level, record.getMessage() ) + + # Process log file for Cronitor -def process_logfile(LOG_FILENAME, terms=None): - with open(LOG_FILENAME) as logs: +def process_logfile(logfile, terms=None): + with open(logfile) as logs: lines = logs.readlines() log_list = [] for line in lines: if any(term in line for term in terms): log_list.append(line) - return "".join(log_list) \ No newline at end of file + return "".join(log_list) From 1e8d221440941b71bbad906028237068aecdaa53 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 13:57:20 -0500 Subject: [PATCH 19/25] Moved testing dir to main dir --- autobooks/AutoBooks.py | 350 ++++++++--------- {autobookstesting => autobooks}/DiscordBot.py | 0 {autobookstesting => autobooks}/main.py | 0 {autobookstesting => autobooks}/utils.py | 0 autobooksold/AutoBooks.py | 351 ++++++++++++++++++ .../AutoBooksDiscord.py | 0 .../__init__.py | 0 autobookstesting/AutoBooks.py | 327 ---------------- setup.py | 3 +- 9 files changed, 515 insertions(+), 516 deletions(-) rename {autobookstesting => autobooks}/DiscordBot.py (100%) rename {autobookstesting => autobooks}/main.py (100%) rename {autobookstesting => autobooks}/utils.py (100%) create mode 100644 autobooksold/AutoBooks.py rename {autobooks => autobooksold}/AutoBooksDiscord.py (100%) rename {autobookstesting => autobooksold}/__init__.py (100%) delete mode 100644 autobookstesting/AutoBooks.py diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 3d61029..5868cef 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -1,275 +1,257 @@ -from selenium import webdriver -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options +import glob +import os +import shutil +import sys from configparser import ConfigParser -from unittest.mock import patch from datetime import datetime -from time import sleep from pathlib import Path +from time import sleep +from unittest.mock import patch + +import cronitor import odmpy.odm as odmpy import pandas as pd -import selenium -import cronitor -import glob -import sys -import shutil -import logging -import os import requests +import selenium +from loguru import logger +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys - -# Logging Redacting formatter from https://relaxdiego.com/2014/07/logging-in-python.html#configuring-your-loggers -class RedactingFormatter(object): - def __init__(self, orig_formatter, patterns): - self.orig_formatter = orig_formatter - self._patterns = patterns - - def format(self, record): - msg = self.orig_formatter.format(record) - for pattern in self._patterns: - msg = msg.replace(pattern, "") - return msg - - def __getattr__(self, attr): - return getattr(self.orig_formatter, attr) - +from utils import InterceptHandler, RedactingFormatter, process_logfile # Set Vars -scriptver = "0.2.1" # Version number of script +version = "0.2.1" # Version number of script error_count = 0 -good_odm_list = [] -bad_odm_list = [] -log_list = [] -library_list = [] -book_id_list = [] -book_title_list = [] -book_odm_list = [] -scriptdir = os.path.join(Path.home(), "AutoBooks") -csv_path = os.path.join(scriptdir, 'web_known_files.csv') +good_odm_list, bad_odm_list, library_list, book_id_list, book_title_list, book_odm_list = ([ +] for i in range(6)) +script_dir = os.path.join(Path.home(), "AutoBooks") +csv_path = os.path.join(script_dir, 'web_known_files.csv') # Check paths, and if not found do first time setup -if os.path.exists(scriptdir): - os.chdir(scriptdir) +if os.path.exists(script_dir): + os.chdir(script_dir) else: - os.mkdir(scriptdir) - main_conf = requests.get('https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') - odmpy_conf = requests.get("https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") - folders = ['log', 'web_downloads', 'chrome_profile', 'sourcefiles'] + os.mkdir(script_dir) + main_conf = requests.get( + 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') + # odmpy_conf = requests.get( + # "https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") + folders = ['log', 'downloads', 'profile', 'source_backup'] for folder in folders: - os.mkdir(os.path.join(scriptdir, folder)) - with open(os.path.join(scriptdir, "autobooks.conf"), mode='wb') as localfile: - localfile.write(main_conf.content) - with open(os.path.join(scriptdir, "odmpydl.conf"), mode='wb') as localfile: - localfile.write(odmpy_conf.content) - print("Finished setup please configure settings in file: ", os.path.join(scriptdir, "autobooks.conf")) + os.mkdir(os.path.join(script_dir, folder)) + with open(os.path.join(script_dir, "autobooks.conf"), mode='wb') as local_file: + local_file.write(main_conf.content) + # with open(os.path.join(script_dir, "odmpydl.conf"), mode='wb') as local_file: + # local_file.write(odmpy_conf.content) + print("Finished setup please configure settings in file: ", + os.path.join(script_dir, "autobooks.conf")) sys.exit(1) # Logging Config -LOG_FILENAME = os.path.join(scriptdir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}-Main.log'.format(datetime.now())) -fh = logging.FileHandler(LOG_FILENAME) -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s [%(name)s] %(levelname)s: %(message)s', - datefmt='%I:%M:%S %p', - handlers=[ - fh, - logging.StreamHandler(sys.stdout) - ]) -fh.setFormatter(RedactingFormatter(fh.formatter, patterns=["", "", "", ""])) -odmpy.logger = logging.getLogger('AutoBooks.odmpy') -process_logger = logging.getLogger("AutoBooks.main") -web_logger = logging.getLogger("AutoBooks.web") -discord_logger = logging.getLogger("AutoBooks.discord") +LOG_FILENAME = os.path.join( + script_dir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}.log'.format(datetime.now())) +patterns = ['', '', '', ''] +console_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {message}\n{exception}" +cronitor_log_format = "[{name}:{function}] {level}: {message}\n{exception}" +file_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {extra[scrubbed]}\n{exception}" +redacting_formatter = RedactingFormatter(patterns=patterns, source_fmt=file_log_format) +logger.configure(handlers=[ + {'sink': sys.stderr, "format": console_log_format}, + {'sink': LOG_FILENAME, "format": redacting_formatter.format}, + # {'sink': process_logfile(), "format": cronitor_log_format.format} +]) +odmpy.logger.handlers.clear() +odmpy.logger.addHandler(InterceptHandler()) # Read config file parser = ConfigParser() -parser.read(os.path.join(scriptdir, "autobooks.conf")) -odmdir = parser.get("DEFAULT", - "odm_folder") # Folder that contains the .odm files to process. For windows use / slashes -outdir = parser.get("DEFAULT", - "out_folder") # Folder where the finished audiobooks will be moved to. For windows use / slashes - -# Cronitor Setup -cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") # Cronitor API key https://cronitor.io/ -monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) # Set Cronitor monitor name -web_monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_web")) # Set Cronitor monitor name +parser.read(os.path.join(script_dir, "autobooks.conf")) +odm_dir = parser.get("DEFAULT", "odm_folder") +out_dir = parser.get("DEFAULT", "out_folder") +library_count = len(parser.sections()) +# Cronitor Setup https://cronitor.io/ +cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") +monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) +web_monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_web")) # Function to process the books. -def process_books(odm_list): +def process(odm_list): global error_count - process_logger.info('Begin processing booklist: %s', " ".join(odm_list)) + logger.info('Begin processing book list: {}', " ".join(odm_list)) for x in odm_list: if parser.get('DEFAULT', "test_args") == "true": odmpy_args = ["odmpy", "dl", x] else: - odmpy_args = ["odmpy", "dl", "@" + os.path.join(scriptdir, "odmpydl.conf"), x] + odmpy_args = ["odmpy", "dl", "-c", "-m", "--mergeformat", "m4b", "--nobookfolder", x] with patch.object(sys, 'argv', odmpy_args): try: odmpy.run() except FileNotFoundError: - process_logger.error("Could not find odm file %s", x) + logger.error("Could not find odm file {}", x) except FileExistsError: - process_logger.error("FileAlreadyExists, likely from m4b creation attempt") + logger.error("FileAlreadyExists, likely from m4b creation attempt") except SystemExit as e: bad_odm_list.append(x) - try: + if os.path.isfile("cover.jpg"): os.remove("cover.jpg") - except FileNotFoundError: - process_logger.debug("Could not remove cover.jpg, moving on") - else: - process_logger.debug("Removed cover.jpg to prep for next attempt") error_count += 1 else: good_odm_list.append(x) - process_logger.info("Book Processing Finished") + logger.info("Book Processing Finished") -# Function to cleanup in and out files. -def cleanup(m4bs, odms, odmfolder): +# Function to clean up in and out files. +def cleanup(m4b_list, odm_list, odm_folder): global error_count - # Move m4b files to outdir - for x in m4bs: - exists = os.path.isfile(os.path.join(outdir + x)) - if exists: - process_logger.error("Book %s already exists in outdir skipped", x) + # Move m4b files to out_dir + for x in m4b_list: + if os.path.isfile(os.path.join(out_dir + x)): + logger.error("Book {} already exists in out dir skipped", x) error_count += 1 else: - shutil.move(os.path.join(odmfolder, x), os.path.join(outdir, x)) - process_logger.info("Moved book %s to outdir", x) + shutil.move(os.path.join(odm_folder, x), os.path.join(out_dir, x)) + logger.info("Moved book {} to out_dir", x) # Backup source files - sourcefiles = odms + glob.glob("*.license") - for x in sourcefiles: - if os.path.isfile(os.path.join(scriptdir, "sourcefiles", x)): - process_logger.error("File %s already exists in sourcefiles dir skipped", x) + for x in odm_list: + if os.path.isfile(os.path.join(script_dir, "source_files", x)): + logger.error( + "File {} already exists in source files dir skipped", x) error_count += 1 else: - shutil.move(x, os.path.join(scriptdir, "sourcefiles", x)) - process_logger.info("Moved file %s to sourcefiles", x) + license_file = x.replace(".odm", ".license") + shutil.move(x, os.path.join(script_dir, "source_files", x)) + shutil.move(license_file, os.path.join(script_dir, "source_files", license_file)) + logger.info("Moved file pair {} to source files", x) # Function for login -def web_login(driver, name, cardno, pin, select): +def web_login(driver, name, card_num, pin, select): global error_count - web_logger.info("web_login: Logging into library: %s", name) + logger.info("Logging into library: {}", name) # Attempt selecting library from dropdown if select != "false": - select_box = driver.find_element(By.XPATH, '//input[@id="signin-options"]') - webdriver.ActionChains(driver).move_to_element(select_box).click().send_keys(select).perform() + select_box = driver.find_element( + By.XPATH, '//input[@id="signin-options"]') + webdriver.ActionChains(driver).move_to_element( + select_box).click().send_keys(select).perform() sleep(1) - webdriver.ActionChains(driver).send_keys(Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform() + webdriver.ActionChains(driver).send_keys( + Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform() # Attempt sending card number try: - driver.find_element(By.ID, "username").send_keys(cardno) + driver.find_element(By.ID, "username").send_keys(card_num) except selenium.common.exceptions.NoSuchElementException: - web_logger.critical("web_login: Can't find card number field skipped library %s", ) + logger.critical("Can't find card number field skipped library {}", name) error_count += 1 # Attempt sending pin Note:Some pages don't have pin input if pin != "false": driver.find_element(By.ID, "password").send_keys(pin) - driver.find_element(By.CSS_SELECTOR, "button.signin-button.button.secondary").click() + driver.find_element( + By.CSS_SELECTOR, "button.signin-button.button.secondary").click() sleep(5) # Function to download loans from OverDrive page def web_dl(driver, df, name): global error_count - #Gather all book title elements and check if any found + # Gather all book title elements and check if any found books = driver.find_elements(By.XPATH, '//a[@tabindex="0"][@role="link"]') if len(books) == 0: - web_logger.warning("Can't find books skipped library: %s", name) + logger.warning("Can't find books skipped library: {}", name) error_count += 1 return () else: - web_logger.info("web_dl: Begin DL from library: %s ", name) - bookcount = 0 + logger.info("Begin DL from library: {} ", name) + book_count = 0 for i in books: - #Fetch info about the book + # Fetch info about the book book_url = i.get_attribute('href') book_info = i.get_attribute('aria-label') book_info_split = book_info.split(". Audiobook. Expires in") - book_dl_url = book_url.replace('/media/', '/media/download/audiobook-mp3/') + book_dl_url = book_url.replace( + '/media/', '/media/download/audiobook-mp3/') book_id = int(''.join(filter(str.isdigit, book_url))) book_title = book_info_split[0] + + # Check if found book is a not known audiobook if "Audiobook." in book_info: if str(book_id) in df['book_id'].to_string(): - web_logger.info('web_dl: Skipped %s found in known books', book_title) + logger.info('Skipped {} found in known books', book_title) else: # Download book driver.get(book_dl_url) - web_logger.info("web_dl: Downloaded book: %s", book_title) + logger.info("Downloaded book: {}", book_title) book_odm = max(glob.glob("*.odm"), key=os.path.getmtime) - bookcount += 1 + book_count += 1 - #Add book data to vars + # Add book data to vars library_list.append(name) book_id_list.append(book_id) book_title_list.append(book_title) book_odm_list.append(book_odm) sleep(1) - - sleep(1) - web_logger.info("web_dl: Finished downloading %s books from library %s", bookcount, name) + logger.info("Finished downloading {} books from library {}", + book_count, name) return () + def main_run(): # AutoBooks - process_logger.info("Started AutoBooks V.%s By:IvyB", scriptver) + logger.info("Started AutoBooks V.{} By:IvyB", version) # Try to change to ODM folder try: - os.chdir(odmdir) + os.chdir(odm_dir) except FileNotFoundError: - process_logger.critical("The provided .odm dir was not found, exiting") + logger.critical("The provided .odm dir was not found, exiting") sys.exit(1) else: odm_list = glob.glob("*.odm") monitor.ping(state='run', - message='AutoBooks by IvyB Version:' + scriptver + '\n odmdir:' + odmdir + '\n outdir:' + outdir + '\n logfile:' + LOG_FILENAME + '\n Found the following books \n' + " ".join( - odm_list)) + message=f'AutoBooks by IvyB Started from web V.{version} \n' + + f'odm_dir: {odm_dir} \n out_dir:{out_dir}\n logfile:{LOG_FILENAME}\n' + + f'odm_list: \n{" ".join(odm_list)}') - # Check if any .odm files exist in odmdir + # Check if any .odm files exist in odm_dir if len(odm_list) == 0: monitor.ping(state='fail', message='Error: No .odm files found, exiting', metrics={'error_count': error_count}) - process_logger.critical("No .odm files found, exiting") + logger.critical("No .odm files found, exiting") sys.exit(1) else: - process_books(odm_list) - # Cleanup input and output files + process(odm_list) + # Cleanup files m4blist = glob.glob("*.m4b") - cleanup(m4blist, good_odm_list, odmdir) - # Process log file for Cronitor - with open(LOG_FILENAME) as logs: - lines = logs.readlines() - log_list = [] - for line in lines: - if any(term in line for term in ("Downloading", "expired", "generating", "merged")): - log_list.append(line) - # Send complete event and log to Cronitor - monitor.ping(state='complete', message="".join(log_list), - metrics={'count': len(odm_list), 'error_count': error_count}) + cleanup(m4blist, good_odm_list, odm_dir) + # Send complete event and log to Cronitor + log_str = process_logfile(LOG_FILENAME, terms=( + "Downloading", "expired", "generating", "merged", "saved")) + monitor.ping(state='complete', message=log_str, + metrics={'count': len(odm_list), 'error_count': error_count}) # AutoBooks Web Code def web_run(): if len(parser.sections()) == 0: - web_logger.critical("No libraries configured!") + logger.critical("No libraries configured!") sys.exit(1) else: - web_logger.info("Started AutoBooks Web V.%s By:IvyB", scriptver) + logger.info("Started AutoBooks Web V.{} By:IvyB", version) monitor.ping(state='run', - message='AutoBooks Web by IvyB Version:' + scriptver + '\n logfile:' + LOG_FILENAME + '\n LibraryCount: ' + str( - len(parser.sections()))) + message=f'AutoBooks Web by IvyB V.{version} \n' + + f'logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}') # Configure WebDriver options options = Options() prefs = { - "download.default_directory": os.path.join(scriptdir, "web_downloads"), + "download.default_directory": os.path.join(script_dir, "downloads"), "download.prompt_for_download": False, "download.directory_upgrade": True } - options.add_argument('user-data-dir=' + os.path.join(scriptdir, "chrome_profile")) + options.add_argument('user-data-dir=' + + os.path.join(script_dir, "profile")) # Headless mode check if parser.get('DEFAULT', "web_headless") == "true": options.add_argument('--headless') @@ -281,70 +263,64 @@ def web_run(): if os.path.exists(csv_path): df = pd.read_csv(csv_path, sep=",") else: - df = pd.DataFrame({ - 'book_id': book_id_list, + # Failing above create an empty df for checking + df = pd.DataFrame({ + 'book_id': book_id_list, }) - os.chdir(os.path.join(scriptdir,"web_downloads")) + os.chdir(os.path.join(script_dir, "downloads")) + # For every library, open site, attempt sign in, and attempt download. for i in range(0, len(parser.sections())): library_index = 'library_' + str(i) library_subdomain = parser.get(library_index, "library_subdomain") library_name = parser.get(library_index, "library_name") - web_logger.info("Started library %s", library_name) + logger.info("Started library {}", library_name) url = "https://" + library_subdomain + ".overdrive.com/" driver.get(url + "account/loans") sleep(3) # Check signed in status and either sign in or move on if "/account/ozone/sign-in" in driver.current_url: web_login(driver, library_name, parser.get(library_index, "card_number"), - parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) + parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) web_dl(driver, df, library_name) sleep(2) - # Output book data to csv + # Output book data to csv df_out = pd.DataFrame({ - 'library_name': library_list, - 'book_id': book_id_list, - 'book_title': book_title_list, - 'book_odm': book_odm_list - }) + 'library_name': library_list, + 'book_id': book_id_list, + 'book_title': book_title_list, + 'book_odm': book_odm_list + }) if os.path.exists(csv_path): df_out.to_csv(csv_path, mode='a', index=False, header=False) else: df_out.to_csv(csv_path, mode='w', index=False, header=True) driver.close() - web_logger.info("AutoBooksWeb Complete") - odmlist = glob.glob("*.odm") + logger.info("AutoBooksWeb Complete") + web_odm_list = glob.glob("*.odm") # Process log file for Cronitor. - with open(LOG_FILENAME) as logs: - lines = logs.readlines() - log_list = [] - for line in lines: - if "AutoBooks.web" in line: - log_list.append(line) - monitor.ping(state='complete', message="".join(odmlist), - metrics={'count': len(odmlist), 'error_count': error_count}) + process_logfile(LOG_FILENAME, terms=("web", "ERROR")) + monitor.ping(state='complete', message="".join(web_odm_list), + metrics={'count': len(web_odm_list), 'error_count': error_count}) - # Call Minimum DL functions - if len(odmlist) != 0: - process_logger.info("Started AutoBooks V.%s By:IvyB", scriptver) + # Call DL to process odm files from web + if len(web_odm_list) != 0: + logger.info("Started AutoBooks V.{} By:IvyB", version) monitor.ping(state='run', - message='AutoBooks by IvyB Started from web Version:' + scriptver + '\n outdir:' + outdir + '\n logfile:' + LOG_FILENAME + '\n Found the following books \n' + " ".join( - odmlist)) - process_books(odmlist) + message=f'AutoBooks by IvyB Started from web V.{version} \n' + + f'out_dir:{out_dir}\n logfile:{LOG_FILENAME}\n odm_list: \n{" ".join(web_odm_list)}') + process(web_odm_list) m4blist = glob.glob("*.m4b") - cleanup(m4blist, good_odm_list, os.path.join(scriptdir, "web_downloads")) + cleanup(m4blist, good_odm_list, os.path.join( + script_dir, "downloads")) # Process log file for Cronitor - with open(LOG_FILENAME) as logs: - lines = logs.readlines() - log_list = [] - for line in lines: - if any(term in line for term in ("Downloading", "expired", "generating", "merged")): - log_list.append(line) - # Send complete event and log to Cronitor - monitor.ping(state='complete', message="".join(log_list), - metrics={'count': len(odmlist), 'error_count': error_count}) - #return["\n".join(title_list), error_count] + log_str = process_logfile(LOG_FILENAME, terms=( + "Downloading", "expired", "generating", "merged")) + # Send complete event and log to Cronitor + monitor.ping(state='complete', message=log_str, + metrics={'count': len(web_odm_list), 'error_count': error_count}) + # return["\n".join(title_list), error_count] if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "true": diff --git a/autobookstesting/DiscordBot.py b/autobooks/DiscordBot.py similarity index 100% rename from autobookstesting/DiscordBot.py rename to autobooks/DiscordBot.py diff --git a/autobookstesting/main.py b/autobooks/main.py similarity index 100% rename from autobookstesting/main.py rename to autobooks/main.py diff --git a/autobookstesting/utils.py b/autobooks/utils.py similarity index 100% rename from autobookstesting/utils.py rename to autobooks/utils.py diff --git a/autobooksold/AutoBooks.py b/autobooksold/AutoBooks.py new file mode 100644 index 0000000..3d61029 --- /dev/null +++ b/autobooksold/AutoBooks.py @@ -0,0 +1,351 @@ +from selenium import webdriver +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.common.by import By +from selenium.webdriver.chrome.options import Options +from configparser import ConfigParser +from unittest.mock import patch +from datetime import datetime +from time import sleep +from pathlib import Path +import odmpy.odm as odmpy +import pandas as pd +import selenium +import cronitor +import glob +import sys +import shutil +import logging +import os +import requests + + +# Logging Redacting formatter from https://relaxdiego.com/2014/07/logging-in-python.html#configuring-your-loggers +class RedactingFormatter(object): + def __init__(self, orig_formatter, patterns): + self.orig_formatter = orig_formatter + self._patterns = patterns + + def format(self, record): + msg = self.orig_formatter.format(record) + for pattern in self._patterns: + msg = msg.replace(pattern, "") + return msg + + def __getattr__(self, attr): + return getattr(self.orig_formatter, attr) + + +# Set Vars +scriptver = "0.2.1" # Version number of script +error_count = 0 +good_odm_list = [] +bad_odm_list = [] +log_list = [] +library_list = [] +book_id_list = [] +book_title_list = [] +book_odm_list = [] +scriptdir = os.path.join(Path.home(), "AutoBooks") +csv_path = os.path.join(scriptdir, 'web_known_files.csv') + +# Check paths, and if not found do first time setup +if os.path.exists(scriptdir): + os.chdir(scriptdir) +else: + os.mkdir(scriptdir) + main_conf = requests.get('https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') + odmpy_conf = requests.get("https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") + folders = ['log', 'web_downloads', 'chrome_profile', 'sourcefiles'] + for folder in folders: + os.mkdir(os.path.join(scriptdir, folder)) + with open(os.path.join(scriptdir, "autobooks.conf"), mode='wb') as localfile: + localfile.write(main_conf.content) + with open(os.path.join(scriptdir, "odmpydl.conf"), mode='wb') as localfile: + localfile.write(odmpy_conf.content) + print("Finished setup please configure settings in file: ", os.path.join(scriptdir, "autobooks.conf")) + sys.exit(1) + +# Logging Config +LOG_FILENAME = os.path.join(scriptdir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}-Main.log'.format(datetime.now())) +fh = logging.FileHandler(LOG_FILENAME) +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s [%(name)s] %(levelname)s: %(message)s', + datefmt='%I:%M:%S %p', + handlers=[ + fh, + logging.StreamHandler(sys.stdout) + ]) +fh.setFormatter(RedactingFormatter(fh.formatter, patterns=["", "", "", ""])) +odmpy.logger = logging.getLogger('AutoBooks.odmpy') +process_logger = logging.getLogger("AutoBooks.main") +web_logger = logging.getLogger("AutoBooks.web") +discord_logger = logging.getLogger("AutoBooks.discord") + +# Read config file +parser = ConfigParser() +parser.read(os.path.join(scriptdir, "autobooks.conf")) +odmdir = parser.get("DEFAULT", + "odm_folder") # Folder that contains the .odm files to process. For windows use / slashes +outdir = parser.get("DEFAULT", + "out_folder") # Folder where the finished audiobooks will be moved to. For windows use / slashes + +# Cronitor Setup +cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") # Cronitor API key https://cronitor.io/ +monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) # Set Cronitor monitor name +web_monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_web")) # Set Cronitor monitor name + + +# Function to process the books. +def process_books(odm_list): + global error_count + process_logger.info('Begin processing booklist: %s', " ".join(odm_list)) + for x in odm_list: + if parser.get('DEFAULT', "test_args") == "true": + odmpy_args = ["odmpy", "dl", x] + else: + odmpy_args = ["odmpy", "dl", "@" + os.path.join(scriptdir, "odmpydl.conf"), x] + with patch.object(sys, 'argv', odmpy_args): + try: + odmpy.run() + except FileNotFoundError: + process_logger.error("Could not find odm file %s", x) + except FileExistsError: + process_logger.error("FileAlreadyExists, likely from m4b creation attempt") + except SystemExit as e: + bad_odm_list.append(x) + try: + os.remove("cover.jpg") + except FileNotFoundError: + process_logger.debug("Could not remove cover.jpg, moving on") + else: + process_logger.debug("Removed cover.jpg to prep for next attempt") + error_count += 1 + else: + good_odm_list.append(x) + process_logger.info("Book Processing Finished") + + +# Function to cleanup in and out files. +def cleanup(m4bs, odms, odmfolder): + global error_count + # Move m4b files to outdir + for x in m4bs: + exists = os.path.isfile(os.path.join(outdir + x)) + if exists: + process_logger.error("Book %s already exists in outdir skipped", x) + error_count += 1 + else: + shutil.move(os.path.join(odmfolder, x), os.path.join(outdir, x)) + process_logger.info("Moved book %s to outdir", x) + # Backup source files + sourcefiles = odms + glob.glob("*.license") + for x in sourcefiles: + if os.path.isfile(os.path.join(scriptdir, "sourcefiles", x)): + process_logger.error("File %s already exists in sourcefiles dir skipped", x) + error_count += 1 + else: + shutil.move(x, os.path.join(scriptdir, "sourcefiles", x)) + process_logger.info("Moved file %s to sourcefiles", x) + + +# Function for login +def web_login(driver, name, cardno, pin, select): + global error_count + web_logger.info("web_login: Logging into library: %s", name) + # Attempt selecting library from dropdown + if select != "false": + select_box = driver.find_element(By.XPATH, '//input[@id="signin-options"]') + webdriver.ActionChains(driver).move_to_element(select_box).click().send_keys(select).perform() + sleep(1) + webdriver.ActionChains(driver).send_keys(Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform() + # Attempt sending card number + try: + driver.find_element(By.ID, "username").send_keys(cardno) + except selenium.common.exceptions.NoSuchElementException: + web_logger.critical("web_login: Can't find card number field skipped library %s", ) + error_count += 1 + # Attempt sending pin Note:Some pages don't have pin input + if pin != "false": + driver.find_element(By.ID, "password").send_keys(pin) + driver.find_element(By.CSS_SELECTOR, "button.signin-button.button.secondary").click() + sleep(5) + + +# Function to download loans from OverDrive page +def web_dl(driver, df, name): + global error_count + #Gather all book title elements and check if any found + books = driver.find_elements(By.XPATH, '//a[@tabindex="0"][@role="link"]') + if len(books) == 0: + web_logger.warning("Can't find books skipped library: %s", name) + error_count += 1 + return () + else: + web_logger.info("web_dl: Begin DL from library: %s ", name) + bookcount = 0 + for i in books: + #Fetch info about the book + book_url = i.get_attribute('href') + book_info = i.get_attribute('aria-label') + book_info_split = book_info.split(". Audiobook. Expires in") + book_dl_url = book_url.replace('/media/', '/media/download/audiobook-mp3/') + book_id = int(''.join(filter(str.isdigit, book_url))) + book_title = book_info_split[0] + if "Audiobook." in book_info: + if str(book_id) in df['book_id'].to_string(): + web_logger.info('web_dl: Skipped %s found in known books', book_title) + else: + # Download book + driver.get(book_dl_url) + web_logger.info("web_dl: Downloaded book: %s", book_title) + book_odm = max(glob.glob("*.odm"), key=os.path.getmtime) + bookcount += 1 + + #Add book data to vars + library_list.append(name) + book_id_list.append(book_id) + book_title_list.append(book_title) + book_odm_list.append(book_odm) + sleep(1) + + + sleep(1) + web_logger.info("web_dl: Finished downloading %s books from library %s", bookcount, name) + return () + +def main_run(): + # AutoBooks + process_logger.info("Started AutoBooks V.%s By:IvyB", scriptver) + # Try to change to ODM folder + try: + os.chdir(odmdir) + except FileNotFoundError: + process_logger.critical("The provided .odm dir was not found, exiting") + sys.exit(1) + else: + odm_list = glob.glob("*.odm") + monitor.ping(state='run', + message='AutoBooks by IvyB Version:' + scriptver + '\n odmdir:' + odmdir + '\n outdir:' + outdir + '\n logfile:' + LOG_FILENAME + '\n Found the following books \n' + " ".join( + odm_list)) + + # Check if any .odm files exist in odmdir + if len(odm_list) == 0: + monitor.ping(state='fail', message='Error: No .odm files found, exiting', + metrics={'error_count': error_count}) + process_logger.critical("No .odm files found, exiting") + sys.exit(1) + else: + process_books(odm_list) + # Cleanup input and output files + m4blist = glob.glob("*.m4b") + cleanup(m4blist, good_odm_list, odmdir) + # Process log file for Cronitor + with open(LOG_FILENAME) as logs: + lines = logs.readlines() + log_list = [] + for line in lines: + if any(term in line for term in ("Downloading", "expired", "generating", "merged")): + log_list.append(line) + # Send complete event and log to Cronitor + monitor.ping(state='complete', message="".join(log_list), + metrics={'count': len(odm_list), 'error_count': error_count}) + + +# AutoBooks Web Code +def web_run(): + if len(parser.sections()) == 0: + web_logger.critical("No libraries configured!") + sys.exit(1) + else: + web_logger.info("Started AutoBooks Web V.%s By:IvyB", scriptver) + monitor.ping(state='run', + message='AutoBooks Web by IvyB Version:' + scriptver + '\n logfile:' + LOG_FILENAME + '\n LibraryCount: ' + str( + len(parser.sections()))) + # Configure WebDriver options + options = Options() + prefs = { + "download.default_directory": os.path.join(scriptdir, "web_downloads"), + "download.prompt_for_download": False, + "download.directory_upgrade": True + } + options.add_argument('user-data-dir=' + os.path.join(scriptdir, "chrome_profile")) + # Headless mode check + if parser.get('DEFAULT', "web_headless") == "true": + options.add_argument('--headless') + options.add_argument('--disable-gpu') + options.add_experimental_option('prefs', prefs) + driver = webdriver.Chrome(options=options) + + # Attempt to read known files csv for checking books + if os.path.exists(csv_path): + df = pd.read_csv(csv_path, sep=",") + else: + df = pd.DataFrame({ + 'book_id': book_id_list, + }) + os.chdir(os.path.join(scriptdir,"web_downloads")) + # For every library, open site, attempt sign in, and attempt download. + for i in range(0, len(parser.sections())): + library_index = 'library_' + str(i) + library_subdomain = parser.get(library_index, "library_subdomain") + library_name = parser.get(library_index, "library_name") + web_logger.info("Started library %s", library_name) + url = "https://" + library_subdomain + ".overdrive.com/" + driver.get(url + "account/loans") + sleep(3) + # Check signed in status and either sign in or move on + if "/account/ozone/sign-in" in driver.current_url: + web_login(driver, library_name, parser.get(library_index, "card_number"), + parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) + web_dl(driver, df, library_name) + sleep(2) + # Output book data to csv + df_out = pd.DataFrame({ + 'library_name': library_list, + 'book_id': book_id_list, + 'book_title': book_title_list, + 'book_odm': book_odm_list + }) + if os.path.exists(csv_path): + df_out.to_csv(csv_path, mode='a', index=False, header=False) + else: + df_out.to_csv(csv_path, mode='w', index=False, header=True) + driver.close() + web_logger.info("AutoBooksWeb Complete") + odmlist = glob.glob("*.odm") + + # Process log file for Cronitor. + with open(LOG_FILENAME) as logs: + lines = logs.readlines() + log_list = [] + for line in lines: + if "AutoBooks.web" in line: + log_list.append(line) + monitor.ping(state='complete', message="".join(odmlist), + metrics={'count': len(odmlist), 'error_count': error_count}) + + # Call Minimum DL functions + if len(odmlist) != 0: + process_logger.info("Started AutoBooks V.%s By:IvyB", scriptver) + monitor.ping(state='run', + message='AutoBooks by IvyB Started from web Version:' + scriptver + '\n outdir:' + outdir + '\n logfile:' + LOG_FILENAME + '\n Found the following books \n' + " ".join( + odmlist)) + process_books(odmlist) + m4blist = glob.glob("*.m4b") + cleanup(m4blist, good_odm_list, os.path.join(scriptdir, "web_downloads")) + # Process log file for Cronitor + with open(LOG_FILENAME) as logs: + lines = logs.readlines() + log_list = [] + for line in lines: + if any(term in line for term in ("Downloading", "expired", "generating", "merged")): + log_list.append(line) + # Send complete event and log to Cronitor + monitor.ping(state='complete', message="".join(log_list), + metrics={'count': len(odmlist), 'error_count': error_count}) + #return["\n".join(title_list), error_count] + + +if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "true": + web_run() diff --git a/autobooks/AutoBooksDiscord.py b/autobooksold/AutoBooksDiscord.py similarity index 100% rename from autobooks/AutoBooksDiscord.py rename to autobooksold/AutoBooksDiscord.py diff --git a/autobookstesting/__init__.py b/autobooksold/__init__.py similarity index 100% rename from autobookstesting/__init__.py rename to autobooksold/__init__.py diff --git a/autobookstesting/AutoBooks.py b/autobookstesting/AutoBooks.py deleted file mode 100644 index a808ece..0000000 --- a/autobookstesting/AutoBooks.py +++ /dev/null @@ -1,327 +0,0 @@ -import glob -import os -import shutil -import sys -from configparser import ConfigParser -from datetime import datetime -from pathlib import Path -from time import sleep -from unittest.mock import patch - -import cronitor -import odmpy.odm as odmpy -import pandas as pd -import requests -import selenium -from loguru import logger -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.common.by import By -from selenium.webdriver.common.keys import Keys - -from utils import InterceptHandler, RedactingFormatter, process_logfile - -# Set Vars -version = "0.2.1" # Version number of script -error_count = 0 -good_odm_list, bad_odm_list, library_list, book_id_list, book_title_list, book_odm_list = ([ -] for i in range(6)) -script_dir = os.path.join(Path.home(), "AutoBooks") -csv_path = os.path.join(script_dir, 'web_known_files.csv') - -# Check paths, and if not found do first time setup -if os.path.exists(script_dir): - os.chdir(script_dir) -else: - os.mkdir(script_dir) - main_conf = requests.get( - 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') - # odmpy_conf = requests.get( - # "https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") - folders = ['log', 'downloads', 'profile', 'source_backup'] - for folder in folders: - os.mkdir(os.path.join(script_dir, folder)) - with open(os.path.join(script_dir, "autobooks.conf"), mode='wb') as local_file: - local_file.write(main_conf.content) - # with open(os.path.join(script_dir, "odmpydl.conf"), mode='wb') as local_file: - # local_file.write(odmpy_conf.content) - print("Finished setup please configure settings in file: ", - os.path.join(script_dir, "autobooks.conf")) - sys.exit(1) - -# Logging Config -LOG_FILENAME = os.path.join( - script_dir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}.log'.format(datetime.now())) -patterns = ['', '', '', ''] -console_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {message}\n{exception}" -cronitor_log_format = "[{name}:{function}] {level}: {message}\n{exception}" -file_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {extra[scrubbed]}\n{exception}" -redacting_formatter = RedactingFormatter(patterns=patterns, source_fmt=file_log_format) -logger.configure(handlers=[ - {'sink': sys.stderr, "format": console_log_format}, - {'sink': LOG_FILENAME, "format": redacting_formatter.format}, - # {'sink': process_logfile(), "format": cronitor_log_format.format} -]) -odmpy.logger.handlers.clear() -odmpy.logger.addHandler(InterceptHandler()) - -# Read config file -parser = ConfigParser() -parser.read(os.path.join(script_dir, "autobooks.conf")) -odm_dir = parser.get("DEFAULT", "odm_folder") -out_dir = parser.get("DEFAULT", "out_folder") -library_count = len(parser.sections()) -# Cronitor Setup https://cronitor.io/ -cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") -monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) -web_monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_web")) - - -# Function to process the books. -def process(odm_list): - global error_count - logger.info('Begin processing book list: {}', " ".join(odm_list)) - for x in odm_list: - if parser.get('DEFAULT', "test_args") == "true": - odmpy_args = ["odmpy", "dl", x] - else: - odmpy_args = ["odmpy", "dl", "-c", "-m", "--mergeformat", "m4b", "--nobookfolder", x] - with patch.object(sys, 'argv', odmpy_args): - try: - odmpy.run() - except FileNotFoundError: - logger.error("Could not find odm file {}", x) - except FileExistsError: - logger.error("FileAlreadyExists, likely from m4b creation attempt") - except SystemExit as e: - bad_odm_list.append(x) - if os.path.isfile("cover.jpg"): - os.remove("cover.jpg") - error_count += 1 - else: - good_odm_list.append(x) - logger.info("Book Processing Finished") - - -# Function to clean up in and out files. -def cleanup(m4b_list, odm_list, odm_folder): - global error_count - # Move m4b files to out_dir - for x in m4b_list: - if os.path.isfile(os.path.join(out_dir + x)): - logger.error("Book {} already exists in out dir skipped", x) - error_count += 1 - else: - shutil.move(os.path.join(odm_folder, x), os.path.join(out_dir, x)) - logger.info("Moved book {} to out_dir", x) - # Backup source files - for x in odm_list: - if os.path.isfile(os.path.join(script_dir, "source_files", x)): - logger.error( - "File {} already exists in source files dir skipped", x) - error_count += 1 - else: - license_file = x.replace(".odm", ".license") - shutil.move(x, os.path.join(script_dir, "source_files", x)) - shutil.move(license_file, os.path.join(script_dir, "source_files", license_file)) - logger.info("Moved file pair {} to source files", x) - - -# Function for login -def web_login(driver, name, card_num, pin, select): - global error_count - logger.info("Logging into library: {}", name) - # Attempt selecting library from dropdown - if select != "false": - select_box = driver.find_element( - By.XPATH, '//input[@id="signin-options"]') - webdriver.ActionChains(driver).move_to_element( - select_box).click().send_keys(select).perform() - sleep(1) - webdriver.ActionChains(driver).send_keys( - Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform() - # Attempt sending card number - try: - driver.find_element(By.ID, "username").send_keys(card_num) - except selenium.common.exceptions.NoSuchElementException: - logger.critical("Can't find card number field skipped library {}", name) - error_count += 1 - # Attempt sending pin Note:Some pages don't have pin input - if pin != "false": - driver.find_element(By.ID, "password").send_keys(pin) - driver.find_element( - By.CSS_SELECTOR, "button.signin-button.button.secondary").click() - sleep(5) - - -# Function to download loans from OverDrive page -def web_dl(driver, df, name): - global error_count - # Gather all book title elements and check if any found - books = driver.find_elements(By.XPATH, '//a[@tabindex="0"][@role="link"]') - if len(books) == 0: - logger.warning("Can't find books skipped library: {}", name) - error_count += 1 - return () - else: - logger.info("Begin DL from library: {} ", name) - book_count = 0 - for i in books: - # Fetch info about the book - book_url = i.get_attribute('href') - book_info = i.get_attribute('aria-label') - book_info_split = book_info.split(". Audiobook. Expires in") - book_dl_url = book_url.replace( - '/media/', '/media/download/audiobook-mp3/') - book_id = int(''.join(filter(str.isdigit, book_url))) - book_title = book_info_split[0] - - # Check if found book is a not known audiobook - if "Audiobook." in book_info: - if str(book_id) in df['book_id'].to_string(): - logger.info('Skipped {} found in known books', book_title) - else: - # Download book - driver.get(book_dl_url) - logger.info("Downloaded book: {}", book_title) - book_odm = max(glob.glob("*.odm"), key=os.path.getmtime) - book_count += 1 - - # Add book data to vars - library_list.append(name) - book_id_list.append(book_id) - book_title_list.append(book_title) - book_odm_list.append(book_odm) - sleep(1) - sleep(1) - logger.info("Finished downloading {} books from library {}", - book_count, name) - return () - - -def main_run(): - # AutoBooks - logger.info("Started AutoBooks V.{} By:IvyB", version) - # Try to change to ODM folder - try: - os.chdir(odm_dir) - except FileNotFoundError: - logger.critical("The provided .odm dir was not found, exiting") - sys.exit(1) - else: - odm_list = glob.glob("*.odm") - monitor.ping(state='run', - message=f'AutoBooks by IvyB Started from web V.{version} \n' + - f'odm_dir: {odm_dir} \n out_dir:{out_dir}\n logfile:{LOG_FILENAME}\n' + - f'odm_list: \n{" ".join(odm_list)}') - - # Check if any .odm files exist in odm_dir - if len(odm_list) == 0: - monitor.ping(state='fail', message='Error: No .odm files found, exiting', - metrics={'error_count': error_count}) - logger.critical("No .odm files found, exiting") - sys.exit(1) - else: - process(odm_list) - # Cleanup files - m4blist = glob.glob("*.m4b") - cleanup(m4blist, good_odm_list, odm_dir) - # Send complete event and log to Cronitor - log_str = process_logfile(LOG_FILENAME, terms=( - "Downloading", "expired", "generating", "merged", "saved")) - monitor.ping(state='complete', message=log_str, - metrics={'count': len(odm_list), 'error_count': error_count}) - - -# AutoBooks Web Code -def web_run(): - if len(parser.sections()) == 0: - logger.critical("No libraries configured!") - sys.exit(1) - else: - logger.info("Started AutoBooks Web V.{} By:IvyB", version) - monitor.ping(state='run', - message=f'AutoBooks Web by IvyB V.{version} \n' + - f'logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}') - # Configure WebDriver options - options = Options() - prefs = { - "download.default_directory": os.path.join(script_dir, "downloads"), - "download.prompt_for_download": False, - "download.directory_upgrade": True - } - options.add_argument('user-data-dir=' + - os.path.join(script_dir, "profile")) - # Headless mode check - if parser.get('DEFAULT', "web_headless") == "true": - options.add_argument('--headless') - options.add_argument('--disable-gpu') - options.add_experimental_option('prefs', prefs) - driver = webdriver.Chrome(options=options) - - # Attempt to read known files csv for checking books - if os.path.exists(csv_path): - df = pd.read_csv(csv_path, sep=",") - else: - # Failing above create an empty df for checking - df = pd.DataFrame({ - 'book_id': book_id_list, - }) - os.chdir(os.path.join(script_dir, "downloads")) - - # For every library, open site, attempt sign in, and attempt download. - for i in range(0, len(parser.sections())): - library_index = 'library_' + str(i) - library_subdomain = parser.get(library_index, "library_subdomain") - library_name = parser.get(library_index, "library_name") - logger.info("Started library {}", library_name) - url = "https://" + library_subdomain + ".overdrive.com/" - driver.get(url + "account/loans") - sleep(3) - # Check signed in status and either sign in or move on - if "/account/ozone/sign-in" in driver.current_url: - web_login(driver, library_name, parser.get(library_index, "card_number"), - parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) - web_dl(driver, df, library_name) - sleep(2) - # Output book data to csv - df_out = pd.DataFrame({ - 'library_name': library_list, - 'book_id': book_id_list, - 'book_title': book_title_list, - 'book_odm': book_odm_list - }) - if os.path.exists(csv_path): - df_out.to_csv(csv_path, mode='a', index=False, header=False) - else: - df_out.to_csv(csv_path, mode='w', index=False, header=True) - driver.close() - logger.info("AutoBooksWeb Complete") - web_odm_list = glob.glob("*.odm") - - # Process log file for Cronitor. - process_logfile(LOG_FILENAME, terms=("web", "ERROR")) - monitor.ping(state='complete', message="".join(web_odm_list), - metrics={'count': len(web_odm_list), 'error_count': error_count}) - - # Call DL to process files from web - if len(web_odm_list) != 0: - logger.info("Started AutoBooks V.{} By:IvyB", version) - monitor.ping(state='run', - message=f'AutoBooks by IvyB Started from web V.{version} \n' + - f'out_dir:{out_dir}\n logfile:{LOG_FILENAME}\n odm_list: \n{" ".join(web_odm_list)}') - process(web_odm_list) - m4blist = glob.glob("*.m4b") - cleanup(m4blist, good_odm_list, os.path.join( - script_dir, "downloads")) - # Process log file for Cronitor - log_str = process_logfile(LOG_FILENAME, terms=( - "Downloading", "expired", "generating", "merged")) - # Send complete event and log to Cronitor - monitor.ping(state='complete', message=log_str, - metrics={'count': len(web_odm_list), 'error_count': error_count}) - # return["\n".join(title_list), error_count] - - -if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "true": - web_run() diff --git a/setup.py b/setup.py index 00c0b98..9b63ffa 100644 --- a/setup.py +++ b/setup.py @@ -13,11 +13,10 @@ url="https://github.com/ivybowman/AutoBooks", description=DESCRIPTION, long_description=LONG_DESCRIPTION, - packages=["autobooks", "autobookstesting"], + packages=["autobooks"], entry_points={ "console_scripts": [ "autobooks = autobooks.AutoBooks:main_run", - "autobooks-web-test = autobookstesting.AutoBooks:web_run", "autobooks-web = autobooks.AutoBooks:web_run", "autobooks-discord = autobooks.AutoBooksDiscord:run" ] From af0e6b610b9ff8c5400a751789b18298dbbbc6f1 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 13:58:51 -0500 Subject: [PATCH 20/25] Update AutoBooks.py --- autobooks/AutoBooks.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 5868cef..539bcdc 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -36,15 +36,11 @@ os.mkdir(script_dir) main_conf = requests.get( 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') - # odmpy_conf = requests.get( - # "https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") folders = ['log', 'downloads', 'profile', 'source_backup'] for folder in folders: os.mkdir(os.path.join(script_dir, folder)) with open(os.path.join(script_dir, "autobooks.conf"), mode='wb') as local_file: local_file.write(main_conf.content) - # with open(os.path.join(script_dir, "odmpydl.conf"), mode='wb') as local_file: - # local_file.write(odmpy_conf.content) print("Finished setup please configure settings in file: ", os.path.join(script_dir, "autobooks.conf")) sys.exit(1) From 4fd51d08d3a8a3fce04a330081018ca926d62f26 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 14:02:20 -0500 Subject: [PATCH 21/25] Update AutoBooks.py --- autobooks/AutoBooks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 539bcdc..db0bb10 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -78,7 +78,7 @@ def process(odm_list): global error_count logger.info('Begin processing book list: {}', " ".join(odm_list)) for x in odm_list: - if parser.get('DEFAULT', "test_args") == "true": + if parser.get('DEFAULT', "test_args") is True: odmpy_args = ["odmpy", "dl", x] else: odmpy_args = ["odmpy", "dl", "-c", "-m", "--mergeformat", "m4b", "--nobookfolder", x] @@ -249,7 +249,7 @@ def web_run(): options.add_argument('user-data-dir=' + os.path.join(script_dir, "profile")) # Headless mode check - if parser.get('DEFAULT', "web_headless") == "true": + if parser.get('DEFAULT', "web_headless") is True: options.add_argument('--headless') options.add_argument('--disable-gpu') options.add_experimental_option('prefs', prefs) @@ -319,5 +319,5 @@ def web_run(): # return["\n".join(title_list), error_count] -if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "true": +if __name__ == "__main__" and parser.get('DEFAULT', "test_run") is True: web_run() From 53f889d709dd2f882e2d9410581d3a51fa80b7c9 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 17:40:46 -0500 Subject: [PATCH 22/25] Update AutoBooks.py --- autobooks/AutoBooks.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index db0bb10..0a57bd8 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -78,8 +78,8 @@ def process(odm_list): global error_count logger.info('Begin processing book list: {}', " ".join(odm_list)) for x in odm_list: - if parser.get('DEFAULT', "test_args") is True: - odmpy_args = ["odmpy", "dl", x] + if parser.get('DEFAULT', "test_args") == "True": + odmpy_args = ["odmpy", "dl","--nobookfolder", x] else: odmpy_args = ["odmpy", "dl", "-c", "-m", "--mergeformat", "m4b", "--nobookfolder", x] with patch.object(sys, 'argv', odmpy_args): @@ -112,14 +112,14 @@ def cleanup(m4b_list, odm_list, odm_folder): logger.info("Moved book {} to out_dir", x) # Backup source files for x in odm_list: - if os.path.isfile(os.path.join(script_dir, "source_files", x)): + if os.path.isfile(os.path.join(script_dir, "source_backup", x)): logger.error( - "File {} already exists in source files dir skipped", x) + "File pair {} already exists in source backup dir skipped", x) error_count += 1 else: license_file = x.replace(".odm", ".license") - shutil.move(x, os.path.join(script_dir, "source_files", x)) - shutil.move(license_file, os.path.join(script_dir, "source_files", license_file)) + shutil.move(x, os.path.join(script_dir, "source_backup", x)) + shutil.move(license_file, os.path.join(script_dir, "source_backup", license_file)) logger.info("Moved file pair {} to source files", x) @@ -128,7 +128,7 @@ def web_login(driver, name, card_num, pin, select): global error_count logger.info("Logging into library: {}", name) # Attempt selecting library from dropdown - if select != "false": + if select != "False": select_box = driver.find_element( By.XPATH, '//input[@id="signin-options"]') webdriver.ActionChains(driver).move_to_element( @@ -143,7 +143,7 @@ def web_login(driver, name, card_num, pin, select): logger.critical("Can't find card number field skipped library {}", name) error_count += 1 # Attempt sending pin Note:Some pages don't have pin input - if pin != "false": + if pin != "False": driver.find_element(By.ID, "password").send_keys(pin) driver.find_element( By.CSS_SELECTOR, "button.signin-button.button.secondary").click() @@ -207,9 +207,9 @@ def main_run(): else: odm_list = glob.glob("*.odm") monitor.ping(state='run', - message=f'AutoBooks by IvyB Started from web V.{version} \n' + - f'odm_dir: {odm_dir} \n out_dir:{out_dir}\n logfile:{LOG_FILENAME}\n' + - f'odm_list: \n{" ".join(odm_list)}') + message=f"AutoBooks by IvyB v.{version} \n" + f"logfile: {LOG_FILENAME}\n odm_dir: {odm_dir} \n out_dir: {out_dir} \n" + f"odm_list:{odm_list}") # Check if any .odm files exist in odm_dir if len(odm_list) == 0: @@ -237,8 +237,9 @@ def web_run(): else: logger.info("Started AutoBooks Web V.{} By:IvyB", version) monitor.ping(state='run', - message=f'AutoBooks Web by IvyB V.{version} \n' + - f'logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}') + message=(f'AutoBooks Web by IvyB V.{version} \n' + f'logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}')) + # Configure WebDriver options options = Options() prefs = { @@ -249,7 +250,7 @@ def web_run(): options.add_argument('user-data-dir=' + os.path.join(script_dir, "profile")) # Headless mode check - if parser.get('DEFAULT', "web_headless") is True: + if parser.get('DEFAULT', "web_headless") == "True": options.add_argument('--headless') options.add_argument('--disable-gpu') options.add_experimental_option('prefs', prefs) @@ -304,8 +305,9 @@ def web_run(): if len(web_odm_list) != 0: logger.info("Started AutoBooks V.{} By:IvyB", version) monitor.ping(state='run', - message=f'AutoBooks by IvyB Started from web V.{version} \n' + - f'out_dir:{out_dir}\n logfile:{LOG_FILENAME}\n odm_list: \n{" ".join(web_odm_list)}') + message=f"AutoBooks by IvyB v.{version} \n" + f"logfile: {LOG_FILENAME}\n odm_dir: {odm_dir} \n out_dir: {out_dir} \n" + f"odm_list:{web_odm_list}") process(web_odm_list) m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, os.path.join( @@ -319,5 +321,5 @@ def web_run(): # return["\n".join(title_list), error_count] -if __name__ == "__main__" and parser.get('DEFAULT', "test_run") is True: +if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "True": web_run() From 243de23c03443cf52013258fa39d5198fef1e69c Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 18:06:47 -0500 Subject: [PATCH 23/25] Update AutoBooks.py --- autobooks/AutoBooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 0a57bd8..a96d53a 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -56,7 +56,7 @@ logger.configure(handlers=[ {'sink': sys.stderr, "format": console_log_format}, {'sink': LOG_FILENAME, "format": redacting_formatter.format}, - # {'sink': process_logfile(), "format": cronitor_log_format.format} + # {'sink': process_logfile(), "format": cronitor_log_format} ]) odmpy.logger.handlers.clear() odmpy.logger.addHandler(InterceptHandler()) @@ -79,7 +79,7 @@ def process(odm_list): logger.info('Begin processing book list: {}', " ".join(odm_list)) for x in odm_list: if parser.get('DEFAULT', "test_args") == "True": - odmpy_args = ["odmpy", "dl","--nobookfolder", x] + odmpy_args = ["odmpy", "dl", "--nobookfolder", x] else: odmpy_args = ["odmpy", "dl", "-c", "-m", "--mergeformat", "m4b", "--nobookfolder", x] with patch.object(sys, 'argv', odmpy_args): From 2734d06af664af80dffc462e7db94904800a9639 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 22:24:24 -0500 Subject: [PATCH 24/25] Remove old code --- autobooks/AutoBooks.py | 2 +- autobooks/DiscordBot.py | 4 +- autobooksold/AutoBooks.py | 351 ------------------------------- autobooksold/AutoBooksDiscord.py | 104 --------- autobooksold/__init__.py | 0 5 files changed, 3 insertions(+), 458 deletions(-) delete mode 100644 autobooksold/AutoBooks.py delete mode 100644 autobooksold/AutoBooksDiscord.py delete mode 100644 autobooksold/__init__.py diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index a96d53a..8c3c96d 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -22,7 +22,7 @@ from utils import InterceptHandler, RedactingFormatter, process_logfile # Set Vars -version = "0.2.1" # Version number of script +version = "0.3" # Version number of script error_count = 0 good_odm_list, bad_odm_list, library_list, book_id_list, book_title_list, book_odm_list = ([ ] for i in range(6)) diff --git a/autobooks/DiscordBot.py b/autobooks/DiscordBot.py index 7de3707..2ee1b9e 100644 --- a/autobooks/DiscordBot.py +++ b/autobooks/DiscordBot.py @@ -26,7 +26,7 @@ async def on_ready(): async def hello(ctx): # Send starting embed and running web embed_start = discord.Embed(title="Running AutoBooks Web. This may take awhile....", - description="Version: " + version + " \nLogfile: " + LOG_FILENAME, color=0xFFAFCC) + description=f"V.{version} \n Logfile: {LOG_FILENAME}", color=0xFFAFCC) embed_start.set_image(url="https://raw.githubusercontent.com/ivybowman/AutoBooks/main/img/logo/small_pink.png") embed_start.set_footer(text="OS: " + platform.platform() + " Host: " + platform.node()) await ctx.channel.send(embed=embed_start) @@ -42,7 +42,7 @@ async def hello(ctx): embed_end.add_field(name="Book List", value=str(web_info[0]), inline=False) await ctx.channel.send(embed=embed_end) # Logfile fetching - files = glob.glob(os.path.join(script_dir, "log", "*-Main.log")) + files = glob.glob(os.path.join(script_dir, "log", "*.log")) files2 = sorted(files, key=os.path.getmtime, reverse=True) print(files2[0]) await ctx.channel.send(file=discord.File(files2[0])) diff --git a/autobooksold/AutoBooks.py b/autobooksold/AutoBooks.py deleted file mode 100644 index 3d61029..0000000 --- a/autobooksold/AutoBooks.py +++ /dev/null @@ -1,351 +0,0 @@ -from selenium import webdriver -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options -from configparser import ConfigParser -from unittest.mock import patch -from datetime import datetime -from time import sleep -from pathlib import Path -import odmpy.odm as odmpy -import pandas as pd -import selenium -import cronitor -import glob -import sys -import shutil -import logging -import os -import requests - - -# Logging Redacting formatter from https://relaxdiego.com/2014/07/logging-in-python.html#configuring-your-loggers -class RedactingFormatter(object): - def __init__(self, orig_formatter, patterns): - self.orig_formatter = orig_formatter - self._patterns = patterns - - def format(self, record): - msg = self.orig_formatter.format(record) - for pattern in self._patterns: - msg = msg.replace(pattern, "") - return msg - - def __getattr__(self, attr): - return getattr(self.orig_formatter, attr) - - -# Set Vars -scriptver = "0.2.1" # Version number of script -error_count = 0 -good_odm_list = [] -bad_odm_list = [] -log_list = [] -library_list = [] -book_id_list = [] -book_title_list = [] -book_odm_list = [] -scriptdir = os.path.join(Path.home(), "AutoBooks") -csv_path = os.path.join(scriptdir, 'web_known_files.csv') - -# Check paths, and if not found do first time setup -if os.path.exists(scriptdir): - os.chdir(scriptdir) -else: - os.mkdir(scriptdir) - main_conf = requests.get('https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') - odmpy_conf = requests.get("https://raw.githubusercontent.com/ivybowman/AutoBooks/main/odmpydl.conf") - folders = ['log', 'web_downloads', 'chrome_profile', 'sourcefiles'] - for folder in folders: - os.mkdir(os.path.join(scriptdir, folder)) - with open(os.path.join(scriptdir, "autobooks.conf"), mode='wb') as localfile: - localfile.write(main_conf.content) - with open(os.path.join(scriptdir, "odmpydl.conf"), mode='wb') as localfile: - localfile.write(odmpy_conf.content) - print("Finished setup please configure settings in file: ", os.path.join(scriptdir, "autobooks.conf")) - sys.exit(1) - -# Logging Config -LOG_FILENAME = os.path.join(scriptdir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d-%Y}-Main.log'.format(datetime.now())) -fh = logging.FileHandler(LOG_FILENAME) -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s [%(name)s] %(levelname)s: %(message)s', - datefmt='%I:%M:%S %p', - handlers=[ - fh, - logging.StreamHandler(sys.stdout) - ]) -fh.setFormatter(RedactingFormatter(fh.formatter, patterns=["", "", "", ""])) -odmpy.logger = logging.getLogger('AutoBooks.odmpy') -process_logger = logging.getLogger("AutoBooks.main") -web_logger = logging.getLogger("AutoBooks.web") -discord_logger = logging.getLogger("AutoBooks.discord") - -# Read config file -parser = ConfigParser() -parser.read(os.path.join(scriptdir, "autobooks.conf")) -odmdir = parser.get("DEFAULT", - "odm_folder") # Folder that contains the .odm files to process. For windows use / slashes -outdir = parser.get("DEFAULT", - "out_folder") # Folder where the finished audiobooks will be moved to. For windows use / slashes - -# Cronitor Setup -cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") # Cronitor API key https://cronitor.io/ -monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) # Set Cronitor monitor name -web_monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_web")) # Set Cronitor monitor name - - -# Function to process the books. -def process_books(odm_list): - global error_count - process_logger.info('Begin processing booklist: %s', " ".join(odm_list)) - for x in odm_list: - if parser.get('DEFAULT', "test_args") == "true": - odmpy_args = ["odmpy", "dl", x] - else: - odmpy_args = ["odmpy", "dl", "@" + os.path.join(scriptdir, "odmpydl.conf"), x] - with patch.object(sys, 'argv', odmpy_args): - try: - odmpy.run() - except FileNotFoundError: - process_logger.error("Could not find odm file %s", x) - except FileExistsError: - process_logger.error("FileAlreadyExists, likely from m4b creation attempt") - except SystemExit as e: - bad_odm_list.append(x) - try: - os.remove("cover.jpg") - except FileNotFoundError: - process_logger.debug("Could not remove cover.jpg, moving on") - else: - process_logger.debug("Removed cover.jpg to prep for next attempt") - error_count += 1 - else: - good_odm_list.append(x) - process_logger.info("Book Processing Finished") - - -# Function to cleanup in and out files. -def cleanup(m4bs, odms, odmfolder): - global error_count - # Move m4b files to outdir - for x in m4bs: - exists = os.path.isfile(os.path.join(outdir + x)) - if exists: - process_logger.error("Book %s already exists in outdir skipped", x) - error_count += 1 - else: - shutil.move(os.path.join(odmfolder, x), os.path.join(outdir, x)) - process_logger.info("Moved book %s to outdir", x) - # Backup source files - sourcefiles = odms + glob.glob("*.license") - for x in sourcefiles: - if os.path.isfile(os.path.join(scriptdir, "sourcefiles", x)): - process_logger.error("File %s already exists in sourcefiles dir skipped", x) - error_count += 1 - else: - shutil.move(x, os.path.join(scriptdir, "sourcefiles", x)) - process_logger.info("Moved file %s to sourcefiles", x) - - -# Function for login -def web_login(driver, name, cardno, pin, select): - global error_count - web_logger.info("web_login: Logging into library: %s", name) - # Attempt selecting library from dropdown - if select != "false": - select_box = driver.find_element(By.XPATH, '//input[@id="signin-options"]') - webdriver.ActionChains(driver).move_to_element(select_box).click().send_keys(select).perform() - sleep(1) - webdriver.ActionChains(driver).send_keys(Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform() - # Attempt sending card number - try: - driver.find_element(By.ID, "username").send_keys(cardno) - except selenium.common.exceptions.NoSuchElementException: - web_logger.critical("web_login: Can't find card number field skipped library %s", ) - error_count += 1 - # Attempt sending pin Note:Some pages don't have pin input - if pin != "false": - driver.find_element(By.ID, "password").send_keys(pin) - driver.find_element(By.CSS_SELECTOR, "button.signin-button.button.secondary").click() - sleep(5) - - -# Function to download loans from OverDrive page -def web_dl(driver, df, name): - global error_count - #Gather all book title elements and check if any found - books = driver.find_elements(By.XPATH, '//a[@tabindex="0"][@role="link"]') - if len(books) == 0: - web_logger.warning("Can't find books skipped library: %s", name) - error_count += 1 - return () - else: - web_logger.info("web_dl: Begin DL from library: %s ", name) - bookcount = 0 - for i in books: - #Fetch info about the book - book_url = i.get_attribute('href') - book_info = i.get_attribute('aria-label') - book_info_split = book_info.split(". Audiobook. Expires in") - book_dl_url = book_url.replace('/media/', '/media/download/audiobook-mp3/') - book_id = int(''.join(filter(str.isdigit, book_url))) - book_title = book_info_split[0] - if "Audiobook." in book_info: - if str(book_id) in df['book_id'].to_string(): - web_logger.info('web_dl: Skipped %s found in known books', book_title) - else: - # Download book - driver.get(book_dl_url) - web_logger.info("web_dl: Downloaded book: %s", book_title) - book_odm = max(glob.glob("*.odm"), key=os.path.getmtime) - bookcount += 1 - - #Add book data to vars - library_list.append(name) - book_id_list.append(book_id) - book_title_list.append(book_title) - book_odm_list.append(book_odm) - sleep(1) - - - sleep(1) - web_logger.info("web_dl: Finished downloading %s books from library %s", bookcount, name) - return () - -def main_run(): - # AutoBooks - process_logger.info("Started AutoBooks V.%s By:IvyB", scriptver) - # Try to change to ODM folder - try: - os.chdir(odmdir) - except FileNotFoundError: - process_logger.critical("The provided .odm dir was not found, exiting") - sys.exit(1) - else: - odm_list = glob.glob("*.odm") - monitor.ping(state='run', - message='AutoBooks by IvyB Version:' + scriptver + '\n odmdir:' + odmdir + '\n outdir:' + outdir + '\n logfile:' + LOG_FILENAME + '\n Found the following books \n' + " ".join( - odm_list)) - - # Check if any .odm files exist in odmdir - if len(odm_list) == 0: - monitor.ping(state='fail', message='Error: No .odm files found, exiting', - metrics={'error_count': error_count}) - process_logger.critical("No .odm files found, exiting") - sys.exit(1) - else: - process_books(odm_list) - # Cleanup input and output files - m4blist = glob.glob("*.m4b") - cleanup(m4blist, good_odm_list, odmdir) - # Process log file for Cronitor - with open(LOG_FILENAME) as logs: - lines = logs.readlines() - log_list = [] - for line in lines: - if any(term in line for term in ("Downloading", "expired", "generating", "merged")): - log_list.append(line) - # Send complete event and log to Cronitor - monitor.ping(state='complete', message="".join(log_list), - metrics={'count': len(odm_list), 'error_count': error_count}) - - -# AutoBooks Web Code -def web_run(): - if len(parser.sections()) == 0: - web_logger.critical("No libraries configured!") - sys.exit(1) - else: - web_logger.info("Started AutoBooks Web V.%s By:IvyB", scriptver) - monitor.ping(state='run', - message='AutoBooks Web by IvyB Version:' + scriptver + '\n logfile:' + LOG_FILENAME + '\n LibraryCount: ' + str( - len(parser.sections()))) - # Configure WebDriver options - options = Options() - prefs = { - "download.default_directory": os.path.join(scriptdir, "web_downloads"), - "download.prompt_for_download": False, - "download.directory_upgrade": True - } - options.add_argument('user-data-dir=' + os.path.join(scriptdir, "chrome_profile")) - # Headless mode check - if parser.get('DEFAULT', "web_headless") == "true": - options.add_argument('--headless') - options.add_argument('--disable-gpu') - options.add_experimental_option('prefs', prefs) - driver = webdriver.Chrome(options=options) - - # Attempt to read known files csv for checking books - if os.path.exists(csv_path): - df = pd.read_csv(csv_path, sep=",") - else: - df = pd.DataFrame({ - 'book_id': book_id_list, - }) - os.chdir(os.path.join(scriptdir,"web_downloads")) - # For every library, open site, attempt sign in, and attempt download. - for i in range(0, len(parser.sections())): - library_index = 'library_' + str(i) - library_subdomain = parser.get(library_index, "library_subdomain") - library_name = parser.get(library_index, "library_name") - web_logger.info("Started library %s", library_name) - url = "https://" + library_subdomain + ".overdrive.com/" - driver.get(url + "account/loans") - sleep(3) - # Check signed in status and either sign in or move on - if "/account/ozone/sign-in" in driver.current_url: - web_login(driver, library_name, parser.get(library_index, "card_number"), - parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) - web_dl(driver, df, library_name) - sleep(2) - # Output book data to csv - df_out = pd.DataFrame({ - 'library_name': library_list, - 'book_id': book_id_list, - 'book_title': book_title_list, - 'book_odm': book_odm_list - }) - if os.path.exists(csv_path): - df_out.to_csv(csv_path, mode='a', index=False, header=False) - else: - df_out.to_csv(csv_path, mode='w', index=False, header=True) - driver.close() - web_logger.info("AutoBooksWeb Complete") - odmlist = glob.glob("*.odm") - - # Process log file for Cronitor. - with open(LOG_FILENAME) as logs: - lines = logs.readlines() - log_list = [] - for line in lines: - if "AutoBooks.web" in line: - log_list.append(line) - monitor.ping(state='complete', message="".join(odmlist), - metrics={'count': len(odmlist), 'error_count': error_count}) - - # Call Minimum DL functions - if len(odmlist) != 0: - process_logger.info("Started AutoBooks V.%s By:IvyB", scriptver) - monitor.ping(state='run', - message='AutoBooks by IvyB Started from web Version:' + scriptver + '\n outdir:' + outdir + '\n logfile:' + LOG_FILENAME + '\n Found the following books \n' + " ".join( - odmlist)) - process_books(odmlist) - m4blist = glob.glob("*.m4b") - cleanup(m4blist, good_odm_list, os.path.join(scriptdir, "web_downloads")) - # Process log file for Cronitor - with open(LOG_FILENAME) as logs: - lines = logs.readlines() - log_list = [] - for line in lines: - if any(term in line for term in ("Downloading", "expired", "generating", "merged")): - log_list.append(line) - # Send complete event and log to Cronitor - monitor.ping(state='complete', message="".join(log_list), - metrics={'count': len(odmlist), 'error_count': error_count}) - #return["\n".join(title_list), error_count] - - -if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "true": - web_run() diff --git a/autobooksold/AutoBooksDiscord.py b/autobooksold/AutoBooksDiscord.py deleted file mode 100644 index 9439a34..0000000 --- a/autobooksold/AutoBooksDiscord.py +++ /dev/null @@ -1,104 +0,0 @@ -from configparser import ConfigParser -from unittest.mock import patch -from datetime import datetime -from discord.ext import commands -from pathlib import Path -import os -import discord -import logging -import glob -import sys -import shutil -import platform -from AutoBooks import web_run, main_run, scriptver, scriptdir, parser, csv_path, fh, discord_logger, LOG_FILENAME -import pandas as pd - -# Log Settings -# DISCORD_LOGFILE = os.path.join(AutoBooks.scriptdir,'log','AutoBooks-{:%H-%M-%S_%m-%d-%Y}-Discord.log'.format(datetime.now())) -# formatter = logging.Formatter('%(asctime)s [%(name)s] %(levelname)s: %(message)s', datefmt='%I:%M:%S %p',) -# discord_fh = logging.FileHandler(DISCORD_LOGFILE) -# discord_fh.setFormatter(formatter) -# AutoBooks.discord_logger.removeHandler(AutoBooks.fh) -# AutoBooks.discord_logger.addHandler(discord_fh) -logger = logging.getLogger('discord') -logger.removeHandler(fh) -# logger.addHandler(discord_fh) -# Bot Settings -try: - token = parser.get("DEFAULT", "discord_bot_token") -except KeyError: - discord_logger.critical("Bot token not found in config file, exiting.") -bot = commands.Bot(command_prefix='?') - - -@bot.event -async def on_ready(): - discord_logger.info(f'{bot.user} has connected to Discord!') - - -@bot.command(name='web') -async def hello(ctx): - #Starting embed and running web - embed_start = discord.Embed(title="Running AutoBooks Web. This may take awhile....", description="Version: "+scriptver+" \nLogfile: "+LOG_FILENAME, color=0xFFAFCC) - embed_start.set_image(url="https://raw.githubusercontent.com/ivybowman/AutoBooks/main/img/logo/small_pink.png") - embed_start.set_footer(text="OS: "+ platform.platform()+" Host: "+platform.node()) - await ctx.channel.send(embed=embed_start) - web_info = web_run() - - #Ending Embed - embed_end = discord.Embed(title="AutoBooks Web Finished", description="See log info below for details. ErrorCount: "+str(web_info[1]), color=0xFFAFCC) - embed_end.set_thumbnail(url="https://raw.githubusercontent.com/ivybowman/AutoBooks/main/img/icon_pink.png") - if web_info[0] != "": - embed_end.add_field(name="Book List", value=str(web_info[0]), inline=False) - await ctx.channel.send(embed=embed_end) - #Logfile fetching - files = glob.glob(os.path.join(scriptdir, "log", "*-Main.log")) - files2 = sorted(files, key=os.path.getmtime, reverse=True) - print(files2[0]) - await ctx.channel.send(file=discord.File(files2[0])) - - -@bot.command(name='main') -async def hello(ctx): - embedVar = discord.Embed(title="Title", description="Desc", color=0xFFAFCC) - embedVar.add_field(name="Field1", value="hi", inline=False) - embedVar.add_field(name="Field2", value="hi2", inline=False) - - await ctx.channel.send(embed=embedVar) - main_run() - - -@bot.command(name='log') -async def hello(ctx): - files = glob.glob(os.path.join(scriptdir, "log", "*-Main.log")) - max_file = max(files, key=os.path.getmtime) - print(max_file) - await ctx.channel.send("Fetched latest AutoBooks logfile: \n" + max_file) - await ctx.channel.send(file=discord.File(max_file)) - - -@bot.command(name='csv') -async def hello(ctx): - try: - df = pd.read_csv(csv_path, sep=",") - embedVar = discord.Embed(title="Autobooks Known Books", description=df['audiobook_title'].to_string(index=False), color=0xFFAFCC) - embedVar.set_footer(text="OS: "+ platform.platform()+" Host: "+os.uname()) - await ctx.channel.send(embed=embedVar) - except FileNotFoundError: - await ctx.channel.send("Known Books CSV not found.") - - # await ctx.channel.send(file=discord.File(max_file)) - - -def discord_run(): - if token == "": - discord_logger.critical("Bot token not found in config file, exiting.") - else: - bot.run(token) - - -if __name__ == "__main__": - try: - discord_run() - except KeyboardInterrupt: - sys.exit(1) diff --git a/autobooksold/__init__.py b/autobooksold/__init__.py deleted file mode 100644 index e69de29..0000000 From fc75856956ccc3f75f3b17eea49b1e934f7b6ef3 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Mon, 7 Mar 2022 22:30:05 -0500 Subject: [PATCH 25/25] Fix version numbers --- README.md | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3859de6..cce599f 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,10 @@ The intention of this script is solely to automate downloading and converting of # Features -- AutoBooks Web: Uses selenium and chromedriver to download the odms from overdrive without user interaction. -- Uses odmpy to fulfill and convert odms to chapterized m4b audiobook files. +- AutoBooks Web: Uses selenium and chromedriver to download the odm files from overdrive without user interaction. +- Uses odmpy to fulfill and convert odm files to chapterized m4b audiobooks. - Moves the generated audiobooks to a chosen folder. -- Backs up the download files in case you need to redownload the books. +- Backs up the download files in case you need to re-download the books. - Logs to console and timestamped logfile. - Reports execution status and some logs to a [Cronitor](https://cronitor.io/) monitor. - Can be controlled via included Discord bot or terminal. diff --git a/setup.py b/setup.py index 9b63ffa..f840a05 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup -VERSION = '0.2.1' +VERSION = '0.3' DESCRIPTION = 'Python tool to automate processing a batch of OverDrive audiobooks.' LONG_DESCRIPTION = 'Python tool to automate processing a batch of OverDrive audiobooks.'