From 660f8899e573d1a508e0f902d98abb8dd13fea5c Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 11:03:46 -0500 Subject: [PATCH 01/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 67c787e..0b9569b 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -14,10 +14,6 @@ import requests import selenium from loguru import logger -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.common.by import By -from selenium.webdriver.common.keys import Keys from utils import InterceptHandler, RedactingFormatter, process_logfile From da2ae4e09b5977fc378754796de4fe2a5f44ffc3 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 11:11:42 -0500 Subject: [PATCH 02/18] Begin testing move to requests --- autobooks/AutoBooks.py | 58 ++++++++++-------------------------------- autobooks/utils.py | 29 ++++++++++++++++++++- setup.py | 2 +- 3 files changed, 43 insertions(+), 46 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 0b9569b..708b31b 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -118,32 +118,20 @@ def cleanup(m4b_list, odm_list, odm_folder): logger.info("Moved file pair {} to source files", x) -# Function for login -def web_login(driver, name, card_num, pin, select): - global error_count - logger.info("Logging into library: {}", name) - # Attempt selecting library from dropdown - if select != "False": - select_box = driver.find_element( - By.XPATH, '//input[@id="signin-options"]') - webdriver.ActionChains(driver).move_to_element( - select_box).click().send_keys(select).perform() - sleep(1) - webdriver.ActionChains(driver).send_keys( - Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform() - # Attempt sending card number - try: - driver.find_element(By.ID, "username").send_keys(card_num) - except selenium.common.exceptions.NoSuchElementException: - logger.critical("Can't find card number field skipped library {}", name) - error_count += 1 - # Attempt sending pin Note:Some pages don't have pin input - if pin != "False": - driver.find_element(By.ID, "password").send_keys(pin) - driver.find_element( - By.CSS_SELECTOR, "button.signin-button.button.secondary").click() - sleep(5) - +def login(card_num, pin): + x = 0 + login_session = requests.Session() + box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') + login_form = parse_form(box, "loginForms") + auth = login_session.post(f'https://{subdomain}.overdrive.com/account/signInOzone', params=params, + data={ + 'ilsName': login_form['forms'][x]['ilsName'], + 'authType': 'Local', + 'libraryName': '', + 'username': card_num, + 'password': pin + }) + return login_session, auth.url, login_form['forms'][x]['ilsName'] # Function to download loans from OverDrive page def web_dl(driver, df, name): @@ -235,22 +223,6 @@ def web_run(): message=(f'AutoBooks Web by IvyB V.{version} \n' f'logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}')) - # Configure WebDriver options - options = Options() - prefs = { - "download.default_directory": os.path.join(script_dir, "downloads"), - "download.prompt_for_download": False, - "download.directory_upgrade": True - } - options.add_argument('user-data-dir=' + - os.path.join(script_dir, "profile")) - # Headless mode check - if parser.get('DEFAULT', "web_headless") == "True": - options.add_argument('--headless') - options.add_argument('--disable-gpu') - options.add_experimental_option('prefs', prefs) - driver = webdriver.Chrome(options=options) - # Attempt to read known files csv for checking books if os.path.exists(csv_path): df = pd.read_csv(csv_path, sep=",") @@ -267,8 +239,6 @@ def web_run(): library_subdomain = parser.get(library_index, "library_subdomain") library_name = parser.get(library_index, "library_name") logger.info("Started library {}", library_name) - url = "https://" + library_subdomain + ".overdrive.com/" - driver.get(url + "account/loans") sleep(3) # Check signed in status and either sign in or move on if "/account/ozone/sign-in" in driver.current_url: diff --git a/autobooks/utils.py b/autobooks/utils.py index 43c98f8..f1899d8 100644 --- a/autobooks/utils.py +++ b/autobooks/utils.py @@ -1,5 +1,5 @@ import logging - +import lxml.etree from loguru import logger @@ -45,3 +45,30 @@ def process_logfile(logfile, terms=None): if any(term in line for term in terms): log_list.append(line) return "".join(log_list) + + +def parse_form(box, sort): + form_dict = {} + txt = lxml.etree.HTML(box.content) + js = str(txt.xpath(f"//script[contains(text(), 'window.OverDrive.{sort} =')]/text()")[0]).strip() + split_1 = js.split(sep=" = ") + for i in range(0, len(split_1)): + if "window.OverDrive." + sort in split_1[i]: + form_dict = split_1[i + 1].strip().split(';')[0] + break + return dict(json.loads(form_dict)) + +def craft_booklist(loans_page): + book_dict = parse_form(loans_page, "mediaItems") + book_list_parse = [] + for i in book_dict: + book_format = book_dict[i]['overDriveFormat']['id'] + book_title = book_dict[i]['title'] + book_id = book_dict[i]['id'] + book_parse = { + 'id': book_id, + 'title': book_title, + 'format': book_format, + } + book_list_parse.append(book_parse) + return book_list_parse \ No newline at end of file diff --git a/setup.py b/setup.py index f840a05..ddf1244 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ ] }, install_requires=['odmpy @ git+https://git@github.com/ping/odmpy.git', "cronitor", "pandas", "discord.py", - "selenium", "requests", "loguru"], + "selenium", "requests", "loguru", "lxml"], include_package_data=True, platforms="any", keywords=['python', 'AutoBooks'], From 5d51045dab94a4001f9f5ecc13d419204c29c2e7 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 11:32:36 -0500 Subject: [PATCH 03/18] Remove webdriver code --- autobooks/AutoBooks.py | 19 +++++++++++-------- autobooks/testconf.py | 34 ++++++++++++++++++++++++++++++++++ autobooks/utils.py | 4 +++- setup.py | 4 ++-- 4 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 autobooks/testconf.py diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 708b31b..bdc738f 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -15,7 +15,7 @@ import selenium from loguru import logger -from utils import InterceptHandler, RedactingFormatter, process_logfile +from utils import InterceptHandler, RedactingFormatter, process_logfile, parse_form, craft_booklist # Set Vars version = "0.3" # Version number of script @@ -24,6 +24,9 @@ ] for i in range(6)) script_dir = os.path.join(Path.home(), "AutoBooks") csv_path = os.path.join(script_dir, 'web_known_files.csv') +params = ( + ('forwardUrl', '/'), +) # Check paths, and if not found do first time setup if os.path.exists(script_dir): @@ -118,7 +121,7 @@ def cleanup(m4b_list, odm_list, odm_folder): logger.info("Moved file pair {} to source files", x) -def login(card_num, pin): +def web_login(subdomain, card_num, pin, select): x = 0 login_session = requests.Session() box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') @@ -133,6 +136,7 @@ def login(card_num, pin): }) return login_session, auth.url, login_form['forms'][x]['ilsName'] +''' # Function to download loans from OverDrive page def web_dl(driver, df, name): global error_count @@ -176,7 +180,7 @@ def web_dl(driver, df, name): logger.info("Finished downloading {} books from library {}", book_count, name) return () - +''' def main_run(): # AutoBooks @@ -240,11 +244,10 @@ def web_run(): library_name = parser.get(library_index, "library_name") logger.info("Started library {}", library_name) sleep(3) - # Check signed in status and either sign in or move on - if "/account/ozone/sign-in" in driver.current_url: - web_login(driver, library_name, parser.get(library_index, "card_number"), - parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) - web_dl(driver, df, library_name) + + session, base_url, library_name = web_login(library_subdomain, parser.get(library_index, "card_number"), + parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) + # web_dl(driver, df, library_name) sleep(2) # Output book data to csv df_out = pd.DataFrame({ diff --git a/autobooks/testconf.py b/autobooks/testconf.py new file mode 100644 index 0000000..0eab608 --- /dev/null +++ b/autobooks/testconf.py @@ -0,0 +1,34 @@ +import glob +import os +import shutil +import sys +from configparser import ConfigParser +from configobj import ConfigObj +from datetime import datetime +from pathlib import Path +from time import sleep +from unittest.mock import patch +import json +import cronitor +import odmpy.odm as odmpy +import pandas as pd +import requests +import selenium +from loguru import logger + +script_dir = os.path.join(Path.home(), "AutoBooks") +#config = ConfigObj(os.path.join(script_dir, "autobooks.ini")) +# Read config file +parser = ConfigParser() +parser.read(os.path.join(script_dir, "autobooks.ini")) +print(parser.sections()) +section = parser.default_section +print(section.get('odm_folder')) +print(parser) +#print(config.keys()) +#odm_dir = parser.get("DEFAULT", "odm_folder") +#out_dir = parser.get("DEFAULT", "out_folder") +#library_count = len(parser.sections()) +# Cronitor Setup https://cronitor.io/ +#cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") +#monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) \ No newline at end of file diff --git a/autobooks/utils.py b/autobooks/utils.py index f1899d8..33d3e22 100644 --- a/autobooks/utils.py +++ b/autobooks/utils.py @@ -1,6 +1,7 @@ import logging import lxml.etree from loguru import logger +import json # Formatter to remove patterns from log output @@ -58,6 +59,7 @@ def parse_form(box, sort): break return dict(json.loads(form_dict)) + def craft_booklist(loans_page): book_dict = parse_form(loans_page, "mediaItems") book_list_parse = [] @@ -71,4 +73,4 @@ def craft_booklist(loans_page): 'format': book_format, } book_list_parse.append(book_parse) - return book_list_parse \ No newline at end of file + return book_list_parse diff --git a/setup.py b/setup.py index ddf1244..7afa35e 100644 --- a/setup.py +++ b/setup.py @@ -21,8 +21,8 @@ "autobooks-discord = autobooks.AutoBooksDiscord:run" ] }, - install_requires=['odmpy @ git+https://git@github.com/ping/odmpy.git', "cronitor", "pandas", "discord.py", - "selenium", "requests", "loguru", "lxml"], + install_requires=["odmpy @ git+https://git@github.com/ping/odmpy.git", "cronitor", "pandas", "discord.py", + "requests", "loguru", "lxml", "configobj"], include_package_data=True, platforms="any", keywords=['python', 'AutoBooks'], From df5089917ec218e3d9120974b1c367152c3744d2 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 12:21:10 -0500 Subject: [PATCH 04/18] Cleanup code, build new requests --- autobooks/AutoBooks.py | 125 ++++++++---------- autobooks/testconf.py | 6 +- ...ks_template.conf => autobooks_template.ini | 0 3 files changed, 55 insertions(+), 76 deletions(-) rename autobooks_template.conf => autobooks_template.ini (100%) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index bdc738f..6955b9b 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -12,7 +12,6 @@ import odmpy.odm as odmpy import pandas as pd import requests -import selenium from loguru import logger from utils import InterceptHandler, RedactingFormatter, process_logfile, parse_form, craft_booklist @@ -20,13 +19,10 @@ # Set Vars version = "0.3" # Version number of script error_count = 0 -good_odm_list, bad_odm_list, library_list, book_id_list, book_title_list, book_odm_list = ([ -] for i in range(6)) +good_odm_list = [] +bad_odm_list = [] script_dir = os.path.join(Path.home(), "AutoBooks") csv_path = os.path.join(script_dir, 'web_known_files.csv') -params = ( - ('forwardUrl', '/'), -) # Check paths, and if not found do first time setup if os.path.exists(script_dir): @@ -35,7 +31,7 @@ os.mkdir(script_dir) main_conf = requests.get( 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') - folders = ['log', 'downloads', 'profile', 'source_backup'] + folders = ['log', 'downloads', 'source_backup'] for folder in folders: os.mkdir(os.path.join(script_dir, folder)) with open(os.path.join(script_dir, "autobooks.conf"), mode='wb') as local_file: @@ -62,13 +58,14 @@ # Read config file parser = ConfigParser() -parser.read(os.path.join(script_dir, "autobooks.conf")) -odm_dir = parser.get("DEFAULT", "odm_folder") -out_dir = parser.get("DEFAULT", "out_folder") +parser.read(os.path.join(script_dir, "autobooks.ini")) +config = parser['DEFAULT'] +odm_dir = config["odm_folder"] +out_dir = config["out_folder"] library_count = len(parser.sections()) # Cronitor Setup https://cronitor.io/ -cronitor.api_key = parser.get("DEFAULT", "cronitor_apikey") -monitor = cronitor.Monitor(parser.get("DEFAULT", "cronitor_name_main")) +cronitor.api_key = config['cronitor_apikey'] +monitor = cronitor.Monitor(config['cronitor_monitor']) # Function to process the books. @@ -126,7 +123,8 @@ def web_login(subdomain, card_num, pin, select): login_session = requests.Session() box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') login_form = parse_form(box, "loginForms") - auth = login_session.post(f'https://{subdomain}.overdrive.com/account/signInOzone', params=params, + auth = login_session.post(f'https://{subdomain}.overdrive.com/account/signInOzone', + params=(('forwardUrl', '/'),), data={ 'ilsName': login_form['forms'][x]['ilsName'], 'authType': 'Local', @@ -136,51 +134,39 @@ def web_login(subdomain, card_num, pin, select): }) return login_session, auth.url, login_form['forms'][x]['ilsName'] -''' + # Function to download loans from OverDrive page -def web_dl(driver, df, name): +def web_dl(df, session, base_url, name, book_list): global error_count - # Gather all book title elements and check if any found - books = driver.find_elements(By.XPATH, '//a[@tabindex="0"][@role="link"]') - if len(books) == 0: + df_out = pd.DataFrame() + logger.info("Begin DL from library: {} ", name) + book_count = 0 + if len(book_list) == 0: logger.warning("Can't find books skipped library: {}", name) error_count += 1 - return () - else: - logger.info("Begin DL from library: {} ", name) - book_count = 0 - for i in books: - # Fetch info about the book - book_url = i.get_attribute('href') - book_info = i.get_attribute('aria-label') - book_info_split = book_info.split(". Audiobook. Expires in") - book_dl_url = book_url.replace( - '/media/', '/media/download/audiobook-mp3/') - book_id = int(''.join(filter(str.isdigit, book_url))) - book_title = book_info_split[0] + return df_out + for book in book_list: + if book['format'] != "ebook-overdrive": + print("AudioBook Info: ", book['title'], "-", book['id'], "-", book['format']) + id_query = df.query('book_id == ' + book['id']) + if id_query.empty is False: + logger.info('Skipped {} found in known books', book['title']) + else: + df_book = pd.DataFrame([[name, book['id'], book['title']]], + columns=['library_name', 'book_id', 'book_title']) + df_out = pd.concat([df_out, df_book]) + sleep(0.5) + odm = session.get(f'{base_url}media/download/audiobook-mp3/{book["id"]}') + odm_filename = odm.url.split("/")[-1].split('?')[0] + print(odm.content) + print(odm_filename) + logger.info("Downloaded book: {} as {}", book['title'], odm_filename) - # Check if found book is a not known audiobook - if "Audiobook." in book_info: - if str(book_id) in df['book_id'].to_string(): - logger.info('Skipped {} found in known books', book_title) - else: - # Download book - driver.get(book_dl_url) - logger.info("Downloaded book: {}", book_title) - book_odm = max(glob.glob("*.odm"), key=os.path.getmtime) - book_count += 1 + sleep(1) + logger.info("Finished downloading {} books from library {}", + book_count, name) + return df_out - # Add book data to vars - library_list.append(name) - book_id_list.append(book_id) - book_title_list.append(book_title) - book_odm_list.append(book_odm) - sleep(1) - sleep(1) - logger.info("Finished downloading {} books from library {}", - book_count, name) - return () -''' def main_run(): # AutoBooks @@ -238,29 +224,22 @@ def web_run(): os.chdir(os.path.join(script_dir, "downloads")) # For every library, open site, attempt sign in, and attempt download. - for i in range(0, len(parser.sections())): - library_index = 'library_' + str(i) - library_subdomain = parser.get(library_index, "library_subdomain") - library_name = parser.get(library_index, "library_name") - logger.info("Started library {}", library_name) + for i in range(0, library_count): + lib_conf = parser['library_' + str(i)] + logger.info("Started library {}", lib_conf['library_name']) sleep(3) - - session, base_url, library_name = web_login(library_subdomain, parser.get(library_index, "card_number"), - parser.get(library_index, "card_pin"), parser.get(library_index, "library_select")) - # web_dl(driver, df, library_name) + session, base_url, library_name = web_login(lib_conf['library_subdomain'], + lib_conf['card_number'], + lib_conf['card_pin'], + lib_conf['library_select']) + loans = session.get(f'{base_url}account/loans') + book_list = craft_booklist(loans) + df_out = web_dl(df, session, base_url, library_name, book_list) sleep(2) - # Output book data to csv - df_out = pd.DataFrame({ - 'library_name': library_list, - 'book_id': book_id_list, - 'book_title': book_title_list, - 'book_odm': book_odm_list - }) - if os.path.exists(csv_path): - df_out.to_csv(csv_path, mode='a', index=False, header=False) - else: - df_out.to_csv(csv_path, mode='w', index=False, header=True) - driver.close() + if os.path.exists(csv_path) and not df_out.empty: + df_out.to_csv(csv_path, mode='a', index=False, header=False) + elif df_out: + df_out.to_csv(csv_path, mode='w', index=False, header=True) logger.info("AutoBooksWeb Complete") web_odm_list = glob.glob("*.odm") diff --git a/autobooks/testconf.py b/autobooks/testconf.py index 0eab608..e8702f4 100644 --- a/autobooks/testconf.py +++ b/autobooks/testconf.py @@ -13,7 +13,6 @@ import odmpy.odm as odmpy import pandas as pd import requests -import selenium from loguru import logger script_dir = os.path.join(Path.home(), "AutoBooks") @@ -22,9 +21,10 @@ parser = ConfigParser() parser.read(os.path.join(script_dir, "autobooks.ini")) print(parser.sections()) +config = parser['DEFAULT'] +print(section['cronitor_apikey']) section = parser.default_section -print(section.get('odm_folder')) -print(parser) + #print(config.keys()) #odm_dir = parser.get("DEFAULT", "odm_folder") #out_dir = parser.get("DEFAULT", "out_folder") diff --git a/autobooks_template.conf b/autobooks_template.ini similarity index 100% rename from autobooks_template.conf rename to autobooks_template.ini From d93cf122c1afd587a0897bda9794afc51be20d65 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 12:47:23 -0500 Subject: [PATCH 05/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 105 +++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 47 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 6955b9b..91f68db 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -1,4 +1,5 @@ import glob +import logging import os import shutil import sys @@ -53,6 +54,8 @@ {'sink': LOG_FILENAME, "format": redacting_formatter.format, "retention": 10, "rotation": "1 day"}, ]) +logging.getLogger().setLevel('DEBUG') +logging.getLogger().addHandler(InterceptHandler()) odmpy.logger.handlers.clear() odmpy.logger.addHandler(InterceptHandler()) @@ -73,10 +76,11 @@ def process(odm_list): global error_count logger.info('Begin processing book list: {}', " ".join(odm_list)) for x in odm_list: - if parser.get('DEFAULT', "test_args") == "True": - odmpy_args = ["odmpy", "dl", "--nobookfolder", x] + odmpy_args = ["odmpy", "dl", "--nobookfolder"] + if config['odmpy_test_args'] == "True": + odmpy_args.extend([x]) else: - odmpy_args = ["odmpy", "dl", "-c", "-m", "--mergeformat", "m4b", "--nobookfolder", x] + odmpy_args.extend(["-c", "-m", "--mergeformat", "m4b", x]) with patch.object(sys, 'argv', odmpy_args): try: odmpy.run() @@ -141,6 +145,7 @@ def web_dl(df, session, base_url, name, book_list): df_out = pd.DataFrame() logger.info("Begin DL from library: {} ", name) book_count = 0 + if len(book_list) == 0: logger.warning("Can't find books skipped library: {}", name) error_count += 1 @@ -152,9 +157,11 @@ def web_dl(df, session, base_url, name, book_list): if id_query.empty is False: logger.info('Skipped {} found in known books', book['title']) else: + # Save book info to dataframe df_book = pd.DataFrame([[name, book['id'], book['title']]], columns=['library_name', 'book_id', 'book_title']) df_out = pd.concat([df_out, df_book]) + # Short wait then download ODM sleep(0.5) odm = session.get(f'{base_url}media/download/audiobook-mp3/{book["id"]}') odm_filename = odm.url.split("/")[-1].split('?')[0] @@ -168,42 +175,9 @@ def web_dl(df, session, base_url, name, book_list): return df_out -def main_run(): - # AutoBooks - logger.info("Started AutoBooks V.{} By:IvyB", version) - # Try to change to ODM folder - try: - os.chdir(odm_dir) - except FileNotFoundError: - logger.critical("The provided .odm dir was not found, exiting") - sys.exit(1) - else: - odm_list = glob.glob("*.odm") - monitor.ping(state='run', - message=f"AutoBooks by IvyB v.{version} \n" - f"logfile: {LOG_FILENAME}\n odm_dir: {odm_dir} \n out_dir: {out_dir} \n" - f"odm_list:{odm_list}") - - # Check if any .odm files exist in odm_dir - if len(odm_list) == 0: - monitor.ping(state='fail', message='Error: No .odm files found, exiting', - metrics={'error_count': error_count}) - logger.critical("No .odm files found, exiting") - sys.exit(1) - else: - process(odm_list) - # Cleanup files - m4blist = glob.glob("*.m4b") - cleanup(m4blist, good_odm_list, odm_dir) - # Send complete event and log to Cronitor - log_str = process_logfile(LOG_FILENAME, terms=( - "Downloading", "expired", "generating", "merged", "saved")) - monitor.ping(state='complete', message=log_str, - metrics={'count': len(odm_list), 'error_count': error_count}) - - # AutoBooks Web Code def web_run(): + global error_count if len(parser.sections()) == 0: logger.critical("No libraries configured!") sys.exit(1) @@ -219,7 +193,7 @@ def web_run(): else: # Failing above create an empty df for checking df = pd.DataFrame({ - 'book_id': book_id_list, + 'book_id': [''], }) os.chdir(os.path.join(script_dir, "downloads")) @@ -234,12 +208,16 @@ def web_run(): lib_conf['library_select']) loans = session.get(f'{base_url}account/loans') book_list = craft_booklist(loans) - df_out = web_dl(df, session, base_url, library_name, book_list) - sleep(2) - if os.path.exists(csv_path) and not df_out.empty: - df_out.to_csv(csv_path, mode='a', index=False, header=False) - elif df_out: - df_out.to_csv(csv_path, mode='w', index=False, header=True) + if len(book_list) != 0: + df_out = web_dl(df, session, base_url, library_name, book_list) + sleep(2) + if os.path.isfile(csv_path): + df_out.to_csv(csv_path, mode='a', index=False, header=False) + else: + df_out.to_csv(csv_path, mode='w', index=False, header=True) + else: + logger.warning("Can't find books skipped library: {}", lib_conf['library_name']) + error_count += 1 logger.info("AutoBooksWeb Complete") web_odm_list = glob.glob("*.odm") @@ -249,7 +227,7 @@ def web_run(): metrics={'count': len(web_odm_list), 'error_count': error_count}) # Call DL to process odm files from web - if len(web_odm_list) != 0: + if len(web_odm_list) != 0 and 4+5 == 10: logger.info("Started AutoBooks V.{} By:IvyB", version) monitor.ping(state='run', message=f"AutoBooks by IvyB v.{version} \n" @@ -265,8 +243,41 @@ def web_run(): # Send complete event and log to Cronitor monitor.ping(state='complete', message=log_str, metrics={'count': len(web_odm_list), 'error_count': error_count}) - # return["\n".join(title_list), error_count] -if __name__ == "__main__" and parser.get('DEFAULT', "test_run") == "True": +def main_run(): + # AutoBooks + logger.info("Started AutoBooks V.{} By:IvyB", version) + # Try to change to ODM folder + try: + os.chdir(odm_dir) + except FileNotFoundError: + logger.critical("The provided .odm dir was not found, exiting") + sys.exit(1) + else: + odm_list = glob.glob("*.odm") + monitor.ping(state='run', + message=f"AutoBooks by IvyB v.{version} \n" + f"logfile: {LOG_FILENAME}\n odm_dir: {odm_dir} \n out_dir: {out_dir} \n" + f"odm_list:{odm_list}") + + # Check if any .odm files exist in odm_dir + if len(odm_list) == 0: + monitor.ping(state='fail', message='Error: No .odm files found, exiting', + metrics={'error_count': error_count}) + logger.critical("No .odm files found, exiting") + sys.exit(1) + else: + process(odm_list) + # Cleanup files + m4blist = glob.glob("*.m4b") + cleanup(m4blist, good_odm_list, odm_dir) + # Send complete event and log to Cronitor + log_str = process_logfile(LOG_FILENAME, terms=( + "Downloading", "expired", "generating", "merged", "saved")) + monitor.ping(state='complete', message=log_str, + metrics={'count': len(odm_list), 'error_count': error_count}) + + +if __name__ == "__main__" and config["test_run"] == "True": web_run() From 7d4fd2f29ac51245fdad92b164fc620fa3be1f39 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 12:58:43 -0500 Subject: [PATCH 06/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 91f68db..78a253b 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -4,7 +4,6 @@ import shutil import sys from configparser import ConfigParser -from datetime import datetime from pathlib import Path from time import sleep from unittest.mock import patch @@ -122,11 +121,19 @@ def cleanup(m4b_list, odm_list, odm_folder): logger.info("Moved file pair {} to source files", x) -def web_login(subdomain, card_num, pin, select): - x = 0 +def web_login(subdomain): login_session = requests.Session() box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') login_form = parse_form(box, "loginForms") + print(login_form) + + return login_session, login_form['forms'] + + +def web_auth(login_session, subdomain, card_num, pin, select, x): + box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') + login_form = parse_form(box, "loginForms") + print(login_form) auth = login_session.post(f'https://{subdomain}.overdrive.com/account/signInOzone', params=(('forwardUrl', '/'),), data={ @@ -136,7 +143,7 @@ def web_login(subdomain, card_num, pin, select): 'username': card_num, 'password': pin }) - return login_session, auth.url, login_form['forms'][x]['ilsName'] + return auth.url, login_form['forms'][x]['ilsName'] # Function to download loans from OverDrive page @@ -202,13 +209,14 @@ def web_run(): lib_conf = parser['library_' + str(i)] logger.info("Started library {}", lib_conf['library_name']) sleep(3) - session, base_url, library_name = web_login(lib_conf['library_subdomain'], - lib_conf['card_number'], - lib_conf['card_pin'], - lib_conf['library_select']) + session, login_form = web_login(lib_conf['library_subdomain']) + base_url, library_name = web_auth(lib_conf['library_subdomain'], + lib_conf['card_number'], + lib_conf['card_pin'], + lib_conf['library_select'], 0) loans = session.get(f'{base_url}account/loans') book_list = craft_booklist(loans) - if len(book_list) != 0: + if len(book_list) != 0 and 2+2 == 5: df_out = web_dl(df, session, base_url, library_name, book_list) sleep(2) if os.path.isfile(csv_path): @@ -227,7 +235,7 @@ def web_run(): metrics={'count': len(web_odm_list), 'error_count': error_count}) # Call DL to process odm files from web - if len(web_odm_list) != 0 and 4+5 == 10: + if len(web_odm_list) != 0 and 4 + 5 == 10: logger.info("Started AutoBooks V.{} By:IvyB", version) monitor.ping(state='run', message=f"AutoBooks by IvyB v.{version} \n" From 4342e37f58a7070be5a5bf793878134ef2419cf9 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 14:54:21 -0500 Subject: [PATCH 07/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 78a253b..9a18edf 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -130,20 +130,24 @@ def web_login(subdomain): return login_session, login_form['forms'] -def web_auth(login_session, subdomain, card_num, pin, select, x): +def web_auth(login_session, subdomain, card_num, pin, select, login_forms): box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') - login_form = parse_form(box, "loginForms") - print(login_form) + x = 0 + if len(login_forms) != 0: + for form in login_forms: + print(form['type']) + print(form['ilsName']) + auth = login_session.post(f'https://{subdomain}.overdrive.com/account/signInOzone', params=(('forwardUrl', '/'),), data={ - 'ilsName': login_form['forms'][x]['ilsName'], + 'ilsName': login_forms[x]['ilsName'], 'authType': 'Local', 'libraryName': '', 'username': card_num, 'password': pin }) - return auth.url, login_form['forms'][x]['ilsName'] + return auth.url # Function to download loans from OverDrive page @@ -210,10 +214,11 @@ def web_run(): logger.info("Started library {}", lib_conf['library_name']) sleep(3) session, login_form = web_login(lib_conf['library_subdomain']) - base_url, library_name = web_auth(lib_conf['library_subdomain'], + base_url, library_name = web_auth(session, lib_conf['library_subdomain'], lib_conf['card_number'], lib_conf['card_pin'], - lib_conf['library_select'], 0) + lib_conf['library_select'], + login_form['forms']) loans = session.get(f'{base_url}account/loans') book_list = craft_booklist(loans) if len(book_list) != 0 and 2+2 == 5: From 09473304dfe6ca096a6981a38b4a754765646e3b Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 15:32:49 -0500 Subject: [PATCH 08/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 47 ++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 9a18edf..023e61f 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -121,33 +121,29 @@ def cleanup(m4b_list, odm_list, odm_folder): logger.info("Moved file pair {} to source files", x) -def web_login(subdomain): +def web_login(subdomain, card_num, pin, select): login_session = requests.Session() box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') - login_form = parse_form(box, "loginForms") - print(login_form) - - return login_session, login_form['forms'] - - -def web_auth(login_session, subdomain, card_num, pin, select, login_forms): - box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') - x = 0 - if len(login_forms) != 0: - for form in login_forms: - print(form['type']) - print(form['ilsName']) + form_list = parse_form(box, "loginForms") + if len(form_list) == 0: + x = 0 + else: + for form in form_list: + if select in form['displayName']: + print(select, "Matches: ", form['displayName'], "ils:", form['ilsName']) + x = form_list.index(form) + break auth = login_session.post(f'https://{subdomain}.overdrive.com/account/signInOzone', params=(('forwardUrl', '/'),), data={ - 'ilsName': login_forms[x]['ilsName'], - 'authType': 'Local', - 'libraryName': '', + 'ilsName': form_list[x]['ilsName'], + 'authType': form_list[x]['type'], + 'libraryName': form_list[x]['displayName'], 'username': card_num, 'password': pin }) - return auth.url + return auth.url, form_list[x]['ilsName'], login_session # Function to download loans from OverDrive page @@ -179,7 +175,6 @@ def web_dl(df, session, base_url, name, book_list): print(odm.content) print(odm_filename) logger.info("Downloaded book: {} as {}", book['title'], odm_filename) - sleep(1) logger.info("Finished downloading {} books from library {}", book_count, name) @@ -213,16 +208,14 @@ def web_run(): lib_conf = parser['library_' + str(i)] logger.info("Started library {}", lib_conf['library_name']) sleep(3) - session, login_form = web_login(lib_conf['library_subdomain']) - base_url, library_name = web_auth(session, lib_conf['library_subdomain'], - lib_conf['card_number'], - lib_conf['card_pin'], - lib_conf['library_select'], - login_form['forms']) + base_url, ils_name, session = web_login(lib_conf['library_subdomain'], + lib_conf['card_number'], + lib_conf['card_pin'], + lib_conf['library_select'], ) loans = session.get(f'{base_url}account/loans') book_list = craft_booklist(loans) - if len(book_list) != 0 and 2+2 == 5: - df_out = web_dl(df, session, base_url, library_name, book_list) + if len(book_list) != 0 and 2 + 2 == 5: + df_out = web_dl(df, session, base_url, ils_name, book_list) sleep(2) if os.path.isfile(csv_path): df_out.to_csv(csv_path, mode='a', index=False, header=False) From a2c91c7569fd9b9c6b1745fbffadb0d8ab0c2f9a Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 16:12:50 -0500 Subject: [PATCH 09/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 49 +++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 023e61f..1a1e679 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -30,7 +30,7 @@ else: os.mkdir(script_dir) main_conf = requests.get( - 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.conf') + 'https://raw.githubusercontent.com/ivybowman/AutoBooks/main/autobooks_template.ini') folders = ['log', 'downloads', 'source_backup'] for folder in folders: os.mkdir(os.path.join(script_dir, folder)) @@ -53,8 +53,8 @@ {'sink': LOG_FILENAME, "format": redacting_formatter.format, "retention": 10, "rotation": "1 day"}, ]) -logging.getLogger().setLevel('DEBUG') -logging.getLogger().addHandler(InterceptHandler()) +# logging.getLogger().setLevel('DEBUG') +# logging.getLogger().addHandler(InterceptHandler()) odmpy.logger.handlers.clear() odmpy.logger.addHandler(InterceptHandler()) @@ -124,8 +124,8 @@ def cleanup(m4b_list, odm_list, odm_folder): def web_login(subdomain, card_num, pin, select): login_session = requests.Session() box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') - form_list = parse_form(box, "loginForms") - if len(form_list) == 0: + form_list = parse_form(box, "loginForms")['forms'] + if len(form_list) == 1: x = 0 else: for form in form_list: @@ -133,7 +133,7 @@ def web_login(subdomain, card_num, pin, select): print(select, "Matches: ", form['displayName'], "ils:", form['ilsName']) x = form_list.index(form) break - + sleep(0.5) auth = login_session.post(f'https://{subdomain}.overdrive.com/account/signInOzone', params=(('forwardUrl', '/'),), data={ @@ -143,6 +143,7 @@ def web_login(subdomain, card_num, pin, select): 'username': card_num, 'password': pin }) + # print("AUTH URL: ", auth.url) return auth.url, form_list[x]['ilsName'], login_session @@ -152,7 +153,6 @@ def web_dl(df, session, base_url, name, book_list): df_out = pd.DataFrame() logger.info("Begin DL from library: {} ", name) book_count = 0 - if len(book_list) == 0: logger.warning("Can't find books skipped library: {}", name) error_count += 1 @@ -162,18 +162,20 @@ def web_dl(df, session, base_url, name, book_list): print("AudioBook Info: ", book['title'], "-", book['id'], "-", book['format']) id_query = df.query('book_id == ' + book['id']) if id_query.empty is False: - logger.info('Skipped {} found in known books', book['title']) + logger.info('Skipped "{}" found in known books', book['title']) else: - # Save book info to dataframe - df_book = pd.DataFrame([[name, book['id'], book['title']]], - columns=['library_name', 'book_id', 'book_title']) - df_out = pd.concat([df_out, df_book]) # Short wait then download ODM sleep(0.5) odm = session.get(f'{base_url}media/download/audiobook-mp3/{book["id"]}') odm_filename = odm.url.split("/")[-1].split('?')[0] + with open(odm_filename, "wb") as f: + f.write(odm.content) print(odm.content) print(odm_filename) + # Save book info to dataframe + df_book = pd.DataFrame([[name, book['id'], book['title'], odm_filename]], + columns=['ils_name', 'book_id', 'book_title', 'book_odm']) + df_out = pd.concat([df_out, df_book]) logger.info("Downloaded book: {} as {}", book['title'], odm_filename) sleep(1) logger.info("Finished downloading {} books from library {}", @@ -212,18 +214,21 @@ def web_run(): lib_conf['card_number'], lib_conf['card_pin'], lib_conf['library_select'], ) + if not base_url.endswith('/'): + base_url = base_url+'/' loans = session.get(f'{base_url}account/loans') - book_list = craft_booklist(loans) - if len(book_list) != 0 and 2 + 2 == 5: - df_out = web_dl(df, session, base_url, ils_name, book_list) - sleep(2) - if os.path.isfile(csv_path): - df_out.to_csv(csv_path, mode='a', index=False, header=False) + if loans.status_code == 200: + book_list = craft_booklist(loans) + if len(book_list) != 0: + df_out = web_dl(df, session, base_url, ils_name, book_list) + sleep(2) + if os.path.isfile(csv_path): + df_out.to_csv(csv_path, mode='a', index=False, header=False) + else: + df_out.to_csv(csv_path, mode='w', index=False, header=True) else: - df_out.to_csv(csv_path, mode='w', index=False, header=True) - else: - logger.warning("Can't find books skipped library: {}", lib_conf['library_name']) - error_count += 1 + logger.warning("Can't find books skipped library: {}", lib_conf['library_name']) + error_count += 1 logger.info("AutoBooksWeb Complete") web_odm_list = glob.glob("*.odm") From 7b06e7011415a407ed0bd08a1fdb6f0d2d1cbf69 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 16:33:39 -0500 Subject: [PATCH 10/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 1a1e679..0f18ba2 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -152,6 +152,7 @@ def web_dl(df, session, base_url, name, book_list): global error_count df_out = pd.DataFrame() logger.info("Begin DL from library: {} ", name) + odm_list = [] book_count = 0 if len(book_list) == 0: logger.warning("Can't find books skipped library: {}", name) @@ -170,7 +171,7 @@ def web_dl(df, session, base_url, name, book_list): odm_filename = odm.url.split("/")[-1].split('?')[0] with open(odm_filename, "wb") as f: f.write(odm.content) - print(odm.content) + odm_list.append(odm_filename) print(odm_filename) # Save book info to dataframe df_book = pd.DataFrame([[name, book['id'], book['title'], odm_filename]], @@ -180,12 +181,13 @@ def web_dl(df, session, base_url, name, book_list): sleep(1) logger.info("Finished downloading {} books from library {}", book_count, name) - return df_out + return df_out, odm_list # AutoBooks Web Code def web_run(): global error_count + web_odm_list = [] if len(parser.sections()) == 0: logger.critical("No libraries configured!") sys.exit(1) @@ -220,7 +222,14 @@ def web_run(): if loans.status_code == 200: book_list = craft_booklist(loans) if len(book_list) != 0: - df_out = web_dl(df, session, base_url, ils_name, book_list) + df_out, odm_list = web_dl(df, session, base_url, ils_name, book_list) + if web_odm_list == [] and odm_list != []: + web_odm_list = odm_list + else: + web_odm_list.extend(odm_list) + print(odm_list) + print("Web:") + print(web_odm_list) sleep(2) if os.path.isfile(csv_path): df_out.to_csv(csv_path, mode='a', index=False, header=False) @@ -230,7 +239,8 @@ def web_run(): logger.warning("Can't find books skipped library: {}", lib_conf['library_name']) error_count += 1 logger.info("AutoBooksWeb Complete") - web_odm_list = glob.glob("*.odm") + print(web_odm_list) + #web_odm_list = glob.glob("*.odm") # Process log file for Cronitor. process_logfile(LOG_FILENAME, terms=("web", "ERROR")) From 049069e8cb1eb0982a289b129002080581d3e6d7 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 16:52:23 -0500 Subject: [PATCH 11/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 0f18ba2..a3facf6 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -211,7 +211,7 @@ def web_run(): for i in range(0, library_count): lib_conf = parser['library_' + str(i)] logger.info("Started library {}", lib_conf['library_name']) - sleep(3) + sleep(0.5) base_url, ils_name, session = web_login(lib_conf['library_subdomain'], lib_conf['card_number'], lib_conf['card_pin'], @@ -219,18 +219,19 @@ def web_run(): if not base_url.endswith('/'): base_url = base_url+'/' loans = session.get(f'{base_url}account/loans') + sleep(0.5) if loans.status_code == 200: book_list = craft_booklist(loans) if len(book_list) != 0: df_out, odm_list = web_dl(df, session, base_url, ils_name, book_list) + sleep(0.5) if web_odm_list == [] and odm_list != []: web_odm_list = odm_list else: web_odm_list.extend(odm_list) print(odm_list) - print("Web:") - print(web_odm_list) sleep(2) + # Write book data to csv if os.path.isfile(csv_path): df_out.to_csv(csv_path, mode='a', index=False, header=False) else: @@ -240,7 +241,6 @@ def web_run(): error_count += 1 logger.info("AutoBooksWeb Complete") print(web_odm_list) - #web_odm_list = glob.glob("*.odm") # Process log file for Cronitor. process_logfile(LOG_FILENAME, terms=("web", "ERROR")) From 7161fcb58c27979651d552e806fcd24c7ee46e1f Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 17:33:04 -0500 Subject: [PATCH 12/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index a3facf6..55d376c 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -7,7 +7,7 @@ from pathlib import Path from time import sleep from unittest.mock import patch - +from datetime import datetime import cronitor import odmpy.odm as odmpy import pandas as pd @@ -42,7 +42,9 @@ # Logging Config LOG_PATH = os.path.join(script_dir, 'log') -LOG_FILENAME = os.path.join(LOG_PATH, "AutoBooks.log") +LOG_FILENAME = os.path.join( + script_dir, 'log', 'AutoBooks-{:%H-%M-%S_%m-%d}.log'.format(datetime.now())) + patterns = ['', '', '', ''] console_log_format = "{time:HH:mm:ss A} [{name}:{function}] {level}: {message}\n{exception}" cronitor_log_format = "[{name}:{function}] {level}: {message}\n{exception}" @@ -51,7 +53,7 @@ logger.configure(handlers=[ {'sink': sys.stderr, "format": console_log_format}, {'sink': LOG_FILENAME, - "format": redacting_formatter.format, "retention": 10, "rotation": "1 day"}, + "format": redacting_formatter.format, "retention": 10}, ]) # logging.getLogger().setLevel('DEBUG') # logging.getLogger().addHandler(InterceptHandler()) From d79c798c7ad00a40cacfae1767fe1a930ac01dab Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 18:01:28 -0500 Subject: [PATCH 13/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 55d376c..53e8588 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -125,13 +125,16 @@ def cleanup(m4b_list, odm_list, odm_folder): def web_login(subdomain, card_num, pin, select): login_session = requests.Session() + logger.info("Logging into: {}", subdomain) box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') + logger.success('Fetched login page. Status Code: {}', box.status_code) form_list = parse_form(box, "loginForms")['forms'] if len(form_list) == 1: x = 0 else: for form in form_list: if select in form['displayName']: + logger.success('Matched Config: {} to {}', select, form['displayName']) print(select, "Matches: ", form['displayName'], "ils:", form['ilsName']) x = form_list.index(form) break @@ -145,6 +148,7 @@ def web_login(subdomain, card_num, pin, select): 'username': card_num, 'password': pin }) + logger.success("Logged into: {} Status Code: {} ", subdomain, auth.status_code) # print("AUTH URL: ", auth.url) return auth.url, form_list[x]['ilsName'], login_session @@ -174,6 +178,7 @@ def web_dl(df, session, base_url, name, book_list): with open(odm_filename, "wb") as f: f.write(odm.content) odm_list.append(odm_filename) + book_count += 1 print(odm_filename) # Save book info to dataframe df_book = pd.DataFrame([[name, book['id'], book['title'], odm_filename]], @@ -212,7 +217,7 @@ def web_run(): # For every library, open site, attempt sign in, and attempt download. for i in range(0, library_count): lib_conf = parser['library_' + str(i)] - logger.info("Started library {}", lib_conf['library_name']) + logger.info("Begin Processing library: {}", lib_conf['library_name']) sleep(0.5) base_url, ils_name, session = web_login(lib_conf['library_subdomain'], lib_conf['card_number'], @@ -231,6 +236,7 @@ def web_run(): web_odm_list = odm_list else: web_odm_list.extend(odm_list) + print("ODM LIST") print(odm_list) sleep(2) # Write book data to csv @@ -242,6 +248,7 @@ def web_run(): logger.warning("Can't find books skipped library: {}", lib_conf['library_name']) error_count += 1 logger.info("AutoBooksWeb Complete") + print("WEB ODM LIST") print(web_odm_list) # Process log file for Cronitor. @@ -250,7 +257,7 @@ def web_run(): metrics={'count': len(web_odm_list), 'error_count': error_count}) # Call DL to process odm files from web - if len(web_odm_list) != 0 and 4 + 5 == 10: + if len(web_odm_list) != 0: logger.info("Started AutoBooks V.{} By:IvyB", version) monitor.ping(state='run', message=f"AutoBooks by IvyB v.{version} \n" From dad4fc496a3458ad62be876b0000d3c685bfda19 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 18:47:46 -0500 Subject: [PATCH 14/18] Almost done with request move --- autobooks/AutoBooks.py | 57 +++++++++++++++++++---------------------- autobooks/DiscordBot.py | 9 +++---- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 53e8588..64bca4c 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -1,5 +1,5 @@ import glob -import logging +import platform import os import shutil import sys @@ -19,8 +19,6 @@ # Set Vars version = "0.3" # Version number of script error_count = 0 -good_odm_list = [] -bad_odm_list = [] script_dir = os.path.join(Path.home(), "AutoBooks") csv_path = os.path.join(script_dir, 'web_known_files.csv') @@ -64,9 +62,8 @@ parser = ConfigParser() parser.read(os.path.join(script_dir, "autobooks.ini")) config = parser['DEFAULT'] -odm_dir = config["odm_folder"] -out_dir = config["out_folder"] library_count = len(parser.sections()) + # Cronitor Setup https://cronitor.io/ cronitor.api_key = config['cronitor_apikey'] monitor = cronitor.Monitor(config['cronitor_monitor']) @@ -75,9 +72,11 @@ # Function to process the books. def process(odm_list): global error_count + good_odm_list = [] + bad_odm_list = [] + odmpy_args = ["odmpy", "dl", "--nobookfolder"] logger.info('Begin processing book list: {}', " ".join(odm_list)) for x in odm_list: - odmpy_args = ["odmpy", "dl", "--nobookfolder"] if config['odmpy_test_args'] == "True": odmpy_args.extend([x]) else: @@ -85,10 +84,8 @@ def process(odm_list): with patch.object(sys, 'argv', odmpy_args): try: odmpy.run() - except FileNotFoundError: - logger.error("Could not find odm file {}", x) - except FileExistsError: - logger.error("FileAlreadyExists, likely from m4b creation attempt") + except(FileNotFoundError, FileExistsError) as e: + logger.error("Error starting odm {} Message: {}", x, e) except SystemExit as e: bad_odm_list.append(x) if os.path.isfile("cover.jpg"): @@ -97,6 +94,7 @@ def process(odm_list): else: good_odm_list.append(x) logger.info("Book Processing Finished") + return good_odm_list, bad_odm_list # Function to clean up in and out files. @@ -104,11 +102,11 @@ def cleanup(m4b_list, odm_list, odm_folder): global error_count # Move m4b files to out_dir for x in m4b_list: - if os.path.isfile(os.path.join(out_dir + x)): + if os.path.isfile(os.path.join(config["out_folder"] + x)): logger.error("Book {} already exists in out dir skipped", x) error_count += 1 else: - shutil.move(os.path.join(odm_folder, x), os.path.join(out_dir, x)) + shutil.move(os.path.join(odm_folder, x), os.path.join(config["out_folder"], x)) logger.info("Moved book {} to out_dir", x) # Backup source files for x in odm_list: @@ -129,9 +127,8 @@ def web_login(subdomain, card_num, pin, select): box = login_session.get(f'https://{subdomain}.overdrive.com/account/ozone/sign-in?forward=%2F') logger.success('Fetched login page. Status Code: {}', box.status_code) form_list = parse_form(box, "loginForms")['forms'] - if len(form_list) == 1: - x = 0 - else: + x = 0 + if len(form_list) != 1: for form in form_list: if select in form['displayName']: logger.success('Matched Config: {} to {}', select, form['displayName']) @@ -199,9 +196,9 @@ def web_run(): logger.critical("No libraries configured!") sys.exit(1) else: - logger.info("Started AutoBooks Web V.{} By:IvyB", version) + logger.info("Started AutoBooks Web V.{} By:IvyB on Host: {}", version, platform.node()) monitor.ping(state='run', - message=(f'AutoBooks Web by IvyB V.{version} \n' + message=(f'AutoBooks Web by IvyB V.{version} on Host: {platform.node()} \n' f'logfile: {LOG_FILENAME} \n LibraryCount: {str(library_count)}')) # Attempt to read known files csv for checking books @@ -236,8 +233,6 @@ def web_run(): web_odm_list = odm_list else: web_odm_list.extend(odm_list) - print("ODM LIST") - print(odm_list) sleep(2) # Write book data to csv if os.path.isfile(csv_path): @@ -248,22 +243,21 @@ def web_run(): logger.warning("Can't find books skipped library: {}", lib_conf['library_name']) error_count += 1 logger.info("AutoBooksWeb Complete") - print("WEB ODM LIST") - print(web_odm_list) # Process log file for Cronitor. - process_logfile(LOG_FILENAME, terms=("web", "ERROR")) - monitor.ping(state='complete', message="".join(web_odm_list), + web_log = process_logfile(LOG_FILENAME, terms=("web", "ERROR")) + monitor.ping(state='complete', + message=f'{"".join(web_log)}', metrics={'count': len(web_odm_list), 'error_count': error_count}) # Call DL to process odm files from web if len(web_odm_list) != 0: - logger.info("Started AutoBooks V.{} By:IvyB", version) + logger.info("Started AutoBooks V.{} By:IvyB on Host: {}", version, platform.node()) monitor.ping(state='run', - message=f"AutoBooks by IvyB v.{version} \n" - f"logfile: {LOG_FILENAME}\n odm_dir: {odm_dir} \n out_dir: {out_dir} \n" + message=f"AutoBooks by IvyB v.{version} on Host: {platform.node()} \n" + f"logfile: {LOG_FILENAME}\n out_dir: {config['out_folder']} \n" f"odm_list:{web_odm_list}") - process(web_odm_list) + good_odm_list, bad_odm_list = process(web_odm_list) m4blist = glob.glob("*.m4b") cleanup(m4blist, good_odm_list, os.path.join( script_dir, "downloads")) @@ -280,7 +274,7 @@ def main_run(): logger.info("Started AutoBooks V.{} By:IvyB", version) # Try to change to ODM folder try: - os.chdir(odm_dir) + os.chdir(config["odm_folder"]) except FileNotFoundError: logger.critical("The provided .odm dir was not found, exiting") sys.exit(1) @@ -288,7 +282,8 @@ def main_run(): odm_list = glob.glob("*.odm") monitor.ping(state='run', message=f"AutoBooks by IvyB v.{version} \n" - f"logfile: {LOG_FILENAME}\n odm_dir: {odm_dir} \n out_dir: {out_dir} \n" + f"logfile: {LOG_FILENAME}\n odm_dir: {config['odm_folder']} \n " + f"out_dir: {config['out_folder']} \n" f"odm_list:{odm_list}") # Check if any .odm files exist in odm_dir @@ -298,10 +293,10 @@ def main_run(): logger.critical("No .odm files found, exiting") sys.exit(1) else: - process(odm_list) + good_odm_list, bad_odm_list = process(odm_list) # Cleanup files m4blist = glob.glob("*.m4b") - cleanup(m4blist, good_odm_list, odm_dir) + cleanup(m4blist, good_odm_list, config["odm_folder"]) # Send complete event and log to Cronitor log_str = process_logfile(LOG_FILENAME, terms=( "Downloading", "expired", "generating", "merged", "saved")) diff --git a/autobooks/DiscordBot.py b/autobooks/DiscordBot.py index bc2c4af..1803a9f 100644 --- a/autobooks/DiscordBot.py +++ b/autobooks/DiscordBot.py @@ -7,13 +7,10 @@ import pandas as pd from discord.ext import commands -from AutoBooks import web_run, main_run, version, script_dir, parser, csv_path, LOG_FILENAME, logger +from AutoBooks import web_run, main_run, version, script_dir, config, csv_path, LOG_FILENAME, logger # Bot Settings -try: - token = parser.get("DEFAULT", "discord_bot_token") -except KeyError: - logger.critical("Bot token field not found in config file, exiting.") +token = config["discord_bot_token"] bot = commands.Bot(command_prefix='?') @@ -72,7 +69,7 @@ async def hello(ctx): try: df = pd.read_csv(csv_path, sep=",") embed_var = discord.Embed(title="Autobooks Known Books", - description=df['audiobook_title'].to_string(index=False), color=0xFFAFCC) + description=df['book_title'].to_string(index=False), color=0xFFAFCC) embed_var.set_footer(text="OS: " + platform.platform() + " Host: " + os.uname()) await ctx.channel.send(embed=embed_var) except FileNotFoundError: From 5a226b938081a41e793317f382eddc109b42f259 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Fri, 11 Mar 2022 21:27:28 -0500 Subject: [PATCH 15/18] Update AutoBooks.py --- autobooks/AutoBooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 64bca4c..d1868d7 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -242,6 +242,7 @@ def web_run(): else: logger.warning("Can't find books skipped library: {}", lib_conf['library_name']) error_count += 1 + session.close() logger.info("AutoBooksWeb Complete") # Process log file for Cronitor. From c02527c41c412f2d143555a78ec968b1516dc3d7 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Sat, 12 Mar 2022 00:37:41 -0500 Subject: [PATCH 16/18] Formatting and setup guide updates --- README.md | 6 +++--- autobooks/AutoBooks.py | 7 ++++--- setup.md | 27 +++++++++++---------------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9a66fc8..2031358 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ for personal use with your preferred audiobook player during the loan period. # Features -- AutoBooks Web: Uses selenium and chromedriver to download the odm files from overdrive without user interaction. -- Uses odmpy to fulfill and convert odm files to chapterized m4b audiobooks. +- AutoBooks Web: Uses requests to download odm files. +- Uses odmpy to fulfill and convert odm files to m4b audiobooks with chapters. - Moves the generated audiobooks to a chosen folder. - Backs up the download files in case you need to re-download the books. - Logs to console and timestamped logfile. @@ -20,7 +20,7 @@ for personal use with your preferred audiobook player during the loan period. # Prerequisites -- Tools: git, ffmpeg, odmpy, chromedriver (Installed in setup guide.) +- Tools: git, ffmpeg, odmpy (Installed in setup guide.) - Accounts: [Cronitor](https://cronitor.io/) For script monitoring, optional but will display errors if not setup. # Links diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index d1868d7..48dec40 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -1,13 +1,14 @@ import glob -import platform import os +import platform import shutil import sys from configparser import ConfigParser +from datetime import datetime from pathlib import Path from time import sleep from unittest.mock import patch -from datetime import datetime + import cronitor import odmpy.odm as odmpy import pandas as pd @@ -221,7 +222,7 @@ def web_run(): lib_conf['card_pin'], lib_conf['library_select'], ) if not base_url.endswith('/'): - base_url = base_url+'/' + base_url = base_url + '/' loans = session.get(f'{base_url}account/loans') sleep(0.5) if loans.status_code == 200: diff --git a/setup.md b/setup.md index e3c8b8e..50ca715 100644 --- a/setup.md +++ b/setup.md @@ -13,8 +13,8 @@ Open a PowerShell window then follow the steps below. `Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser` 2. Run the Scoop installer, this is a package manager used to install some prerequisites. `iwr -useb get.scoop.sh | iex` -3. Install prerequisites. Note: Chromedriver requires Google Chrome to be installed. -`scoop install ffmpeg chromedriver git` +3. Install prerequisites. +`scoop install ffmpeg git` ## macOS Setup Guide @@ -23,20 +23,15 @@ Open a terminal window then follow the steps below. Works with both M1 and Intel 2. Install Homebrew. Be sure to follow instructions at the end for adding it to path. `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"` 3. Install prerequisites. On trying to use these tools you might see an unideditifed developer pop up, this is normal just open the folder and ctrl+click or right click on the file and click open. -Note: Chromedriver requires Google Chrome to be installed. -`brew install ffmpeg chromedriver` +`brew install ffmpeg` ## Debian/Ubuntu Linux Setup Guide Open a terminal window then follow the steps below. 2. Update package list. `sudo apt-get update` -2. Install most prerequisites. -`sudo apt-get install -y unzip ffmpeg git` -3. Run script to install the other prerequisites. Note: Chrome is required for AutoBooksWeb. -With Google Chrome: `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/ivybowman/AutoBooks/main/ubuntusetupchrome.sh)"` -Without Google Chrome: `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/ivybowman/AutoBooks/main/ubuntusetup.sh)"` -Note: AutoBooks Web is not compatible with WSL, and requires X11 forwarding to run via ssh if headless mode is disabled. +2. Install prerequisites. +`sudo apt-get install -y ffmpeg git` ## AutoBooks Install & Setup (All Operating Systems) @@ -45,17 +40,17 @@ Note: AutoBooks Web is not compatible with WSL, and requires X11 forwarding to r To install from the latest source run the following command. `pip3 install git+https://git@github.com/ivybowman/autobooks.git --upgrade --force-reinstall` To install from a specific version run the following command. -`pip3 install git+https://git@github.com/ivybowman/autobooks.git@v0.2.1-alpha --upgrade` +`pip3 install git+https://git@github.com/ivybowman/autobooks.git@v0.3-alpha --upgrade` To uninstall AutoBooks run the following command. `pip3 uninstall autobooks` ### Configuration 1. Open a terminal and run `autobooks` this will run setup commands to create the data folder. -2. Edit the `autobooks.conf` file using one of the commands below or by browsing to the autobooks folder inside your home directory. -- Windows(GUI) PowerShell: `notepad $env:USERPROFILE\AutoBooks\autobooks.conf` -- Windows(GUI) Command Prompt: `notepad %userprofile%\AutoBooks\autobooks.conf` -- macOS(GUI) Terminal: `open -a TextEdit ~/AutoBooks/autobooks.conf` -- Linux or macOS(CLI): `nano ~/AutoBooks/autobooks.conf` +2. Edit the `autobooks.ini` file using one of the commands below or by browsing to the autobooks folder inside your home directory. +- Windows(GUI) PowerShell: `notepad $env:USERPROFILE\AutoBooks\autobooks.ini` +- Windows(GUI) Command Prompt: `notepad %userprofile%\AutoBooks\autobooks.ini` +- macOS(GUI) Terminal: `open -a TextEdit ~/AutoBooks/autobooks.ini` +- Linux or macOS(CLI): `nano ~/AutoBooks/autobooks.ini` From 5fa225a7d35cfc991d34a152dc523131af5ab074 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Sat, 12 Mar 2022 00:39:01 -0500 Subject: [PATCH 17/18] Fix version number --- autobooks/AutoBooks.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/autobooks/AutoBooks.py b/autobooks/AutoBooks.py index 48dec40..57492ad 100644 --- a/autobooks/AutoBooks.py +++ b/autobooks/AutoBooks.py @@ -18,7 +18,7 @@ from utils import InterceptHandler, RedactingFormatter, process_logfile, parse_form, craft_booklist # Set Vars -version = "0.3" # Version number of script +version = "0.4" # Version number of script error_count = 0 script_dir = os.path.join(Path.home(), "AutoBooks") csv_path = os.path.join(script_dir, 'web_known_files.csv') diff --git a/setup.py b/setup.py index 7afa35e..323e722 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup -VERSION = '0.3' +VERSION = '0.4' DESCRIPTION = 'Python tool to automate processing a batch of OverDrive audiobooks.' LONG_DESCRIPTION = 'Python tool to automate processing a batch of OverDrive audiobooks.' From 8d862ec3530e79424a8f9c574aaa02f954d5e0f4 Mon Sep 17 00:00:00 2001 From: Ivy Bowman Date: Sat, 12 Mar 2022 00:42:01 -0500 Subject: [PATCH 18/18] Fix requirements --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 323e722..4e78d58 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ ] }, install_requires=["odmpy @ git+https://git@github.com/ping/odmpy.git", "cronitor", "pandas", "discord.py", - "requests", "loguru", "lxml", "configobj"], + "requests", "loguru", "lxml"], include_package_data=True, platforms="any", keywords=['python', 'AutoBooks'],