diff --git a/Pipfile.lock b/Pipfile.lock index 12dc9bf..1734979 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -195,11 +195,11 @@ }, "idna": { "hashes": [ - "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", - "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" + "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac", + "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603" ], - "markers": "python_version >= '3.5'", - "version": "==3.7" + "markers": "python_version >= '3.6'", + "version": "==3.8" }, "parsedatetime": { "hashes": [ @@ -219,7 +219,7 @@ }, "praw-wrapper": { "git": "https://github.com/Watchful1/PrawWrapper.git", - "ref": "28f0cdeff345be23a38409192fc631d5739217f8" + "ref": "690d71c7f47488c685f9353f9f87c1c61c101e34" }, "prawcore": { "hashes": [ diff --git a/scripts/explain_parse.py b/scripts/explain_parse.py index fc8affb..5473922 100644 --- a/scripts/explain_parse.py +++ b/scripts/explain_parse.py @@ -12,12 +12,15 @@ cal = parsedatetime.Calendar() -input_string = '''!remindme 30th of February, 2024.''' -base_time_string = "2023-04-27 02:45:19 -0700" +input_string = '''!remindme December 24th :)''' +base_time_string = None#"2024-04-27 02:45:19 -0700" +created_utc = 1723766419 timezone_string = None # "America/New_York" if base_time_string: base_time = utils.datetime_as_timezone(utils.parse_datetime_string(base_time_string, False, '%Y-%m-%d %H:%M:%S %z'), "UTC") +elif created_utc: + base_time = utils.datetime_from_timestamp(created_utc) else: base_time = utils.datetime_now() diff --git a/scripts/update_wiki.py b/scripts/update_wiki.py new file mode 100644 index 0000000..90e6d28 --- /dev/null +++ b/scripts/update_wiki.py @@ -0,0 +1,17 @@ +import discord_logging +import traceback +from datetime import timedelta +import praw_wrapper + +log = discord_logging.init_logging() + +import utils +from database import Database +import stats + + +if __name__ == "__main__": + reddit = praw_wrapper.Reddit("Watchful1") + database = Database() + + stats.update_stats(reddit, database) diff --git a/src/classes/stat.py b/src/classes/stat.py index 4743311..592e927 100644 --- a/src/classes/stat.py +++ b/src/classes/stat.py @@ -12,8 +12,13 @@ class DbStat(Base): subreddit = Column(String(80), nullable=False) thread_id = Column(String(12), nullable=False) comment_id = Column(String(12)) - initial_date = Column(UtcDateTime, nullable=False) + initial_date = Column(UtcDateTime) count_reminders = Column(Integer, nullable=False) + #thread_title = Column(String(200)) + + title = None + answered = False + count_pending_reminders = None def __init__( self, @@ -26,4 +31,7 @@ def __init__( self.thread_id = thread_id self.comment_id = comment_id self.count_reminders = count_reminders - self.initial_date = utils.datetime_now() + + def __str__(self): + return f"{self.id}:{self.subreddit}:{self.thread_id}:{self.comment_id}:" \ + f": {utils.get_datetime_string(self.initial_date)}:{self.count_reminders}" diff --git a/src/comments.py b/src/comments.py index 78fb233..56979f4 100644 --- a/src/comments.py +++ b/src/comments.py @@ -1,6 +1,5 @@ import discord_logging import traceback -import praw import utils import static diff --git a/src/counters.py b/src/counters.py index 1f25fe3..69ed035 100644 --- a/src/counters.py +++ b/src/counters.py @@ -5,6 +5,7 @@ queue = prometheus_client.Gauge('bot_queue', "Current queue size") objects = prometheus_client.Gauge('bot_objects', "Total number of objects by type", ['type']) errors = prometheus_client.Counter('bot_errors', "Count of errors", ['type']) +run_time = prometheus_client.Summary('bot_run_seconds', "How long a full loop takes") def init(port): diff --git a/src/database/UtcDateTime.py b/src/database/UtcDateTime.py index f80f438..f38393e 100644 --- a/src/database/UtcDateTime.py +++ b/src/database/UtcDateTime.py @@ -9,4 +9,7 @@ class UtcDateTime(types.TypeDecorator): cache_ok = True def process_result_value(self, value, dialect): - return utils.datetime_force_utc(value) + if value is not None: + return utils.datetime_force_utc(value) + else: + return None diff --git a/src/database/_reminders.py b/src/database/_reminders.py index 02c9639..034e5a8 100644 --- a/src/database/_reminders.py +++ b/src/database/_reminders.py @@ -72,6 +72,17 @@ def get_user_reminders(self, user_name): log.debug(f"Found reminders: {len(regular_reminders)} : {len(recurring_reminders)}") return regular_reminders, recurring_reminders + def get_reminders_with_keyword(self, search_key, earliest_date): + log.debug(f"Searching for reminders with {search_key}") + + count_reminders = self.session.query(Reminder)\ + .filter(Reminder.target_date > earliest_date)\ + .filter(Reminder.message.like(f"%{search_key}%"))\ + .count() + + log.debug(f"Found reminders with keyword: {count_reminders}") + return count_reminders + def get_reminder(self, reminder_id): log.debug(f"Fetching reminder by id: {reminder_id}") diff --git a/src/database/_stats.py b/src/database/_stats.py index cd546f1..d1b9733 100644 --- a/src/database/_stats.py +++ b/src/database/_stats.py @@ -51,13 +51,33 @@ def get_stats_for_ids(self, subreddit, thread_id, comment_id=None): return stat - def get_stats_for_subreddit(self, subreddit, earliest_date): + def get_stats_for_subreddit(self, subreddit, earliest_date, min_reminders=0, thread_only=False): log.debug("Fetching stats for subreddit") + if thread_only: + stats = self.session.query(DbStat)\ + .filter(DbStat.subreddit == subreddit)\ + .filter(DbStat.comment_id == None)\ + .filter(DbStat.initial_date > earliest_date)\ + .filter(DbStat.count_reminders >= min_reminders)\ + .order_by(DbStat.initial_date.desc())\ + .all() + else: + stats = self.session.query(DbStat)\ + .filter(DbStat.subreddit == subreddit)\ + .filter(DbStat.initial_date > earliest_date)\ + .filter(DbStat.count_reminders >= min_reminders)\ + .order_by(DbStat.initial_date.desc())\ + .all() + + log.debug(f"{len(stats)} stats found") + return stats + + def get_stats_without_date(self): + log.debug("Fetching stats without a date") + stats = self.session.query(DbStat)\ - .filter(DbStat.subreddit == subreddit)\ - .filter(DbStat.initial_date > earliest_date)\ - .order_by(DbStat.initial_date.asc())\ + .filter(DbStat.initial_date == None)\ .all() log.debug(f"{len(stats)} stats found") diff --git a/src/main.py b/src/main.py index 7e1a5c5..5a4a0fb 100644 --- a/src/main.py +++ b/src/main.py @@ -20,6 +20,7 @@ import notifications import utils import static +import stats database = None @@ -79,6 +80,7 @@ def signal_handler(signal, frame): last_backup = None last_comments = None + last_stats = None while True: startTime = time.perf_counter() log.debug("Starting run") @@ -118,6 +120,14 @@ def signal_handler(signal, frame): utils.process_error(f"Error updating comments", err, traceback.format_exc()) errors += 1 + if utils.time_offset(last_stats, minutes=60): + try: + stats.update_stats(reddit, database) + last_stats = utils.datetime_now() + except Exception as err: + utils.process_error(f"Error updating stats", err, traceback.format_exc()) + errors += 1 + if not args.no_backup and utils.time_offset(last_backup, hours=12): try: database.backup() @@ -126,7 +136,11 @@ def signal_handler(signal, frame): utils.process_error(f"Error backing up database", err, traceback.format_exc()) errors += 1 - log.debug("Run complete after: %d", int(time.perf_counter() - startTime)) + database.commit() + + run_time = time.perf_counter() - startTime + counters.run_time.observe(round(run_time, 2)) + log.debug(f"Run complete after: {int(run_time)}") discord_logging.flush_discord() diff --git a/src/stats.py b/src/stats.py new file mode 100644 index 0000000..17e5b73 --- /dev/null +++ b/src/stats.py @@ -0,0 +1,80 @@ +import discord_logging +from datetime import timedelta + +import utils + + +log = discord_logging.get_logger() + + +def update_stat_dates(reddit, database): + empty_stats = database.get_stats_without_date() + if empty_stats: + full_names = {} + for stat in empty_stats: + if stat.comment_id is not None: + full_names[f"t1_{stat.comment_id}"] = stat + else: + full_names[f"t3_{stat.thread_id}"] = stat + + reddit_objects = reddit.call_info(full_names.keys()) + count_updated = 0 + for reddit_object in reddit_objects: + stat = full_names[reddit_object.name] + stat.initial_date = utils.datetime_from_timestamp(reddit_object.created_utc) + count_updated += 1 + + if count_updated != 0: + log.info(f"Updated {count_updated} stats") + if count_updated != len(empty_stats): + for stat in empty_stats: + if stat.initial_date is None: + log.warning(f"Unable to retrieve date for stat: {stat}") + + +def update_ask_historians(reddit, database, min_reminders=10, days_back=7): + earliest_date = utils.datetime_now() - timedelta(days=days_back) + stats = database.get_stats_for_subreddit("AskHistorians", earliest_date, min_reminders=min_reminders, thread_only=True) + + bldr = utils.str_bldr() + bldr.append("Thread | Thread date | Words in top answer | Total reminders | Pending reminders\n") + bldr.append("---|---|----|----|----|----\n") + + for stat in stats: + reddit_submission = reddit.get_submission(stat.thread_id) + bldr.append(f"[{utils.truncate_string(reddit_submission.title, 60)}](https://www.reddit.com/{reddit_submission.permalink})|") + bldr.append(f"{utils.get_datetime_string(utils.datetime_from_timestamp(reddit_submission.created_utc), '%Y-%m-%d %H:%M %Z')}|") + + top_comment = None + for comment in reddit_submission.comments: + if comment.author is not None and comment.author.name != "AutoModerator" and comment.distinguished is None: + top_comment = comment + break + #utils.datetime_from_timestamp(comment.created_utc) + if top_comment is None: + bldr.append(f"|") + else: + bldr.append(f"{utils.surround_int_over_threshold(len(top_comment.body.split(' ')), '**', 350)}|") + + bldr.append(f"{utils.surround_int_over_threshold(stat.count_reminders, '**', 50)}|") + bldr.append(f"{utils.surround_int_over_threshold(database.get_reminders_with_keyword(stat.thread_id, earliest_date), '**', 50)}") + bldr.append(f"\n") + + old_wiki_content = reddit.get_subreddit_wiki_page("SubTestBot1", "remindme") + new_wiki_content = ''.join(bldr) + log.debug(new_wiki_content) + if old_wiki_content == new_wiki_content: + log.debug("Wiki content unchanged") + else: + reddit.update_subreddit_wiki_page("SubTestBot1", "remindme", new_wiki_content) + + +def update_stats(reddit, database): + update_stat_dates(reddit, database) + + update_ask_historians(reddit, database) + + + + + diff --git a/src/utils.py b/src/utils.py index 8817285..215b7e2 100644 --- a/src/utils.py +++ b/src/utils.py @@ -326,3 +326,19 @@ def check_append_context_to_link(link): return link + "?context=3" else: return link + + +def truncate_string(string, total_characters): + if string is not None and len(string) > total_characters: + return f"{string[:total_characters - 3]}..." + else: + return string + + +def surround_int_over_threshold(val, surround, threshold): + if val >= threshold: + return f"{surround}{val}{surround}" + elif val == 0: + return "" + else: + return f"{val}" diff --git a/test/stat_test.py b/test/stat_test.py index dd9dcf3..a0426ac 100644 --- a/test/stat_test.py +++ b/test/stat_test.py @@ -1,45 +1,12 @@ import utils -import notifications import static from praw_wrapper import reddit_test -import messages from datetime import timedelta from classes.reminder import Reminder +import stats -def test_add_stat(database, reddit): - utils.debug_time = utils.parse_datetime_string("2019-01-05 12:00:00") - reminder = Reminder( - source="https://www.reddit.com/message/messages/XXXXX", - message="""[https://www.reddit.com/r/AskHistorians/comments/1emshj8/___/] -RemindMe! 2 days""", - user=database.get_or_add_user("Watchful1"), - requested_date=utils.parse_datetime_string("2019-01-01 04:00:00"), - target_date=utils.parse_datetime_string("2019-01-05 05:00:00") - ) - database.add_reminder(reminder) - - stat = database.get_stats_for_ids("AskHistorians", "1emshj8") - assert stat.count_reminders == 1 - assert stat.initial_date == utils.debug_time - - reminder = Reminder( - source="https://www.reddit.com/message/messages/YYYYY", - message="""[https://www.reddit.com/r/AskHistorians/comments/1emshj8/___/] -RemindMe! 2 days""", - user=database.get_or_add_user("Watchful1"), - requested_date=utils.parse_datetime_string("2019-01-01 04:00:00"), - target_date=utils.parse_datetime_string("2019-01-05 05:00:00") - ) - database.add_reminder(reminder) - - stat = database.get_stats_for_ids("AskHistorians", "1emshj8") - assert stat.count_reminders == 2 - assert stat.initial_date == utils.debug_time - - -def test_add_stats(database, reddit): - utils.debug_time = utils.parse_datetime_string("2019-01-05 12:00:00") +def add_sample_stats(database, reddit): reminders = [ Reminder( source="https://www.reddit.com/message/messages/XXXXX", @@ -59,14 +26,14 @@ def test_add_stats(database, reddit): source="https://www.reddit.com/message/messages/XXXXX", message="[https://www.reddit.com/r/AskHistorians/comments/1emshk6/___/]", user=database.get_or_add_user("Watchful1"), - requested_date=utils.parse_datetime_string("2019-01-01 04:00:00"), + requested_date=utils.parse_datetime_string("2019-01-02 04:00:00"), target_date=utils.parse_datetime_string("2019-01-07 05:00:00") ), Reminder( source="https://www.reddit.com/message/messages/XXXXX", message="[https://www.reddit.com/r/AskHistorians/comments/1emshk6/___/]", user=database.get_or_add_user("Watchful1"), - requested_date=utils.parse_datetime_string("2019-01-01 04:00:00"), + requested_date=utils.parse_datetime_string("2019-01-02 04:00:00"), target_date=utils.parse_datetime_string("2019-01-07 05:00:00") ), Reminder( @@ -80,14 +47,114 @@ def test_add_stats(database, reddit): source="https://www.reddit.com/message/messages/XXXXX", message="[https://www.reddit.com/r/AskHistorians/comments/1emshj8/___/]", user=database.get_or_add_user("Watchful1"), - requested_date=utils.parse_datetime_string("2019-01-01 04:00:00"), + requested_date=utils.parse_datetime_string("2019-01-03 04:00:00"), + target_date=utils.parse_datetime_string("2019-01-09 05:00:00") + ), + Reminder( + source="https://www.reddit.com/message/messages/XXXXX", + message="[https://www.reddit.com/r/AskHistorians/comments/1emshj2/___/]", + user=database.get_or_add_user("Watchful1"), + requested_date=utils.parse_datetime_string("2019-01-03 04:00:00"), target_date=utils.parse_datetime_string("2019-01-09 05:00:00") ) ] for reminder in reminders: database.add_reminder(reminder) - stats = database.get_stats_for_subreddit("AskHistorians", utils.debug_time - timedelta(days=1)) - assert len(stats) == 2 - assert stats[0].count_reminders == 2 - assert stats[1].count_reminders == 3 + submissions = [ + {"created": utils.parse_datetime_string("2018-01-01 04:00:00"), "id": "1emshj2", "subreddit": "AskHistorians", "title": "Title1"}, + {"created": utils.parse_datetime_string("2019-01-01 04:00:00"), "id": "1emshj8", "subreddit": "AskHistorians", "title": "Title2"}, + {"created": utils.parse_datetime_string("2019-01-01 04:00:00"), "id": "1emshk6", "subreddit": "AskHistorians", "title": "Title3"}, + {"created": utils.parse_datetime_string("2019-01-01 04:00:00"), "id": "1emshf5", "subreddit": "AskHistorians", "title": "Title4"}, + ] + for submission in submissions: + submission_obj = reddit_test.RedditObject( + body=f"blank", + author="blank", + title=submission["title"], + created=submission["created"], + id=submission["id"], + permalink=f"/r/{submission['subreddit']}/comments/{submission['id']}/___/", + subreddit=submission["subreddit"], + prefix="t3", + ) + reddit.add_submission(submission_obj) + + +def test_add_stat(database, reddit): + utils.debug_time = utils.parse_datetime_string("2019-01-05 12:00:00") + reminder = Reminder( + source="https://www.reddit.com/message/messages/XXXXX", + message="""[https://www.reddit.com/r/AskHistorians/comments/1emshj8/___/] +RemindMe! 2 days""", + user=database.get_or_add_user("Watchful1"), + requested_date=utils.parse_datetime_string("2019-01-01 04:00:00"), + target_date=utils.parse_datetime_string("2019-01-05 05:00:00") + ) + database.add_reminder(reminder) + + stat = database.get_stats_for_ids("AskHistorians", "1emshj8") + assert stat.count_reminders == 1 + assert stat.initial_date == utils.debug_time + + reminder = Reminder( + source="https://www.reddit.com/message/messages/YYYYY", + message="""[https://www.reddit.com/r/AskHistorians/comments/1emshj8/___/] +RemindMe! 2 days""", + user=database.get_or_add_user("Watchful1"), + requested_date=utils.parse_datetime_string("2019-01-01 04:00:00"), + target_date=utils.parse_datetime_string("2019-01-05 05:00:00") + ) + database.add_reminder(reminder) + + stat = database.get_stats_for_ids("AskHistorians", "1emshj8") + assert stat.count_reminders == 2 + assert stat.initial_date == utils.debug_time + + +def test_add_stats(database, reddit): + utils.debug_time = utils.parse_datetime_string("2019-01-05 12:00:00") + add_sample_stats(database, reddit) + + sub_stats = database.get_stats_for_subreddit("AskHistorians", utils.debug_time - timedelta(days=1)) + assert len(sub_stats) == 2 + assert sub_stats[0].count_reminders == 2 + assert sub_stats[1].count_reminders == 3 + + +def test_update_dates(database, reddit): + utils.debug_time = utils.parse_datetime_string("2019-01-05 12:00:00") + add_sample_stats(database, reddit) + + stats.update_stat_dates(reddit, database) + + count_empty_stats = len(database.get_stats_without_date()) + assert count_empty_stats == 0 + + post_stat = database.get_stats_for_ids("AskHistorians", "1emshj8") + assert post_stat.initial_date == utils.parse_datetime_string("2019-01-01 04:00:00") + + +def test_update_stat_wiki(database, reddit): + utils.debug_time = utils.parse_datetime_string("2019-01-05 12:00:00") + add_sample_stats(database, reddit) + + reddit.reply_submission( + reddit.get_submission("1emshj2"), + "1234567890" * 30 + ) + reddit.reply_submission( + reddit.get_submission("1emshk6"), + "1234567890" + ) + + stats.update_stat_dates(reddit, database) + stats.update_ask_historians(reddit, database, min_reminders=0) + + wiki_content = reddit.get_subreddit_wiki_page("SubTestBot1", "remindme") + + assert wiki_content == """Thread | Thread date | Words in top answer | Total reminders | Pending reminders +---|---|----|----|----|---- +[Title2](https://www.reddit.com//r/AskHistorians/comments/1emshj8/___/)|2019-01-01 04:00:00||2|2 +[Title3](https://www.reddit.com//r/AskHistorians/comments/1emshk6/___/)|2019-01-01 04:00:00||3|3 +"""