Add stat feature

scripts/explain_parse.py

@@ -12,12 +12,15 @@
 
 cal = parsedatetime.Calendar()
 
-input_string = '''!remindme 30th of February, 2024.'''
-base_time_string = "2023-04-27 02:45:19 -0700"
+input_string = '''!remindme December 24th :)'''
+base_time_string = None#"2024-04-27 02:45:19 -0700"
+created_utc = 1723766419
 timezone_string = None  # "America/New_York"
 
 if base_time_string:
 	base_time = utils.datetime_as_timezone(utils.parse_datetime_string(base_time_string, False, '%Y-%m-%d %H:%M:%S %z'), "UTC")
+elif created_utc:
+	base_time = utils.datetime_from_timestamp(created_utc)
 else:
 	base_time = utils.datetime_now()
 

scripts/update_wiki.py

@@ -0,0 +1,17 @@
+import discord_logging
+import traceback
+from datetime import timedelta
+import praw_wrapper
+
+log = discord_logging.init_logging()
+
+import utils
+from database import Database
+import stats
+
+
+if __name__ == "__main__":
+	reddit = praw_wrapper.Reddit("Watchful1")
+	database = Database()
+
+	stats.update_stats(reddit, database)

src/classes/stat.py

@@ -12,8 +12,13 @@ class DbStat(Base):
 	subreddit = Column(String(80), nullable=False)
 	thread_id = Column(String(12), nullable=False)
 	comment_id = Column(String(12))
-	initial_date = Column(UtcDateTime, nullable=False)
+	initial_date = Column(UtcDateTime)
 	count_reminders = Column(Integer, nullable=False)
+	#thread_title = Column(String(200))
+
+	title = None
+	answered = False
+	count_pending_reminders = None
 
 	def __init__(
 		self,
@@ -26,4 +31,7 @@ def __init__(
 		self.thread_id = thread_id
 		self.comment_id = comment_id
 		self.count_reminders = count_reminders
-		self.initial_date = utils.datetime_now()
+
+	def __str__(self):
+		return f"{self.id}:{self.subreddit}:{self.thread_id}:{self.comment_id}:" \
+			f": {utils.get_datetime_string(self.initial_date)}:{self.count_reminders}"

src/comments.py

@@ -1,6 +1,5 @@
 import discord_logging
 import traceback
-import praw
 
 import utils
 import static

src/counters.py

@@ -5,6 +5,7 @@
 queue = prometheus_client.Gauge('bot_queue', "Current queue size")
 objects = prometheus_client.Gauge('bot_objects', "Total number of objects by type", ['type'])
 errors = prometheus_client.Counter('bot_errors', "Count of errors", ['type'])
+run_time = prometheus_client.Summary('bot_run_seconds', "How long a full loop takes")
 
 
 def init(port):

src/database/UtcDateTime.py

@@ -9,4 +9,7 @@ class UtcDateTime(types.TypeDecorator):
     cache_ok = True
 
     def process_result_value(self, value, dialect):
-        return utils.datetime_force_utc(value)
+        if value is not None:
+            return utils.datetime_force_utc(value)
+        else:
+            return None

src/database/_reminders.py

@@ -72,6 +72,17 @@ def get_user_reminders(self, user_name):
 		log.debug(f"Found reminders: {len(regular_reminders)} : {len(recurring_reminders)}")
 		return regular_reminders, recurring_reminders
 
+	def get_reminders_with_keyword(self, search_key, earliest_date):
+		log.debug(f"Searching for reminders with {search_key}")
+
+		count_reminders = self.session.query(Reminder)\
+			.filter(Reminder.target_date > earliest_date)\
+			.filter(Reminder.message.like(f"%{search_key}%"))\
+			.count()
+
+		log.debug(f"Found reminders with keyword: {count_reminders}")
+		return count_reminders
+
 	def get_reminder(self, reminder_id):
 		log.debug(f"Fetching reminder by id: {reminder_id}")
 

src/database/_stats.py

@@ -51,13 +51,33 @@ def get_stats_for_ids(self, subreddit, thread_id, comment_id=None):
 
 		return stat
 
-	def get_stats_for_subreddit(self, subreddit, earliest_date):
+	def get_stats_for_subreddit(self, subreddit, earliest_date, min_reminders=0, thread_only=False):
 		log.debug("Fetching stats for subreddit")
 
+		if thread_only:
+			stats = self.session.query(DbStat)\
+				.filter(DbStat.subreddit == subreddit)\
+				.filter(DbStat.comment_id == None)\
+				.filter(DbStat.initial_date > earliest_date)\
+				.filter(DbStat.count_reminders >= min_reminders)\
+				.order_by(DbStat.initial_date.desc())\
+				.all()
+		else:
+			stats = self.session.query(DbStat)\
+				.filter(DbStat.subreddit == subreddit)\
+				.filter(DbStat.initial_date > earliest_date)\
+				.filter(DbStat.count_reminders >= min_reminders)\
+				.order_by(DbStat.initial_date.desc())\
+				.all()
+
+		log.debug(f"{len(stats)} stats found")
+		return stats
+
+	def get_stats_without_date(self):
+		log.debug("Fetching stats without a date")
+
 		stats = self.session.query(DbStat)\
-			.filter(DbStat.subreddit == subreddit)\
-			.filter(DbStat.initial_date > earliest_date)\
-			.order_by(DbStat.initial_date.asc())\
+			.filter(DbStat.initial_date == None)\
 			.all()
 
 		log.debug(f"{len(stats)} stats found")

src/main.py

@@ -20,6 +20,7 @@
 import notifications
 import utils
 import static
+import stats
 
 
 database = None
@@ -79,6 +80,7 @@ def signal_handler(signal, frame):
 
 	last_backup = None
 	last_comments = None
+	last_stats = None
 	while True:
 		startTime = time.perf_counter()
 		log.debug("Starting run")
@@ -118,6 +120,14 @@ def signal_handler(signal, frame):
 				utils.process_error(f"Error updating comments", err, traceback.format_exc())
 				errors += 1
 
+		if utils.time_offset(last_stats, minutes=60):
+			try:
+				stats.update_stats(reddit, database)
+				last_stats = utils.datetime_now()
+			except Exception as err:
+				utils.process_error(f"Error updating stats", err, traceback.format_exc())
+				errors += 1
+
 		if not args.no_backup and utils.time_offset(last_backup, hours=12):
 			try:
 				database.backup()
@@ -126,7 +136,11 @@ def signal_handler(signal, frame):
 				utils.process_error(f"Error backing up database", err, traceback.format_exc())
 				errors += 1
 
-		log.debug("Run complete after: %d", int(time.perf_counter() - startTime))
+		database.commit()
+
+		run_time = time.perf_counter() - startTime
+		counters.run_time.observe(round(run_time, 2))
+		log.debug(f"Run complete after: {int(run_time)}")
 
 		discord_logging.flush_discord()
 

src/stats.py

@@ -0,0 +1,80 @@
+import discord_logging
+from datetime import timedelta
+
+import utils
+
+
+log = discord_logging.get_logger()
+
+
+def update_stat_dates(reddit, database):
+	empty_stats = database.get_stats_without_date()
+	if empty_stats:
+		full_names = {}
+		for stat in empty_stats:
+			if stat.comment_id is not None:
+				full_names[f"t1_{stat.comment_id}"] = stat
+			else:
+				full_names[f"t3_{stat.thread_id}"] = stat
+
+		reddit_objects = reddit.call_info(full_names.keys())
+		count_updated = 0
+		for reddit_object in reddit_objects:
+			stat = full_names[reddit_object.name]
+			stat.initial_date = utils.datetime_from_timestamp(reddit_object.created_utc)
+			count_updated += 1
+
+		if count_updated != 0:
+			log.info(f"Updated {count_updated} stats")
+		if count_updated != len(empty_stats):
+			for stat in empty_stats:
+				if stat.initial_date is None:
+					log.warning(f"Unable to retrieve date for stat: {stat}")
+
+
+def update_ask_historians(reddit, database, min_reminders=10, days_back=7):
+	earliest_date = utils.datetime_now() - timedelta(days=days_back)
+	stats = database.get_stats_for_subreddit("AskHistorians", earliest_date, min_reminders=min_reminders, thread_only=True)
+
+	bldr = utils.str_bldr()
+	bldr.append("Thread | Thread date | Words in top answer | Total reminders | Pending reminders\n")
+	bldr.append("---|---|----|----|----|----\n")
+
+	for stat in stats:
+		reddit_submission = reddit.get_submission(stat.thread_id)
+		bldr.append(f"[{utils.truncate_string(reddit_submission.title, 60)}](https://www.reddit.com/{reddit_submission.permalink})|")
+		bldr.append(f"{utils.get_datetime_string(utils.datetime_from_timestamp(reddit_submission.created_utc), '%Y-%m-%d %H:%M %Z')}|")
+
+		top_comment = None
+		for comment in reddit_submission.comments:
+			if comment.author is not None and comment.author.name != "AutoModerator" and comment.distinguished is None:
+				top_comment = comment
+				break
+		#utils.datetime_from_timestamp(comment.created_utc)
+		if top_comment is None:
+			bldr.append(f"|")
+		else:
+			bldr.append(f"{utils.surround_int_over_threshold(len(top_comment.body.split(' ')), '**', 350)}|")
+
+		bldr.append(f"{utils.surround_int_over_threshold(stat.count_reminders, '**', 50)}|")
+		bldr.append(f"{utils.surround_int_over_threshold(database.get_reminders_with_keyword(stat.thread_id, earliest_date), '**', 50)}")
+		bldr.append(f"\n")
+
+	old_wiki_content = reddit.get_subreddit_wiki_page("SubTestBot1", "remindme")
+	new_wiki_content = ''.join(bldr)
+	log.debug(new_wiki_content)
+	if old_wiki_content == new_wiki_content:
+		log.debug("Wiki content unchanged")
+	else:
+		reddit.update_subreddit_wiki_page("SubTestBot1", "remindme", new_wiki_content)
+
+
+def update_stats(reddit, database):
+	update_stat_dates(reddit, database)
+
+	update_ask_historians(reddit, database)
+
+
+
+
+

src/utils.py

@@ -326,3 +326,19 @@ def check_append_context_to_link(link):
 		return link + "?context=3"
 	else:
 		return link
+
+
+def truncate_string(string, total_characters):
+	if string is not None and len(string) > total_characters:
+		return f"{string[:total_characters - 3]}..."
+	else:
+		return string
+
+
+def surround_int_over_threshold(val, surround, threshold):
+	if val >= threshold:
+		return f"{surround}{val}{surround}"
+	elif val == 0:
+		return ""
+	else:
+		return f"{val}"