Merge pull request #5 from etiennecallies/refacto-schedules

Refacto schedules

ansible/prod/group_vars/all

@@ -51,4 +51,4 @@ whitelisted_ip1: "{{ lookup('ansible.builtin.env', 'WHITELISTED_IP1') }}"
 whitelisted_user1: "{{ lookup('ansible.builtin.env', 'WHITELISTED_USER1') }}"
 
 # API keys
-openai_api_key: "{{ lookup('ansible.builtin.env', 'OPENAI_API_KEY') }}"
+openai_api_key: "{{ lookup('ansible.builtin.env', 'PROD_OPENAI_API_KEY') }}"

env.sample

@@ -21,6 +21,7 @@ export DB_NAME=confessio
 export PROD_DB_PASSWORD=secret
 export PROD_SECRET_KEY=secret
 export PROD_LETSENCRYPT_EMAIL=secret
+export PROD_OPENAI_API_KEY=secret
 export AWS_ACCESS_KEY=secret
 export AWS_SECRET_KEY=secret
 export ADDITIONAL_PUB_KEY1="type public name"

home/management/commands/test_explain_schedule.py → home/management/commands/explain_all_schedules.py

@@ -6,7 +6,7 @@
 
 
 class Command(AbstractCommand):
-    help = "Test get_parsing_schedules_list on all schedules."
+    help = "Test get_explanation_from_schedule on all schedules."
 
     def handle(self, *args, **options):
         counter = 0

home/migrations/0121_rename_is_exception_rule_historicaloneoffschedule_is_cancellation_and_more.py

@@ -0,0 +1,137 @@
+# Generated by Django 5.0.9 on 2024-11-01 21:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('home', '0120_rename_weekday_historicaloneoffschedule_weekday_iso8601_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='historicaloneoffschedule',
+            old_name='is_exception_rule',
+            new_name='is_cancellation',
+        ),
+        migrations.RenameField(
+            model_name='historicalregularschedule',
+            old_name='is_exception_rule',
+            new_name='is_cancellation',
+        ),
+        migrations.RenameField(
+            model_name='oneoffschedule',
+            old_name='is_exception_rule',
+            new_name='is_cancellation',
+        ),
+        migrations.RenameField(
+            model_name='regularschedule',
+            old_name='is_exception_rule',
+            new_name='is_cancellation',
+        ),
+        migrations.RemoveField(
+            model_name='historicaloneoffschedule',
+            name='start_isoformat',
+        ),
+        migrations.RemoveField(
+            model_name='historicalparsing',
+            name='current_year',
+        ),
+        migrations.RemoveField(
+            model_name='oneoffschedule',
+            name='start_isoformat',
+        ),
+        migrations.AlterUniqueTogether(
+            name='parsing',
+            unique_together={('truncated_html_hash', 'church_desc_by_id')},
+        ),
+        migrations.AddField(
+            model_name='historicaloneoffschedule',
+            name='day',
+            field=models.SmallIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicaloneoffschedule',
+            name='end_time_iso8601',
+            field=models.CharField(blank=True, max_length=8, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicaloneoffschedule',
+            name='liturgical_day',
+            field=models.CharField(blank=True, choices=[('ash_wednesday', 'ASH_WEDNESDAY'), ('palm_sunday', 'PALM_SUNDAY'), ('holy_monday', 'HOLY_MONDAY'), ('holy_tuesday', 'HOLY_TUESDAY'), ('holy_wednesday', 'HOLY_WEDNESDAY'), ('maundy_thursday', 'MAUNDY_THURSDAY'), ('good_friday', 'GOOD_FRIDAY'), ('holy_saturday', 'HOLY_SATURDAY'), ('easter_sunday', 'EASTER_SUNDAY'), ('ascension', 'ASCENSION'), ('pentecost', 'PENTECOST')], max_length=16, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicaloneoffschedule',
+            name='month',
+            field=models.SmallIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicaloneoffschedule',
+            name='start_time_iso8601',
+            field=models.CharField(default='', max_length=8),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='historicaloneoffschedule',
+            name='year',
+            field=models.SmallIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalregularschedule',
+            name='end_time_iso8601',
+            field=models.CharField(blank=True, max_length=8, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalregularschedule',
+            name='start_time_iso8601',
+            field=models.CharField(default='', max_length=8),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='oneoffschedule',
+            name='day',
+            field=models.SmallIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='oneoffschedule',
+            name='end_time_iso8601',
+            field=models.CharField(blank=True, max_length=8, null=True),
+        ),
+        migrations.AddField(
+            model_name='oneoffschedule',
+            name='liturgical_day',
+            field=models.CharField(blank=True, choices=[('ash_wednesday', 'ASH_WEDNESDAY'), ('palm_sunday', 'PALM_SUNDAY'), ('holy_monday', 'HOLY_MONDAY'), ('holy_tuesday', 'HOLY_TUESDAY'), ('holy_wednesday', 'HOLY_WEDNESDAY'), ('maundy_thursday', 'MAUNDY_THURSDAY'), ('good_friday', 'GOOD_FRIDAY'), ('holy_saturday', 'HOLY_SATURDAY'), ('easter_sunday', 'EASTER_SUNDAY'), ('ascension', 'ASCENSION'), ('pentecost', 'PENTECOST')], max_length=16, null=True),
+        ),
+        migrations.AddField(
+            model_name='oneoffschedule',
+            name='month',
+            field=models.SmallIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='oneoffschedule',
+            name='start_time_iso8601',
+            field=models.CharField(default='', max_length=8),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='oneoffschedule',
+            name='year',
+            field=models.SmallIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='regularschedule',
+            name='end_time_iso8601',
+            field=models.CharField(blank=True, max_length=8, null=True),
+        ),
+        migrations.AddField(
+            model_name='regularschedule',
+            name='start_time_iso8601',
+            field=models.CharField(default='', max_length=8),
+            preserve_default=False,
+        ),
+        migrations.RemoveField(
+            model_name='parsing',
+            name='current_year',
+        ),
+    ]

home/models/base_models.py

@@ -9,7 +9,7 @@
 
 from home.models.custom_fields import ChoiceArrayField
 from home.utils.hash_utils import hash_string_to_hex
-from scraping.parse.periods import PeriodEnum
+from scraping.parse.periods import PeriodEnum, LiturgicalDayEnum
 
 
 class TimeStampMixin(models.Model):
@@ -214,7 +214,6 @@ class Parsing(TimeStampMixin):
     truncated_html = models.TextField(editable=False)
     truncated_html_hash = models.CharField(max_length=32, editable=False)
     church_desc_by_id = models.JSONField(editable=False)
-    current_year = models.SmallIntegerField(editable=False)
 
     websites = models.ManyToManyField('Website', related_name='parsings')
     prunings = models.ManyToManyField('Pruning', related_name='parsings')
@@ -232,15 +231,17 @@ class Parsing(TimeStampMixin):
     history = HistoricalRecords()
 
     class Meta:
-        unique_together = ('truncated_html_hash', 'church_desc_by_id', 'current_year')
+        unique_together = ('truncated_html_hash', 'church_desc_by_id')
 
     def get_schedules(self) -> list['Schedule']:
         return list(self.one_off_schedules.all()) + list(self.regular_schedules.all())
 
 
 class Schedule(TimeStampMixin):
     church_id = models.SmallIntegerField(null=True, blank=True)
-    is_exception_rule = models.BooleanField()
+    is_cancellation = models.BooleanField()
+    start_time_iso8601 = models.CharField(max_length=8)
+    end_time_iso8601 = models.CharField(max_length=8, null=True, blank=True)
     duration_in_minutes = models.SmallIntegerField(null=True, blank=True)
 
     class Meta:
@@ -250,8 +251,12 @@ class Meta:
 class OneOffSchedule(Schedule):
     parsing = models.ForeignKey('Parsing', on_delete=models.CASCADE,
                                 related_name='one_off_schedules')
-    start_isoformat = models.CharField(max_length=19)
+    year = models.SmallIntegerField(null=True, blank=True)
+    month = models.SmallIntegerField(null=True, blank=True)
+    day = models.SmallIntegerField(null=True, blank=True)
     weekday_iso8601 = models.SmallIntegerField(null=True, blank=True)
+    liturgical_day = models.CharField(max_length=16, null=True, blank=True,
+                                      choices=LiturgicalDayEnum.choices())
     history = HistoricalRecords()
 
 

home/services/events_service.py

@@ -3,6 +3,7 @@
 from typing import Optional
 
 from home.models import Church, Parsing, Website
+from home.utils.date_utils import get_current_year
 from scraping.parse.schedules import Event, ScheduleItem, SchedulesList, get_merged_schedules_list
 from scraping.parse.rrule_utils import get_events_from_schedule_items
 from scraping.services.parse_pruning_service import get_parsing_schedules_list, get_church_by_id
@@ -76,7 +77,8 @@ def from_parsing(cls, parsing: Parsing, website: Website) -> Optional['ChurchSch
     def get_church_events(self) -> list[ChurchEvent]:
         start_date = datetime.now()
         end_date = start_date + timedelta(days=365)
-        events = get_events_from_schedule_items(self.schedules_list.schedules, start_date, end_date)
+        events = get_events_from_schedule_items(self.schedules_list.schedules, start_date, end_date,
+                                                get_current_year())
         church_by_id = {cs.schedule_item.church_id: cs.church for cs in self.church_schedules}
 
         return [ChurchEvent.from_event(event, church_by_id) for event in events[:7]]

home/templates/partials/schedules_display.html

@@ -20,9 +20,10 @@
     <div id="tabs-nice-display">
         {% for schedule in schedules_list.schedules %}
             church_id: {{ schedule.church_id }}<br>
-            rule: {{ schedule.rule }}<br>
-            is_exception_rule: {% display_bool schedule.is_exception_rule %}<br>
-            duration_in_minutes: {{ schedule.duration_in_minutes }}<br>
+            date_rule: {{ schedule.date_rule }}<br>
+            is_cancellation: {% display_bool schedule.is_cancellation %}<br>
+            start_time_iso8601: {{ schedule.start_time_iso8601 }}<br>
+            end_time_iso8601: {{ schedule.end_time_iso8601 }}<br>
             {% with events=schedule|get_schedule_item_events %}
                 {% for event in events %}
                     <p>{% display_event event %}</p>

home/templatetags/custom_tags.py

@@ -7,6 +7,7 @@
     PruningModeration, SentenceModeration, ParsingModeration, Website, ModerationMixin
 from home.services.events_service import ChurchSchedulesList, \
     get_merged_church_schedules_list
+from home.utils.date_utils import get_current_year
 from home.utils.list_utils import enumerate_with_and
 from scraping.parse.rrule_utils import get_events_from_schedule_item
 from scraping.parse.schedules import ScheduleItem, Event, SchedulesList
@@ -34,8 +35,9 @@ def get_church_schedules_list(website: Website) -> ChurchSchedulesList:
 def get_schedule_item_events(schedule_item: ScheduleItem) -> list[Event]:
     start_date = datetime(2000, 1, 1)
     end_date = datetime(2040, 1, 1)
+    default_year = get_current_year()
 
-    return get_events_from_schedule_item(schedule_item, start_date, end_date)[:7]
+    return get_events_from_schedule_item(schedule_item, start_date, end_date, default_year)[:7]
 
 
 @register.filter

home/templatetags/display_tags.py

@@ -8,9 +8,9 @@
 from home.services.map_service import (get_map_with_single_location,
                                        get_map_with_multiple_locations,
                                        get_map_with_alternative_locations)
+from home.utils.list_utils import group_consecutive_indices
 from scraping.parse.explain_schedule import get_explanation_from_schedule
 from scraping.parse.schedules import SchedulesList, Event, ScheduleItem
-from home.utils.list_utils import group_consecutive_indices
 
 
 @register.simple_tag

home/utils/date_utils.py

@@ -36,6 +36,25 @@ def format_datetime_with_locale(dt: datetime, dt_format: str, locale_name: str)
     return formatted_date
 
 
+def guess_year_from_weekday(default_year: int, month: int, day: int, weekday_iso8601: int) -> int:
+    start_date = datetime(default_year, month, day)
+    weekday_python = weekday_iso8601 - 1
+
+    for year in [default_year, default_year + 1]:
+        if start_date.replace(year=year).weekday() == weekday_python:
+            return year
+
+    max_count = 28  # 28 years is the maximum difference between two years with the same weekday
+    for year_count in range(1, max_count + 1):
+        year = default_year - year_count
+        if start_date.replace(year=year).weekday() == weekday_python:
+            return year
+
+    raise ValueError(f"Could not find a date between year {default_year - max_count} "
+                     f"and year {default_year + 1} "
+                     f"for month {month}, day {day}, weekday {weekday_iso8601}")
+
+
 if __name__ == '__main__':
     ts_us = datetime_to_ts_us(datetime.now())
     print(ts_us)