Support for drupal data sync

.env.dist

@@ -23,3 +23,8 @@ DJANGO_EMAIL_PORT=
 DJANGO_EMAIL_HOST_USER=
 DJANGO_EMAIL_HOST_PASSWORD=
 DJANGO_EMAIL_USE_TLS=
+
+# drupal api
+DRUPAL_API_CLIENT_ID=
+DRUPAL_API_CLIENT_SECRET=
+DRUPAL_API_REL_PATH=

config/settings/base.py

@@ -390,3 +390,9 @@
 # Specify the subject for AnVIL account verification emails.
 ANVIL_ACCOUNT_LINK_EMAIL_SUBJECT = "Verify your AnVIL account email"
 ANVIL_ACCOUNT_VERIFY_NOTIFICATION_EMAIL = "gregorconsortium@uw.edu"
+
+DRUPAL_API_CLIENT_ID = env("DRUPAL_API_CLIENT_ID", default="")
+DRUPAL_API_CLIENT_SECRET = env("DRUPAL_API_CLIENT_SECRET", default="")
+DRUPAL_API_REL_PATH = env("DRUPAL_API_REL_PATH", default="mockapi")
+DRUPAL_DATA_AUDIT_DEACTIVATE_USERS = env("DRUPAL_DATA_AUDIT_DEACTIVATE_USERS", default=False)
+DRUPAL_DATA_AUDIT_REMOVE_USER_SITES = env("DRUPAL_DATA_AUDIT_REMOVE_USER_SITES", default=False)

gregor_apps.cron

@@ -7,3 +7,6 @@ MAILTO="gregorweb@uw.edu"
 
 # sunday night at 02:00
 0 2 * * SUN . /var/www/django/gregor_apps/gregor-apps-activate.sh; python manage.py run_anvil_audit --email gregorconsortium@uw.edu >> cron.log
+
+# Nightly user data audit
+0 2 * * * . /var/www/django/gregor_apps/gregor-apps-activate.sh; python manage.py sync-drupal-data --update --email gregorweb@uw.edu >> cron.log

gregor_apps_dev.cron

@@ -1,7 +1,10 @@
 # GREGOR_APPS_DEV crontab - disabled by default
 # can be enabled for testing.
 # nightly except sunday at 00:00
-# 0 0 * * MON-SAT . /var/www/django/gregor_apps/gregor-apps-dev-activate.sh; python manage.py run_anvil_audit --email gregorconsortium@uw.edu --errors-only >> cron.log 2>&1
+# 0 0 * * MON-SAT . /var/www/django/gregor_apps_dev/gregor-apps-dev-activate.sh; python manage.py run_anvil_audit --email gregorconsortium@uw.edu --errors-only >> cron.log 2>&1
 
 # sunday night at 00:00
-# 0 0 * * SUN . /var/www/django/gregor_apps/gregor-apps-dev-activate.sh; python manage.py run_anvil_audit --email gregorconsortium@uw.edu >> cron.log 2>&1
+# 0 0 * * SUN . /var/www/django/gregor_apps_dev/gregor-apps-dev-activate.sh; python manage.py run_anvil_audit --email gregorconsortium@uw.edu >> cron.log 2>&1
+
+# sunday night at 01:00
+# 0 1 * * SUN . /var/www/django/gregor_apps_dev/gregor-apps-dev-activate.sh; python manage.py run_anvil_audit --email gregorconsortium@uw.edu >> cron.log

gregor_django/gregor_anvil/audit.py

@@ -0,0 +1,135 @@
+from abc import ABC, abstractmethod, abstractproperty
+
+
+class GREGORAuditResult(ABC):
+    """Abstract base class to hold an audit result for a single check.
+
+    Subclasses of this class are typically also dataclasses. They can define any number of
+    fields that track information about an audit and its result. The companion RPIMEDAudit
+    class `verified`, `needs_action`, and `errors` attributes should store lists of
+    GREGORAuditResult instances.
+
+    Typical usage:
+        @dataclass
+        class MyAuditResult(GREGORAuditResult):
+
+            some_value: str
+
+            def get_table_dictionary(self):
+                return {"some_value": self.some_value}
+
+        audit_result = MyAuditResult(some_value="the value for this result")
+    """
+
+    @abstractmethod
+    def get_table_dictionary(self):
+        """Return a dictionary representation of the result."""
+        ...  # pragma: no cover
+
+
+class GREGORAudit(ABC):
+    """Abstract base class for GREGOR audit classes.
+
+    This class is intended to be subclassed in order to store all results for a GREGOR audit.
+    Subclasses should implement the _run_audit class method to perform the audit. To run the
+    audit itself, one can use the run_audit method, which calls the _run_audit method in
+    addition to performs completion checks. Typically, _run_audit should loop over a set of
+    instances or checks, and store the results in the `verified`, `needs_action`, and `errors`
+    attributes.
+
+    Attributes:
+        verified: A list of GREGORAuditResult subclasses instances that have been verified.
+        needs_action: A list of GREGORAuditResult subclasses instances that some sort of need action.
+        errors: A list of GREGORAuditResult subclasses instances where an error has been detected.
+        completed: A boolean indicator of whether the audit has been run.
+    """
+
+    # TODO: Add add_verified_result, add_needs_action_result, add_error_result methods. They should
+    # verify that the result is an instance of GREGORAuditResult (subclass).
+
+    @abstractproperty
+    def results_table_class(self):
+        return ...  # pragma: no cover
+
+    def __init__(self):
+        self.completed = False
+        # Set up lists to hold audit results.
+        self.verified = []
+        self.needs_action = []
+        self.errors = []
+        self.completed = False
+
+    @abstractmethod
+    def _run_audit(self):
+        """Run the audit and store results in `verified`, `needs_action`, and `errors` lists.
+
+        This method should typically loop over a set of instances or checks, and store the
+        results in the `verified`, `needs_action`, and `errors` attributes. The results should
+        be instances of GREGORAuditResult subclasses. This method should not be called directly.
+
+        When deciding which list to store a result in, consider the following:
+        - verified: The result is as expected and no action is needed.
+        - needs_action: The result is expected for some reason, but action is needed.
+        - errors: The result is not expected and action is likely needed.
+        """
+        ...  # pragma: no cover
+
+    def run_audit(self):
+        """Run the audit and mark it as completed."""
+        self._run_audit()
+        self.completed = True
+
+    def get_all_results(self):
+        """Return all results in a list, regardless of type.
+
+        Returns:
+            list: A combined list of `verified`, `needs_action`, and `errors` results.
+        """
+        self._check_completed()
+        return self.verified + self.needs_action + self.errors
+
+    def _check_completed(self):
+        if not self.completed:
+            raise ValueError("Audit has not been completed. Use run_audit() to run the audit.")
+
+    def get_verified_table(self):
+        """Return a table of verified audit results.
+
+        The subclass of the table will be the specified `results_table_class`.
+
+        Returns:
+            results_table_class: A table of verified results.
+        """
+        self._check_completed()
+        return self.results_table_class([x.get_table_dictionary() for x in self.verified])
+
+    def get_needs_action_table(self):
+        """Return a table of needs_action audit results.
+
+        The subclass of the table will be the specified `results_table_class`.
+
+        Returns:
+            results_table_class: A table of need action results.
+        """
+        self._check_completed()
+        return self.results_table_class([x.get_table_dictionary() for x in self.needs_action])
+
+    def get_errors_table(self):
+        """Return a table of error audit results.
+
+        The subclass of the table will be the specified `results_table_class`.
+
+        Returns:
+            results_table_class: A table of error results.
+        """
+        self._check_completed()
+        return self.results_table_class([x.get_table_dictionary() for x in self.errors])
+
+    def ok(self):
+        """Check audit results to see if action is needed.
+
+        Returns:
+            bool: True if no action is needed, False otherwise.
+        """
+        self._check_completed()
+        return len(self.errors) + len(self.needs_action) == 0

gregor_django/gregor_anvil/migrations/0026_historicalpartnergroup_drupal_node_id_and_more.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.2.15 on 2024-08-13 23:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gregor_anvil', '0025_releaseworkspace_meta_change_verbose_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='historicalpartnergroup',
+            name='drupal_node_id',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalresearchcenter',
+            name='drupal_node_id',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='partnergroup',
+            name='drupal_node_id',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='researchcenter',
+            name='drupal_node_id',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

gregor_django/gregor_anvil/models.py

@@ -42,6 +42,9 @@ class ResearchCenter(TimeStampedModel, models.Model):
     full_name = models.CharField(max_length=255, unique=True)
     """The full name of the Research Center."""
 
+    drupal_node_id = models.IntegerField(blank=True, null=True)
+    """Reference node ID for entity in drupal"""
+
     member_group = models.OneToOneField(
         ManagedGroup,
         on_delete=models.PROTECT,
@@ -90,6 +93,9 @@ class PartnerGroup(TimeStampedModel, models.Model):
     full_name = models.CharField(max_length=255, unique=True)
     """The full name of the Partner Group"""
 
+    drupal_node_id = models.IntegerField(blank=True, null=True)
+    """Reference node ID for entity in drupal"""
+
     member_group = models.OneToOneField(
         ManagedGroup,
         on_delete=models.PROTECT,

gregor_django/templates/gregor_anvil/partnergroup_detail.html

@@ -70,7 +70,26 @@ <h2 class="accordion-header" id="headingWorkspacesOne">
 </div>
 
 
-<h3>Partner Group Users</h3>
-  {% render_table tables.0 %}
-  <p class='alert alert-warning'>Partner Group user list only contains those group users who have created an account on this website.</p>
+<div class="my-3">
+  <div class="accordion" id="accordionPartnerGroupUsers">
+    <div class="accordion-item">
+      <h2 class="accordion-header" id="headingPartnerGroupUsersOne">
+        <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapsePartnerGroupUsersOne" aria-expanded="false" aria-controls="collapsePartnerGroupUsersOne">
+          <span class="fa-solid fa-cloud-arrow-up mx-2"></span>
+          Partner Group members
+          <span class="badge mx-2 bg-secondary pill"> {{ tables.0.rows|length }}</span>
+        </button>
+      </h2>
+      <div id="collapsePartnerGroupUsersOne" class="accordion-collapse collapse" aria-labelledby="headingPartnerGroupUsersOne" data-bs-parent="#accordionPartnerGroupUsers">
+        <div class="accordion-body">
+          <p>
+            This table shows users who are associated with this Partner Group.
+          </p>
+        {% render_table tables.0 %}
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
 {% endblock after_panel %}

gregor_django/templates/gregor_anvil/researchcenter_detail.html

@@ -69,7 +69,26 @@ <h2 class="accordion-header" id="headingWorkspacesOne">
   </div>
 </div>
 
-<h3>Research Center Users</h3>
-  {% render_table tables.0 %}
-  <p class='alert alert-warning'>Research center user list only contains those site users who have created an account on this website.</p>
+<div class="my-3">
+  <div class="accordion" id="accordionResearchCenterUsers">
+    <div class="accordion-item">
+      <h2 class="accordion-header" id="headingResearchCenterUsersOne">
+        <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapseResearchCenterUsersOne" aria-expanded="false" aria-controls="collapseResearchCenterUsersOne">
+          <span class="fa-solid fa-cloud-arrow-up mx-2"></span>
+          Partner Group members
+          <span class="badge mx-2 bg-secondary pill"> {{ tables.0.rows|length }}</span>
+        </button>
+      </h2>
+      <div id="collapseResearchCenterUsersOne" class="accordion-collapse collapse" aria-labelledby="headingResearchCenterUsersOne" data-bs-parent="#accordionResearchCenterUsers">
+        <div class="accordion-body">
+          <p>
+            This table shows users who are associated with this Partner Group.
+          </p>
+        {% render_table tables.0 %}
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
 {% endblock after_panel %}

gregor_django/templates/users/drupal_data_audit_email.html

@@ -0,0 +1,65 @@
+
+{% load static i18n %}<!DOCTYPE html>
+{% load render_table from django_tables2 %}
+
+<html>
+  <head>
+    <title>Drupal Data Audit Report</title>
+  </head>
+
+  <div class="container">
+
+{% block content %}
+
+      <h1>Drupal Data Audit - [applying_changes={{ apply_changes }}]</h1>
+      <h2>User Audit</h2>
+
+      <h3>Verified Users - {{ user_audit.verified|length }} record(s)</h3>
+
+      <h3>Needs action - {{user_audit.needs_action|length }} record(s)</h3>
+
+      {% if user_audit.needs_action %}
+      {% render_table user_audit.get_needs_action_table %}
+      {% endif %}
+
+      <h3>Errors - {{user_audit.errors|length }} record(s)</h3>
+      {% if user_audit.errors %}
+      {% render_table user_audit.get_errors_table %}
+      {% endif %}
+
+      <h2>Site Audit</h2>
+
+      <h3>Verified sites - {{ site_audit.verified|length }} record(s)</h3>
+
+      <h3>Sites that need action - {{site_audit.needs_action|length }} record(s)</h3>
+      {% if site_audit.needs_action %}
+      {% render_table site_audit.get_needs_action_table %}
+      {% endif %}
+
+      <h3>Sites with errors - {{site_audit.errors|length }} record(s)</h3>
+      {% if site_audit.errors %}
+      {% render_table site_audit.get_errors_table %}
+      {% endif %}
+
+      <h2>Partner Group Audit</h2>
+
+      <h3>Verified partner groups - {{ partner_group_audit.verified|length }} record(s)</h3>
+
+      <h3>PartnerGroups that need action - {{ partner_group_audit.needs_action|length }} record(s)</h3>
+      {% if partner_group_audit.needs_action %}
+      {% render_table partner_group_audit.get_needs_action_table %}
+      {% endif %}
+
+      <h3>PartnerGroups with errors - {{partner_group_audit.errors|length }} record(s)</h3>
+      {% if partner_group_audit.errors %}
+      {% render_table partner_group_audit.get_errors_table %}
+      {% endif %}
+
+      <p>* Users, sites, partner groups that <b>Need Action</b> will be resolved by this script if in update mode</p>
+      <p>* Users, sites, partner groups listed as <b>Error</b> need manual intervention to resolve</p>
+
+
+{% endblock content %}
+
+  </div>
+</html>