add scheduled run models to housewatch

docker-compose.dev.yml

@@ -84,6 +84,8 @@ services:
     restart: on-failure
     depends_on:
       - zookeeper
+    ports:
+      - "9000:9000"
 
   zookeeper:
     image: zookeeper:3.7.0

housewatch/migrations/0007_scheduledbackup_scheduledbackuprun.py

@@ -0,0 +1,41 @@
+# Generated by Django 4.1.1 on 2023-08-17 01:06
+
+from django.db import migrations, models
+import django.db.models.deletion
+import housewatch.utils.encrypted_fields.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('housewatch', '0006_savedquery'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ScheduledBackup',
+            fields=[
+                ('id', models.UUIDField(primary_key=True, serialize=False)),
+                ('schedule', models.CharField(max_length=255)),
+                ('table', models.CharField(max_length=255, null=True)),
+                ('database', models.CharField(max_length=255)),
+                ('bucket', models.CharField(max_length=255)),
+                ('path', models.CharField(max_length=255)),
+                ('aws_access_key_id', housewatch.utils.encrypted_fields.fields.EncryptedCharField(max_length=255, null=True)),
+                ('aws_secret_access_key', housewatch.utils.encrypted_fields.fields.EncryptedCharField(max_length=255, null=True)),
+                ('aws_region', housewatch.utils.encrypted_fields.fields.EncryptedCharField(max_length=255, null=True)),
+                ('aws_endpoint_url', housewatch.utils.encrypted_fields.fields.EncryptedCharField(max_length=255, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ScheduledBackupRun',
+            fields=[
+                ('id', models.UUIDField(primary_key=True, serialize=False)),
+                ('started_at', models.DateTimeField(auto_now_add=True)),
+                ('finished_at', models.DateTimeField(null=True)),
+                ('success', models.BooleanField(default=False)),
+                ('error', models.TextField(null=True)),
+                ('scheduled_backup', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='housewatch.scheduledbackup')),
+            ],
+        ),
+    ]

housewatch/models/__init__.py

@@ -1,3 +1,4 @@
 from .instance import Instance
+from .backup import ScheduledBackup, ScheduledBackupRun
 
 __all__ = ["Instance"]

housewatch/models/backup.py

@@ -0,0 +1,27 @@
+from django.db import models
+from housewatch.utils.encrypted_fields.fields import EncryptedCharField
+
+
+class ScheduledBackup(models.Model):
+    id: models.UUIDField = models.UUIDField(primary_key=True)
+    # This will be a CRON expression for the job
+    schedule: models.CharField = models.CharField(max_length=255)
+    table: models.CharField = models.CharField(max_length=255, null=True)
+    database: models.CharField = models.CharField(max_length=255)
+    bucket: models.CharField = models.CharField(max_length=255)
+    path: models.CharField = models.CharField(max_length=255)
+    # if set these will override the defaults from settings
+    # raw keys will not be stored here but will obfuscated
+    aws_access_key_id: models.CharField = EncryptedCharField(max_length=255, null=True)
+    aws_secret_access_key: models.CharField = EncryptedCharField(max_length=255, null=True)
+    aws_region: models.CharField = EncryptedCharField(max_length=255, null=True)
+    aws_endpoint_url: models.CharField = EncryptedCharField(max_length=255, null=True)
+
+
+class ScheduledBackupRun(models.Model):
+    id: models.UUIDField = models.UUIDField(primary_key=True)
+    scheduled_backup: models.ForeignKey = models.ForeignKey(ScheduledBackup, on_delete=models.CASCADE)
+    started_at: models.DateTimeField = models.DateTimeField(auto_now_add=True)
+    finished_at: models.DateTimeField = models.DateTimeField(null=True)
+    success: models.BooleanField = models.BooleanField(default=False)
+    error: models.TextField = models.TextField(null=True)

housewatch/utils/encrypted_fields/fields.py

@@ -0,0 +1,115 @@
+from cryptography.fernet import Fernet, MultiFernet
+from django.conf import settings
+from django.core.exceptions import FieldError, ImproperlyConfigured
+from django.db import models
+from django.utils.encoding import force_bytes, force_str
+from django.utils.functional import cached_property
+
+from . import hkdf
+
+
+__all__ = [
+    "EncryptedField",
+    "EncryptedTextField",
+    "EncryptedCharField",
+    "EncryptedEmailField",
+    "EncryptedIntegerField",
+    "EncryptedDateField",
+    "EncryptedDateTimeField",
+]
+
+
+class EncryptedField(models.Field):
+    """A field that encrypts values using Fernet symmetric encryption."""
+
+    _internal_type = "BinaryField"
+
+    def __init__(self, *args, **kwargs):
+        if kwargs.get("primary_key"):
+            raise ImproperlyConfigured("%s does not support primary_key=True." % self.__class__.__name__)
+        if kwargs.get("unique"):
+            raise ImproperlyConfigured("%s does not support unique=True." % self.__class__.__name__)
+        if kwargs.get("db_index"):
+            raise ImproperlyConfigured("%s does not support db_index=True." % self.__class__.__name__)
+        super(EncryptedField, self).__init__(*args, **kwargs)
+
+    @cached_property
+    def keys(self):
+        keys = getattr(settings, "FERNET_KEYS", None)
+        if keys is None:
+            keys = [settings.SECRET_KEY]
+        return keys
+
+    @cached_property
+    def fernet_keys(self):
+        if getattr(settings, "FERNET_USE_HKDF", True):
+            return [hkdf.derive_fernet_key(k) for k in self.keys]
+        return self.keys
+
+    @cached_property
+    def fernet(self):
+        if len(self.fernet_keys) == 1:
+            return Fernet(self.fernet_keys[0])
+        return MultiFernet([Fernet(k) for k in self.fernet_keys])
+
+    def get_internal_type(self):
+        return self._internal_type
+
+    def get_db_prep_save(self, value, connection):
+        value = super(EncryptedField, self).get_db_prep_save(value, connection)
+        if value is not None:
+            retval = self.fernet.encrypt(force_bytes(value))
+            return connection.Database.Binary(retval)
+
+    def from_db_value(self, value, expression, connection, *args):
+        if value is not None:
+            value = bytes(value)
+            return self.to_python(force_str(self.fernet.decrypt(value)))
+
+    @cached_property
+    def validators(self):
+        # Temporarily pretend to be whatever type of field we're masquerading
+        # as, for purposes of constructing validators (needed for
+        # IntegerField and subclasses).
+        self.__dict__["_internal_type"] = super(EncryptedField, self).get_internal_type()
+        try:
+            return super(EncryptedField, self).validators
+        finally:
+            del self.__dict__["_internal_type"]
+
+
+def get_prep_lookup(self):
+    """Raise errors for unsupported lookups"""
+    raise FieldError("{} '{}' does not support lookups".format(self.lhs.field.__class__.__name__, self.lookup_name))
+
+
+# Register all field lookups (except 'isnull') to our handler
+for name, lookup in models.Field.class_lookups.items():
+    # Dynamically create classes that inherit from the right lookups
+    if name != "isnull":
+        lookup_class = type("EncryptedField" + name, (lookup,), {"get_prep_lookup": get_prep_lookup})
+        EncryptedField.register_lookup(lookup_class)
+
+
+class EncryptedTextField(EncryptedField, models.TextField):
+    pass
+
+
+class EncryptedCharField(EncryptedField, models.CharField):
+    pass
+
+
+class EncryptedEmailField(EncryptedField, models.EmailField):
+    pass
+
+
+class EncryptedIntegerField(EncryptedField, models.IntegerField):
+    pass
+
+
+class EncryptedDateField(EncryptedField, models.DateField):
+    pass
+
+
+class EncryptedDateTimeField(EncryptedField, models.DateTimeField):
+    pass

housewatch/utils/encrypted_fields/hkdf.py

@@ -0,0 +1,23 @@
+import base64
+
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from cryptography.hazmat.backends import default_backend
+from django.utils.encoding import force_bytes
+
+backend = default_backend()
+info = b"django-fernet-fields"
+# We need reproducible key derivation, so we can't use a random salt
+salt = b"django-fernet-fields-hkdf-salt"
+
+
+def derive_fernet_key(input_key):
+    """Derive a 32-bit b64-encoded Fernet key from arbitrary input key."""
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=32,
+        salt=salt,
+        info=info,
+        backend=backend,
+    )
+    return base64.urlsafe_b64encode(hkdf.derive(force_bytes(input_key)))

requirements.in

@@ -14,6 +14,7 @@ click-plugins==1.1.1
 click-repl==0.2.0
 clickhouse-driver==0.2.5
 clickhouse-pool==0.5.3
+cryptography>=0.9
 dj-database-url==1.0.0
 Django==4.1.1
 django-cors-headers==3.13.0

requirements.txt

@@ -41,6 +41,8 @@ certifi==2022.12.7
     #   -r requirements.in
     #   requests
     #   sentry-sdk
+cffi==1.15.1
+    # via cryptography
 charset-normalizer==3.1.0
     # via
     #   -r requirements.in
@@ -71,6 +73,8 @@ clickhouse-driver==0.2.5
     #   clickhouse-pool
 clickhouse-pool==0.5.3
     # via -r requirements.in
+cryptography==41.0.3
+    # via -r requirements.in
 dj-database-url==1.0.0
     # via -r requirements.in
 django==4.1.1
@@ -141,6 +145,8 @@ prompt-toolkit==3.0.38
     #   click-repl
 psycopg2-binary==2.9.7
     # via -r requirements.in
+pycparser==2.21
+    # via cffi
 pyrsistent==0.19.3
     # via
     #   -r requirements.in