diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index c78dd4d..cbe6990 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -84,6 +84,8 @@ services: restart: on-failure depends_on: - zookeeper + ports: + - "9000:9000" zookeeper: image: zookeeper:3.7.0 diff --git a/housewatch/migrations/0007_scheduledbackup_scheduledbackuprun.py b/housewatch/migrations/0007_scheduledbackup_scheduledbackuprun.py new file mode 100644 index 0000000..2356b9b --- /dev/null +++ b/housewatch/migrations/0007_scheduledbackup_scheduledbackuprun.py @@ -0,0 +1,41 @@ +# Generated by Django 4.1.1 on 2023-08-17 01:06 + +from django.db import migrations, models +import django.db.models.deletion +import housewatch.utils.encrypted_fields.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('housewatch', '0006_savedquery'), + ] + + operations = [ + migrations.CreateModel( + name='ScheduledBackup', + fields=[ + ('id', models.UUIDField(primary_key=True, serialize=False)), + ('schedule', models.CharField(max_length=255)), + ('table', models.CharField(max_length=255, null=True)), + ('database', models.CharField(max_length=255)), + ('bucket', models.CharField(max_length=255)), + ('path', models.CharField(max_length=255)), + ('aws_access_key_id', housewatch.utils.encrypted_fields.fields.EncryptedCharField(max_length=255, null=True)), + ('aws_secret_access_key', housewatch.utils.encrypted_fields.fields.EncryptedCharField(max_length=255, null=True)), + ('aws_region', housewatch.utils.encrypted_fields.fields.EncryptedCharField(max_length=255, null=True)), + ('aws_endpoint_url', housewatch.utils.encrypted_fields.fields.EncryptedCharField(max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='ScheduledBackupRun', + fields=[ + ('id', models.UUIDField(primary_key=True, serialize=False)), + ('started_at', models.DateTimeField(auto_now_add=True)), + ('finished_at', models.DateTimeField(null=True)), + ('success', models.BooleanField(default=False)), + ('error', models.TextField(null=True)), + ('scheduled_backup', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='housewatch.scheduledbackup')), + ], + ), + ] diff --git a/housewatch/models/__init__.py b/housewatch/models/__init__.py index b934e56..31a0c71 100644 --- a/housewatch/models/__init__.py +++ b/housewatch/models/__init__.py @@ -1,3 +1,4 @@ from .instance import Instance +from .backup import ScheduledBackup, ScheduledBackupRun __all__ = ["Instance"] diff --git a/housewatch/models/backup.py b/housewatch/models/backup.py new file mode 100644 index 0000000..8559411 --- /dev/null +++ b/housewatch/models/backup.py @@ -0,0 +1,27 @@ +from django.db import models +from housewatch.utils.encrypted_fields.fields import EncryptedCharField + + +class ScheduledBackup(models.Model): + id: models.UUIDField = models.UUIDField(primary_key=True) + # This will be a CRON expression for the job + schedule: models.CharField = models.CharField(max_length=255) + table: models.CharField = models.CharField(max_length=255, null=True) + database: models.CharField = models.CharField(max_length=255) + bucket: models.CharField = models.CharField(max_length=255) + path: models.CharField = models.CharField(max_length=255) + # if set these will override the defaults from settings + # raw keys will not be stored here but will obfuscated + aws_access_key_id: models.CharField = EncryptedCharField(max_length=255, null=True) + aws_secret_access_key: models.CharField = EncryptedCharField(max_length=255, null=True) + aws_region: models.CharField = EncryptedCharField(max_length=255, null=True) + aws_endpoint_url: models.CharField = EncryptedCharField(max_length=255, null=True) + + +class ScheduledBackupRun(models.Model): + id: models.UUIDField = models.UUIDField(primary_key=True) + scheduled_backup: models.ForeignKey = models.ForeignKey(ScheduledBackup, on_delete=models.CASCADE) + started_at: models.DateTimeField = models.DateTimeField(auto_now_add=True) + finished_at: models.DateTimeField = models.DateTimeField(null=True) + success: models.BooleanField = models.BooleanField(default=False) + error: models.TextField = models.TextField(null=True) diff --git a/housewatch/utils.py b/housewatch/utils/__init__.py similarity index 100% rename from housewatch/utils.py rename to housewatch/utils/__init__.py diff --git a/housewatch/utils/encrypted_fields/fields.py b/housewatch/utils/encrypted_fields/fields.py new file mode 100644 index 0000000..2c94416 --- /dev/null +++ b/housewatch/utils/encrypted_fields/fields.py @@ -0,0 +1,115 @@ +from cryptography.fernet import Fernet, MultiFernet +from django.conf import settings +from django.core.exceptions import FieldError, ImproperlyConfigured +from django.db import models +from django.utils.encoding import force_bytes, force_str +from django.utils.functional import cached_property + +from . import hkdf + + +__all__ = [ + "EncryptedField", + "EncryptedTextField", + "EncryptedCharField", + "EncryptedEmailField", + "EncryptedIntegerField", + "EncryptedDateField", + "EncryptedDateTimeField", +] + + +class EncryptedField(models.Field): + """A field that encrypts values using Fernet symmetric encryption.""" + + _internal_type = "BinaryField" + + def __init__(self, *args, **kwargs): + if kwargs.get("primary_key"): + raise ImproperlyConfigured("%s does not support primary_key=True." % self.__class__.__name__) + if kwargs.get("unique"): + raise ImproperlyConfigured("%s does not support unique=True." % self.__class__.__name__) + if kwargs.get("db_index"): + raise ImproperlyConfigured("%s does not support db_index=True." % self.__class__.__name__) + super(EncryptedField, self).__init__(*args, **kwargs) + + @cached_property + def keys(self): + keys = getattr(settings, "FERNET_KEYS", None) + if keys is None: + keys = [settings.SECRET_KEY] + return keys + + @cached_property + def fernet_keys(self): + if getattr(settings, "FERNET_USE_HKDF", True): + return [hkdf.derive_fernet_key(k) for k in self.keys] + return self.keys + + @cached_property + def fernet(self): + if len(self.fernet_keys) == 1: + return Fernet(self.fernet_keys[0]) + return MultiFernet([Fernet(k) for k in self.fernet_keys]) + + def get_internal_type(self): + return self._internal_type + + def get_db_prep_save(self, value, connection): + value = super(EncryptedField, self).get_db_prep_save(value, connection) + if value is not None: + retval = self.fernet.encrypt(force_bytes(value)) + return connection.Database.Binary(retval) + + def from_db_value(self, value, expression, connection, *args): + if value is not None: + value = bytes(value) + return self.to_python(force_str(self.fernet.decrypt(value))) + + @cached_property + def validators(self): + # Temporarily pretend to be whatever type of field we're masquerading + # as, for purposes of constructing validators (needed for + # IntegerField and subclasses). + self.__dict__["_internal_type"] = super(EncryptedField, self).get_internal_type() + try: + return super(EncryptedField, self).validators + finally: + del self.__dict__["_internal_type"] + + +def get_prep_lookup(self): + """Raise errors for unsupported lookups""" + raise FieldError("{} '{}' does not support lookups".format(self.lhs.field.__class__.__name__, self.lookup_name)) + + +# Register all field lookups (except 'isnull') to our handler +for name, lookup in models.Field.class_lookups.items(): + # Dynamically create classes that inherit from the right lookups + if name != "isnull": + lookup_class = type("EncryptedField" + name, (lookup,), {"get_prep_lookup": get_prep_lookup}) + EncryptedField.register_lookup(lookup_class) + + +class EncryptedTextField(EncryptedField, models.TextField): + pass + + +class EncryptedCharField(EncryptedField, models.CharField): + pass + + +class EncryptedEmailField(EncryptedField, models.EmailField): + pass + + +class EncryptedIntegerField(EncryptedField, models.IntegerField): + pass + + +class EncryptedDateField(EncryptedField, models.DateField): + pass + + +class EncryptedDateTimeField(EncryptedField, models.DateTimeField): + pass diff --git a/housewatch/utils/encrypted_fields/hkdf.py b/housewatch/utils/encrypted_fields/hkdf.py new file mode 100644 index 0000000..4fb5bd7 --- /dev/null +++ b/housewatch/utils/encrypted_fields/hkdf.py @@ -0,0 +1,23 @@ +import base64 + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from cryptography.hazmat.backends import default_backend +from django.utils.encoding import force_bytes + +backend = default_backend() +info = b"django-fernet-fields" +# We need reproducible key derivation, so we can't use a random salt +salt = b"django-fernet-fields-hkdf-salt" + + +def derive_fernet_key(input_key): + """Derive a 32-bit b64-encoded Fernet key from arbitrary input key.""" + hkdf = HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + info=info, + backend=backend, + ) + return base64.urlsafe_b64encode(hkdf.derive(force_bytes(input_key))) diff --git a/requirements.in b/requirements.in index 4ee4202..df4952f 100644 --- a/requirements.in +++ b/requirements.in @@ -14,6 +14,7 @@ click-plugins==1.1.1 click-repl==0.2.0 clickhouse-driver==0.2.5 clickhouse-pool==0.5.3 +cryptography>=0.9 dj-database-url==1.0.0 Django==4.1.1 django-cors-headers==3.13.0 diff --git a/requirements.txt b/requirements.txt index cc9a9fe..03a2d1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,6 +41,8 @@ certifi==2022.12.7 # -r requirements.in # requests # sentry-sdk +cffi==1.15.1 + # via cryptography charset-normalizer==3.1.0 # via # -r requirements.in @@ -71,6 +73,8 @@ clickhouse-driver==0.2.5 # clickhouse-pool clickhouse-pool==0.5.3 # via -r requirements.in +cryptography==41.0.3 + # via -r requirements.in dj-database-url==1.0.0 # via -r requirements.in django==4.1.1 @@ -141,6 +145,8 @@ prompt-toolkit==3.0.38 # click-repl psycopg2-binary==2.9.7 # via -r requirements.in +pycparser==2.21 + # via cffi pyrsistent==0.19.3 # via # -r requirements.in