From 68184c91f853226df3917bd30c7a08e7c0563f0a Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 30 Sep 2024 14:06:37 -0700 Subject: [PATCH 1/6] Migrate JSONField to Django builtin JSONField This runs a migration that converts from text to jsonb. In my local testing it converted the field properly: ``` ethicaladserver=# select targeting_parameters from adserver_flight; targeting_parameters ---------------------------- { + "niche_targeting": 0.9+ } (1 row) ethicaladserver=# select targeting_parameters from adserver_flight; targeting_parameters -------------------------- {"niche_targeting": 0.9} (1 row) ``` --- .../migrations/0008_migrate_jsonfield.py | 34 ++++++ adserver/analyzer/models.py | 3 +- adserver/migrations/0097_migrate_jsonfield.py | 110 ++++++++++++++++++ adserver/models.py | 13 ++- requirements/analyzer.txt | 33 ++---- requirements/base.txt | 15 +-- requirements/development.txt | 15 +-- requirements/production.txt | 17 +-- requirements/testing.txt | 8 +- requirements/update.sh | 30 +++-- 10 files changed, 201 insertions(+), 77 deletions(-) create mode 100644 adserver/analyzer/migrations/0008_migrate_jsonfield.py create mode 100644 adserver/migrations/0097_migrate_jsonfield.py diff --git a/adserver/analyzer/migrations/0008_migrate_jsonfield.py b/adserver/analyzer/migrations/0008_migrate_jsonfield.py new file mode 100644 index 00000000..152d935c --- /dev/null +++ b/adserver/analyzer/migrations/0008_migrate_jsonfield.py @@ -0,0 +1,34 @@ +# Generated by Django 5.0.8 on 2024-09-30 21:04 + +import adserver.analyzer.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("adserver_analyzer", "0007_add_advertiser_flights"), + ] + + operations = [ + migrations.AlterField( + model_name="analyzedadvertiserurl", + name="keywords", + field=models.JSONField( + blank=True, + null=True, + validators=[adserver.analyzer.validators.KeywordsValidator()], + verbose_name="Keywords for this URL", + ), + ), + migrations.AlterField( + model_name="analyzedurl", + name="keywords", + field=models.JSONField( + blank=True, + null=True, + validators=[adserver.analyzer.validators.KeywordsValidator()], + verbose_name="Keywords for this URL", + ), + ), + ] diff --git a/adserver/analyzer/models.py b/adserver/analyzer/models.py index 1f682d5f..a269ab7d 100644 --- a/adserver/analyzer/models.py +++ b/adserver/analyzer/models.py @@ -3,7 +3,6 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from django_extensions.db.models import TimeStampedModel -from jsonfield import JSONField from ..models import Advertiser from ..models import Flight @@ -21,7 +20,7 @@ class BaseAnalyzedUrl(TimeStampedModel): ) # Fields below are updated by the analyzer - keywords = JSONField( + keywords = models.JSONField( _("Keywords for this URL"), blank=True, null=True, diff --git a/adserver/migrations/0097_migrate_jsonfield.py b/adserver/migrations/0097_migrate_jsonfield.py new file mode 100644 index 00000000..e264d0de --- /dev/null +++ b/adserver/migrations/0097_migrate_jsonfield.py @@ -0,0 +1,110 @@ +# Generated by Django 5.0.8 on 2024-09-30 21:04 + +import adserver.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("adserver", "0096_simple_history_upgrade"), + ] + + operations = [ + migrations.AlterField( + model_name="click", + name="keywords", + field=models.JSONField( + blank=True, null=True, verbose_name="Keyword targeting for this view" + ), + ), + migrations.AlterField( + model_name="flight", + name="targeting_parameters", + field=models.JSONField( + blank=True, + null=True, + validators=[adserver.validators.TargetingParametersValidator()], + verbose_name="Targeting parameters", + ), + ), + migrations.AlterField( + model_name="flight", + name="traffic_cap", + field=models.JSONField( + blank=True, + default=None, + null=True, + validators=[adserver.validators.TrafficFillValidator()], + verbose_name="Traffic cap", + ), + ), + migrations.AlterField( + model_name="flight", + name="traffic_fill", + field=models.JSONField( + blank=True, + default=None, + null=True, + validators=[adserver.validators.TrafficFillValidator()], + verbose_name="Traffic fill", + ), + ), + migrations.AlterField( + model_name="historicalflight", + name="targeting_parameters", + field=models.JSONField( + blank=True, + null=True, + validators=[adserver.validators.TargetingParametersValidator()], + verbose_name="Targeting parameters", + ), + ), + migrations.AlterField( + model_name="historicalflight", + name="traffic_cap", + field=models.JSONField( + blank=True, + default=None, + null=True, + validators=[adserver.validators.TrafficFillValidator()], + verbose_name="Traffic cap", + ), + ), + migrations.AlterField( + model_name="historicalflight", + name="traffic_fill", + field=models.JSONField( + blank=True, + default=None, + null=True, + validators=[adserver.validators.TrafficFillValidator()], + verbose_name="Traffic fill", + ), + ), + migrations.AlterField( + model_name="offer", + name="keywords", + field=models.JSONField( + blank=True, null=True, verbose_name="Keyword targeting for this view" + ), + ), + migrations.AlterField( + model_name="region", + name="prices", + field=models.JSONField( + blank=True, + help_text="Topic pricing matrix for this region", + null=True, + validators=[adserver.validators.TopicPricingValidator()], + verbose_name="Topic prices", + ), + ), + migrations.AlterField( + model_name="view", + name="keywords", + field=models.JSONField( + blank=True, null=True, verbose_name="Keyword targeting for this view" + ), + ), + ] diff --git a/adserver/models.py b/adserver/models.py index 9e54d74a..35444fa1 100644 --- a/adserver/models.py +++ b/adserver/models.py @@ -33,7 +33,6 @@ from django_countries.fields import CountryField from django_extensions.db.models import TimeStampedModel from djstripe.enums import InvoiceStatus -from jsonfield import JSONField from simple_history.models import HistoricalRecords from user_agents import parse @@ -195,7 +194,7 @@ class Region(TimeStampedModel, models.Model): help_text=_("Whether advertisers can select this region for new flights"), ) - prices = JSONField( + prices = models.JSONField( _("Topic prices"), blank=True, null=True, @@ -831,7 +830,7 @@ class Flight(TimeStampedModel, IndestructibleModel): Campaign, related_name="flights", on_delete=models.PROTECT ) - targeting_parameters = JSONField( + targeting_parameters = models.JSONField( _("Targeting parameters"), blank=True, null=True, @@ -866,7 +865,7 @@ class Flight(TimeStampedModel, IndestructibleModel): # "countries": {"US": 0.1, "CA": 0.05, "DE": 0.05}, # "regions": {"us-ca": 0.25, "eu": 0.5}, # } - traffic_fill = JSONField( + traffic_fill = models.JSONField( _("Traffic fill"), blank=True, null=True, @@ -877,7 +876,7 @@ class Flight(TimeStampedModel, IndestructibleModel): # If set, any publisher, country, or region whose `traffic_fill` exceeds the cap # will not be eligible to show on this campaign until they're below the cap. # Format is the same as `traffic_fill` but this is set manually - traffic_cap = JSONField( + traffic_cap = models.JSONField( _("Traffic cap"), blank=True, null=True, @@ -2540,7 +2539,9 @@ class AdBase(TimeStampedModel, IndestructibleModel): ) # Client data - keywords = JSONField(_("Keyword targeting for this view"), blank=True, null=True) + keywords = models.JSONField( + _("Keyword targeting for this view"), blank=True, null=True + ) div_id = models.CharField( _("Div id"), blank=True, null=True, max_length=DIV_MAXLENGTH ) diff --git a/requirements/analyzer.txt b/requirements/analyzer.txt index e8f2be9f..5c8ef4aa 100644 --- a/requirements/analyzer.txt +++ b/requirements/analyzer.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile analyzer.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile analyzer.in -o analyzer.txt babel==2.15.0 # via courlan beautifulsoup4==4.12.3 @@ -32,9 +28,7 @@ click==8.1.7 # nltk # typer confection==0.1.5 - # via - # thinc - # weasel + # via thinc courlan==1.2.0 # via trafilatura cymem==2.0.8 @@ -53,7 +47,6 @@ filelock==3.15.4 # huggingface-hub # torch # transformers - # triton floret==0.10.5 # via textacy fsspec==2024.6.1 @@ -87,7 +80,7 @@ langdetect==1.0.9 # via -r analyzer.in language-data==1.2.0 # via langcodes -lxml[html-clean,html_clean]==5.2.2 +lxml==5.2.2 # via # -r analyzer.in # htmldate @@ -135,7 +128,6 @@ packaging==24.1 # spacy # thinc # transformers - # weasel pathlib-abc==0.1.1 # via pathy pathy==0.11.0 @@ -153,7 +145,6 @@ pydantic==1.10.17 # confection # spacy # thinc - # weasel pyphen==0.15.0 # via textacy python-dateutil==2.9.0.post0 @@ -177,7 +168,6 @@ requests==2.32.3 # spacy # textacy # transformers - # weasel scikit-learn==1.5.1 # via # sentence-transformers @@ -191,6 +181,11 @@ sentence-transformers==2.2.2 # via -r analyzer.in sentencepiece==0.2.0 # via sentence-transformers +setuptools==75.1.0 + # via + # marisa-trie + # spacy + # thinc six==1.16.0 # via # langdetect @@ -199,7 +194,6 @@ smart-open==6.4.0 # via # pathy # spacy - # weasel soupsieve==2.5 # via beautifulsoup4 spacy==3.4.4 @@ -222,7 +216,6 @@ srsly==2.4.8 # spacy # spacy-transformers # thinc - # weasel sympy==1.13.0 # via torch textacy==0.13.0 @@ -258,16 +251,12 @@ transformers==4.26.1 # via # sentence-transformers # spacy-transformers -triton==2.3.1 - # via torch typer==0.7.0 # via # pathy # spacy - # weasel typing-extensions==4.12.2 # via - # cloudpathlib # huggingface-hub # pydantic # torch @@ -283,7 +272,3 @@ wasabi==0.10.1 # via # spacy # thinc - # weasel - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/base.txt b/requirements/base.txt index f6824556..cb9f3775 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile base.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile base.in -o base.txt aiohttp==3.9.5 # via geoip2 aiosignal==1.3.1 @@ -25,7 +21,7 @@ billiard==4.2.0 # via celery bleach==6.1.0 # via -r base.in -celery[redis]==5.4.0 +celery==5.4.0 # via -r base.in certifi==2024.7.4 # via requests @@ -125,6 +121,8 @@ requests==2.32.3 # django-slack # geoip2 # stripe +setuptools==75.1.0 + # via geoip2 six==1.16.0 # via # bleach @@ -160,6 +158,3 @@ whitenoise==6.7.0 # via -r base.in yarl==1.9.4 # via aiohttp - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/development.txt b/requirements/development.txt index 7fd78e6e..c66dbf46 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile development.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile development.in -o development.txt aiohttp==3.9.5 # via geoip2 aiosignal==1.3.1 @@ -42,7 +38,7 @@ bleach==6.1.0 # via -r base.in cattrs==23.2.3 # via requests-cache -celery[redis]==5.4.0 +celery==5.4.0 # via -r base.in certifi==2024.7.4 # via requests @@ -257,6 +253,8 @@ responses==0.25.3 # via -r development.in ruff==0.5.2 # via -r development.in +setuptools==75.1.0 + # via geoip2 six==1.16.0 # via # asttokens @@ -368,6 +366,3 @@ whitenoise==6.7.0 # via -r base.in yarl==1.9.4 # via aiohttp - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/production.txt b/requirements/production.txt index 58e0efc1..d6ac7b0e 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile production.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile production.in -o production.txt aiohttp==3.9.5 # via geoip2 aiosignal==1.3.1 @@ -31,7 +27,7 @@ billiard==4.2.0 # via celery bleach==6.1.0 # via -r base.in -celery[redis]==5.4.0 +celery==5.4.0 # via -r base.in certifi==2024.7.4 # via @@ -101,7 +97,7 @@ django-simple-history==3.7.0 # via -r base.in django-slack==5.19.0 # via -r base.in -django-storages[azure]==1.14.4 +django-storages==1.14.4 # via -r production.in djangorestframework==3.15.2 # via -r base.in @@ -164,6 +160,8 @@ requests==2.32.3 # stripe sentry-sdk==2.10.0 # via -r production.in +setuptools==75.1.0 + # via geoip2 six==1.16.0 # via # azure-core @@ -206,6 +204,3 @@ whitenoise==6.7.0 # via -r base.in yarl==1.9.4 # via aiohttp - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/testing.txt b/requirements/testing.txt index db9b1a8a..a33fc5f7 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile testing.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile testing.in -o testing.txt beautifulsoup4==4.12.3 # via -r testing.in distlib==0.3.8 diff --git a/requirements/update.sh b/requirements/update.sh index d6711efa..154a1b76 100755 --- a/requirements/update.sh +++ b/requirements/update.sh @@ -3,18 +3,32 @@ # Exit immediately if a command exits with a non-zero status set -e +# Check if uv is installed +if command -v uv &> /dev/null; then + compile_command="uv pip compile" +else + compile_command="pip-compile" +fi + +# Flag to check if any .in files exist +found_files=false + # Loop through all .in files in the current directory -for file in *.in -do +for file in *.in; do # Check if the file exists to avoid errors if there are no .in files if [[ -f "$file" ]]; then - echo "Compiling $file..." - # Call pip-compile for each .in file - pip-compile -U "$file" - else - echo "No .in files found in the directory." - break + found_files=true + output_file="${file%.in}.txt" + echo "Compiling $file to $output_file..." + echo "Running command: $compile_command $file -o $output_file" + # Call the compile command for each .in file, outputting to the corresponding .txt file + $compile_command "$file" -o "$output_file" fi done +# If no .in files were found, notify the user +if [[ $found_files == false ]]; then + echo "No .in files found in the directory." +fi + echo "All .in files have been processed." From 680799b67bf8a4ccbf526ce7ce887005976ff2c9 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 30 Sep 2024 14:12:06 -0700 Subject: [PATCH 2/6] Back to pip-compile --- requirements/analyzer.txt | 18 ++++++++++-------- requirements/base.in | 1 - requirements/base.txt | 18 ++++++++++-------- requirements/development.txt | 18 ++++++++++-------- requirements/production.txt | 20 +++++++++++--------- requirements/testing.txt | 8 ++++++-- 6 files changed, 47 insertions(+), 36 deletions(-) diff --git a/requirements/analyzer.txt b/requirements/analyzer.txt index 5c8ef4aa..f2dcc06c 100644 --- a/requirements/analyzer.txt +++ b/requirements/analyzer.txt @@ -1,5 +1,9 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile analyzer.in -o analyzer.txt +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=analyzer.txt analyzer.in +# babel==2.15.0 # via courlan beautifulsoup4==4.12.3 @@ -80,7 +84,7 @@ langdetect==1.0.9 # via -r analyzer.in language-data==1.2.0 # via langcodes -lxml==5.2.2 +lxml[html_clean]==5.2.2 # via # -r analyzer.in # htmldate @@ -181,11 +185,6 @@ sentence-transformers==2.2.2 # via -r analyzer.in sentencepiece==0.2.0 # via sentence-transformers -setuptools==75.1.0 - # via - # marisa-trie - # spacy - # thinc six==1.16.0 # via # langdetect @@ -272,3 +271,6 @@ wasabi==0.10.1 # via # spacy # thinc + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/base.in b/requirements/base.in index 69566dab..3d77f203 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -44,7 +44,6 @@ IP2Proxy # Countries helper used in ad targeting django-countries -jsonfield bleach # Security features diff --git a/requirements/base.txt b/requirements/base.txt index cb9f3775..ce1119cb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,9 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile base.in -o base.txt +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=base.txt base.in +# aiohttp==3.9.5 # via geoip2 aiosignal==1.3.1 @@ -21,7 +25,7 @@ billiard==4.2.0 # via celery bleach==6.1.0 # via -r base.in -celery==5.4.0 +celery[redis]==5.4.0 # via -r base.in certifi==2024.7.4 # via requests @@ -55,7 +59,6 @@ django==5.0.8 # django-simple-history # django-slack # djangorestframework - # jsonfield django-allauth==0.63.6 # via -r base.in django-cors-headers==4.4.0 @@ -94,8 +97,6 @@ idna==3.7 # yarl ip2proxy==3.4.0 # via -r base.in -jsonfield==3.1.0 - # via -r base.in kombu==5.3.7 # via celery maxminddb==2.6.2 @@ -121,8 +122,6 @@ requests==2.32.3 # django-slack # geoip2 # stripe -setuptools==75.1.0 - # via geoip2 six==1.16.0 # via # bleach @@ -158,3 +157,6 @@ whitenoise==6.7.0 # via -r base.in yarl==1.9.4 # via aiohttp + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/development.txt b/requirements/development.txt index c66dbf46..b5b2e54f 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,5 +1,9 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile development.in -o development.txt +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=development.txt development.in +# aiohttp==3.9.5 # via geoip2 aiosignal==1.3.1 @@ -38,7 +42,7 @@ bleach==6.1.0 # via -r base.in cattrs==23.2.3 # via requests-cache -celery==5.4.0 +celery[redis]==5.4.0 # via -r base.in certifi==2024.7.4 # via requests @@ -88,7 +92,6 @@ django==5.0.8 # django-simple-history # django-slack # djangorestframework - # jsonfield django-allauth==0.63.6 # via -r base.in django-cors-headers==4.4.0 @@ -170,8 +173,6 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via sphinx -jsonfield==3.1.0 - # via -r base.in kombu==5.3.7 # via celery markupsafe==2.1.5 @@ -253,8 +254,6 @@ responses==0.25.3 # via -r development.in ruff==0.5.2 # via -r development.in -setuptools==75.1.0 - # via geoip2 six==1.16.0 # via # asttokens @@ -366,3 +365,6 @@ whitenoise==6.7.0 # via -r base.in yarl==1.9.4 # via aiohttp + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/production.txt b/requirements/production.txt index d6ac7b0e..e7ece31b 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,5 +1,9 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile production.in -o production.txt +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=production.txt production.in +# aiohttp==3.9.5 # via geoip2 aiosignal==1.3.1 @@ -27,7 +31,7 @@ billiard==4.2.0 # via celery bleach==6.1.0 # via -r base.in -celery==5.4.0 +celery[redis]==5.4.0 # via -r base.in certifi==2024.7.4 # via @@ -70,7 +74,6 @@ django==5.0.8 # django-slack # django-storages # djangorestframework - # jsonfield django-allauth==0.63.6 # via -r base.in django-anymail==11.0.1 @@ -97,7 +100,7 @@ django-simple-history==3.7.0 # via -r base.in django-slack==5.19.0 # via -r base.in -django-storages==1.14.4 +django-storages[azure]==1.14.4 # via -r production.in djangorestframework==3.15.2 # via -r base.in @@ -119,8 +122,6 @@ ip2proxy==3.4.0 # via -r base.in isodate==0.6.1 # via azure-storage-blob -jsonfield==3.1.0 - # via -r base.in kombu==5.3.7 # via celery maxminddb==2.6.2 @@ -160,8 +161,6 @@ requests==2.32.3 # stripe sentry-sdk==2.10.0 # via -r production.in -setuptools==75.1.0 - # via geoip2 six==1.16.0 # via # azure-core @@ -204,3 +203,6 @@ whitenoise==6.7.0 # via -r base.in yarl==1.9.4 # via aiohttp + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/testing.txt b/requirements/testing.txt index a33fc5f7..cbc2ed1f 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,5 +1,9 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile testing.in -o testing.txt +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=testing.txt testing.in +# beautifulsoup4==4.12.3 # via -r testing.in distlib==0.3.8 From dbaea5a054f3102e3a4d513f37efbecd38b19613 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 30 Sep 2024 14:18:52 -0700 Subject: [PATCH 3/6] Revert uv changes --- requirements/update.sh | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/requirements/update.sh b/requirements/update.sh index 154a1b76..d6711efa 100755 --- a/requirements/update.sh +++ b/requirements/update.sh @@ -3,32 +3,18 @@ # Exit immediately if a command exits with a non-zero status set -e -# Check if uv is installed -if command -v uv &> /dev/null; then - compile_command="uv pip compile" -else - compile_command="pip-compile" -fi - -# Flag to check if any .in files exist -found_files=false - # Loop through all .in files in the current directory -for file in *.in; do +for file in *.in +do # Check if the file exists to avoid errors if there are no .in files if [[ -f "$file" ]]; then - found_files=true - output_file="${file%.in}.txt" - echo "Compiling $file to $output_file..." - echo "Running command: $compile_command $file -o $output_file" - # Call the compile command for each .in file, outputting to the corresponding .txt file - $compile_command "$file" -o "$output_file" + echo "Compiling $file..." + # Call pip-compile for each .in file + pip-compile -U "$file" + else + echo "No .in files found in the directory." + break fi done -# If no .in files were found, notify the user -if [[ $found_files == false ]]; then - echo "No .in files found in the directory." -fi - echo "All .in files have been processed." From c6470249b3f023085d79e550dfe6eebf71397977 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 30 Sep 2024 14:34:13 -0700 Subject: [PATCH 4/6] Revert requirements removal --- requirements/analyzer.txt | 19 ++++++++++++++++--- requirements/base.in | 1 + requirements/base.txt | 5 ++++- requirements/development.txt | 5 ++++- requirements/production.txt | 5 ++++- requirements/testing.txt | 2 +- 6 files changed, 30 insertions(+), 7 deletions(-) diff --git a/requirements/analyzer.txt b/requirements/analyzer.txt index f2dcc06c..e8f2be9f 100644 --- a/requirements/analyzer.txt +++ b/requirements/analyzer.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --output-file=analyzer.txt analyzer.in +# pip-compile analyzer.in # babel==2.15.0 # via courlan @@ -32,7 +32,9 @@ click==8.1.7 # nltk # typer confection==0.1.5 - # via thinc + # via + # thinc + # weasel courlan==1.2.0 # via trafilatura cymem==2.0.8 @@ -51,6 +53,7 @@ filelock==3.15.4 # huggingface-hub # torch # transformers + # triton floret==0.10.5 # via textacy fsspec==2024.6.1 @@ -84,7 +87,7 @@ langdetect==1.0.9 # via -r analyzer.in language-data==1.2.0 # via langcodes -lxml[html_clean]==5.2.2 +lxml[html-clean,html_clean]==5.2.2 # via # -r analyzer.in # htmldate @@ -132,6 +135,7 @@ packaging==24.1 # spacy # thinc # transformers + # weasel pathlib-abc==0.1.1 # via pathy pathy==0.11.0 @@ -149,6 +153,7 @@ pydantic==1.10.17 # confection # spacy # thinc + # weasel pyphen==0.15.0 # via textacy python-dateutil==2.9.0.post0 @@ -172,6 +177,7 @@ requests==2.32.3 # spacy # textacy # transformers + # weasel scikit-learn==1.5.1 # via # sentence-transformers @@ -193,6 +199,7 @@ smart-open==6.4.0 # via # pathy # spacy + # weasel soupsieve==2.5 # via beautifulsoup4 spacy==3.4.4 @@ -215,6 +222,7 @@ srsly==2.4.8 # spacy # spacy-transformers # thinc + # weasel sympy==1.13.0 # via torch textacy==0.13.0 @@ -250,12 +258,16 @@ transformers==4.26.1 # via # sentence-transformers # spacy-transformers +triton==2.3.1 + # via torch typer==0.7.0 # via # pathy # spacy + # weasel typing-extensions==4.12.2 # via + # cloudpathlib # huggingface-hub # pydantic # torch @@ -271,6 +283,7 @@ wasabi==0.10.1 # via # spacy # thinc + # weasel # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/base.in b/requirements/base.in index 3d77f203..69566dab 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -44,6 +44,7 @@ IP2Proxy # Countries helper used in ad targeting django-countries +jsonfield bleach # Security features diff --git a/requirements/base.txt b/requirements/base.txt index ce1119cb..f6824556 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --output-file=base.txt base.in +# pip-compile base.in # aiohttp==3.9.5 # via geoip2 @@ -59,6 +59,7 @@ django==5.0.8 # django-simple-history # django-slack # djangorestframework + # jsonfield django-allauth==0.63.6 # via -r base.in django-cors-headers==4.4.0 @@ -97,6 +98,8 @@ idna==3.7 # yarl ip2proxy==3.4.0 # via -r base.in +jsonfield==3.1.0 + # via -r base.in kombu==5.3.7 # via celery maxminddb==2.6.2 diff --git a/requirements/development.txt b/requirements/development.txt index b5b2e54f..7fd78e6e 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --output-file=development.txt development.in +# pip-compile development.in # aiohttp==3.9.5 # via geoip2 @@ -92,6 +92,7 @@ django==5.0.8 # django-simple-history # django-slack # djangorestframework + # jsonfield django-allauth==0.63.6 # via -r base.in django-cors-headers==4.4.0 @@ -173,6 +174,8 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via sphinx +jsonfield==3.1.0 + # via -r base.in kombu==5.3.7 # via celery markupsafe==2.1.5 diff --git a/requirements/production.txt b/requirements/production.txt index e7ece31b..58e0efc1 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --output-file=production.txt production.in +# pip-compile production.in # aiohttp==3.9.5 # via geoip2 @@ -74,6 +74,7 @@ django==5.0.8 # django-slack # django-storages # djangorestframework + # jsonfield django-allauth==0.63.6 # via -r base.in django-anymail==11.0.1 @@ -122,6 +123,8 @@ ip2proxy==3.4.0 # via -r base.in isodate==0.6.1 # via azure-storage-blob +jsonfield==3.1.0 + # via -r base.in kombu==5.3.7 # via celery maxminddb==2.6.2 diff --git a/requirements/testing.txt b/requirements/testing.txt index cbc2ed1f..db9b1a8a 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --output-file=testing.txt testing.in +# pip-compile testing.in # beautifulsoup4==4.12.3 # via -r testing.in From 47c2ee10b20de258102b38652adf26e8aeba8451 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 30 Sep 2024 14:35:27 -0700 Subject: [PATCH 5/6] Add note about jsonfield --- requirements/base.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements/base.in b/requirements/base.in index 69566dab..75a1348f 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -44,7 +44,6 @@ IP2Proxy # Countries helper used in ad targeting django-countries -jsonfield bleach # Security features @@ -62,3 +61,6 @@ PyJWT # CORS headers django-cors-headers + +# Deprecated, but still used in migrations +jsonfield From 630f2e30d5e72045fdf9345e348193c2d10c9685 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Wed, 9 Oct 2024 10:19:14 -0700 Subject: [PATCH 6/6] Remove simple history change reason This is a temporary fix until the underlying issue is fixed. --- adserver/forms.py | 4 ++-- adserver/staff/forms.py | 35 ++++++++++++++++++++++------------- adserver/tasks.py | 8 ++++---- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/adserver/forms.py b/adserver/forms.py index bf7ee1af..829cc3c5 100644 --- a/adserver/forms.py +++ b/adserver/forms.py @@ -28,7 +28,6 @@ from django.utils.text import slugify from django.utils.translation import gettext from django.utils.translation import gettext_lazy as _ -from simple_history.utils import update_change_reason from .models import Advertisement from .models import Campaign @@ -1358,7 +1357,8 @@ def save(self, commit=True): user.invite_user() # Track who added this user - update_change_reason(user, "Invited via authorized users view") + # See: https://github.com/jazzband/django-simple-history/issues/1181 + # update_change_reason(user, "Invited via authorized users view") # You will need to add the user to the publisher/advertiser in the view return user diff --git a/adserver/staff/forms.py b/adserver/staff/forms.py index d8c60571..e942348f 100644 --- a/adserver/staff/forms.py +++ b/adserver/staff/forms.py @@ -17,7 +17,6 @@ from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ from djstripe.models import Customer -from simple_history.utils import update_change_reason from ..constants import EMAILED from ..constants import PUBLISHER_HOUSE_CAMPAIGN @@ -110,7 +109,10 @@ def create_user(self): user_email = self.cleaned_data["user_email"] user = User.objects.create_user(name=user_name, email=user_email, password="") - update_change_reason(user, self.message) + + # See: https://github.com/jazzband/django-simple-history/issues/1181 + # update_change_reason(user, self.message) + if hasattr(user, "invite_user"): user.invite_user() return user @@ -134,7 +136,7 @@ def create_advertiser(self): campaign.publisher_groups.add(pub_group) flight_name = f"{advertiser_name} Initial Flight" - flight = Flight.objects.create( + Flight.objects.create( campaign=campaign, name=flight_name, slug=slugify(flight_name), @@ -145,9 +147,10 @@ def create_advertiser(self): }, ) - update_change_reason(advertiser, self.message) - update_change_reason(campaign, self.message) - update_change_reason(flight, self.message) + # See https://github.com/jazzband/django-simple-history/issues/1181 + # update_change_reason(advertiser, self.message) + # update_change_reason(campaign, self.message) + # update_change_reason(flight, self.message) return advertiser @@ -256,7 +259,10 @@ def create_user(self): user_email = self.cleaned_data["user_email"] user = User.objects.create_user(name=user_name, email=user_email, password="") - update_change_reason(user, self.message) + + # See https://github.com/jazzband/django-simple-history/issues/1181 + # update_change_reason(user, self.message) + if hasattr(user, "invite_user"): user.invite_user() return user @@ -277,7 +283,8 @@ def create_publisher(self): if group_obj: group_obj.publishers.add(publisher) - update_change_reason(publisher, self.message) + # See: https://github.com/jazzband/django-simple-history/issues/1181 + # update_change_reason(publisher, self.message) # Create this publisher's advertiser account self.create_publisher_advertiser_account(publisher) @@ -305,7 +312,7 @@ def create_publisher_advertiser_account(self, publisher): campaign.publisher_groups.add(pub_group) flight_name = f"{publisher.name} House Ads" - flight = Flight.objects.create( + Flight.objects.create( campaign=campaign, name=flight_name, slug=slugify(flight_name), @@ -316,9 +323,10 @@ def create_publisher_advertiser_account(self, publisher): }, ) - update_change_reason(advertiser, self.message) - update_change_reason(campaign, self.message) - update_change_reason(flight, self.message) + # See: https://github.com/jazzband/django-simple-history/issues/1181 + # update_change_reason(advertiser, self.message) + # update_change_reason(campaign, self.message) + # update_change_reason(flight, self.message) def save(self): """Create the publisher and associated objects. Send the invitation to the user account.""" @@ -408,6 +416,7 @@ def save(self): status=EMAILED, ) - update_change_reason(payout, "Payout via staff interface") + # See: https://github.com/jazzband/django-simple-history/issues/1181 + # update_change_reason(payout, "Payout via staff interface") return payout diff --git a/adserver/tasks.py b/adserver/tasks.py index 5356959e..ead9638d 100644 --- a/adserver/tasks.py +++ b/adserver/tasks.py @@ -18,7 +18,6 @@ from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ from django_slack import slack_message -from simple_history.utils import update_change_reason from config.celery_app import app @@ -886,9 +885,10 @@ def notify_of_completed_flights(): flight.save() # Store the change reason in the history - update_change_reason( - flight, f"Hard stopped with ${value_remaining} value remaining." - ) + # See: https://github.com/jazzband/django-simple-history/issues/1181 + # update_change_reason( + # flight, f"Hard stopped with ${value_remaining} value remaining." + # ) completed_flights_by_advertiser[flight.campaign.advertiser.slug].append( flight