Restructure utils.py to its own package

Makefile

@@ -89,5 +89,6 @@ help: ## Print this message and exit
 		| sort \
 		| column -s ':' -t
 
+.PHONY: attach
 attach:
 	docker-compose exec postgres psql -h localhost -U openoversight openoversight-dev

OpenOversight/app/auth/forms.py

@@ -10,7 +10,7 @@
 from wtforms.validators import DataRequired, Email, EqualTo, Length, Optional, Regexp
 
 from ..models import User
-from ..utils import dept_choices
+from ..util.db import dept_choices
 
 
 class LoginForm(Form):

OpenOversight/app/auth/views.py

@@ -11,15 +11,13 @@
 )
 from flask_login import current_user, login_required, login_user, logout_user
 
+from OpenOversight.app.util.constants import HTTP_METHOD_GET, HTTP_METHOD_POST
+from OpenOversight.app.util.forms import set_dynamic_default
+from OpenOversight.app.util.general import validate_redirect_url
+
 from .. import sitemap
 from ..email import send_email
 from ..models import User, db
-from ..utils import (
-    HTTP_METHOD_GET,
-    HTTP_METHOD_POST,
-    set_dynamic_default,
-    validate_redirect_url,
-)
 from . import auth
 from .forms import (
     ChangeDefaultDepartmentForm,

OpenOversight/app/commands.py

@@ -12,15 +12,12 @@
 from flask import current_app
 from flask.cli import with_appcontext
 
+from OpenOversight.app.util.constants import ENCODING_UTF_8
+from OpenOversight.app.util.db import get_officer
+from OpenOversight.app.util.general import normalize_gender, prompt_yes_no, str_is_true
+
 from .csv_imports import import_csv_files
 from .models import Assignment, Department, Job, Officer, Salary, User, db
-from .utils import (
-    ENCODING_UTF_8,
-    get_officer,
-    normalize_gender,
-    prompt_yes_no,
-    str_is_true,
-)
 
 
 @click.command()

OpenOversight/app/main/forms.py

@@ -29,9 +29,10 @@
     ValidationError,
 )
 
+from OpenOversight.app.util.db import dept_choices, unit_choices
+
 from ..formfields import TimeField
 from ..models import Officer
-from ..utils import dept_choices, unit_choices
 from ..widgets import BootstrapListWidget, FormFieldWidget
 from .choices import (
     AGE_CHOICES,

OpenOversight/app/main/model_view.py

@@ -7,14 +7,12 @@
 from flask_sqlalchemy.model import DefaultMeta
 from flask_wtf import FlaskForm as Form
 
+from OpenOversight.app.util.constants import HTTP_METHOD_GET, HTTP_METHOD_POST
+from OpenOversight.app.util.db import add_department_query
+from OpenOversight.app.util.forms import set_dynamic_default
+
 from ..auth.utils import ac_or_admin_required
 from ..models import db
-from ..utils import (
-    HTTP_METHOD_GET,
-    HTTP_METHOD_POST,
-    add_department_query,
-    set_dynamic_default,
-)
 
 
 class ModelView(MethodView):

OpenOversight/app/main/views.py

@@ -21,6 +21,36 @@
 from sqlalchemy.orm import contains_eager, joinedload, selectinload
 from sqlalchemy.orm.exc import NoResultFound
 
+from OpenOversight.app.util.cloud import crop_image, upload_image_to_s3_and_store_in_db
+from OpenOversight.app.util.constants import HTTP_METHOD_GET, HTTP_METHOD_POST
+from OpenOversight.app.util.db import (
+    add_department_query,
+    add_unit_query,
+    compute_leaderboard_stats,
+    dept_choices,
+    unit_choices,
+)
+from OpenOversight.app.util.forms import (
+    add_new_assignment,
+    add_officer_profile,
+    create_description,
+    create_incident,
+    create_note,
+    edit_existing_assignment,
+    edit_officer_profile,
+    filter_by_form,
+    set_dynamic_default,
+)
+from OpenOversight.app.util.general import (
+    ac_can_edit_officer,
+    allowed_file,
+    get_or_create,
+    get_random_image,
+    replace_list,
+    serve_image,
+    validate_redirect_url,
+)
+
 from .. import limiter, sitemap
 from ..auth.forms import LoginForm
 from ..auth.utils import ac_or_admin_required, admin_required
@@ -42,33 +72,6 @@
     User,
     db,
 )
-from ..utils import (
-    HTTP_METHOD_GET,
-    HTTP_METHOD_POST,
-    ac_can_edit_officer,
-    add_department_query,
-    add_new_assignment,
-    add_officer_profile,
-    add_unit_query,
-    allowed_file,
-    compute_leaderboard_stats,
-    create_description,
-    create_incident,
-    create_note,
-    crop_image,
-    dept_choices,
-    edit_existing_assignment,
-    edit_officer_profile,
-    filter_by_form,
-    get_or_create,
-    get_random_image,
-    replace_list,
-    serve_image,
-    set_dynamic_default,
-    unit_choices,
-    upload_image_to_s3_and_store_in_db,
-    validate_redirect_url,
-)
 from . import downloads, main
 from .choices import AGE_CHOICES, GENDER_CHOICES, RACE_CHOICES
 from .forms import (
@@ -153,7 +156,8 @@ def get_officer():
                 else None,
                 rank=form.data["rank"] if form.data["rank"] != "Not Sure" else None,
                 unit=form.data["unit"] if form.data["unit"] != "Not Sure" else None,
-                current_job=form.data["current_job"] or None,  # set to None if False
+                current_job=form.data["current_job"] or None,
+                # set to None if False
                 min_age=form.data["min_age"],
                 max_age=form.data["max_age"],
                 first_name=form.data["first_name"],

OpenOversight/app/model_imports.py

@@ -2,6 +2,8 @@
 
 import dateutil.parser
 
+from OpenOversight.app.util.general import get_or_create, str_is_true
+
 from .main import choices
 from .models import (
     Assignment,
@@ -13,7 +15,6 @@
     Salary,
     db,
 )
-from .utils import get_or_create, str_is_true
 from .validators import state_validator, url_validator
 
 

OpenOversight/app/models.py

@@ -12,11 +12,11 @@
 from werkzeug.security import check_password_hash, generate_password_hash
 
 from . import login_manager
+from .util.constants import ENCODING_UTF_8
 from .validators import state_validator, url_validator
 
 
 db = SQLAlchemy()
-model_encoding = "utf-8"
 jwt = JsonWebToken("HS512")
 
 BaseModel = db.Model  # type: DefaultMeta
@@ -552,7 +552,7 @@ def verify_password(self, password):
 
     def generate_confirmation_token(self, expiration=3600):
         payload = {"confirm": self.id}
-        return self._jwt_encode(payload, expiration).decode(model_encoding)
+        return self._jwt_encode(payload, expiration).decode(ENCODING_UTF_8)
 
     def confirm(self, token):
         try:
@@ -572,7 +572,7 @@ def confirm(self, token):
 
     def generate_reset_token(self, expiration=3600):
         payload = {"reset": self.id}
-        return self._jwt_encode(payload, expiration).decode(model_encoding)
+        return self._jwt_encode(payload, expiration).decode(ENCODING_UTF_8)
 
     def reset_password(self, token, new_password):
         try:
@@ -588,7 +588,7 @@ def reset_password(self, token, new_password):
 
     def generate_email_change_token(self, new_email, expiration=3600):
         payload = {"change_email": self.id, "new_email": new_email}
-        return self._jwt_encode(payload, expiration).decode(model_encoding)
+        return self._jwt_encode(payload, expiration).decode(ENCODING_UTF_8)
 
     def change_email(self, token):
         try:

OpenOversight/app/util/cloud.py

@@ -0,0 +1,154 @@
+import datetime
+import hashlib
+import imghdr as imghdr
+import os
+import sys
+from io import BytesIO
+from traceback import format_exc
+from urllib.request import urlopen
+
+import boto3
+import botocore
+from botocore.exceptions import ClientError
+from flask import current_app
+from flask_login import current_user
+from PIL import Image as Pimage
+from PIL.PngImagePlugin import PngImageFile
+
+from ..models import Image, db
+
+
+def compute_hash(data_to_hash):
+    return hashlib.sha256(data_to_hash).hexdigest()
+
+
+def crop_image(image, crop_data=None, department_id=None):
+    """Crops an image to given dimensions and shrinks it to fit within a configured
+    bounding box if the cropped image is still too big.
+    """
+    # Cropped officer face image size
+    THUMBNAIL_SIZE = 1000, 1000
+
+    if "http" in image.filepath:
+        with urlopen(image.filepath) as response:
+            image_buf = BytesIO(response.read())
+    else:
+        image_buf = open(os.path.abspath(current_app.root_path) + image.filepath, "rb")
+
+    pimage = Pimage.open(image_buf)
+
+    if (
+        not crop_data
+        and pimage.size[0] < THUMBNAIL_SIZE[0]
+        and pimage.size[1] < THUMBNAIL_SIZE[1]
+    ):
+        return image
+
+    # Crops image to face and resizes to bounding box if still too big
+    if crop_data:
+        pimage = pimage.crop(crop_data)
+    if pimage.size[0] > THUMBNAIL_SIZE[0] or pimage.size[1] > THUMBNAIL_SIZE[1]:
+        pimage.thumbnail(THUMBNAIL_SIZE)
+
+    # JPEG doesn't support alpha channel, convert to RGB
+    if pimage.mode in ("RGBA", "P"):
+        pimage = pimage.convert("RGB")
+
+    # Save preview image as JPEG to save bandwidth for mobile users
+    cropped_image_buf = BytesIO()
+    pimage.save(cropped_image_buf, "jpeg", quality=95, optimize=True, progressive=True)
+
+    return upload_image_to_s3_and_store_in_db(
+        cropped_image_buf, current_user.get_id(), department_id
+    )
+
+
+def find_date_taken(pimage):
+    if isinstance(pimage, PngImageFile):
+        return None
+
+    exif = hasattr(pimage, "_getexif") and pimage._getexif()
+    if exif:
+        # 36867 in the exif tags holds the date and the original image was taken
+        # https://www.awaresystems.be/imaging/tiff/tifftags/privateifd/exif.html
+        if 36867 in exif:
+            return exif[36867]
+    else:
+        return None
+
+
+def upload_obj_to_s3(file_obj, dest_filename):
+    s3_client = boto3.client("s3")
+
+    # Folder to store files in on S3 is first two chars of dest_filename
+    s3_folder = dest_filename[0:2]
+    s3_filename = dest_filename[2:]
+    file_ending = imghdr.what(None, h=file_obj.read())
+    file_obj.seek(0)
+    s3_content_type = "image/%s" % file_ending
+    s3_path = "{}/{}".format(s3_folder, s3_filename)
+    s3_client.upload_fileobj(
+        file_obj,
+        current_app.config["S3_BUCKET_NAME"],
+        s3_path,
+        ExtraArgs={"ContentType": s3_content_type, "ACL": "public-read"},
+    )
+
+    config = s3_client._client_config
+    config.signature_version = botocore.UNSIGNED
+    url = boto3.resource("s3", config=config).meta.client.generate_presigned_url(
+        "get_object",
+        Params={"Bucket": current_app.config["S3_BUCKET_NAME"], "Key": s3_path},
+    )
+
+    return url
+
+
+def upload_image_to_s3_and_store_in_db(image_buf, user_id, department_id=None):
+    """
+    Just a quick explaination of the order of operations here...
+    we have to scrub the image before we do anything else like hash it
+    but we also have to get the date for the image before we scrub it.
+    """
+    image_buf.seek(0)
+    image_type = imghdr.what(image_buf)
+    if image_type not in current_app.config["ALLOWED_EXTENSIONS"]:
+        raise ValueError("Attempted to pass invalid data type: {}".format(image_type))
+    image_buf.seek(0)
+    pimage = Pimage.open(image_buf)
+    date_taken = find_date_taken(pimage)
+    if date_taken:
+        date_taken = datetime.datetime.strptime(date_taken, "%Y:%m:%d %H:%M:%S")
+    pimage.getexif().clear()
+    scrubbed_image_buf = BytesIO()
+    pimage.save(scrubbed_image_buf, image_type)
+    pimage.close()
+    scrubbed_image_buf.seek(0)
+    image_data = scrubbed_image_buf.read()
+    hash_img = compute_hash(image_data)
+    existing_image = Image.query.filter_by(hash_img=hash_img).first()
+    if existing_image:
+        return existing_image
+    try:
+        new_filename = "{}.{}".format(hash_img, image_type)
+        scrubbed_image_buf.seek(0)
+        url = upload_obj_to_s3(scrubbed_image_buf, new_filename)
+        new_image = Image(
+            filepath=url,
+            hash_img=hash_img,
+            date_image_inserted=datetime.datetime.now(),
+            department_id=department_id,
+            date_image_taken=date_taken,
+            user_id=user_id,
+        )
+        db.session.add(new_image)
+        db.session.commit()
+        return new_image
+    except ClientError:
+        exception_type, value, full_tback = sys.exc_info()
+        current_app.logger.error(
+            "Error uploading to S3: {}".format(
+                " ".join([str(exception_type), str(value), format_exc()])
+            )
+        )
+        return None

OpenOversight/app/util/constants.py

@@ -0,0 +1,14 @@
+import os
+
+
+# Encoding constants
+ENCODING_UTF_8 = "utf-8"
+
+# HTTP Method constants
+# TODO: Remove these constants and use HTTPMethod in http package when we
+#  migrate to version 3.11
+HTTP_METHOD_GET = "GET"
+HTTP_METHOD_POST = "POST"
+
+# Ensure the file is read/write by the creator only
+SAVED_UMASK = os.umask(0o077)