Merge pull request #353 from RTIInternational/add_categories_for_labels

Add metadata to labels and add project category which filters suggestions

.gitlab-ci.yml

@@ -10,7 +10,7 @@ build:
   stage: build
   script:
     - cd envs/dev
-    - docker-compose build
+    - docker compose build
 
 setup:
   tags:
@@ -19,32 +19,32 @@ setup:
   script:
     - cd envs/dev
     - docker volume create vol_smart_pgdata && docker volume create vol_smart_data
-    - docker-compose up -d
+    - docker compose up -d
   after_script:
     - cd envs/dev
-    - docker-compose stop
+    - docker compose stop
 
 test_frontend:
   tags:
     - docker-shell
   script:
     - cd envs/dev
-    - docker-compose start
+    - docker compose start
     - docker exec -i smart_frontend bash -c "./run_tests.sh"
   after_script:
     - cd envs/dev
-    - docker-compose stop
+    - docker compose stop
 
 test_backend:
   tags:
     - docker-shell
   script:
     - cd envs/dev
-    - docker-compose start
+    - docker compose start
     - docker exec -i backend bash -c "./run_tests.sh"
   after_script:
     - cd envs/dev
-    - docker-compose stop
+    - docker compose stop
 
 cleanup:
   tags:
@@ -53,7 +53,7 @@ cleanup:
   allow_failure: true
   script:
     - cd envs/dev
-    - docker-compose down
+    - docker compose down
     - docker volume remove vol_smart_pgdata
     - docker volume remove vol_smart_data
   when: always

.pre-commit-config.yaml

@@ -3,30 +3,30 @@
     - id: isort
       name: isort
       pass_filenames: false
-      entry: docker-compose -f envs/dev/docker-compose.yml run --rm backend bash -c "/code/scripts/isort.sh --format"
+      entry: docker compose -f envs/dev/docker-compose.yml run --rm backend bash -c "/code/scripts/isort.sh --format"
       language: system
       types: [python]
     - id: docformatter
       name: docformatter
       pass_filenames: false
-      entry: docker-compose -f envs/dev/docker-compose.yml run --rm backend bash -c "/code/scripts/docformatter.sh --format"
+      entry: docker compose -f envs/dev/docker-compose.yml run --rm backend bash -c "/code/scripts/docformatter.sh --format"
       language: system
       types: [python]
     - id: black
       name: black
       pass_filenames: false
-      entry: docker-compose -f envs/dev/docker-compose.yml run --rm backend bash -c "/code/scripts/black.sh --format"
+      entry: docker compose -f envs/dev/docker-compose.yml run --rm backend bash -c "/code/scripts/black.sh --format"
       language: system
       types: [python]
     - id: flake8
       name: flake8
       pass_filenames: false
-      entry: docker-compose -f envs/dev/docker-compose.yml run --rm backend bash -c "/code/scripts/flake8.sh"
+      entry: docker compose -f envs/dev/docker-compose.yml run --rm backend bash -c "/code/scripts/flake8.sh"
       language: system
       types: [python]
     - id: eslint-js
       name: eslint-js
       pass_filenames: false
-      entry: docker-compose -f envs/dev/docker-compose.yml run --rm smart_frontend yarn lint
+      entry: docker compose -f envs/dev/docker-compose.yml run --rm smart_frontend yarn lint
       language: system
       types: [javascript]

.travis.yml

@@ -5,22 +5,22 @@ services:
 
 before_install:
   - docker -v
-  - docker-compose -v
+  - docker compose -v
 
   # Disable services enabled by default
   - sudo /etc/init.d/mysql stop
   - sudo /etc/init.d/postgresql stop
 
 before_script:
   - cd envs/dev
-  - docker-compose build
+  - docker compose build
   - docker volume create vol_smart_pgdata && docker volume create vol_smart_data
 
 script:
-  - docker-compose up -d
-  - docker-compose exec smart_frontend bash -c "./run_tests.sh"
-  - docker-compose exec backend bash -c "./run_tests.sh"
+  - docker compose up -d
+  - docker compose exec smart_frontend bash -c "./run_tests.sh"
+  - docker compose exec backend bash -c "./run_tests.sh"
 
 after_script:
-  - docker-compose down
+  - docker compose down
   - docker volume remove vol_smart_pgdata && docker volume remove vol_smart_data

README.md

@@ -28,7 +28,7 @@ This project uses `docker` containers organized by `docker-compose` to ease depe
 First, install docker and docker-compose. Then navigate to `envs/dev` and to build all the images run:
 
 ```bash
-docker-compose build
+docker compose build
 ```
 
 Next, create the docker volumes where persistent data will be stored.
@@ -41,37 +41,37 @@ docker volume create --name=vol_smart_data
 Then, migrate the database to ensure the schema is prepared for the application. 
 
 ```bash
-docker-compose run --rm backend ./migrate.sh
+docker compose run --rm backend ./migrate.sh
 ```
 
 ### Workflow During Development
 
-Run `docker-compose up` to start all docker containers.  This will start up the containers in the foreground so you can see the logs.  If you prefer to run the containers in the background use `docker-compose up -d`. When switching between branches there is no need to run any additional commands (except build if there is dependency change).
+Run `docker compose up` to start all docker containers.  This will start up the containers in the foreground so you can see the logs.  If you prefer to run the containers in the background use `docker compose up -d`. When switching between branches there is no need to run any additional commands (except build if there is dependency change).
 
 ### Dependency Changes
 
 If there is ever a dependency change than you will need to re-build the containers using the following commands:
 
 ```shell
-docker-compose build <container with new dependency>
-docker-compose rm <container with new dependency>
-docker-compose up
+docker compose build <container with new dependency>
+docker compose rm <container with new dependency>
+docker compose up
 ```
 
 If your database is blank, you will need to run migrations to initialize all the required schema objects; you can start a blank backend container and run the migration django management command with the following command:
 
 ```shell
-docker-compose run --rm backend ./migrate.sh
+docker compose run --rm backend ./migrate.sh
 ```
 
 #### Dependency management in Python
 
 We use [pip-tools](https://github.com/jazzband/pip-tools) to manage Python dependencies. To change the dependencies:
 
 1. Edit [requirements.in](./backend/docker/requirements.in) file to add, remove, or update dependencies as needed. Include only the primary dependencies—those directly required by our source code—in this file. pip-tools will automatically manage and incorporate any transitive dependencies. For routine maintenance, you may specify both primary and transitive dependencies with pinned versions to ensure consistent updates.
-1. Run `docker-compose run --rm backend pip-compile docker/requirements.in` to generate a new [requirements.txt](./backend/docker/requirements.txt). Note that pip-tools uses the existing `requirements.txt` file when building a new one, so that it can maintain existing versions. To upgrade a package to the newest version compatible with the other libraries, just remove it from the existing `requirements.txt` before running pip-compile.
+1. Run `docker compose run --rm backend pip-compile docker/requirements.in` to generate a new [requirements.txt](./backend/docker/requirements.txt). Note that pip-tools uses the existing `requirements.txt` file when building a new one, so that it can maintain existing versions. To upgrade a package to the newest version compatible with the other libraries, just remove it from the existing `requirements.txt` before running pip-compile.
     - If you encounter an error while running with docker, a possible workaround is to create a virtual environment within `backend/docker`, install `pip-tools` within that environment, and then run `pip-compile requirements.in` directly from there. This method bypasses network issues that might occur within Docker.
-1. Run `docker-compose build backend` to install the updated requirements into the Docker image.
+1. Run `docker compose build backend` to install the updated requirements into the Docker image.
 
 ### Custom Environment Variables
 
@@ -94,10 +94,10 @@ All date-times in the SMART backend and database are set to UTC (Coordinated Uni
 
 ### Running tests
 
-Backend tests use [py.test](https://docs.pytest.org/en/latest/) and [flake8](http://flake8.pycqa.org/en/latest/).  To run them, use the following `docker-compose` command from the `env/dev` directory:
+Backend tests use [py.test](https://docs.pytest.org/en/latest/) and [flake8](http://flake8.pycqa.org/en/latest/).  To run them, use the following `docker compose` command from the `env/dev` directory:
 
 ```
-docker-compose run --rm backend ./run_tests.sh <args>
+docker compose run --rm backend ./run_tests.sh <args>
 ```
 
 Where `<args>` are arguments to be passed to py.test.  Use `py.test -h` to see all the options, but a few useful ones are highlighted below:
@@ -108,10 +108,10 @@ Where `<args>` are arguments to be passed to py.test.  Use `py.test -h` to see a
  - `--reuse-db`: Don't drop/recreate the database between test runs.  This is useful for for reducing test runtime.  You must not pass this flag if the schema has changed since the last test run.
 
 
-Frontend tests use [mocha](https://mochajs.org/api/mocha.js.html) and [eslint](https://eslint.org/docs/user-guide/getting-started).  To run them, use the following `docker-compose` command from the `env/dev` directory:
+Frontend tests use [mocha](https://mochajs.org/api/mocha.js.html) and [eslint](https://eslint.org/docs/user-guide/getting-started).  To run them, use the following `docker compose` command from the `env/dev` directory:
 
 ```
-docker-compose run --rm smart_frontend ./run_tests.sh
+docker compose run --rm smart_frontend ./run_tests.sh
 ```
 
 ### Contributing

backend/django/core/forms.py

@@ -4,7 +4,7 @@
 import pandas as pd
 from django import forms
 from django.core.exceptions import ValidationError
-from django.forms.widgets import RadioSelect, Select, Textarea, TextInput
+from django.forms.widgets import RadioSelect, Select
 from pandas.errors import ParserError
 
 from core.utils.utils_external_db import (
@@ -14,9 +14,9 @@
     test_connection,
     test_login,
 )
-from core.utils.utils_form import clean_data_helper
+from core.utils.utils_form import clean_data_helper, clean_label_data_helper
 
-from .models import ExternalDatabase, Label, Project, ProjectPermissions
+from .models import ExternalDatabase, Project, ProjectPermissions
 
 
 def read_data_file(data_file):
@@ -35,6 +35,9 @@ def read_data_file(data_file):
     if data_file is None:
         raise ValidationError("ERROR: no file specified.")
 
+    if data_file.size == 0:
+        raise ValidationError("File is empty")
+
     if data_file.size > MAX_FILE_SIZE:
         raise ValidationError(
             "File is too large.  Received {0} but max size is {1}.".format(
@@ -141,45 +144,32 @@ class Meta:
         model = Project
         fields = ["allow_coders_view_labels"]
 
+    category = forms.ChoiceField(
+        required=True,
+    )
+
     def __init__(self, *args, **kwargs):
+        project = Project.objects.get(pk=kwargs.pop("project"))
+        if hasattr(project, "category"):
+            start_val = project.category.field_name
+        else:
+            start_val = "None"
+        data_metadata = project.metadatafields.values_list("field_name", flat=True)
+        label_metadata = project.labelmetadatafields.values_list(
+            "field_name", flat=True
+        )
+        options = list(set(data_metadata) & set(label_metadata)) + ["None"]
+
         percentage_irr = kwargs.pop("percentage_irr")
         super(ProjectUpdateAdvancedForm, self).__init__(*args, **kwargs)
+        self.fields["category"].choices = [(opt, opt) for opt in options]
+        self.fields["category"].initial = start_val
         if percentage_irr > 0:
             self.fields["allow_coders_view_labels"].widget.attrs[
                 "disabled"
             ] = "disabled"
 
 
-class LabelForm(forms.ModelForm):
-    class Meta:
-        model = Label
-        fields = "__all__"
-
-    name = forms.CharField(widget=TextInput(attrs={"class": "form-control"}))
-    description = forms.CharField(
-        required=False,
-        initial="",
-        widget=Textarea(attrs={"class": "form-control", "rows": "5"}),
-    )
-
-
-class LabelDescriptionForm(forms.ModelForm):
-    class Meta:
-        model = Label
-        fields = ["name", "description"]
-
-    name = forms.CharField(
-        disabled=True, widget=TextInput(attrs={"class": "form-control"})
-    )
-    description = forms.CharField(
-        required=False, widget=Textarea(attrs={"class": "form-control", "rows": "5"})
-    )
-
-    def __init__(self, *args, **kwargs):
-        self.action = kwargs.pop("action", None)
-        super(LabelDescriptionForm, self).__init__(*args, **kwargs)
-
-
 class ProjectPermissionsForm(forms.ModelForm):
     class Meta:
         model = ProjectPermissions
@@ -213,24 +203,6 @@ def __init__(self, *args, **kwargs):
         )
 
 
-LabelFormSet = forms.inlineformset_factory(
-    Project,
-    Label,
-    form=LabelForm,
-    min_num=2,
-    validate_min=True,
-    extra=0,
-    can_delete=True,
-    absolute_max=55000,
-)
-LabelDescriptionFormSet = forms.inlineformset_factory(
-    Project,
-    Label,
-    form=LabelDescriptionForm,
-    can_delete=False,
-    extra=0,
-    absolute_max=55000,
-)
 PermissionsFormSet = forms.inlineformset_factory(
     Project, ProjectPermissions, form=ProjectPermissionsForm, extra=1, can_delete=True
 )
@@ -406,6 +378,44 @@ def clean_data(self):
             return ""
 
 
+class LabelWizardForm(forms.Form):
+    label_data_file = forms.FileField(required=True)
+
+    def __init__(self, *args, **kwargs):
+        super(LabelWizardForm, self).__init__(*args, **kwargs)
+
+    def clean_label_data_file(self):
+        data = self.cleaned_data.get("label_data_file", False)
+        if data:
+            try:
+                data_file = read_data_file(data)
+                return clean_label_data_helper(data_file)
+            except pd.errors.EmptyDataError:
+                return pd.DataFrame({"Label": []})
+        else:
+            raise ValidationError("ERROR: no file provided")
+
+
+class LabelUpdateWizardForm(forms.Form):
+    data = forms.FileField()
+
+    def __init__(self, *args, **kwargs):
+        self.supplied_labels = kwargs.pop("labels", None)
+        super(LabelUpdateWizardForm, self).__init__(*args, **kwargs)
+
+    def clean(self):
+        data = self.cleaned_data.get("data", False)
+        if data:
+            try:
+                data_file = read_data_file(data)
+                existing_labels = self.supplied_labels
+                return clean_label_data_helper(data_file, existing_labels)
+            except pd.errors.EmptyDataError:
+                return pd.DataFrame({"Label": []})
+        else:
+            raise ValidationError("ERROR: no file provided")
+
+
 class ExternalDatabaseWizardForm(forms.ModelForm):
     class Meta:
         model = ExternalDatabase