From 48ad5610504f0d276a105baf6c64c2fa8ce49f0d Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Sat, 17 Aug 2024 01:00:39 +0200 Subject: [PATCH 1/7] Remove version from docker-compose.yml --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b631fa52..f29166ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3.3' services: postgres: From d660e5a09092049b3d3b27ee4970d625d1bd1c8b Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Sat, 17 Aug 2024 01:06:16 +0200 Subject: [PATCH 2/7] Move requirements files to top level Move requirements from conda to pip. Update dependabot config. --- .github/dependabot.yml | 8 -------- Makefile | 4 ++-- .../roles/publicdb/files/requirements-conda.txt | 8 -------- provisioning/roles/publicdb/tasks/main.yml | 14 ++------------ requirements-conda.txt | 2 ++ .../files/requirements-pip.txt => requirements.txt | 9 ++++++--- 6 files changed, 12 insertions(+), 33 deletions(-) delete mode 100644 provisioning/roles/publicdb/files/requirements-conda.txt create mode 100644 requirements-conda.txt rename provisioning/roles/publicdb/files/requirements-pip.txt => requirements.txt (50%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 420e9449..4953af16 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,14 +5,6 @@ updates: schedule: interval: "weekly" - - package-ecosystem: 'pip' - directory: '/provisioning/roles/publicdb/files' - schedule: - interval: 'weekly' - ignore: - - dependency-name: 'Django' - update-types: ['version-update:semver-major'] - - package-ecosystem: 'pip' directory: '/' schedule: diff --git a/Makefile b/Makefile index 3787abb9..0625cf50 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: devinstall devinstall: pip install --upgrade --upgrade-strategy eager -r requirements-dev.txt - conda install --quiet --yes --channel conda-forge --file provisioning/roles/publicdb/files/requirements-conda.txt - pip install -r provisioning/roles/publicdb/files/requirements-pip.txt + conda install --quiet --yes --channel conda-forge --file requirements-conda.txt + pip install -r requirements.txt .PHONY: test test: coveragetests linttest doctest ansibletest diff --git a/provisioning/roles/publicdb/files/requirements-conda.txt b/provisioning/roles/publicdb/files/requirements-conda.txt deleted file mode 100644 index 0d6bf6b4..00000000 --- a/provisioning/roles/publicdb/files/requirements-conda.txt +++ /dev/null @@ -1,8 +0,0 @@ -pip -ipython==8.25.0 -numpy==1.26.4 -pytables==3.7.0 -scipy==1.10.1 -psycopg2==2.9.9 -supervisor==4.2.5 -uwsgi==2.0.20 diff --git a/provisioning/roles/publicdb/tasks/main.yml b/provisioning/roles/publicdb/tasks/main.yml index 839af0a3..870ed094 100644 --- a/provisioning/roles/publicdb/tasks/main.yml +++ b/provisioning/roles/publicdb/tasks/main.yml @@ -59,16 +59,6 @@ path: "{{ publicdb_media }}/raw_data" state: directory -- name: Copy conda requirements files - ansible.builtin.copy: - src: requirements-conda.txt - dest: "{{ publicdb_path }}requirements-conda.txt" - -- name: Copy pip requirements files - ansible.builtin.copy: - src: requirements-pip.txt - dest: "{{ publicdb_path }}requirements-pip.txt" - - name: Create conda environment ansible.builtin.shell: cmd: "{{ miniconda_path }}/bin/conda create -n publicdb_venv python={{ publicdb_python_version }} --yes" @@ -76,13 +66,13 @@ - name: Install conda packages in conda environment ansible.builtin.command: - cmd: "{{ miniconda_path }}/bin/conda install -n publicdb_venv --channel conda-forge --file {{ publicdb_path }}requirements-conda.txt --yes" + cmd: "{{ miniconda_path }}/bin/conda install -n publicdb_venv --channel conda-forge --file {{ publicdb_code }}requirements-conda.txt --yes" register: conda_result changed_when: "'Executing transaction' in conda_result.stdout" - name: Install packages in conda environment using pip ansible.builtin.pip: - requirements: "{{ publicdb_path }}requirements-pip.txt" + requirements: "{{ publicdb_code }}requirements.txt" executable: "{{ publicdb_venv }}/bin/pip" - name: Setup the publicdb database in PostgreSQL diff --git a/requirements-conda.txt b/requirements-conda.txt new file mode 100644 index 00000000..01b54950 --- /dev/null +++ b/requirements-conda.txt @@ -0,0 +1,2 @@ +pytables==3.7.0 +uwsgi==2.0.20 diff --git a/provisioning/roles/publicdb/files/requirements-pip.txt b/requirements.txt similarity index 50% rename from provisioning/roles/publicdb/files/requirements-pip.txt rename to requirements.txt index 57077ee2..2d2b9dc2 100644 --- a/provisioning/roles/publicdb/files/requirements-pip.txt +++ b/requirements.txt @@ -1,6 +1,9 @@ Django~=4.2.11 +hisparc-sapphire==3.1.0 +ipython==8.25.0 +numpy==1.26.4 +psycopg2-binary==2.9.9 python-dateutil==2.9.0.post0 - +scipy==1.10.1 sentry-sdk~=2.5.1 - -hisparc-sapphire==3.1.0 +supervisor==4.2.5 From d84d53344aff35eca4e6a8b0eef6275f2930b3b9 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Sat, 17 Aug 2024 01:13:46 +0200 Subject: [PATCH 3/7] Update locations of requirements in Dockerfile --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 851bac70..51462a82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,11 +24,11 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1 RUN conda update --yes conda # Install requirements -COPY provisioning/roles/publicdb/files/requirements-conda.txt ./ +COPY requirements-conda.txt ./ RUN conda install --yes --file requirements-conda.txt -COPY provisioning/roles/publicdb/files/requirements-pip.txt ./ -RUN pip install -r requirements-pip.txt +COPY requirements.txt ./ +RUN pip install -r requirements.txt COPY requirements-dev.txt ./ RUN pip install -r requirements-dev.txt From 041bfcfd938a1a3ad4fdfc76bb7e71328eb3d4b8 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Sat, 17 Aug 2024 01:14:06 +0200 Subject: [PATCH 4/7] Bump scipy version to 1.14.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2d2b9dc2..34fcc559 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ ipython==8.25.0 numpy==1.26.4 psycopg2-binary==2.9.9 python-dateutil==2.9.0.post0 -scipy==1.10.1 +scipy==1.14.0 sentry-sdk~=2.5.1 supervisor==4.2.5 From fba6f0ac94f3f05e3a9c653f6155420c0375e974 Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Sun, 18 Aug 2024 15:22:46 +0200 Subject: [PATCH 5/7] Install hdf5 using conda, move (py)tables to requirements for pip --- requirements-conda.txt | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-conda.txt b/requirements-conda.txt index 01b54950..f9bc7574 100644 --- a/requirements-conda.txt +++ b/requirements-conda.txt @@ -1,2 +1,2 @@ -pytables==3.7.0 +hdf5==1.14.3 uwsgi==2.0.20 diff --git a/requirements.txt b/requirements.txt index 34fcc559..23aa4092 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ python-dateutil==2.9.0.post0 scipy==1.14.0 sentry-sdk~=2.5.1 supervisor==4.2.5 +tables==3.7.0 From 207b56cca24d16dcf0b4e4f618abd6116957957b Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Sun, 18 Aug 2024 15:33:06 +0200 Subject: [PATCH 6/7] Bump used tables version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 23aa4092..a5563e48 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ python-dateutil==2.9.0.post0 scipy==1.14.0 sentry-sdk~=2.5.1 supervisor==4.2.5 -tables==3.7.0 +tables==3.10.1 From c5fe0658118c2b0a3ef1f0d21adc5ea437e7d91f Mon Sep 17 00:00:00 2001 From: Arne de Laat Date: Mon, 19 Aug 2024 23:20:23 +0200 Subject: [PATCH 7/7] Downgrade tables until issue related to copying nodes is fixed --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a5563e48..d33a89fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ python-dateutil==2.9.0.post0 scipy==1.14.0 sentry-sdk~=2.5.1 supervisor==4.2.5 -tables==3.10.1 +tables==3.9.2