Dedicated data project (#227)

* aggregating all data generation projects into one

* updating environments

* adding tests

* hanning -> hann

* getting datagen tests working

* cherry-picking in changes from old timeslide injections

* specifying torch cpu source

* updating lockfile

* updating lockfile

* adding conda prefix echo for testing

* adding conda prefix to environment

* updating export to use CPU torch

* building in separate workflow step

.github/filters.yaml

@@ -15,18 +15,12 @@ workflow: &workflow
 # if workflow file is changed?
 
 # libraries
-
 libs/io/:  
   - *workflow
   - 'libs/io/**' 
-libs/injection/:
-  - *workflow
-  - 'libs/injection/**'
-  - 'libs/base/**' 
 libs/data/:
   - *workflow
   - 'libs/data/**'
-  - 'libs/injection/**'
   - 'ml4gw/**' 
 libs/architectures/:
   - *workflow
@@ -50,7 +44,6 @@ libs/trainer/:
 
 
 # projects
-
 projects/sandbox/export/:
   - *workflow 
   - 'projects/sandbox/export/**'
@@ -61,39 +54,18 @@ projects/sandbox/export/:
   - 'ml4gw/**'
   - 'hermes/hermes/hermes.quiver/**'
 
-projects/sandbox/generate_glitches/:
+projects/sandbox/datagen/:
   - *workflow
-  - 'projects/sandbox/generate_glitches/**'
-  - 'libs/base/**'
+  - 'projects/sandbox/datagen/**'
   - 'libs/logging/**'
-
-projects/sandbox/generate_waveforms/:
-  - *workflow
-  - 'projects/sandbox/generate_waveforms/**'
-  - 'libs/logging/**'
-  - 'libs/injection/**'
-
-projects/sandbox/generate_background/:
-  - *workflow
-  - 'projects/sandbox/generate_background/**'
-  - 'libs/logging/**'
-  - 'libs/base/**'
+  - 'ml4gw/**'
 
 projects/sandbox/infer/:
   - *workflow
   - 'projects/sandbox/infer/**'
   - 'hermes/hermes/hermes.stillwater/**'
   - 'hermes/hermes/hermes.aerial/**'
 
-projects/sandbox/timeslide_injections/:
-  - *workflow
-  - 'projects/sandbox/timeslide_injections/**'
-  - 'libs/injection/**'
-  - 'libs/io/**'
-  - 'libs/logging/**' 
-  - 'libs/parallelize/**'
-  - 'ml4gw/**'
-
 projects/sandbox/train/:
   - *workflow
   - 'projects/sandbox/train/**'

.github/workflows/ci-tests.yaml

@@ -12,6 +12,8 @@ on:
       - main
 
 jobs:
+  # first job users filters to output which libraries
+  # and projects have updates that need testing
   changes:
     runs-on: ubuntu-latest
     outputs:
@@ -22,7 +24,10 @@ jobs:
       id: filter 
       with:
         filters: .github/filters.yaml
-    if: github.event.pull_request.draft == false  
+    if: github.event.pull_request.draft == false
+
+  # second job takes those outputs and runs
+  # unit tests on these libs and projects
   tests:
     runs-on: ubuntu-latest
     needs: changes
@@ -37,9 +42,6 @@ jobs:
         exclude:
           # don't run non-existent .github/workflow tests
           - library: 'workflow'
-
-
-
     permissions:
       packages: read
     container:
@@ -61,16 +63,27 @@ jobs:
         name: install gcc
         run: apt update && apt install -y build-essential
 
-
-      # run tests
-
-      - name: run tests 
+      # build the project's environment
+      -
+        name: build environment
         env:
           test_dir: /github/workspace/${{ matrix.library }}
+          CONDA_PREFIX: /opt/conda
+        # TODO: use case statement here, or add separate matrix.build_args
         run: |
             if [[ "${{matrix.library}}" == "libs/architectures/" ]]; then
-                pinto -p $test_dir build -E wrapper
-            fi    
-            pinto -p $test_dir run pytest $test_dir/tests ${{matrix.args}}
+                FLAGS="-E wrapper"
+            else
+                FLAGS=""
+            fi
+            pinto -p $test_dir build $FLAGS
+        shell: bash
+
+      # run its unit tests inside that environment
+      - 
+        name: run tests 
+        env:
+          test_dir: /github/workspace/${{ matrix.library }}
+          CONDA_PREFIX: /opt/conda
+        run: pinto -p $test_dir run pytest $test_dir/tests ${{matrix.args}}
         shell: bash
-

libs/analysis/pyproject.toml

@@ -13,8 +13,6 @@ packages = [
 python = ">=3.8,<3.11"
 scipy = "^1.7.3"
 "bbhnet.io" = {path = "../io", develop = true}
-"bbhnet.parallelize" = {path = "../parallelize", develop = true}
-"bbhnet.base" = {path = "../base", develop = true, extras = ["numpy"]}
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2"