Merge branch 'main' into debug_dandi

NeurodataWithoutBorders · Aug 31, 2024 · 17ec80f · 17ec80f
2 parents 68b52cf + 01908f7
commit 17ec80f
Show file tree

Hide file tree

Showing 75 changed files with 755 additions and 2,074 deletions.
diff --git a/.github/workflows/daily_tests.yml b/.github/workflows/daily_tests.yml
@@ -20,6 +20,11 @@ jobs:
     uses: ./.github/workflows/testing_dev_with_live_services.yml
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  E2ELiveServices:
+    uses: ./.github/workflows/testing_dev_e2e_with_live_services.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       DANDI_STAGING_API_KEY: ${{ secrets.DANDI_STAGING_API_KEY }}
 
   BuildTests:
@@ -38,7 +43,7 @@ jobs:
 
   NotifyOnAnyFailure:
     runs-on: ubuntu-latest
-    needs: [DevTests, LiveServices, BuildTests, ExampleDataCache, ExampleDataTests]
+    needs: [DevTests, LiveServices, E2ELiveServices, BuildTests, ExampleDataCache, ExampleDataTests]
     if: failure()
     steps:
       - uses: dawidd6/action-send-mail@v3
@@ -48,6 +53,6 @@ jobs:
           username: ${{ secrets.MAIL_USERNAME }}
           password: ${{ secrets.MAIL_PASSWORD }}
           subject: NWB GUIDE Daily Test Failure
-          to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }}
+          to: cody.c.baker.phd@gmail.com,rly@lbl.gov
           from: NWB Guide Daily Tests
           body: "The daily tests workflow failed, please check the status at https://github.com/NeurodataWithoutBorders/nwb-guide/actions/workflows/daily_tests.yml"
diff --git a/.github/workflows/deploy_tests_on_pull_request.yml b/.github/workflows/deploy_tests_on_pull_request.yml
@@ -18,6 +18,11 @@ jobs:
     uses: ./.github/workflows/testing_dev_with_live_services.yml
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  E2ELiveServices:
+    uses: ./.github/workflows/testing_dev_e2e_with_live_services.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       DANDI_STAGING_API_KEY: ${{ secrets.DANDI_STAGING_API_KEY }}
 
   BuildTests:

diff --git a/.github/workflows/example_data_cache.yml b/.github/workflows/example_data_cache.yml
@@ -21,7 +21,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.12"]
-        os: [ubuntu-latest, macos-latest, macos-13, windows-latest]
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
 
     steps:
 

diff --git a/.github/workflows/testing_dev.yml b/.github/workflows/testing_dev.yml
@@ -29,8 +29,8 @@ jobs:
           - os: macos-13  # Mac x64 runner
             label: environments/environment-MAC-intel.yml
 
-#          - os: windows-latest
-#            label: environments/environment-Windows.yml
+          - os: windows-latest
+            label: environments/environment-Windows.yml
 
     steps:
       - uses: actions/checkout@v4
@@ -69,6 +69,12 @@ jobs:
       - name: Install GUIDE
         run: npm ci --verbose
 
+      - name: Install testing dependencies
+        run: pip install pytest pytest-cov
+
+      - name: Manually remove matplotlib
+        run: pip uninstall matplotlib --yes
+
       - if: matrix.os != 'ubuntu-latest'
         name: Run tests
         run: npm run test:coverage
@@ -77,16 +83,6 @@ jobs:
         name: Run tests with xvfb
         run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test:coverage
 
-      - name: Archive E2E Test Screenshots
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-screenshots-${{ matrix.os }}
-          path: docs/assets/tutorials
-          retention-days: 1
-          overwrite: true
-
-
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4
         env:

diff --git a/.github/workflows/testing_dev_e2e_with_live_services.yml b/.github/workflows/testing_dev_e2e_with_live_services.yml
@@ -0,0 +1,106 @@
+name: End-to-end tests with live services
+on:
+  workflow_call:
+    secrets:
+      CODECOV_TOKEN:
+        required: true
+      DANDI_STAGING_API_KEY:
+        required: true
+
+jobs:
+
+  run:
+    # Will read on PR dashboard as 'Deploy / E2ELiveServices / {os}'
+    # Action dashboard identified by 'End-to-end tests with live services'
+    # Requirement settings identified as 'E2ELiveServices / {os}'
+    name: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            label: environments/environment-Linux.yml
+
+          - os: macos-latest  # Mac arm64 runner
+            label: environments/environment-MAC-apple-silicon.yml
+
+          - os: macos-13  # Mac x64 runner
+            label: environments/environment-MAC-intel.yml
+
+          - os: windows-latest
+            label: environments/environment-Windows.yml
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: git fetch --prune --unshallow --tags
+
+      # see https://github.com/conda-incubator/setup-miniconda#caching-environments
+      - name: Setup Mambaforge
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          activate-environment: nwb-guide
+          use-mamba: true
+
+      - name: Set cache date
+        id: get-date
+        run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Cache Conda env
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.CONDA }}/envs
+          key: conda-${{ runner.os }}-${{ runner.arch }}-${{steps.get-date.outputs.today }}-${{ hashFiles(matrix.label) }}-${{ env.CACHE_NUMBER }}
+        id: cache
+
+      - if: steps.cache.outputs.cache-hit != 'true'
+        name: Create and activate environment
+        run: mamba env update -n nwb-guide -f ${{ matrix.label }}
+
+      - name: Use Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install GUIDE
+        run: npm ci --verbose
+
+      - name: Install testing dependencies
+        run: pip install pytest pytest-cov
+
+      - name: Manually remove matplotlib
+        run: pip uninstall matplotlib --yes
+
+      - name: Create env file
+        run: |
+          touch .env
+          echo DANDI_STAGING_API_KEY=${{ secrets.DANDI_STAGING_API_KEY }} >> .env
+
+      - if: matrix.os != 'ubuntu-latest'
+        name: Run tests
+        run: npm run coverage:tutorial
+
+      - if: matrix.os == 'ubuntu-latest'
+        name: Run tests with xvfb
+        run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run coverage:tutorial
+
+      - name: Archive E2E Test Screenshots
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-screenshots-${{ matrix.os }}
+          path: docs/assets/tutorials
+          retention-days: 1
+          overwrite: true
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          fail_ci_if_error: true
diff --git a/.github/workflows/testing_dev_with_live_services.yml b/.github/workflows/testing_dev_with_live_services.yml
@@ -4,8 +4,6 @@ on:
     secrets:
       CODECOV_TOKEN:
         required: true
-      DANDI_STAGING_API_KEY:
-        required: true
 
 jobs:
 
@@ -31,8 +29,8 @@ jobs:
           - os: macos-13  # Mac x64 runner
             label: environments/environment-MAC-intel.yml
 
-#          - os: windows-latest
-#            label: environments/environment-Windows.yml
+          - os: windows-latest
+            label: environments/environment-Windows.yml
 
     steps:
       - uses: actions/checkout@v4
@@ -71,10 +69,11 @@ jobs:
       - name: Install GUIDE
         run: npm ci --verbose
 
-      - name: Create env file
-        run: |
-          touch .env
-          echo DANDI_STAGING_API_KEY=${{ secrets.DANDI_STAGING_API_KEY }} >> .env
+      - name: Install testing dependencies
+        run: pip install pytest pytest-cov
+
+      - name: Manually remove matplotlib
+        run: pip uninstall matplotlib --yes
 
       - if: matrix.os != 'ubuntu-latest'
         name: Run tests

diff --git a/.github/workflows/testing_flask_build_and_dist.yml b/.github/workflows/testing_flask_build_and_dist.yml
@@ -17,26 +17,16 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # linux installation instructions use dev mode instead of distributable
-          # - python-version: "3.9"
-          #   os: ubuntu-latest
-          #   label: environments/environment-Linux.yml
-          #   prefix: /usr/share/miniconda3/envs/nwb-guide
-
-          - python-version: "3.9"
-            os: macos-latest  # Mac arm64 runner
+          # No linux in this matrix since installation instructions use dev mode instead of distributable
+
+          - os: macos-latest  # Mac arm64 runner
             label: environments/environment-MAC-apple-silicon.yml
-            prefix: /Users/runner/miniconda3/envs/nwb-guide
 
-          - python-version: "3.9"
-            os: macos-13  # Mac x64 runner
+          - os: macos-13  # Mac x64 runner
             label: environments/environment-MAC-intel.yml
-            prefix: /Users/runner/miniconda3/envs/nwb-guide
 
-          - python-version: "3.9"
-            os: windows-latest
+          - os: windows-latest
             label: environments/environment-Windows.yml
-            prefix: C:\Miniconda3\envs\nwb-guide
 
     steps:
       - uses: actions/checkout@v4
@@ -78,7 +68,14 @@ jobs:
         with:
           node-version: "20"
 
-      - run: npm ci --verbose
+      - name: Install GUIDE
+        run: npm ci --verbose
+
+      - name: Install testing dependencies
+        run: pip install pytest pytest-cov
+
+      - name: Manually remove matplotlib
+        run: pip uninstall matplotlib --yes
 
       # Fix for macos build - remove bad sonpy file
       - if: matrix.os == 'macos-latest' || matrix.os == 'macos-13'

diff --git a/.github/workflows/testing_pipelines.yml b/.github/workflows/testing_pipelines.yml
@@ -30,7 +30,6 @@ jobs:
           - os: windows-latest
             label: environments/environment-Windows.yml
 
-
     steps:
       - uses: actions/checkout@v4
       - run: git fetch --prune --unshallow --tags
@@ -67,6 +66,12 @@ jobs:
       - name: Install GUIDE
         run: npm ci --verbose
 
+      - name: Install testing dependencies
+        run: pip install pytest pytest-cov
+
+      - name: Manually remove matplotlib
+        run: pip uninstall matplotlib --yes
+
       # Load example data caches
       - name: Get ephy_testing_data current head hash
         id: ephys

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,7 +8,7 @@ repos:
     -   id: trailing-whitespace
 
 -   repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 24.8.0
     hooks:
     -   id: black
         exclude: ^docs/

diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css
@@ -11,7 +11,7 @@ button.btn.version-switcher__button {
 .version-switcher__container.dropdown:hover .dropdown-menu {
     display: block;
     left: 0;
-    margin-top: var(--bs-dropdown-spacer);
+    margin-top: 0 !important;
     top: 100%;
 }
 

diff --git a/docs/_static/switcher.json b/docs/_static/switcher.json
@@ -4,11 +4,23 @@
         "url": "https://nwb-guide.readthedocs.io/en/latest/",
         "preferred": false
     },
+    {
+        "name": "1.0.3",
+        "version": "v1.0.3",
+        "url": "https://nwb-guide.readthedocs.io/en/v1.0.3/",
+        "preferred": true
+    },
+    {
+        "name": "1.0.2",
+        "version": "v1.0.2",
+        "url": "https://nwb-guide.readthedocs.io/en/v1.0.2/",
+        "preferred": false
+    },
     {
         "name": "1.0.1",
         "version": "v1.0.1",
         "url": "https://nwb-guide.readthedocs.io/en/v1.0.1/",
-        "preferred": true
+        "preferred": false
     },
     {
         "name": "1.0.0",

diff --git a/docs/assets/tutorials/dandi/create-dandiset.png b/docs/assets/tutorials/dandi/create-dandiset.png
diff --git a/docs/assets/tutorials/dandi/dandiset-id.png b/docs/assets/tutorials/dandi/dandiset-id.png
diff --git a/docs/assets/tutorials/dandi/review-page.png b/docs/assets/tutorials/dandi/review-page.png
diff --git a/docs/assets/tutorials/dataset-created.png b/docs/assets/tutorials/dataset-created.png
diff --git a/docs/assets/tutorials/dataset-creation.png b/docs/assets/tutorials/dataset-creation.png
diff --git a/docs/assets/tutorials/home-page.png b/docs/assets/tutorials/home-page.png
diff --git a/docs/assets/tutorials/multiple/conversion-results-page.png b/docs/assets/tutorials/multiple/conversion-results-page.png
diff --git a/docs/assets/tutorials/multiple/fail-name.png b/docs/assets/tutorials/multiple/fail-name.png
diff --git a/docs/assets/tutorials/multiple/home-page-complete.png b/docs/assets/tutorials/multiple/home-page-complete.png
diff --git a/docs/assets/tutorials/multiple/inspect-page.png b/docs/assets/tutorials/multiple/inspect-page.png
diff --git a/docs/assets/tutorials/multiple/metadata-ecephys.png b/docs/assets/tutorials/multiple/metadata-ecephys.png
diff --git a/docs/assets/tutorials/multiple/metadata-nwbfile.png b/docs/assets/tutorials/multiple/metadata-nwbfile.png
diff --git a/docs/assets/tutorials/multiple/pathexpansion-autocomplete-filled.png b/docs/assets/tutorials/multiple/pathexpansion-autocomplete-filled.png
diff --git a/docs/assets/tutorials/multiple/pathexpansion-autocomplete-open.png b/docs/assets/tutorials/multiple/pathexpansion-autocomplete-open.png
diff --git a/docs/assets/tutorials/multiple/pathexpansion-autocomplete-submitted.png b/docs/assets/tutorials/multiple/pathexpansion-autocomplete-submitted.png
diff --git a/docs/assets/tutorials/multiple/pathexpansion-completed.png b/docs/assets/tutorials/multiple/pathexpansion-completed.png
diff --git a/docs/assets/tutorials/multiple/pathexpansion-page.png b/docs/assets/tutorials/multiple/pathexpansion-page.png
diff --git a/docs/assets/tutorials/multiple/preview-page.png b/docs/assets/tutorials/multiple/preview-page.png
diff --git a/docs/assets/tutorials/multiple/subject-complete.png b/docs/assets/tutorials/multiple/subject-complete.png
diff --git a/docs/assets/tutorials/multiple/subject-error.png b/docs/assets/tutorials/multiple/subject-error.png
diff --git a/docs/assets/tutorials/multiple/subject-invalid.png b/docs/assets/tutorials/multiple/subject-invalid.png
diff --git a/docs/assets/tutorials/multiple/subject-page.png b/docs/assets/tutorials/multiple/subject-page.png
diff --git a/docs/assets/tutorials/multiple/workflow-page.png b/docs/assets/tutorials/multiple/workflow-page.png
diff --git a/docs/assets/tutorials/single/conversion-results-page.png b/docs/assets/tutorials/single/conversion-results-page.png
diff --git a/docs/assets/tutorials/single/home-page-complete.png b/docs/assets/tutorials/single/home-page-complete.png
diff --git a/docs/assets/tutorials/single/inspect-page.png b/docs/assets/tutorials/single/inspect-page.png
diff --git a/docs/assets/tutorials/single/metadata-ecephys.png b/docs/assets/tutorials/single/metadata-ecephys.png
diff --git a/docs/assets/tutorials/single/metadata-nwbfile.png b/docs/assets/tutorials/single/metadata-nwbfile.png
diff --git a/docs/assets/tutorials/single/preview-page.png b/docs/assets/tutorials/single/preview-page.png
diff --git a/docs/assets/tutorials/single/sourcedata-page-specified.png b/docs/assets/tutorials/single/sourcedata-page-specified.png