Merge branch 'master' into patch-1

.github/PULL_REQUEST_TEMPLATE/new_loader.md

@@ -11,7 +11,7 @@ Please include the following information at the top level docstring for the data
 - [ ] Mention the origin of the dataset (e.g. creator, institution)
 - [ ] Describe the type of music included in the dataset
 - [ ] Indicate any relevant papers related to the dataset
-- [ ] Include a description about how the data can be accessed and the license it uses (if applicable)
+- [ ] Include a description about how the data can be accessed and the license it uses (if applicable) 
 
 #### Dataset loaders checklist:
 
@@ -20,12 +20,14 @@ Please include the following information at the top level docstring for the data
 - [ ] Create a module in mirdata, e.g. `mirdata/my_dataset.py`
 - [ ] Create tests for your loader in `tests/datasets/`, e.g. `test_my_dataset.py`
 - [ ] Add your module to `docs/source/mirdata.rst` and `docs/source/table.rst`
+- [ ] Run `black`, `flake8` and `mypy` (see [Running your tests locally](https://mirdata.readthedocs.io/en/stable/source/contributing.html?highlight=contributing#running-your-tests-locally)).
 - [ ] Run `tests/test_full_dataset.py` on your dataset.
+- [ ] Check that codecov coverage does not decrease.
 
 If your dataset **is not fully downloadable** there are two extra steps you should follow:
 - [ ] Contacting the mirdata organizers by opening an issue or PR so we can discuss how to proceed with the closed dataset.
 - [ ] Show that the version used to create the checksum is the "canonical" one, either by getting the version from the dataset creator, or by verifying equivalence with several other copies of the dataset.
-- [ ] Make sure someone has run `pytest -s tests/test_full_dataset.py --local --dataset my_dataset` once on your dataset locally and confirmed it passes
+- [ ] Make sure someone has run `pytest -s tests/test_full_dataset.py --local --dataset my_dataset` once on your dataset locally and confirmed it passes.
 
 #### Please-do-not-edit flag
 To reduce friction, we will make commits on top of contributor's pull requests by default unless they use the `please-do-not-edit` flag. If you don't want this to happen don't forget to add the flag when you start your pull request.

.github/environment-ci.yml

@@ -0,0 +1,41 @@
+name: mirdata-dev
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  # required
+  - pip
+  - chardet>=5.0.0
+  - Deprecated>=1.2.14
+  - h5py>=3.7.0
+  - librosa>=0.10.1
+  - numpy>=1.21.6
+  - openpyxl>=3.0.10
+  - pandas>=1.3.5
+  - pyyaml>=6.0
+  - requests>=2.31.0
+  - scipy>=1.7.3
+  - tqdm>=4.66.1
+  #######
+  - libvorbis
+
+  # optional, but required for testing
+  - pytest>=7.4.3
+  - pytest-pep8>=1.0.6
+  - pytest-cov>=4.1.0
+  - pytest-mock>=3.11.1
+  - pytest-localserver>=0.8.1
+  - pytest-xdist>=3.3.1
+
+  - pip:
+    - attrs>=23.1.0
+    - black>=23.3.0
+    - coveralls>=3.3.1
+    - dali-dataset>=1.1
+    - decorator>=5.1.1
+    - future>=0.18.3
+    - jams>=0.3.4
+    - music21>=7.3.3
+    - pretty_midi>=0.2.10
+    - smart_open[all]>=6.4.0
+    - testcontainers>=3.7.1

.github/environment-lint.yml

@@ -0,0 +1,22 @@
+name: lint
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  # required.
+  - pip
+  - Deprecated >= 1.2.14
+  - flake8>=5.0.4
+  - librosa>=0.10.1
+  - mypy>=0.982
+  - numpy>=1.21.6
+  - pandas>=1.3.5
+  - pytest>=7.4.3
+  - requests>=2.31.0
+  - tqdm>=4.66.1
+  - types-decorator>=5.1.8.3
+
+  - pip:
+    - jams>=0.3.4
+    - types-Deprecated
+    - types-PyYAML

.github/workflows/ci.yml

@@ -0,0 +1,109 @@
+name: CI
+
+on: pull_request
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: True
+
+jobs:
+    test:
+        name: "Python ${{ matrix.python-version }} on ${{ matrix.os }}"
+        runs-on: ${{ matrix.os }}
+
+        strategy:
+            fail-fast: false
+            matrix:
+                include:
+                    - os: ubuntu-latest
+                      python-version: "3.7"
+                      envfile: ".github/environment-ci.yml"
+                      channel-priority: "strict"
+
+                    - os: ubuntu-latest
+                      python-version: "3.8"
+                      channel-priority: "strict"
+                      envfile: ".github/environment-ci.yml"
+
+                    - os: ubuntu-latest
+                      python-version: "3.9"
+                      channel-priority: "strict"
+                      envfile: ".github/environment-ci.yml"
+
+                    - os: ubuntu-latest
+                      python-version: "3.10"
+                      channel-priority: "strict"
+                      envfile: ".github/environment-ci.yml"
+
+                    - os: macos-latest
+                      python-version: "3.10"
+                      channel-priority: "strict"
+                      envfile: ".github/environment-ci.yml"
+
+        steps:
+        - uses: actions/checkout@v3
+          with:
+            submodules: true
+
+        - name: Install OS dependencies
+          shell: bash -l {0}
+          run: |
+            case "${{ runner.os }}" in
+            Linux)
+              sudo apt-get update -yy
+              sudo apt-get install -yy wget libsndfile-dev libsox-fmt-mp3 ffmpeg zip
+              ;; 
+            macOS)
+              brew install libsamplerate wget libsndfile sox ffmpeg zip
+              ;;
+            esac
+
+        - name: Cache conda
+          uses: actions/cache@v3
+          env:
+            CACHE_NUMBER: 2      
+          with:
+            path: ~/conda_pkgs_dir
+            key: ${{ runner.os }}-${{ matrix.python-version }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles( matrix.envfile ) }}
+            restore-keys: |
+              ${{ runner.os }}-${{ matrix.python-version }}-conda-${{ env.CACHE_NUMBER }}-
+              ${{ runner.os }}-${{ matrix.python-version }}-conda-
+              ${{ runner.os }}-conda-
+              
+        - name: Setup conda
+          uses: conda-incubator/setup-miniconda@v2
+          with:
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            use-mamba: true
+            channel-priority: ${{ matrix.channel-priority }}
+            activate-environment: test
+            auto-update-conda: false
+            python-version: ${{ matrix.python-version }}
+            use-only-tar-bz2: false
+
+        - name: Install conda dependencies
+          run: mamba env update -n test -f ${{ matrix.envfile }}
+          if: steps.cache.outputs.cache-hit != 'true'
+
+        - name: Conda info
+          shell: bash -l {0}
+          run: |
+            conda info -a
+            conda list
+
+        - name: Run pytest
+          shell: bash -l {0}
+          run: pytest -n auto -vv --cov-report term-missing --cov-report=xml --cov=mirdata tests
+
+        - name: Upload coverage to Codecov
+          uses: codecov/codecov-action@v3
+          with:
+            token: ${{ secrets.CODECOV_TOKEN }}
+            files: ./coverage.xml
+            directory: ./coverage/reports/
+            flags: unittests
+            env_vars: OS,PYTHON
+            name: codecov-umbrella
+            fail_ci_if_error: true
+            verbose: true

.github/workflows/formatting.yml

@@ -0,0 +1,19 @@
+name: Formatting
+
+on: pull_request
+
+jobs:
+  black:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: black on mirdata
+        uses: psf/black@stable
+        with:
+            options: "--check --verbose"
+            src: "mirdata"
+      - name: black on tests
+        uses: psf/black@stable
+        with:
+            options: "--check --verbose"
+            src: "tests"

.github/workflows/lint-python.yml

@@ -0,0 +1,54 @@
+name: Linting
+
+on: pull_request
+jobs:
+  lint_python:
+    name: "Lint and code analysis"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - os: ubuntu-latest
+            python-version: "3.8"
+            channel-priority: "flexible"
+            envfile: ".github/environment-lint.yml"
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Cache conda
+      uses: actions/cache@v3
+      env:
+        CACHE_NUMBER: 0
+      with:
+        path: ~/conda_pkgs_dir
+        key: ${{ runner.os }}-${{ matrix.python-version }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles( matrix.envfile ) }}
+    - name: Install conda environmnent
+      uses: conda-incubator/setup-miniconda@v2
+      with: 
+        auto-update-conda: true
+        python-version: ${{ matrix.python-version }}
+        add-pip-as-python-dependency: true
+        auto-activate-base: false
+        activate-environment: lint
+        # mamba-version: "*"
+        channel-priority: ${{ matrix.channel-priority }}
+        environment-file: ${{ matrix.envfile }}
+        use-only-tar-bz2: false
+
+    - name: Conda info
+      shell: bash -l {0}
+      run: |
+        conda info -a
+        conda list
+
+    - name: (flake8) Style check
+      shell: bash -l {0}
+      run: flake8 mirdata --count --select=E9,F63,F7,F82 --show-source --statistics
+      # Info about error codes: https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes
+      # and https://flake8.pycqa.org/en/latest/user/error-codes.html
+
+    - name: (mypy) Type check package
+      shell: bash -l {0}
+      run: python -m mypy mirdata --ignore-missing-imports --allow-subclassing-any

.readthedocs.yaml

@@ -1,13 +1,27 @@
+# .readthedocs.yml
 # Read the Docs configuration file
 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
 version: 2
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.9"
 
+# Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
   fail_on_warning: true
 
+# Build documentation with MkDocs
+#mkdocs:
+#  configuration: mkdocs.yml
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats: all
+
 # Optionally set the version of Python and requirements required to build your docs
 python:
-   version: 3.7
-   install:
-   - requirements: docs/requirements.txt
+  install:
+    - requirements: docs/requirements.txt