Streaming datasets V2

.github/actions/isort/action.yaml

@@ -0,0 +1,27 @@
+name: ISort
+description: Sort imports
+inputs:
+  working_directory:
+    required: true
+    description: Root of the repo. Should be set to GITHUB_WORKSPACE
+  args:
+    description: ISort args
+    default: ". -c -v"
+    required: false
+runs:
+  using: composite
+  steps:
+    - name: Install ISort
+      shell: bash
+      working-directory: ${{ inputs.working_directory }}
+      run: |
+        set -ex
+        pip install isort
+    - name: Run ISort
+      shell: bash
+      working-directory: ${{ inputs.working_directory }}
+      env:
+        ISORT_ARGS: ${{ inputs.args }}
+      run: |
+        set -ex
+        isort ${ISORT_ARGS}

.github/actions/pyright/action.yaml

@@ -0,0 +1,21 @@
+name: Pyright
+inputs:
+  working_directory:
+    required: true
+    description: Directory containing the Makefile
+runs:
+  using: composite
+  steps:
+    - name: Install Pyright
+      shell: bash
+      working-directory: ${{ inputs.working_directory }}
+      run: |
+        set -ex
+        sudo npm install -g pyright
+
+    - name: Run Pyright
+      shell: bash
+      working-directory: ${{ inputs.working_directory }}
+      run: |
+        set -ex
+        pyright

.github/actions/yapf/action.yaml

@@ -0,0 +1,27 @@
+name: YAPF
+description: Format with YAPF
+inputs:
+  working_directory:
+    required: true
+    description: Root of the repo. Should be set to GITHUB_WORKSPACE
+  args:
+    description: YAPF args
+    default: "-drp ."
+    required: false
+runs:
+  using: composite
+  steps:
+    - name: Install YAPF
+      shell: bash
+      working-directory: ${{ inputs.working_directory }}
+      run: |
+        set -ex
+        pip install yapf
+    - name: Run YAPF
+      shell: bash
+      working-directory: ${{ inputs.working_directory }}
+      env:
+        YAPF_ARGS: ${{ inputs.args }}
+      run: |
+        set -ex
+        yapf ${YAPF_ARGS}

.github/workflows/formatting.yaml

@@ -0,0 +1,38 @@
+
+name: Formatting
+on:
+  pull_request: {}
+  workflow_dispatch: {}
+defaults:
+  run:
+    working-directory: .
+jobs:
+  formatting:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py', 'requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: Setup
+        run: |
+          set -ex
+          python -m pip install --upgrade pip wheel
+          python -m pip install -e .[dev]
+      - uses: ./.github/actions/yapf
+        with:
+          working_directory: ${{ github.workspace }}
+      - uses: ./.github/actions/isort
+        with:
+          working_directory: "${{ github.workspace }}"

.github/workflows/install.yaml

@@ -0,0 +1,47 @@
+name: streaming-install
+on:
+  push:
+    branches:
+      - main
+      - release/*
+  pull_request:
+    branches:
+      - main
+      - release/*
+  workflow_dispatch: {}
+jobs:
+  streaming-Pip-Install:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python_version:
+          - "3.8"
+          - "3.9"
+        install_version:
+          - ""
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python_version }}
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: Setup
+        run: |
+          set -exuo pipefail
+          python -m pip install --upgrade pip wheel
+          python -m pip install .${{ matrix.install_version }}
+      - name: Run import
+        id: tests
+        run: |
+          set -exuo pipefail
+          python -c "import streaming"

.github/workflows/pyright.yaml

@@ -0,0 +1,34 @@
+name: Pyright
+on:
+  pull_request: {}
+  workflow_dispatch: {}
+defaults:
+  run:
+    working-directory: .
+jobs:
+  pyright:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py', 'extra-requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: Setup
+        run: |
+          set -ex
+          python -m pip install --upgrade pip wheel
+          python -m pip install -e .[dev]
+      - uses: ./.github/actions/pyright
+        with:
+          working_directory: ${{ github.workspace }}

.github/workflows/pytest.yaml

@@ -0,0 +1,33 @@
+name: Pytest
+on:
+  pull_request: {}
+  workflow_dispatch: {}
+jobs:
+  pytest:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: Setup
+        run: |
+          set -ex
+          python -m pip install --upgrade pip wheel
+          python -m pip install -e .[dev]
+      - name: Run Tests
+        id: tests
+        run: |
+          set -ex
+          python3 -m pytest

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+*.swo
+*.swp

Makefile

@@ -0,0 +1,25 @@
+# several pytest settings
+PYTHON ?= python  # Python command
+PYTEST ?= pytest  # Pytest command
+PYRIGHT ?= pyright  # Pyright command. Pyright must be installed seperately -- e.g. `node install -g pyright`
+EXTRA_ARGS ?=  # extra arguments for pytest
+
+dirs := streaming tests examples
+
+# run this to autoformat your code
+style:
+	$(PYTHON) -m isort $(dirs)
+	$(PYTHON) -m yapf -rip $(dirs)
+	$(PYTHON) -m docformatter -ri $(dirs)
+
+# this only checks for style & pyright, makes no code changes
+lint:
+	$(PYTHON) -m isort -c --diff $(dirs)
+	$(PYTHON) -m yapf -dr $(dirs)
+	$(PYTHON) -m docformatter -r $(dirs)
+	$(PYRIGHT) $(dirs)
+
+test:
+	$(PYTHON) -m $(PYTEST) $(EXTRA_ARGS)
+
+.PHONY: test lint style