0.1.0 release patch (#38)

* use old stype Typer signature to account for python3.8

* update message

* use pyproject.toml instead of setup.py

* use tox for test and linter

* use build for release

* update README

.github/workflows/continuous_integration.yml

@@ -11,24 +11,35 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-      - name: Set up Python
+      - name: Set up Python 3.8
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
-      - name: Lint Check
+          python-version: '3.8'
+      - name: Install dependencies
         run: |
-          chmod +x ./.github/workflow_scripts/lint_check.sh && ./.github/workflow_scripts/lint_check.sh
+          python -m pip install --upgrade pip
+          python -m pip install tox
+      - name: Lint Check
+        run: tox -e lint
 
   run_tests:
     needs: lint_check
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-      - name: Set up Python
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
-      - name: Run Bench Tests
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Run Unit Tests
         run: |
-          chmod +x ./.github/workflow_scripts/test_bench.sh && ./.github/workflow_scripts/test_bench.sh
+          tox -e py${{ matrix.python-version }}

.github/workflows/pypi_release.yml

@@ -15,20 +15,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: '3.8'
     - name: Install dependencies
       run: |
         python3 -m pip install --upgrade pip
-        pip install setuptools wheel twine
+        pip install build twine
     - name: Build and publish
       env:
         TWINE_USERNAME: ${{ secrets. PYPI_USERNAME }}
         TWINE_PASSWORD: ${{ secrets. PYPI_PASSWORD }}
-        RELEASE: 1
       run: |
-        python3 setup.py sdist bdist_wheel
+        python -m build
         twine upload dist/* --verbose

README.md

@@ -11,9 +11,10 @@ Welcome to AutoGluon-Bench, a suite for benchmarking your AutoML frameworks.
 Follow the steps below to set up autogluon-bench:
 
 ```bash
-# create virtual env
+# create virtual env and update pip
 python3 -m venv .venv_agbench
 source .venv_agbench/bin/activate
+python3 -m pip install --upgrade pip
 ```
 
 Install `autogloun-bench` from PyPI:
@@ -22,14 +23,14 @@ Install `autogloun-bench` from PyPI:
 python3 -m pip install autogluon.bench
 ```
 
-Or install `autogluon-bench` from source:
+Install `autogluon-bench` from source for development:
 
 ```bash
 git clone https://github.com/autogluon/autogluon-bench.git
 cd autogluon-bench
 
 # install from source in editable mode
-pip install -e .
+pip install -e ".[tests]"
 ```
 
 For development, please be aware that `autogluon.bench` is installed as a dependency in certain places, such as the [Dockerfile](https://github.com/autogluon/autogluon-bench/blob/master/src/autogluon/bench/Dockerfile) and [Multimodal Setup](https://github.com/autogluon/autogluon-bench/blob/master/src/autogluon/bench/frameworks/multimodal/setup.sh). We made it possible to reflect the development changes by pushing the changes to a remote GitHub branch, and providing the URI when testing on benchmark runs:
@@ -169,6 +170,13 @@ where:
 To override these configurations, use the `cdk_context` key in your custom config file. See our [sample cloud config](https://github.com/autogluon/autogluon-bench/blob/master/sample_configs/tabular_cloud_configs.yaml) for reference.
 
 
+### Monitoring metrics for your instances on AWS
+
+A variety of metrics are available for the EC2 instances that are launched during benchmarking. These can be accessed through the AWS Console by following this navigation path: `CloudWatch` -> `All metrics` -> `AWS namespaces` -> `EC2`. For a comprehensive list of these metrics, refer to the [official AWS documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/viewing_metrics_with_cloudwatch.html).
+
+In addition to the standard metrics, we also provide a custom metric for `GPUUtilization`. This can be found in the `CloudWatch` section under `All metrics` -> `Custom namespaces` -> `EC2`. Please note that the `GPUUtilization` metric is also updated every five minutes.
+
+
 ## Evaluating bechmark runs
 
 Tabular benchmark results can be evaluated using the tools in `src/autogluon/bench/eval/`. The evaluation logic will aggregate, clean, and produce evaluation results for runs stored in S3.

pyproject.toml

@@ -1,6 +1,69 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "autogluon.bench"
+dynamic = ["version"]
+description = "A benchmarking tool for AutoML"
+authors = [{name = "AutoGluon Community"}]
+readme = "README.md"
+requires-python = ">=3.8, <3.11"
+license = {file = "LICENSE"}
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Education",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Customer Service",
+    "Intended Audience :: Financial and Insurance Industry",
+    "Intended Audience :: Healthcare Industry",
+    "Intended Audience :: Telecommunications Industry",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: MacOS",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: POSIX",
+    "Operating System :: Unix",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Topic :: Software Development",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Topic :: Scientific/Engineering :: Image Recognition"
+]
+dependencies = [
+    "autogluon.common >=0.7,<1.0",
+    "aws-cdk-lib >=2.0.0,<2.70.0",
+    "aws-cdk.aws-batch-alpha >=2.0.0a1,<2.70.0a0",
+    "awscliv2 >=2.2.0,<2.3.0",
+    "boto3 >=1.26.0,<1.26.99",
+    "constructs >=10.0.0,<10.1.289",
+    "fsspec >=2023.5.0,<=2023.6.0",
+    "matplotlib >=3.4,<3.7",
+    "pandas >=1.2.5,<2.0",
+    "pyyaml >=5.4,<=6.0",
+    "ray >=2.3.0,<2.4.0",
+    "s3fs >=2023.5.0,<=2023.6.0",
+    "typer >=0.9.0,<1.0.0",
+    "wheel >0.38.0,<=0.40.0",
+]
+
+[project.optional-dependencies]
+tests = ["pytest", "pytest-mock", "tox", "black>=23.1.9, <=23.7.0", "isort>=5.11.0, <=5.12.0"]
+
+[project.urls]
+"Homepage" = "https://github.com/autogluon/autogluon-bench"
+"Bug Reports" = "https://github.com/autogluon/autogluon-bench/issues"
+
+[project.scripts]
+agbench = "autogluon.bench.main:app"
+
+# Testing and formatting configurations
 [tool.black]
 line-length = 119
-target-version = ['py38', 'py39']
+target-version = ['py38', 'py39', 'py310']
 
 [tool.isort]
 known_first_party = "autogluon"
@@ -13,4 +76,34 @@ known_third_party = [
     "boto3"
 ]
 line_length = 119
-profile = "black"
+profile = "black"
+
+[tool.tox]
+legacy_tox_ini = """
+    [tox]
+    min_version = 4.0
+    envlist = py38, py39, py310
+    isolated_build = True
+
+    [testenv]
+    deps = 
+        pytest
+        pytest-mock
+    commands = pytest
+
+    [testenv:lint]
+    deps = 
+        black>=23.1.9, <=23.7.0
+        isort>=5.11.0, <=5.12.0
+    commands = 
+        black --check --diff src/ tests/
+        isort --check --diff src/ tests/
+"""
+
+[tool.pytest.ini_options]
+testpaths = "tests"
+addopts = "--strict-markers"
+xfail_strict = true
+
+[tool.setuptools_scm]
+write_to = "src/autogluon/bench/version.py"