3D Object Detection Evaluation (#31)

* Add evaluation.

* Small fixes.

* Add unit tests + eval mask fixes.

* Update detection unit tests.

* Fix typing + update pyproject.

* Run autoflake.

* Add ROI pruning.

* Remove arg.

* Fix typo.

* Speed up argoverse maps.

* Speed up evaluation.

* Small fixes.

* Fix lint.

* Small lint fixes.

* Fix filtering.

* Small fixes.

* Fix enums.

* Remove unused lines.

* Mypy fixes.

* Fix click mypy error.

* Pytype fixes.

* Fix pytype.

* Remove pytype.

* Small typing fixes.

* Add unit tests.

* Fix typing.

* Remove click typing issue.

* Fix mypy.

* Detection eval speed up.

* Rewrite detection eval for major speedup.

* Typing fixes.

* Typing fixes.

* Switch from record arrays to numpy arrays.

* Temp changes.

* Improve readability.

* Add comments.

* Modularize evaluate.

* Additional speedups.

* Cleanup code.

* Additional speedup.

* Add roi pruning back.

* Add multiprocessing.

* Add verbosity.

* Mypy fixes.

* Update cuboid fields.

* Lint fixes.

* Fix map tutorial issues.

* Add test log.

* Revert strings.

* Remove outputs.

* Address missing detection edge cases.

* Address jhony comments.

* Update docstring.

* Clean docstrings.

* Change roi method.

* Clean up roi method.

* Update roi returns.

* Autoflake.:

* Fix lint.

* Fix lint.

* Update detection limiting logic.

* Fix indexing.

* Fix tuple return.

* Update CI.

* Add ROI unit tests.

* Remove val identity.

* Fix import.

* Remove unused import.

* Update column names.

* Update eval.py

Co-authored-by: Benjamin Wilson <benjaminrwilson@noreply.github.com>

.github/workflows/ci.yml

@@ -48,24 +48,3 @@ jobs:
           --wheel
           --outdir dist/
           .
-          
-      - name: Publish package to Test PyPI.
-        if: matrix.os == 'ubuntu-latest'
-        uses: pypa/gh-action-pypi-publish@master
-        with:
-          user: __token__
-          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
-          repository_url: https://test.pypi.org/legacy/
-          print_hash: true
-          skip_existing: true
-          verbose: true
-
-      - name: Publish package to PyPI.
-        if: matrix.os == 'ubuntu-latest' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-        uses: pypa/gh-action-pypi-publish@master
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          print_hash: true
-          skip_existing: true
-          verbose: true

noxfile.py

@@ -3,7 +3,6 @@
 """Test automation using `nox`."""
 
 from pathlib import Path
-from sys import platform
 from typing import Dict, Final, List, Union
 
 import nox

pyproject.toml

@@ -22,3 +22,17 @@ ignore_missing_imports = true
 disallow_untyped_decorators = false
 plugins = "numpy.typing.mypy_plugin"
 strict = true
+
+[tool.pyright]
+include = ["src"]
+
+reportMissingTypeStubs = false
+reportUnknownMemberType = false
+reportUntypedFunctionDecorator = false
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = "--cov-report term-missing:skip-covered --cov av2"
+testpaths = [
+    "tests",
+]

src/av2/evaluation/__init__.py

@@ -0,0 +1,3 @@
+# <Copyright 2022, Argo AI, LLC. Released under the MIT license.>
+
+"""Dataset evaluation subpackage."""

src/av2/evaluation/detection/__init__.py

@@ -0,0 +1,3 @@
+# <Copyright 2022, Argo AI, LLC. Released under the MIT license.>
+
+"""Detection evaluation subpackage."""

src/av2/evaluation/detection/constants.py

@@ -0,0 +1,111 @@
+# <Copyright 2022, Argo AI, LLC. Released under the MIT license.>
+
+"""3D object detection evaluation constants."""
+
+from enum import Enum, unique
+from typing import Final, List
+
+from av2.utils.constants import PI
+
+MAX_SCALE_ERROR: Final[float] = 1.0
+MAX_YAW_RAD_ERROR: Final[float] = PI
+
+# Higher is better.
+MIN_AP: Final[float] = 0.0
+MIN_CDS: Final[float] = 0.0
+
+# Lower is better.
+MAX_NORMALIZED_ATE: Final[float] = 1.0
+MAX_NORMALIZED_ASE: Final[float] = 1.0
+MAX_NORMALIZED_AOE: Final[float] = 1.0
+
+# Max number of boxes considered per class per scene.
+MAX_NUM_BOXES: Final[int] = 500
+
+NUM_DECIMALS: Final[int] = 3
+
+TRANSLATION_COLS: Final[List[str]] = ["tx_m", "ty_m", "tz_m"]
+DIMENSION_COLS: Final[List[str]] = ["length_m", "width_m", "height_m"]
+QUAT_WXYZ_COLS: Final[List[str]] = ["qw", "qx", "qy", "qz"]
+
+
+@unique
+class TruePositiveErrorNames(str, Enum):
+    """True positive error names."""
+
+    ATE = "ATE"
+    ASE = "ASE"
+    AOE = "AOE"
+
+
+@unique
+class MetricNames(str, Enum):
+    """Metric names."""
+
+    AP = "AP"
+    ATE = TruePositiveErrorNames.ATE.value
+    ASE = TruePositiveErrorNames.ASE.value
+    AOE = TruePositiveErrorNames.AOE.value
+    CDS = "CDS"
+
+
+@unique
+class CompetitionCategories(str, Enum):
+    """Sensor dataset annotation categories."""
+
+    ARTICULATED_BUS = "ARTICULATED_BUS"
+    BICYCLE = "BICYCLE"
+    BICYCLIST = "BICYCLIST"
+    BOLLARD = "BOLLARD"
+    BOX_TRUCK = "BOX_TRUCK"
+    BUS = "BUS"
+    CONSTRUCTION_BARREL = "CONSTRUCTION_BARREL"
+    CONSTRUCTION_CONE = "CONSTRUCTION_CONE"
+    DOG = "DOG"
+    LARGE_VEHICLE = "LARGE_VEHICLE"
+    MESSAGE_BOARD_TRAILER = "MESSAGE_BOARD_TRAILER"
+    MOBILE_PEDESTRIAN_CROSSING_SIGN = "MOBILE_PEDESTRIAN_CROSSING_SIGN"
+    MOTORCYCLE = "MOTORCYCLE"
+    MOTORCYCLIST = "MOTORCYCLIST"
+    PEDESTRIAN = "PEDESTRIAN"
+    REGULAR_VEHICLE = "REGULAR_VEHICLE"
+    SCHOOL_BUS = "SCHOOL_BUS"
+    SIGN = "SIGN"
+    STOP_SIGN = "STOP_SIGN"
+    STROLLER = "STROLLER"
+    TRUCK = "TRUCK"
+    TRUCK_CAB = "TRUCK_CAB"
+    VEHICULAR_TRAILER = "VEHICULAR_TRAILER"
+    WHEELCHAIR = "WHEELCHAIR"
+    WHEELED_DEVICE = "WHEELED_DEVICE"
+    WHEELED_RIDER = "WHEELED_RIDER"
+
+
+@unique
+class AffinityType(str, Enum):
+    """Affinity types for assigning detections to ground truth cuboids."""
+
+    CENTER = "CENTER"
+
+
+@unique
+class DistanceType(str, Enum):
+    """Distance types for computing metrics on true positive detections."""
+
+    TRANSLATION = "TRANSLATION"
+    SCALE = "SCALE"
+    ORIENTATION = "ORIENTATION"
+
+
+@unique
+class InterpType(str, Enum):
+    """Interpolation type for interpolating precision over recall samples."""
+
+    ALL = "ALL"
+
+
+@unique
+class FilterMetricType(str, Enum):
+    """Metric used to filter cuboids for evaluation."""
+
+    EUCLIDEAN = "EUCLIDEAN"