JSON reporting

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# pytest-json-report
+.report.json

array_api_tests/stubs.py

@@ -34,6 +34,10 @@
     f for n, f in inspect.getmembers(array, predicate=inspect.isfunction)
     if n != "__init__"  # probably exists for Sphinx
 ]
+array_attributes = [
+    n for n, f in inspect.getmembers(array, predicate=lambda x: not inspect.isfunction(x))
+    if n != "__init__"  # probably exists for Sphinx
+]
 
 category_to_funcs: Dict[str, List[FunctionType]] = {}
 for name, mod in name_to_mod.items():

array_api_tests/test_has_names.py

@@ -0,0 +1,37 @@
+"""
+This is a very basic test to see what names are defined in a library. It
+does not even require functioning hypothesis array_api support.
+"""
+
+import pytest
+
+from ._array_module import mod as xp, mod_name
+from .stubs import (array_attributes, array_methods, category_to_funcs,
+                    extension_to_funcs, EXTENSIONS)
+
+has_name_params = []
+for ext, stubs in extension_to_funcs.items():
+    for stub in stubs:
+        has_name_params.append(pytest.param(ext, stub.__name__))
+for cat, stubs in category_to_funcs.items():
+    for stub in stubs:
+        has_name_params.append(pytest.param(cat, stub.__name__))
+for meth in array_methods:
+    has_name_params.append(pytest.param('array_method', meth.__name__))
+for attr in array_attributes:
+    has_name_params.append(pytest.param('array_attribute', attr))
+
+@pytest.mark.parametrize("category, name", has_name_params)
+def test_has_names(category, name):
+    if category in EXTENSIONS:
+        ext_mod = getattr(xp, category)
+        assert hasattr(ext_mod, name), f"{mod_name} is missing the {category} extension function {name}()"
+    elif category.startswith('array_'):
+        # TODO: This would fail if ones() is missing.
+        arr = xp.ones((1, 1))
+        if category == 'array_attribute':
+            assert hasattr(arr, name), f"The {mod_name} array object is missing the attribute {name}"
+        else:
+            assert hasattr(arr, name), f"The {mod_name} array object is missing the method {name}()"
+    else:
+        assert hasattr(xp, name), f"{mod_name} is missing the {category} function {name}()"

array_api_tests/test_signatures.py

@@ -32,8 +32,7 @@ def squeeze(x, /, axis):
 from . import dtype_helpers as dh
 from . import hypothesis_helpers as hh
 from . import xps
-from ._array_module import _UndefinedStub
-from ._array_module import mod as xp
+from ._array_module import _UndefinedStub, mod as xp
 from .stubs import array_methods, category_to_funcs, extension_to_funcs
 from .typing import Array, DataType
 

conftest.py

@@ -120,8 +120,15 @@ def pytest_collection_modifyitems(config, items):
                         mark.skip(reason="disabled via --disable-data-dependent-shapes")
                     )
                     break
-        # skip if test not appropiate for CI
+        # skip if test not appropriate for CI
         if ci:
             ci_mark = next((m for m in markers if m.name == "ci"), None)
             if ci_mark is None:
                 item.add_marker(mark.skip(reason="disabled via --ci"))
+
+@mark.optionalhook
+def pytest_metadata(metadata):
+    """
+    Additional metadata for --json-report.
+    """
+    metadata['array_api_tests_module'] = xp.mod_name