Issue #4/#14 start with spec for benchmark scenario files

ESA-APEx · Jun 27, 2024 · b020a9e · b020a9e
1 parent f9b5b13
commit b020a9e
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 4 deletions.
diff --git a/benchmark_scenarios/README.md b/benchmark_scenarios/README.md
@@ -0,0 +1,20 @@
+# Benchmark scenarios
+
+Benchmark scenarios are defined as JSON files in this folder.
+
+> [!WARNING]
+> The format of these files is [subject to change](https://github.com/ESA-APEx/apex_algorithms/issues/14).
+
+
+Each of these files should contain an array of benchmark scenario objects,
+which have the following properties:
+
+
+
+| Element         | Type         | Description                          |
+| --------------- | ------------ | ------------------------------------ |
+| id              | string       | **REQUIRED.** Benchmark scenario identifier. |
+| type            | string       | **REQUIRED.** Must be set to "openeo" currently. |
+| description     | string       | Description of the benchmark scenario.   |
+| backend         | string       | **REQUIRED.** The openEO backend URL to be used. |
+| process_graph   | object       | **REQUIRED.** The openEO process graph to be executed. |
diff --git a/qa/benchmarks/tests/test_benchmarks.py b/qa/benchmarks/tests/test_benchmarks.py
@@ -14,12 +14,12 @@
 def test_run_benchmark(scenario: BenchmarkScenario, connection_factory):
     connection: openeo.Connection = connection_factory(url=scenario.backend)
 
-    # TODO: scenario option to use synchronous instead of batch job mode?
+    # TODO #14 scenario option to use synchronous instead of batch job mode?
     job = connection.create_job(
         process_graph=scenario.process_graph,
         title=f"APEx benchmark {scenario.id}",
     )
 
     job.start_and_wait()
 
-    # TODO download job results and inspect
+    # TODO #5 download job results and inspect
diff --git a/qa/tools/apex_algorithm_qa_tools/scenarios.py b/qa/tools/apex_algorithm_qa_tools/scenarios.py
@@ -17,15 +17,15 @@
 
 @dataclasses.dataclass(kw_only=True)
 class BenchmarkScenario:
-    # TODO: need for differentiation between different types of scenarios?
+    # TODO #14 need for differentiation between different types of scenarios?
     id: str
     description: str | None = None
     backend: str
     process_graph: dict
 
     @classmethod
     def from_dict(cls, data: dict) -> BenchmarkScenario:
-        # TODO: standardization of these types? What about other types and how to support them?
+        # TODO #14 standardization of these types? What about other types and how to support them?
         assert data["type"] == "openeo"
         return cls(
             id=data["id"],