Fix android benchmark definitions in new benchmark suites (#13348)

Two fixes in the Android benchmark definitions: - Rename VMVX benchmarks to "experimental" to match the current legacy Android benchmark suite [1] - Remove CPU benchmarks on Moto Edge X30 to match the current legacy Android benchmark suite [2]
iree-org · May 2, 2023 · edeb84c · edeb84c
1 parent f2fc7c5
commit edeb84c
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 26 deletions.
diff --git a/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py b/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py
@@ -21,9 +21,9 @@ class Android_VMVX_Benchmarks(object):
       target_backend=iree_definitions.TargetBackend.VMVX,
       target_architecture=common_definitions.DeviceArchitecture.VMVX_GENERIC,
       target_abi=iree_definitions.TargetABI.VMVX)
-  DEFAULT_COMPILE_CONFIG = iree_definitions.CompileConfig.build(
-      id=unique_ids.IREE_COMPILE_CONFIG_VMVX_GENERIC_DEFAULTS,
-      tags=["default-flags"],
+  EXPERIMENTAL_COMPILE_CONFIG = iree_definitions.CompileConfig.build(
+      id=unique_ids.IREE_COMPILE_CONFIG_VMVX_GENERIC_EXPERIMENTAL,
+      tags=["experimental-flags"],
       compile_targets=[VMVX_CPU_TARGET])
 
   def generate(
@@ -34,7 +34,7 @@ def generate(
 
     gen_configs = [
         iree_definitions.ModuleGenerationConfig.build(
-            compile_config=self.DEFAULT_COMPILE_CONFIG,
+            compile_config=self.EXPERIMENTAL_COMPILE_CONFIG,
             imported_model=iree_definitions.ImportedModel.from_model(model)) for
         model in [tflite_models.MOBILENET_V2, tflite_models.MOBILENET_V3SMALL]
     ]

diff --git a/build_tools/python/e2e_test_framework/device_specs/device_collections.py b/build_tools/python/e2e_test_framework/device_specs/device_collections.py
@@ -54,8 +54,6 @@ def query_device_specs(
     pixel_6_pro_specs.BIG_CORES,
     pixel_6_pro_specs.GPU,
     # Moto Edge X30
-    moto_edge_x30_specs.LITTLE_CORES,
-    moto_edge_x30_specs.BIG_CORES,
     moto_edge_x30_specs.GPU,
     # GCP machines
     gcp_specs.GCP_C2_STANDARD_16,

diff --git a/build_tools/python/e2e_test_framework/device_specs/moto_edge_x30_specs.py b/build_tools/python/e2e_test_framework/device_specs/moto_edge_x30_specs.py
@@ -7,24 +7,9 @@
 
 from e2e_test_framework import unique_ids
 from e2e_test_framework.definitions import common_definitions
-from e2e_test_framework.device_specs import device_parameters
 
 DEVICE_NAME = "XT2201-2"
 
-BIG_CORES = common_definitions.DeviceSpec.build(
-    id=unique_ids.DEVICE_SPEC_MOBILE_MOTO_EDGE_X30 + "-big-core",
-    device_name=DEVICE_NAME,
-    architecture=common_definitions.DeviceArchitecture.ARMV9_A_GENERIC,
-    host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
-    device_parameters=[device_parameters.ARM_BIG_CORES],
-    tags=["big-core"])
-LITTLE_CORES = common_definitions.DeviceSpec.build(
-    id=unique_ids.DEVICE_SPEC_MOBILE_MOTO_EDGE_X30 + "-litte-core",
-    device_name=DEVICE_NAME,
-    architecture=common_definitions.DeviceArchitecture.ARMV9_A_GENERIC,
-    host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A,
-    device_parameters=[device_parameters.ARM_LITTLE_CORES],
-    tags=["little-core"])
 GPU = common_definitions.DeviceSpec.build(
     id=unique_ids.DEVICE_SPEC_MOBILE_MOTO_EDGE_X30 + "-gpu",
     device_name=DEVICE_NAME,

diff --git a/build_tools/python/e2e_test_framework/unique_ids.py b/build_tools/python/e2e_test_framework/unique_ids.py
@@ -151,7 +151,7 @@ def hash_composite_id(keys: Sequence[str]) -> str:
 DEVICE_SPEC_MOBILE_SM_G980F = "0247f709-a300-4d12-b986-bdb0c15c2653"
 
 # IREE benchmarks
-IREE_COMPILE_CONFIG_VMVX_GENERIC_DEFAULTS = "75336abd-8108-462c-9ce3-15443e3f32f4"
+IREE_COMPILE_CONFIG_VMVX_GENERIC_EXPERIMENTAL = "75336abd-8108-462c-9ce3-15443e3f32f4"
 IREE_COMPILE_CONFIG_LINUX_CASCADELAKE = "e7e18b0f-c72d-4f1c-89b1-5afee70df6e9"
 IREE_COMPILE_CONFIG_LINUX_CASCADELAKE_FUSE_PADDING = "6d0d5716-5525-44ad-b71d-8075ee1583a6"
 IREE_COMPILE_CONFIG_LINUX_RV64_GENERIC_DEFAULTS = "cdf579a9-5446-403b-a991-802a6c702e65"

diff --git a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake
@@ -2754,7 +2754,7 @@ iree_bytecode_module(
     "--iree-hal-target-backends=vmvx"
     "--iree-input-type=tosa"
   FRIENDLY_NAME
-    "MobileNetV2_fp32(tflite) [vmvx-generic-vmvx-vmvx][default-flags]"
+    "MobileNetV2_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags]"
   PUBLIC
 )
 
@@ -2769,7 +2769,7 @@ iree_bytecode_module(
     "--iree-hal-target-backends=vmvx"
     "--iree-input-type=tosa"
   FRIENDLY_NAME
-    "MobileNetV3Small_fp32(tflite) [vmvx-generic-vmvx-vmvx][default-flags]"
+    "MobileNetV3Small_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags]"
   PUBLIC
 )
 
@@ -5651,7 +5651,7 @@ iree_bytecode_module(
     "--iree-scheduling-dump-statistics-format=json"
     "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_module_02b72f9538e4dfc9c789e63d722d5eab4333f3f55f8375503f433a790da119cc/scheduling_stats.json"
   FRIENDLY_NAME
-    "MobileNetV2_fp32(tflite) [vmvx-generic-vmvx-vmvx][default-flags,compile-stats]"
+    "MobileNetV2_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags,compile-stats]"
   PUBLIC
 )
 
@@ -5670,7 +5670,7 @@ iree_bytecode_module(
     "--iree-scheduling-dump-statistics-format=json"
     "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_module_e7bd41e564750501f39ac9690c18d1a2e77dc7999da710d0c0bf80751dda84a0/scheduling_stats.json"
   FRIENDLY_NAME
-    "MobileNetV3Small_fp32(tflite) [vmvx-generic-vmvx-vmvx][default-flags,compile-stats]"
+    "MobileNetV3Small_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags,compile-stats]"
   PUBLIC
 )