From edeb84cb1e158ed0946fadb964b3218dd38f7eae Mon Sep 17 00:00:00 2001 From: Jerry Wu Date: Tue, 2 May 2023 22:44:19 +0000 Subject: [PATCH] Fix android benchmark definitions in new benchmark suites (#13348) Two fixes in the Android benchmark definitions: - Rename VMVX benchmarks to "experimental" to match the current legacy Android benchmark suite [1] - Remove CPU benchmarks on Moto Edge X30 to match the current legacy Android benchmark suite [2] --- .../benchmark_suites/iree/vmvx_benchmarks.py | 8 ++++---- .../device_specs/device_collections.py | 2 -- .../device_specs/moto_edge_x30_specs.py | 15 --------------- .../python/e2e_test_framework/unique_ids.py | 2 +- .../generated_e2e_test_iree_artifacts.cmake | 8 ++++---- 5 files changed, 9 insertions(+), 26 deletions(-) diff --git a/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py b/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py index 8977eadec348..0bfd7396067e 100644 --- a/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py @@ -21,9 +21,9 @@ class Android_VMVX_Benchmarks(object): target_backend=iree_definitions.TargetBackend.VMVX, target_architecture=common_definitions.DeviceArchitecture.VMVX_GENERIC, target_abi=iree_definitions.TargetABI.VMVX) - DEFAULT_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_VMVX_GENERIC_DEFAULTS, - tags=["default-flags"], + EXPERIMENTAL_COMPILE_CONFIG = iree_definitions.CompileConfig.build( + id=unique_ids.IREE_COMPILE_CONFIG_VMVX_GENERIC_EXPERIMENTAL, + tags=["experimental-flags"], compile_targets=[VMVX_CPU_TARGET]) def generate( @@ -34,7 +34,7 @@ def generate( gen_configs = [ iree_definitions.ModuleGenerationConfig.build( - compile_config=self.DEFAULT_COMPILE_CONFIG, + compile_config=self.EXPERIMENTAL_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in [tflite_models.MOBILENET_V2, tflite_models.MOBILENET_V3SMALL] ] diff --git a/build_tools/python/e2e_test_framework/device_specs/device_collections.py b/build_tools/python/e2e_test_framework/device_specs/device_collections.py index 9ca488631e83..9ed4629d01fb 100644 --- a/build_tools/python/e2e_test_framework/device_specs/device_collections.py +++ b/build_tools/python/e2e_test_framework/device_specs/device_collections.py @@ -54,8 +54,6 @@ def query_device_specs( pixel_6_pro_specs.BIG_CORES, pixel_6_pro_specs.GPU, # Moto Edge X30 - moto_edge_x30_specs.LITTLE_CORES, - moto_edge_x30_specs.BIG_CORES, moto_edge_x30_specs.GPU, # GCP machines gcp_specs.GCP_C2_STANDARD_16, diff --git a/build_tools/python/e2e_test_framework/device_specs/moto_edge_x30_specs.py b/build_tools/python/e2e_test_framework/device_specs/moto_edge_x30_specs.py index 7628e1869158..7202ea0131c4 100644 --- a/build_tools/python/e2e_test_framework/device_specs/moto_edge_x30_specs.py +++ b/build_tools/python/e2e_test_framework/device_specs/moto_edge_x30_specs.py @@ -7,24 +7,9 @@ from e2e_test_framework import unique_ids from e2e_test_framework.definitions import common_definitions -from e2e_test_framework.device_specs import device_parameters DEVICE_NAME = "XT2201-2" -BIG_CORES = common_definitions.DeviceSpec.build( - id=unique_ids.DEVICE_SPEC_MOBILE_MOTO_EDGE_X30 + "-big-core", - device_name=DEVICE_NAME, - architecture=common_definitions.DeviceArchitecture.ARMV9_A_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - device_parameters=[device_parameters.ARM_BIG_CORES], - tags=["big-core"]) -LITTLE_CORES = common_definitions.DeviceSpec.build( - id=unique_ids.DEVICE_SPEC_MOBILE_MOTO_EDGE_X30 + "-litte-core", - device_name=DEVICE_NAME, - architecture=common_definitions.DeviceArchitecture.ARMV9_A_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - device_parameters=[device_parameters.ARM_LITTLE_CORES], - tags=["little-core"]) GPU = common_definitions.DeviceSpec.build( id=unique_ids.DEVICE_SPEC_MOBILE_MOTO_EDGE_X30 + "-gpu", device_name=DEVICE_NAME, diff --git a/build_tools/python/e2e_test_framework/unique_ids.py b/build_tools/python/e2e_test_framework/unique_ids.py index c5abb8f15abd..34702ccd621e 100644 --- a/build_tools/python/e2e_test_framework/unique_ids.py +++ b/build_tools/python/e2e_test_framework/unique_ids.py @@ -151,7 +151,7 @@ def hash_composite_id(keys: Sequence[str]) -> str: DEVICE_SPEC_MOBILE_SM_G980F = "0247f709-a300-4d12-b986-bdb0c15c2653" # IREE benchmarks -IREE_COMPILE_CONFIG_VMVX_GENERIC_DEFAULTS = "75336abd-8108-462c-9ce3-15443e3f32f4" +IREE_COMPILE_CONFIG_VMVX_GENERIC_EXPERIMENTAL = "75336abd-8108-462c-9ce3-15443e3f32f4" IREE_COMPILE_CONFIG_LINUX_CASCADELAKE = "e7e18b0f-c72d-4f1c-89b1-5afee70df6e9" IREE_COMPILE_CONFIG_LINUX_CASCADELAKE_FUSE_PADDING = "6d0d5716-5525-44ad-b71d-8075ee1583a6" IREE_COMPILE_CONFIG_LINUX_RV64_GENERIC_DEFAULTS = "cdf579a9-5446-403b-a991-802a6c702e65" diff --git a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake index 1226a3d4ab4c..35c2878abeff 100644 --- a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake +++ b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake @@ -2754,7 +2754,7 @@ iree_bytecode_module( "--iree-hal-target-backends=vmvx" "--iree-input-type=tosa" FRIENDLY_NAME - "MobileNetV2_fp32(tflite) [vmvx-generic-vmvx-vmvx][default-flags]" + "MobileNetV2_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags]" PUBLIC ) @@ -2769,7 +2769,7 @@ iree_bytecode_module( "--iree-hal-target-backends=vmvx" "--iree-input-type=tosa" FRIENDLY_NAME - "MobileNetV3Small_fp32(tflite) [vmvx-generic-vmvx-vmvx][default-flags]" + "MobileNetV3Small_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags]" PUBLIC ) @@ -5651,7 +5651,7 @@ iree_bytecode_module( "--iree-scheduling-dump-statistics-format=json" "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_module_02b72f9538e4dfc9c789e63d722d5eab4333f3f55f8375503f433a790da119cc/scheduling_stats.json" FRIENDLY_NAME - "MobileNetV2_fp32(tflite) [vmvx-generic-vmvx-vmvx][default-flags,compile-stats]" + "MobileNetV2_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags,compile-stats]" PUBLIC ) @@ -5670,7 +5670,7 @@ iree_bytecode_module( "--iree-scheduling-dump-statistics-format=json" "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_module_e7bd41e564750501f39ac9690c18d1a2e77dc7999da710d0c0bf80751dda84a0/scheduling_stats.json" FRIENDLY_NAME - "MobileNetV3Small_fp32(tflite) [vmvx-generic-vmvx-vmvx][default-flags,compile-stats]" + "MobileNetV3Small_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags,compile-stats]" PUBLIC )