From 28d0cd3aa19c0e76eeea00dcdd4cae8bc637336d Mon Sep 17 00:00:00 2001 From: Chris Perkins Date: Thu, 6 Oct 2022 10:19:00 -0700 Subject: [PATCH] [SYCL] Add ONEAPI_DEVICE_SELECTOR implementation (#6779) Functionally complete. Needs tests and possible reorganization. Signed-off-by: Chris Perkins --- sycl/doc/EnvironmentVariables.md | 54 ++++- sycl/include/sycl/detail/device_filter.hpp | 60 ++++- sycl/source/detail/config.cpp | 11 +- sycl/source/detail/config.def | 1 + sycl/source/detail/config.hpp | 36 ++- sycl/source/detail/device_filter.cpp | 242 +++++++++++++++++-- sycl/source/detail/filter_selector_impl.cpp | 36 ++- sycl/source/detail/global_handler.cpp | 5 + sycl/source/detail/global_handler.hpp | 3 + sycl/source/detail/pi.cpp | 2 +- sycl/source/detail/platform_impl.cpp | 243 +++++++++++++++++--- sycl/source/device.cpp | 15 +- sycl/tools/sycl-ls/sycl-ls.cpp | 16 ++ sycl/unittests/allowlist/ParseAllowList.cpp | 9 +- 14 files changed, 639 insertions(+), 94 deletions(-) diff --git a/sycl/doc/EnvironmentVariables.md b/sycl/doc/EnvironmentVariables.md index 64012b8dfd3ba..11c591d997450 100644 --- a/sycl/doc/EnvironmentVariables.md +++ b/sycl/doc/EnvironmentVariables.md @@ -7,8 +7,9 @@ compiler and runtime. | Environment variable | Values | Description | | -------------------- | ------ | ----------- | -| `SYCL_BE` (deprecated) | `PI_OPENCL`, `PI_LEVEL_ZERO`, `PI_CUDA` | Force SYCL RT to consider only devices of the specified backend during the device selection. We are planning to deprecate `SYCL_BE` environment variable in the future. The specific grace period is not decided yet. Please use the new env var `SYCL_DEVICE_FILTER` instead. | -| `SYCL_DEVICE_TYPE` (deprecated) | CPU, GPU, ACC, HOST | Force SYCL to use the specified device type. If unset, default selection rules are applied. If set to any unlisted value, this control has no effect. If the requested device type is not found, a `sycl::runtime_error` exception is thrown. If a non-default device selector is used, a device must satisfy both the selector and this control to be chosen. This control only has effect on devices created with a selector. We are planning to deprecate `SYCL_DEVICE_TYPE` environment variable in the future. The specific grace period is not decided yet. Please use the new env var `SYCL_DEVICE_FILTER` instead. | +| `ONEAPI_DEVICE_SELECTOR` | [See below.](#oneapi_device_selector) | This device selection environment variable can be used to limit the choice of devices available when the SYCL-using application is run. Useful for limiting devices to a certain type (like GPUs or accelerators) or backends (like Level Zero or OpenCL). This device selection mechanism is replacing `SYCL_DEVICE_FILTER` . The `ONEAPI_DEVICE_SELECTOR` syntax is shared with OpenMP and also allows sub-devices to be chosen. [See below.](#oneapi_device_selector) for a full description. | +| `SYCL_BE` (deprecated) | `PI_OPENCL`, `PI_LEVEL_ZERO`, `PI_CUDA` | Force SYCL RT to consider only devices of the specified backend during the device selection. The `SYCL_BE` environment variable is deprecated and will be removed soon. Please use the new env var `ONEAPI_DEVICE_SELECTOR` instead. | +| `SYCL_DEVICE_TYPE` (deprecated) | CPU, GPU, ACC, HOST | Force SYCL to use the specified device type. If unset, default selection rules are applied. If set to any unlisted value, this control has no effect. If the requested device type is not found, a `sycl::runtime_error` exception is thrown. If a non-default device selector is used, a device must satisfy both the selector and this control to be chosen. This control only has effect on devices created with a selector. The `SYCL_DEVICE_TYPE` environment variable is deprecated and will be removed soon. Please use the new env var `ONEAPI_DEVICE_SELECTOR` instead. | | `SYCL_DEVICE_FILTER` | `backend:device_type:device_num` | See Section [`SYCL_DEVICE_FILTER`](#sycl_device_filter) below. | | `SYCL_DEVICE_ALLOWLIST` | See [below](#sycl_device_allowlist) | Filter out devices that do not match the pattern specified. `BackendName` accepts `host`, `opencl`, `level_zero` or `cuda`. `DeviceType` accepts `host`, `cpu`, `gpu` or `acc`. `DeviceVendorId` accepts uint32_t in hex form (`0xXYZW`). `DriverVersion`, `PlatformVersion`, `DeviceName` and `PlatformName` accept regular expression. Special characters, such as parenthesis, must be escaped. DPC++ runtime will select only those devices which satisfy provided values above and regex. More than one device can be specified using the piping symbol "\|".| | `SYCL_DISABLE_PARALLEL_FOR_RANGE_ROUNDING` | Any(\*) | Disables automatic rounding-up of `parallel_for` invocation ranges. | @@ -28,6 +29,55 @@ compiler and runtime. `(*) Note: Any means this environment variable is effective when set to any non-null value.` +### `ONEAPI_DEVICE_SELECTOR` + +With no environment variables set to say otherwise, all platforms and devices presently on the machine are available. The default choice will be one of these devices, usually preferring a Level Zero GPU device, if available. The `ONEAPI_DEVICE_SELECTOR` can be used to limit that choice of devices, and to expose GPU sub-devices or sub-sub-devices as individual devices. + +The syntax of this environment variable follows this BNF grammar: +``` +ONEAPI_DEVICE_SELECTOR = + ::= [;...] + ::= : + ::= { * | level_zero | opencl | cuda | hip | esimd_emulator } // case insensitive + ::= [,...] + ::= { * | cpu | gpu | fpga | | . | .* | *.* | .. | ..* | .*.* | *.*.* } // case insensitive +``` + +Each term in the grammar selects a collection of devices from a particular backend. The device names cpu, gpu, and fpga select all devices from that backend with the corresponding type. A backend's device can also be selected by its numeric index (zero-based) or by using `*` which selects all devices in the backend. + +The dot syntax (e.g. `.`) causes one or more GPU sub-devices to be exposed to the application as SYCL root devices. For example, `1.0` exposes the first sub-device of the second device as a SYCL root device. The syntax `.*` exposes all sub-devices of the give device as SYCL root devices. The syntax `*.*` exposes all sub-devices of all GPU devices as SYCL root devices. + +In general, a term with one or more asterisks ( `*` ) matches all backends, devices, or sub-devices with the given pattern. However, a warning is generated if the term does not match anything. For example, `*:gpu` matches all GPU devices in all backends (ignoring backends with no GPU devices), but it generates a warning if there are no GPU devices in any backend. Likewise, `level_zero:*.*` matches all sub-devices of partitionable GPUs in the Level Zero backend, but it generates a warning if there are no Level Zero GPU devices that are partitionable into sub-devices. + +The device indices are zero-based and are unique only within a backend. Therefore, `level_zero:0` is a different device from `cuda:0`. To see the indices of all available devices, run the `sycl-ls` tool. Note that different backends sometimes expose the same hardware as different "devices". For example, the level_zero and opencl backends both expose the Intel GPU devices. + + +Additionally, if a sub-device is chosen (via numeric index or wildcard), then an additional layer of partitioning can be specified. In other words, a sub-sub-device can be selected. Like sub-devices, this is done with a period ( `.` ) and a sub-sub-device specifier which is a wildcard symbol ( `*` ) or a numeric index. Example `ONEAPI_DEVICE_SELECTOR=level_zero:0.*.*` would partition device 0 into sub-devices and then partition each of those into sub-sub-devices. The range of grandchild sub-sub-devices would be the final devices available to the app, neither device 0, nor its child partitions would be in that list. + + +The following examples further illustrate the usage of this environment variable: + +| Example | Result | +-----------|--------- +| `ONEAPI_DEVICE_SELECTOR=opencl:*` | Only the OpenCL devices are available | +| `ONEAPI_DEVICE_SELECTOR=level_zero:gpu` | Only GPU devices on the Level Zero platform are available.| +| `ONEAPI_DEVICE_SELECTOR="opencl:gpu;level_zero:gpu"` | GPU devices from both Level Zero and OpenCL are available. Note that escaping (like quotation marks) will likely be needed when using semi-colon separated entries. | +| `ONEAPI_DEVICE_SELECTOR=opencl:gpu,cpu` | Only CPU and GPU devices on the OpenCL platform are available.| +| `ONEAPI_DEVICE_SELECTOR=opencl:0` | Only the device with index 0 on the OpenCL backend is available. | +| `ONEAPI_DEVICE_SELECTOR=hip:0,2` | Only devices with indices of 0 and 2 from the HIP backend are available. | +| `ONEAPI_DEVICE_SELECTOR=opencl:0.*` | All the sub-devices from the OpenCL device with index 0 are exposed as SYCL root devices. No other devices are available. | +| `ONEAPI_DEVICE_SELECTOR=opencl:0.2` | The third sub-device (2 in zero-based counting) of the OpenCL device with index 0 will be the sole device available. | +| `ONEAPI_DEVICE_SELECTOR=level_zero:*,*.*` | Exposes Level Zero devices to the application in two different ways. Each device (aka "card") is exposed as a SYCL root device and each sub-device is also exposed as a SYCL root device.| + + +Notes: +- The backend argument is always required. An error will be thrown if it is absent. +- Additionally, the backend MUST be followed by colon ( `:` ) and at least one device specifier of some sort, else an error is thrown. +- For sub-devices and sub-sub-devices, the parent device must support partitioning (`info::partition_property::partition_by_affinity_domain` and `info::partition_affinity_domain::next_partitionable`. See the SYCL 2020 specification for a precise definition.) For Intel GPUs, the sub-device and sub-sub-device syntax can be used to expose tiles or CCSs to the SYCL application as root devices. The exact mapping between sub-device, sub-sub-device, tiles, and CCSs is specific to the hardware. +- The semi-colon character ( `;` ) is treated specially by many shells, so you may need to enclose the string in quotes if the selection string contains this character. + + + ### `SYCL_DEVICE_ALLOWLIST` A list of devices and their driver version following the pattern: diff --git a/sycl/include/sycl/detail/device_filter.hpp b/sycl/include/sycl/detail/device_filter.hpp index 903df99c76414..5cc3533baccf4 100644 --- a/sycl/include/sycl/detail/device_filter.hpp +++ b/sycl/include/sycl/detail/device_filter.hpp @@ -13,19 +13,63 @@ #include #include +#include #include namespace sycl { __SYCL_INLINE_VER_NAMESPACE(_V1) { namespace detail { +// --------------------------------------- +// ONEAPI_DEVICE_SELECTOR support + +template +std::ostream &operator<<(std::ostream &os, std::optional const &opt) { + return opt ? os << opt.value() : os << "not set "; +} + +// the ONEAPI_DEVICE_SELECTOR string gets broken down into these targets +// will will match devices. +struct ods_target { +public: + std::optional Backend; + std::optional DeviceType; + + bool HasDeviceWildCard = false; + std::optional DeviceNum; + + bool HasSubDeviceWildCard = false; + std::optional SubDeviceNum; + + bool HasSubSubDeviceWildCard = false; // two levels of sub-devices. + std::optional SubSubDeviceNum; + + ods_target(backend be) { Backend = be; }; + ods_target(){}; + friend std::ostream &operator<<(std::ostream &Out, const ods_target &Target); +}; + +class ods_target_list { + std::vector TargetList; + +public: + ods_target_list() {} + ods_target_list(const std::string &FilterString); + std::vector &get() { return TargetList; } + bool containsHost(); + bool backendCompatible(backend Backend); +}; + +std::ostream &operator<<(std::ostream &Out, const ods_target &Target); +std::vector Parse_ONEAPI_DEVICE_SELECTOR(const std::string &envStr); + +// --------------------------------------- +// SYCL_DEVICE_FILTER support + struct device_filter { - backend Backend = backend::all; - info::device_type DeviceType = info::device_type::all; - int DeviceNum = 0; - bool HasBackend = false; - bool HasDeviceType = false; - bool HasDeviceNum = false; + std::optional Backend; + std::optional DeviceType; + std::optional DeviceNum; int MatchesSeen = 0; device_filter(){}; @@ -66,8 +110,8 @@ inline std::ostream &operator<<(std::ostream &Out, } else { Out << "unknown"; } - if (Filter.HasDeviceNum) { - Out << ":" << Filter.DeviceNum; + if (Filter.DeviceNum) { + Out << ":" << Filter.DeviceNum.value(); } return Out; } diff --git a/sycl/source/detail/config.cpp b/sycl/source/detail/config.cpp index 931f69ec6ae32..bd32be2988280 100644 --- a/sycl/source/detail/config.cpp +++ b/sycl/source/detail/config.cpp @@ -156,19 +156,22 @@ void dumpConfig() { #undef CONFIG } -// Array is used by SYCL_DEVICE_FILTER and SYCL_DEVICE_ALLOWLIST -const std::array, 5> & +// Array is used by SYCL_DEVICE_FILTER and SYCL_DEVICE_ALLOWLIST and +// ONEAPI_DEVICE_SELECTOR +const std::array, 6> & getSyclDeviceTypeMap() { - static const std::array, 5> + static const std::array, 6> SyclDeviceTypeMap = {{{"host", info::device_type::host}, {"cpu", info::device_type::cpu}, {"gpu", info::device_type::gpu}, {"acc", info::device_type::accelerator}, + {"fpga", info::device_type::accelerator}, {"*", info::device_type::all}}}; return SyclDeviceTypeMap; } -// Array is used by SYCL_DEVICE_FILTER and SYCL_DEVICE_ALLOWLIST +// Array is used by SYCL_DEVICE_FILTER and SYCL_DEVICE_ALLOWLIST and +// ONEAPI_DEVICE_SELECTOR const std::array, 7> &getSyclBeMap() { static const std::array, 7> SyclBeMap = { {{"host", backend::host}, diff --git a/sycl/source/detail/config.def b/sycl/source/detail/config.def index 923db8c49ad58..2a0d1b4a8c167 100644 --- a/sycl/source/detail/config.def +++ b/sycl/source/detail/config.def @@ -39,3 +39,4 @@ CONFIG(SYCL_ENABLE_DEFAULT_CONTEXTS, 1, __SYCL_ENABLE_DEFAULT_CONTEXTS) CONFIG(SYCL_QUEUE_THREAD_POOL_SIZE, 4, __SYCL_QUEUE_THREAD_POOL_SIZE) CONFIG(SYCL_RT_WARNING_LEVEL, 4, __SYCL_RT_WARNING_LEVEL) CONFIG(SYCL_REDUCTION_PREFERRED_WORKGROUP_SIZE, 16, __SYCL_REDUCTION_PREFERRED_WORKGROUP_SIZE) +CONFIG(ONEAPI_DEVICE_SELECTOR, 1024, __ONEAPI_DEVICE_SELECTOR) diff --git a/sycl/source/detail/config.hpp b/sycl/source/detail/config.hpp index 231901ca39977..7993b875dc49a 100644 --- a/sycl/source/detail/config.hpp +++ b/sycl/source/detail/config.hpp @@ -270,13 +270,43 @@ template <> class SYCLConfig { } }; -// Array is used by SYCL_DEVICE_FILTER and SYCL_DEVICE_ALLOWLIST -const std::array, 5> & +// Array is used by SYCL_DEVICE_FILTER and SYCL_DEVICE_ALLOWLIST and +// ONEAPI_DEVICE_SELECTOR +const std::array, 6> & getSyclDeviceTypeMap(); -// Array is used by SYCL_DEVICE_FILTER and SYCL_DEVICE_ALLOWLIST +// Array is used by SYCL_DEVICE_FILTER and SYCL_DEVICE_ALLOWLIST and +// ONEAPI_DEVICE_SELECTOR const std::array, 7> &getSyclBeMap(); +// --------------------------------------- +// ONEAPI_DEVICE_SELECTOR support +template <> class SYCLConfig { + using BaseT = SYCLConfigBase; + +public: + static ods_target_list *get() { + // Configuration parameters are processed only once, like reading a string + // from environment and converting it into a typed object. + static bool Initialized = false; + static ods_target_list *DeviceTargets = nullptr; + + if (Initialized) { + return DeviceTargets; + } + const char *ValStr = BaseT::getRawValue(); + if (ValStr) { + DeviceTargets = + &GlobalHandler::instance().getOneapiDeviceSelectorTargets(ValStr); + } + Initialized = true; + return DeviceTargets; + } +}; + +// --------------------------------------- +// SYCL_DEVICE_FILTER support + template <> class SYCLConfig { using BaseT = SYCLConfigBase; diff --git a/sycl/source/detail/device_filter.cpp b/sycl/source/detail/device_filter.cpp index 3b12da90f71ea..c35d6e4fd59a2 100644 --- a/sycl/source/detail/device_filter.cpp +++ b/sycl/source/detail/device_filter.cpp @@ -12,13 +12,14 @@ #include #include +#include #include namespace sycl { __SYCL_INLINE_VER_NAMESPACE(_V1) { namespace detail { -std::vector tokenize(const std::string &Filter, +std::vector tokenize(const std::string_view &Filter, const std::string &Delim) { std::vector Tokens; size_t Pos = 0; @@ -42,6 +43,210 @@ std::vector tokenize(const std::string &Filter, return Tokens; } +// --------------------------------------- +// ONEAPI_DEVICE_SELECTOR support + +static backend Parse_ODS_Backend(const std::string_view &BackendStr, + const std::string_view &FullEntry) { + // Check if the first entry matches with a known backend type + auto SyclBeMap = + getSyclBeMap(); // <-- std::array> + // [{"level_zero", backend::level_zero}, {"*", ::all}, ... + auto It = std::find_if( + std::begin(SyclBeMap), std::end(SyclBeMap), [&](auto BePair) { + return std::string::npos != BackendStr.find(BePair.first); + }); + + if (It == SyclBeMap.end()) { + // backend is required + std::stringstream ss; + ss << "ONEAPI_DEVICE_SELECTOR parsing error. Backend is required but " + "missing from \"" + << FullEntry << "\""; + throw sycl::exception(sycl::make_error_code(errc::invalid), ss.str()); + } else { + return It->second; + } +} + +static void Parse_ODS_Device(ods_target &Target, + const std::string_view &DeviceStr) { + // DeviceStr will be: 'gpu', '*', '0', '0.1', 'gpu.*', '0.*', or 'gpu.2', etc. + std::vector DeviceSubTuple = tokenize(DeviceStr, "."); + std::string_view TopDeviceStr = DeviceSubTuple[0]; + + // Handle explicit device type (e.g. 'gpu'). + auto DeviceTypeMap = + getSyclDeviceTypeMap(); // <-- std::array> + auto It = std::find_if( + std::begin(DeviceTypeMap), std::end(DeviceTypeMap), [&](auto DtPair) { + return std::string::npos != TopDeviceStr.find(DtPair.first); + }); + if (It != DeviceTypeMap.end()) { + Target.DeviceType = It->second; + // Handle wildcard. + if (TopDeviceStr[0] == '*') { + Target.HasDeviceWildCard = true; + Target.DeviceType = {}; + } + } else { // Only thing left is a number. + std::string TDS(TopDeviceStr); + try { + Target.DeviceNum = std::stoi(TDS); + } catch (...) { + throw sycl::exception(sycl::make_error_code(errc::invalid), + "error parsing device number: " + TDS); + } + } + + if (DeviceSubTuple.size() >= 2) { + // We have a subdevice. + // The grammar for sub-devices is ... restrictive. Neither 'gpu.0' nor + // 'gpu.*' are allowed. If wanting a sub-device, then the device itself must + // be specified by a number or a wildcard, and if by wildcard, the only + // allowable sub-device is another wildcard. + + if (Target.DeviceType) + throw sycl::exception( + sycl::make_error_code(errc::invalid), + "sub-devices can only be requested when parent device is specified " + "by number or wildcard, not a device type like 'gpu'"); + + std::string_view SubDeviceStr = DeviceSubTuple[1]; + // SubDeviceStr is wildcard or number. + if (SubDeviceStr[0] == '*') { + Target.HasSubDeviceWildCard = true; + } else { + // sub-device requested by number. So parent device must be a number too + // or it's a parsing error. + if (Target.HasDeviceWildCard) + throw sycl::exception(sycl::make_error_code(errc::invalid), + "sub-device can't be requested by number if " + "parent device is specified by a wildcard."); + + std::string SDS(SubDeviceStr); + try { + Target.SubDeviceNum = std::stoi(SDS); + } catch (...) { + throw sycl::exception(sycl::make_error_code(errc::invalid), + "error parsing sub-device index: " + SDS); + } + } + } + if (DeviceSubTuple.size() == 3) { + // We have a sub-sub-device. + // Similar rules for sub-sub-devices as for sub-devices above. + + std::string_view SubSubDeviceStr = DeviceSubTuple[2]; + if (SubSubDeviceStr[0] == '*') { + Target.HasSubSubDeviceWildCard = true; + } else { + // sub-sub-device requested by number. So partition above must be a number + // too or it's a parsing error. + if (Target.HasSubDeviceWildCard) + throw sycl::exception(sycl::make_error_code(errc::invalid), + "sub-sub-device can't be requested by number if " + "sub-device before is specified by a wildcard."); + + std::string SSDS(SubSubDeviceStr); + try { + Target.SubSubDeviceNum = std::stoi(SSDS); + } catch (...) { + throw sycl::exception(sycl::make_error_code(errc::invalid), + "error parsing sub-sub-device index: " + SSDS); + } + } + } else if (DeviceSubTuple.size() > 3) { + std::stringstream ss; + ss << "error parsing " << DeviceStr + << " Only two levels of sub-devices supported at this time"; + throw sycl::exception(sycl::make_error_code(errc::invalid), ss.str()); + } +} + +std::vector +Parse_ONEAPI_DEVICE_SELECTOR(const std::string &envStr) { + std::vector Result; + if (envStr.empty()) { + ods_target acceptAnything; + Result.push_back(acceptAnything); + return Result; + } + + std::vector Entries = tokenize(envStr, ";"); + // Each entry: "level_zero:gpu" or "opencl:0.0,0.1" or "opencl:*" but NOT just + // "opencl". + for (const auto Entry : Entries) { + std::vector Pair = tokenize(Entry, ":"); + backend be = Parse_ODS_Backend(Pair[0], Entry); // Pair[0] is backend. + + if (Pair.size() == 1) { + std::stringstream ss; + ss << "Incomplete selector! Try '" << Pair[0] + << ":*' if all devices under the backend was original intention."; + throw sycl::exception(sycl::make_error_code(errc::invalid), ss.str()); + } else if (Pair.size() == 2) { + std::vector Targets = tokenize(Pair[1], ","); + for (auto TargetStr : Targets) { + ods_target DeviceTarget(be); + Parse_ODS_Device(DeviceTarget, TargetStr); + Result.push_back(DeviceTarget); + } + } else if (Pair.size() > 2) { + std::stringstream ss; + ss << "Error parsing selector string \"" << Entry + << "\" Too many colons (:)"; + throw sycl::exception(sycl::make_error_code(errc::invalid), ss.str()); + } + } + + return Result; +} + +std::ostream &operator<<(std::ostream &Out, const ods_target &Target) { + Out << Target.Backend; + if (Target.DeviceType) { + auto DeviceTypeMap = getSyclDeviceTypeMap(); + auto Match = std::find_if( + DeviceTypeMap.begin(), DeviceTypeMap.end(), + [&](auto Pair) { return (Pair.second == Target.DeviceType); }); + if (Match != DeviceTypeMap.end()) { + Out << ":" << Match->first; + } else { + Out << ":???"; + } + } + if (Target.HasDeviceWildCard) + Out << ":*"; + if (Target.DeviceNum) + Out << ":" << Target.DeviceNum.value(); + if (Target.HasSubDeviceWildCard) + Out << ".*"; + if (Target.SubDeviceNum) + Out << "." << Target.SubDeviceNum.value(); + + return Out; +} + +ods_target_list::ods_target_list(const std::string &envStr) { + TargetList = Parse_ONEAPI_DEVICE_SELECTOR(envStr); +} + +// Backend is compatible with the SYCL_DEVICE_FILTER in the following cases. +// 1. Filter backend is '*' which means ANY backend. +// 2. Filter backend match exactly with the given 'Backend' +bool ods_target_list::backendCompatible(backend Backend) { + return std::any_of( + TargetList.begin(), TargetList.end(), [&](ods_target &Target) { + backend TargetBackend = Target.Backend.value_or(backend::all); + return (TargetBackend == Backend) || (TargetBackend == backend::all); + }); +} + +// --------------------------------------- +// SYCL_DEVICE_FILTER support + device_filter::device_filter(const std::string &FilterString) { std::vector Tokens = tokenize(FilterString, ":"); size_t TripleValueID = 0; @@ -96,7 +301,6 @@ device_filter::device_filter(const std::string &FilterString) { if (TripleValueID < Tokens.size()) { try { DeviceNum = std::stoi(Tokens[TripleValueID].data()); - HasDeviceNum = true; } catch (...) { std::string Message = std::string("Invalid device filter: ") + FilterString + @@ -141,30 +345,28 @@ void device_filter_list::addFilter(device_filter &Filter) { // 1. Filter backend is '*' which means ANY backend. // 2. Filter backend match exactly with the given 'Backend' bool device_filter_list::backendCompatible(backend Backend) { - for (const device_filter &Filter : FilterList) { - backend FilterBackend = Filter.Backend; - if (FilterBackend == Backend || FilterBackend == backend::all) - return true; - } - return false; + return std::any_of( + FilterList.begin(), FilterList.end(), [&](device_filter &Filter) { + backend FilterBackend = Filter.Backend.value_or(backend::all); + return (FilterBackend == Backend) || (FilterBackend == backend::all); + }); } bool device_filter_list::deviceTypeCompatible(info::device_type DeviceType) { - for (const device_filter &Filter : FilterList) { - info::device_type FilterDevType = Filter.DeviceType; - if (FilterDevType == DeviceType || FilterDevType == info::device_type::all) - return true; - } - return false; + return std::any_of(FilterList.begin(), FilterList.end(), + [&](device_filter &Filter) { + info::device_type FilterDevType = + Filter.DeviceType.value_or(info::device_type::all); + return (FilterDevType == DeviceType) || + (FilterDevType == info::device_type::all); + }); } bool device_filter_list::deviceNumberCompatible(int DeviceNum) { - for (const device_filter &Filter : FilterList) { - int FilterDevNum = Filter.DeviceNum; - if (!Filter.HasDeviceNum || FilterDevNum == DeviceNum) - return true; - } - return false; + return std::any_of( + FilterList.begin(), FilterList.end(), [&](device_filter &Filter) { + return (!Filter.DeviceNum) || (Filter.DeviceNum.value() == DeviceNum); + }); } } // namespace detail diff --git a/sycl/source/detail/filter_selector_impl.cpp b/sycl/source/detail/filter_selector_impl.cpp index 3d89773aa8552..09046c0f6b8b5 100644 --- a/sycl/source/detail/filter_selector_impl.cpp +++ b/sycl/source/detail/filter_selector_impl.cpp @@ -62,34 +62,26 @@ filter create_filter(const std::string &Input) { throw sycl::runtime_error(Error, PI_ERROR_INVALID_VALUE); for (const std::string &Token : Tokens) { - if (Token == "cpu" && !Result.HasDeviceType) { + if (Token == "cpu" && !Result.DeviceType) { Result.DeviceType = sycl::info::device_type::cpu; - Result.HasDeviceType = true; - } else if (Token == "gpu" && !Result.HasDeviceType) { + } else if (Token == "gpu" && !Result.DeviceType) { Result.DeviceType = sycl::info::device_type::gpu; - Result.HasDeviceType = true; - } else if (Token == "accelerator" && !Result.HasDeviceType) { + } else if (Token == "accelerator" && !Result.DeviceType) { Result.DeviceType = sycl::info::device_type::accelerator; - Result.HasDeviceType = true; - } else if (Token == "opencl" && !Result.HasBackend) { + } else if (Token == "opencl" && !Result.Backend) { Result.Backend = backend::opencl; - Result.HasBackend = true; - } else if (Token == "level_zero" && !Result.HasBackend) { + } else if (Token == "level_zero" && !Result.Backend) { Result.Backend = backend::ext_oneapi_level_zero; - Result.HasBackend = true; - } else if (Token == "cuda" && !Result.HasBackend) { + } else if (Token == "cuda" && !Result.Backend) { Result.Backend = backend::ext_oneapi_cuda; - Result.HasBackend = true; - } else if (Token == "hip" && !Result.HasBackend) { + } else if (Token == "hip" && !Result.Backend) { Result.Backend = backend::ext_oneapi_hip; - Result.HasBackend = true; - } else if (std::regex_match(Token, IntegerExpr) && !Result.HasDeviceNum) { + } else if (std::regex_match(Token, IntegerExpr) && !Result.DeviceNum) { try { Result.DeviceNum = std::stoi(Token); } catch (std::logic_error &) { throw sycl::runtime_error(Error, PI_ERROR_INVALID_VALUE); } - Result.HasDeviceNum = true; } else { throw sycl::runtime_error(Error, PI_ERROR_INVALID_VALUE); } @@ -120,15 +112,15 @@ int filter_selector_impl::operator()(const device &Dev) const { bool DeviceTypeOK = true; bool DeviceNumOK = true; - if (Filter.HasBackend) { + if (Filter.Backend) { backend BE = sycl::detail::getSyclObjImpl(Dev)->getPlugin().getBackend(); // Backend is okay if the filter BE is set 'all'. - if (Filter.Backend == backend::all) + if (Filter.Backend.value() == backend::all) BackendOK = true; else - BackendOK = (BE == Filter.Backend); + BackendOK = (BE == Filter.Backend.value()); } - if (Filter.HasDeviceType) { + if (Filter.DeviceType) { sycl::info::device_type DT = Dev.get_info(); // DeviceType is okay if the filter is set 'all'. @@ -137,11 +129,11 @@ int filter_selector_impl::operator()(const device &Dev) const { else DeviceTypeOK = (DT == Filter.DeviceType); } - if (Filter.HasDeviceNum) { + if (Filter.DeviceNum) { // Only check device number if we're good on the previous matches if (BackendOK && DeviceTypeOK) { // Do we match? - DeviceNumOK = (Filter.MatchesSeen == Filter.DeviceNum); + DeviceNumOK = (Filter.MatchesSeen == Filter.DeviceNum.value()); // Safe to increment matches even if we find it Filter.MatchesSeen++; } diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index 1f49ab31e73bf..874ded5862906 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -83,6 +83,11 @@ GlobalHandler::getDeviceFilterList(const std::string &InitValue) { return getOrCreate(MDeviceFilterList, InitValue); } +ods_target_list & +GlobalHandler::getOneapiDeviceSelectorTargets(const std::string &InitValue) { + return getOrCreate(MOneapiDeviceSelectorTargets, InitValue); +} + XPTIRegistry &GlobalHandler::getXPTIRegistry() { return getOrCreate(MXPTIRegistry); } diff --git a/sycl/source/detail/global_handler.hpp b/sycl/source/detail/global_handler.hpp index 9e9016ab218e6..50e0a0f93ef3d 100644 --- a/sycl/source/detail/global_handler.hpp +++ b/sycl/source/detail/global_handler.hpp @@ -24,6 +24,7 @@ class ProgramManager; class Sync; class plugin; class device_filter_list; +class ods_target_list; class XPTIRegistry; class ThreadPool; @@ -66,6 +67,7 @@ class GlobalHandler { std::mutex &getFilterMutex(); std::vector &getPlugins(); device_filter_list &getDeviceFilterList(const std::string &InitValue); + ods_target_list &getOneapiDeviceSelectorTargets(const std::string &InitValue); XPTIRegistry &getXPTIRegistry(); ThreadPool &getHostTaskThreadPool(); @@ -101,6 +103,7 @@ class GlobalHandler { InstWithLock MFilterMutex; InstWithLock> MPlugins; InstWithLock MDeviceFilterList; + InstWithLock MOneapiDeviceSelectorTargets; InstWithLock MXPTIRegistry; // Thread pool for host task and event callbacks execution InstWithLock MHostTaskThreadPool; diff --git a/sycl/source/detail/pi.cpp b/sycl/source/detail/pi.cpp index f81b20bb9d92d..7bf4ab0422a19 100644 --- a/sycl/source/detail/pi.cpp +++ b/sycl/source/detail/pi.cpp @@ -293,7 +293,7 @@ std::vector> findPlugins() { bool EsimdCpuFound = false; bool HIPFound = false; for (const device_filter &Filter : Filters) { - backend Backend = Filter.Backend; + backend Backend = Filter.Backend ? Filter.Backend.value() : backend::all; if (!OpenCLFound && (Backend == backend::opencl || Backend == backend::all)) { PluginNames.emplace_back(__SYCL_OPENCL_PLUGIN_NAME, backend::opencl); diff --git a/sycl/source/detail/platform_impl.cpp b/sycl/source/detail/platform_impl.cpp index 661f6725b4cbb..77d46c6bae692 100644 --- a/sycl/source/detail/platform_impl.cpp +++ b/sycl/source/detail/platform_impl.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -93,6 +94,8 @@ static bool IsBannedPlatform(platform Platform) { return IsNVIDIAOpenCL(Platform); } +// This routine has the side effect of registering each platform's last device +// id into each plugin, which is used for device counting. std::vector platform_impl::get_platforms() { std::vector Platforms; std::vector &Plugins = RT::initialize(); @@ -114,14 +117,18 @@ std::vector platform_impl::get_platforms() { for (const auto &PiPlatform : PiPlatforms) { platform Platform = detail::createSyclObjFromImpl( getOrMakePlatformImpl(PiPlatform, Plugin)); + if (IsBannedPlatform(Platform)) { + continue; // bail as early as possible, otherwise banned platforms may + // mess up device counting + } + { std::lock_guard Guard(*Plugin.getPluginMutex()); // insert PiPlatform into the Plugin Plugin.getPlatformId(PiPlatform); } // Skip platforms which do not contain requested device types - if (!Platform.get_devices(ForcedType).empty() && - !IsBannedPlatform(Platform)) + if (!Platform.get_devices(ForcedType).empty()) Platforms.push_back(Platform); } } @@ -140,17 +147,13 @@ std::vector platform_impl::get_platforms() { return Platforms; } -// Filter out the devices that are not compatible with SYCL_DEVICE_FILTER. -// All three entries (backend:device_type:device_num) are optional. -// The missing entries are constructed using '*', which means 'any' | 'all' -// by the device_filter constructor. -// This function matches devices in the order of backend, device_type, and -// device_num. -static void filterDeviceFilter(std::vector &PiDevices, - RT::PiPlatform Platform) { - device_filter_list *FilterList = SYCLConfig::get(); - if (!FilterList) - return; +// Filter out the devices that are not compatible with SYCL_DEVICE_FILTER or +// ONEAPI_DEVICE_SELECTOR This function matches devices in the order of backend, +// device_type, and device_num. The device_filter and ods_target structs pun for +// each other, as do device_filter_list and ods_target_list. +template +static int filterDeviceFilter(std::vector &PiDevices, + RT::PiPlatform Platform, ListT *FilterList) { std::vector &Plugins = RT::initialize(); auto It = @@ -158,7 +161,7 @@ static void filterDeviceFilter(std::vector &PiDevices, return Plugin.containsPiPlatform(Platform); }); if (It == Plugins.end()) - return; + return -1; plugin &Plugin = *It; backend Backend = Plugin.getBackend(); @@ -167,6 +170,7 @@ static void filterDeviceFilter(std::vector &PiDevices, // backend std::lock_guard Guard(*Plugin.getPluginMutex()); int DeviceNum = Plugin.getStartingDeviceId(Platform); + int StartingNum = DeviceNum; for (RT::PiDevice Device : PiDevices) { RT::PiDeviceType PiDevType; Plugin.call(Device, PI_DEVICE_INFO_TYPE, @@ -176,20 +180,21 @@ static void filterDeviceFilter(std::vector &PiDevices, // Sycl device type for GPU, CPU, and ACC. info::device_type DeviceType = pi::cast(PiDevType); - for (const device_filter &Filter : FilterList->get()) { - backend FilterBackend = Filter.Backend; + for (const FilterT &Filter : FilterList->get()) { + backend FilterBackend = Filter.Backend.value_or(backend::all); // First, match the backend entry if (FilterBackend == Backend || FilterBackend == backend::all) { - info::device_type FilterDevType = Filter.DeviceType; + info::device_type FilterDevType = + Filter.DeviceType.value_or(info::device_type::all); // Next, match the device_type entry if (FilterDevType == info::device_type::all) { // Last, match the device_num entry - if (!Filter.HasDeviceNum || DeviceNum == Filter.DeviceNum) { + if (!Filter.DeviceNum || DeviceNum == Filter.DeviceNum.value()) { PiDevices[InsertIDx++] = Device; break; } } else if (FilterDevType == DeviceType) { - if (!Filter.HasDeviceNum || DeviceNum == Filter.DeviceNum) { + if (!Filter.DeviceNum || DeviceNum == Filter.DeviceNum.value()) { PiDevices[InsertIDx++] = Device; break; } @@ -203,6 +208,7 @@ static void filterDeviceFilter(std::vector &PiDevices, // to assign a unique device id number across platforms that belong to // the same backend. For example, opencl:cpu:0, opencl:acc:1, opencl:gpu:2 Plugin.setLastDeviceId(Platform, DeviceNum); + return StartingNum; } std::shared_ptr @@ -226,9 +232,166 @@ std::shared_ptr platform_impl::getOrMakeDeviceImpl( return Result; } +static bool supportsAffinityDomain(const device &dev, + info::partition_property partitionProp, + info::partition_affinity_domain domain) { + if (partitionProp != info::partition_property::partition_by_affinity_domain) { + return true; + } + auto supported = dev.get_info(); + auto It = std::find(std::begin(supported), std::end(supported), domain); + return It != std::end(supported); +} + +static bool supportsPartitionProperty(const device &dev, + info::partition_property partitionProp) { + auto supported = dev.get_info(); + auto It = + std::find(std::begin(supported), std::end(supported), partitionProp); + return It != std::end(supported); +} + +static std::vector amendDeviceAndSubDevices( + backend PlatformBackend, std::vector &DeviceList, + ods_target_list *OdsTargetList, int PlatformDeviceIndex) { + constexpr info::partition_property partitionProperty = + info::partition_property::partition_by_affinity_domain; + constexpr info::partition_affinity_domain affinityDomain = + info::partition_affinity_domain::next_partitionable; + + std::vector FinalResult; + + for (unsigned i = 0; i < DeviceList.size(); i++) { + // device has already been screened. The question is whether it should be a + // top level device and/or is expected to add its sub-devices to the list. + device &dev = DeviceList[i]; + bool deviceAdded = false; + for (ods_target target : OdsTargetList->get()) { + backend TargetBackend = target.Backend.value_or(backend::all); + if (PlatformBackend == TargetBackend || TargetBackend == backend::all) { + bool deviceMatch = target.HasDeviceWildCard; // opencl:* + if (target.DeviceType) { // opencl:gpu + deviceMatch = ((target.DeviceType == info::device_type::all) || + (dev.get_info() == + target.DeviceType)); + + } else if (target.DeviceNum) { // opencl:0 + deviceMatch = + (target.DeviceNum.value() == PlatformDeviceIndex + (int)i); + } + + if (deviceMatch) { + // Top level matches. Do we add it, or subdevices, or sub-sub-devices? + bool wantSubDevice = + target.SubDeviceNum || target.HasSubDeviceWildCard; + bool supportsSubPartitioning = + (supportsPartitionProperty(dev, partitionProperty) && + supportsAffinityDomain(dev, partitionProperty, affinityDomain)); + bool wantSubSubDevice = + target.SubSubDeviceNum || target.HasSubSubDeviceWildCard; + + // -- Add top level device. + if (!wantSubDevice) { + if (!deviceAdded) { + FinalResult.push_back(dev); + deviceAdded = true; + } + } else { + if (!supportsSubPartitioning) { + if (target.DeviceNum || + (target.DeviceType && + (target.DeviceType.value() != info::device_type::all))) { + // This device was specifically requested and yet is not + // partitionable. + std::cout << "device is not partitionable: " << target + << std::endl; + } + continue; + } + // -- Add sub sub device. + if (wantSubSubDevice) { + + auto subDevicesToPartition = dev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + affinityDomain); + if (target.SubDeviceNum) { + if (subDevicesToPartition.size() > + target.SubDeviceNum.value()) { + subDevicesToPartition[0] = + subDevicesToPartition[target.SubDeviceNum.value()]; + subDevicesToPartition.resize(1); + } else { + std::cout << "subdevice index out of bounds: " << target + << std::endl; + continue; + } + } + for (device subDev : subDevicesToPartition) { + bool supportsSubSubPartitioning = + (supportsPartitionProperty(subDev, partitionProperty) && + supportsAffinityDomain(subDev, partitionProperty, + affinityDomain)); + if (!supportsSubSubPartitioning) { + if (target.SubDeviceNum) { + // Parent subdevice was specifically requested, yet is not + // partitionable. + std::cout << "sub-device is not partitionable: " << target + << std::endl; + } + continue; + } + // Allright, lets get them sub-sub-devices. + auto subSubDevices = subDev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + affinityDomain); + if (target.HasSubSubDeviceWildCard) { + FinalResult.insert(FinalResult.end(), subSubDevices.begin(), + subSubDevices.end()); + } else { + if (subSubDevices.size() > target.SubSubDeviceNum.value()) { + FinalResult.push_back( + subSubDevices[target.SubSubDeviceNum.value()]); + } else { + std::cout + << "sub-sub-device index out of bounds: " << target + << std::endl; + } + } + } + } else if (wantSubDevice) { + auto subDevices = dev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + affinityDomain); + if (target.HasSubDeviceWildCard) { + FinalResult.insert(FinalResult.end(), subDevices.begin(), + subDevices.end()); + } else { + if (subDevices.size() > target.SubDeviceNum.value()) { + FinalResult.push_back( + subDevices[target.SubDeviceNum.value()]); + } else { + std::cout << "subdevice index out of bounds: " << target + << std::endl; + } + } + } + } + } // /if deviceMatch + } + } // /for + } // /for + + return FinalResult; +} + std::vector platform_impl::get_devices(info::device_type DeviceType) const { std::vector Res; + // Will we be filtering with SYCL_DEVICE_FILTER or ONEAPI_DEVICE_SELECTOR ? + // We do NOT attempt to support both simultaneously. + ods_target_list *OdsTargetList = SYCLConfig::get(); + device_filter_list *FilterList = SYCLConfig::get(); + if (is_host() && (DeviceType == info::device_type::host || DeviceType == info::device_type::all)) { Res.push_back( @@ -243,8 +406,10 @@ platform_impl::get_devices(info::device_type DeviceType) const { pi_uint32 NumDevices = 0; const detail::plugin &Plugin = getPlugin(); Plugin.call( - MPlatform, pi::cast(DeviceType), 0, + MPlatform, pi::cast(DeviceType), + 0, // CP info::device_type::all pi::cast(nullptr), &NumDevices); + const backend Backend = Plugin.getBackend(); if (NumDevices == 0) { // If platform doesn't have devices (even without filter) @@ -266,17 +431,34 @@ platform_impl::get_devices(info::device_type DeviceType) const { std::vector PiDevices(NumDevices); // TODO catch an exception and put it to list of asynchronous exceptions - Plugin.call(MPlatform, - pi::cast(DeviceType), - NumDevices, PiDevices.data(), nullptr); + Plugin.call( + MPlatform, + pi::cast(DeviceType), // CP info::device_type::all + NumDevices, PiDevices.data(), nullptr); // Filter out devices that are not present in the SYCL_DEVICE_ALLOWLIST if (SYCLConfig::get()) applyAllowList(PiDevices, MPlatform, Plugin); - // Filter out devices that are not compatible with SYCL_DEVICE_FILTER - filterDeviceFilter(PiDevices, MPlatform); + // The first step is to filter out devices that are not compatible with + // SYCL_DEVICE_FILTER or ONEAPI_DEVICE_SELECTOR. This is also the mechanism by + // which top level device ids are assigned. + int PlatformDeviceIndex; + if (OdsTargetList) { + if (FilterList) { + throw sycl::exception(sycl::make_error_code(errc::invalid), + "ONEAPI_DEVICE_SELECTOR cannot be used in " + "conjunction with SYCL_DEVICE_FILTER"); + } + PlatformDeviceIndex = filterDeviceFilter( + PiDevices, MPlatform, OdsTargetList); + } else if (FilterList) { + PlatformDeviceIndex = filterDeviceFilter( + PiDevices, MPlatform, FilterList); + } + // The next step is to inflate the filtered PIDevices into SYCL Device + // objects. PlatformImplPtr PlatformImpl = getOrMakePlatformImpl(MPlatform, Plugin); std::transform( PiDevices.begin(), PiDevices.end(), std::back_inserter(Res), @@ -285,7 +467,16 @@ platform_impl::get_devices(info::device_type DeviceType) const { PlatformImpl->getOrMakeDeviceImpl(PiDevice, PlatformImpl)); }); - return Res; + // If we aren't using ONEAPI_DEVICE_SELECTOR, then we are done. + // and if there are no devices so far, there won't be any need to replace them + // with subdevices. + if (!OdsTargetList || Res.size() == 0) + return Res; + + // Otherwise, our last step is to revisit the devices, possibly replacing + // them with subdevices (which have been ignored until now) + return amendDeviceAndSubDevices(Backend, Res, OdsTargetList, + PlatformDeviceIndex); } bool platform_impl::has_extension(const std::string &ExtensionName) const { diff --git a/sycl/source/device.cpp b/sycl/source/device.cpp index 31b2030acf061..d6169f9863aea 100644 --- a/sycl/source/device.cpp +++ b/sycl/source/device.cpp @@ -52,11 +52,16 @@ std::vector device::get_devices(info::device_type deviceType) { std::vector devices; detail::device_filter_list *FilterList = detail::SYCLConfig::get(); - info::device_type forced_type = detail::get_forced_type(); + detail::ods_target_list *OdsTargetList = + detail::SYCLConfig::get(); + + info::device_type forced_type = + detail::get_forced_type(); // almost always ::all // Exclude devices which do not match requested device type if (detail::match_types(deviceType, forced_type)) { detail::force_type(deviceType, forced_type); - for (const auto &plt : platform::get_platforms()) { + auto thePlatforms = platform::get_platforms(); + for (const auto &plt : thePlatforms) { // If SYCL_BE is set then skip platforms which doesn't have specified // backend. backend *ForcedBackend = detail::SYCLConfig::get(); @@ -66,7 +71,10 @@ std::vector device::get_devices(info::device_type deviceType) { continue; // If SYCL_DEVICE_FILTER is set, skip platforms that is incompatible // with the filter specification. - if (FilterList && !FilterList->backendCompatible(plt.get_backend())) + backend platformBackend = plt.get_backend(); + if (FilterList && !FilterList->backendCompatible(platformBackend)) + continue; + if (OdsTargetList && !OdsTargetList->backendCompatible(platformBackend)) continue; std::vector found_devices(plt.get_devices(deviceType)); @@ -75,6 +83,7 @@ std::vector device::get_devices(info::device_type deviceType) { found_devices.end()); } } + return devices; } diff --git a/sycl/tools/sycl-ls/sycl-ls.cpp b/sycl/tools/sycl-ls/sycl-ls.cpp index d62aff79dc1f9..cbd2303865b6a 100644 --- a/sycl/tools/sycl-ls/sycl-ls.cpp +++ b/sycl/tools/sycl-ls/sycl-ls.cpp @@ -115,6 +115,17 @@ int main(int argc, char **argv) { << std::endl; } + const char *ods_targets = std::getenv("ONEAPI_DEVICE_SELECTOR"); + if (ods_targets) { + std::cerr + << "Warning: ONEAPI_DEVICE_SELECTOR environment variable is set to " + << ods_targets << "." << std::endl; + std::cerr + << "To see the correct device id, please unset ONEAPI_DEVICE_SELECTOR." + << std::endl + << std::endl; + } + const auto &Platforms = platform::get_platforms(); // Keep track of the number of devices per backend @@ -124,6 +135,11 @@ int main(int argc, char **argv) { backend Backend = Platform.get_backend(); auto PlatformName = Platform.get_info(); const auto &Devices = Platform.get_devices(); + + // the device counting done here should have the same result as the counting + // done by SYCL itself. But technically, it is not the same method, as SYCL + // keeps a table of platforms:start_dev_index in each plugin. + for (const auto &Device : Devices) { std::cout << "[" << Backend << ":" << getDeviceTypeName(Device) << ":" << DeviceNums[Backend] << "] "; diff --git a/sycl/unittests/allowlist/ParseAllowList.cpp b/sycl/unittests/allowlist/ParseAllowList.cpp index 01f746a31ca43..2612ad1005d18 100644 --- a/sycl/unittests/allowlist/ParseAllowList.cpp +++ b/sycl/unittests/allowlist/ParseAllowList.cpp @@ -183,11 +183,10 @@ TEST(ParseAllowListTests, CheckAllValidDeviceTypeValuesAreProcessed) { } sycl::detail::AllowListParsedT ActualValue = sycl::detail::parseAllowList(AllowList); - sycl::detail::AllowListParsedT ExpectedValue{{{"DeviceType", "host"}}, - {{"DeviceType", "cpu"}}, - {{"DeviceType", "gpu"}}, - {{"DeviceType", "acc"}}, - {{"DeviceType", "*"}}}; + sycl::detail::AllowListParsedT ExpectedValue{ + {{"DeviceType", "host"}}, {{"DeviceType", "cpu"}}, + {{"DeviceType", "gpu"}}, {{"DeviceType", "acc"}}, + {{"DeviceType", "fpga"}}, {{"DeviceType", "*"}}}; EXPECT_EQ(ExpectedValue, ActualValue); }