Add the CustomTypeMarshallerAttribute type to make it easier to identify marshaller types

docs/design/libraries/LibraryImportGenerator/SpanMarshallers.md

@@ -55,16 +55,28 @@ Span marshalling would still be implemented with similar semantics as mentioned
 
 ### Proposed extension to the custom type marshalling design
 
-Introduce a new attribute named `GenericContiguousCollectionMarshallerAttribute`. This attribute would have the following shape:
+Introduce a marshaller kind named `LinearCollection`.
 
-```csharp
+```diff
 namespace System.Runtime.InteropServices
-{
-    [AttributeUsage(AttributeTargets.Struct | AttributeTargets.Class)]
-    public sealed class GenericContiguousCollectionMarshallerAttribute : Attribute
-    {
-        public GenericContiguousCollectionMarshallerAttribute();
-    }
+{ 
+  [AttributeUsage(AttributeTargets.Struct)]
+  public sealed class CustomTypeMarshallerAttribute : Attribute
+  {
++       /// <summary>
++       /// This type is used as a placeholder for the first generic parameter when generic parameters cannot be used
++       /// to identify the managed type (i.e. when the marshaller type is generic over T and the managed type is T[])
++       /// </summary>
++       public struct GenericPlaceholder
++       {
++       }
+  }
+
+  public enum CustomTypeMarshallerKind
+  {
+      Value,
++     LinearCollection
+  }
 }
 ```
 
@@ -77,21 +89,27 @@ public ref struct Span<T>
   ...
 }
 
-[GenericContiguousCollectionMarshaller]
+[CustomTypeMarshaller(typeof(Span<>), CustomTypeMarshallerKind.LinearCollection)]
 public ref struct DefaultSpanMarshaler<T>
 {
   ...
 }
 ```
 
-The `GenericContiguousCollectionMarshallerAttribute` attribute is applied to a generic marshaler type with the "collection marshaller" shape described below. Since generic parameters cannot be used in attributes, open generic types will be permitted in the `NativeTypeMarshallingAttribute` constructor as long as they have the same arity as the type the attribute is applied to and generic parameters provided to the applied-to type can also be used to construct the type passed as a parameter.
+The `CustomTypeMarshallerKind.LinearCollection` kind is applied to a generic marshaler type with the "LinearCollection marshaller shape" described below.
+
+#### Supporting generics
+
+Since generic parameters cannot be used in attributes, open generic types will be permitted in the `NativeTypeMarshallingAttribute` and the `CustomTypeMarshallerAttribute` as long as they have the same arity as the type the attribute is applied to and generic parameters provided to the applied-to type can also be used to construct the type passed as a parameter.
+
+If a `CustomTypeMarshaller`-attributed type is a marshaller for a type for a pointer, an array, or a combination of pointers and arrays, the `CustomTypeMarshallerAttribute.GenericPlaceholder` type can be used in the place of the first generic parameter of the marshaller type.
 
-#### Generic collection marshaller shape
+#### LinearCollection marshaller shape
 
-A generic collection marshaller would be required to have the following shape, in addition to the requirements for marshaler types used with the `NativeTypeMarshallingAttribute`, excluding the constructors.
+A generic collection marshaller would be required to have the following shape, in addition to the requirements for marshaler types used with the `CustomTypeMarshallerKind.Value` shape, excluding the constructors.
 
 ```csharp
-[GenericContiguousCollectionMarshaller]
+[CustomTypeMarshaller(typeof(GenericCollection<, , ,...>), CustomTypeMarshallerKind.LinearCollection)]
 public struct GenericContiguousCollectionMarshallerImpl<T, U, V,...>
 {
     // this constructor is required if marshalling from native to managed is supported.
@@ -174,7 +192,7 @@ Alternatively, the `MarshalUsingAttribute` could provide a `Type ElementNativeTy
 This design could be used to provide a default marshaller for spans and arrays. Below is an example simple marshaller for `Span<T>`. This design does not include all possible optimizations, such as stack allocation, for simpilicity of the example.
 
 ```csharp
-[GenericContiguousCollectionMarshaller]
+[CustomTypeMarshaller(typeof(Span<>), CustomTypeMarshallerKind.LinearCollection)]
 public ref struct SpanMarshaler<T>
 {
     private Span<T> managedCollection;
@@ -261,11 +279,11 @@ This design could also be applied to support the built-in array marshalling if i
 
 If a managed or native representation of a collection has a non-contiguous element layout, then developers currently will need to convert to or from array/span types at the interop boundary. This section proposes an API that would enable developers to convert directly between a managed and native non-contiguous collection layout as part of marshalling.
 
-A new attribute named `GenericCollectionMarshaller` attribute could be added that would specify that the collection is noncontiguous in either managed or native representations. Then additional methods should be added to the generic collection model, and some methods would be removed:
+A new marshaller kind named `GenericCollection` could be added that would specify that the collection is noncontiguous in either managed or native representations. Then additional methods should be added to the generic collection model, and some methods would be removed:
 
 ```diff
-- [GenericContiguousCollectionMarshaller]
-+ [GenericCollectionMarshaller]
+- [CustomTypeMarshaller(typeof(Span<>), CustomTypeMarshallerKind.LinearCollection)]
++ [CustomTypeMarshaller(typeof(Span<>), CustomTypeMarshallerKind.GenericCollection)]
 public struct GenericContiguousCollectionMarshallerImpl<T, U, V,...>
 {
     // these constructors are required if marshalling from managed to native is supported.
@@ -312,6 +330,6 @@ Cons:
 - Introduces more attribute types into the BCL.
 - Introduces more complexity in the marshalling type model.
   - It may be worth describing the required members (other than constructors) in interfaces just to simplify the mental load of which members are required for which scenarios.
-    - A set of interfaces (one for managed-to-native members, one for native-to-managed members, and one for the sequential-specific members) could replace the `GenericContiguousCollectionMarshaller` attribute.
+    - A set of interfaces (one for managed-to-native members, one for native-to-managed members, and one for the sequential-specific members) could replace the new marshaller kind.
 - The base proposal only supports contiguous collections.
   - The feeling at time of writing is that we are okay asking developers to convert to/from arrays or spans at the interop boundary.

docs/design/libraries/LibraryImportGenerator/StructMarshalling.md

@@ -25,24 +25,45 @@ All design options would use these attributes:
 ```csharp
 
 [AttributeUsage(AttributeTargets.Struct | AttributeTargets.Class)]
-public class GeneratedMarshallingAttribute : Attribute {}
+public sealed class GeneratedMarshallingAttribute : Attribute {}
 
 [AttributeUsage(AttributeTargets.Struct | AttributeTargets.Class)]
-public class NativeMarshallingAttribute : Attribute
+public sealed class NativeMarshallingAttribute : Attribute
 {
      public NativeMarshallingAttribute(Type nativeType) {}
 }
 
 [AttributeUsage(AttributeTargets.Parameter | AttributeTargets.ReturnValue | AttributeTargets.Field)]
-public class MarshalUsingAttribute : Attribute
+public sealed class MarshalUsingAttribute : Attribute
 {
      public MarshalUsingAttribute(Type nativeType) {}
 }
+
+[AttributeUsage(AttributeTargets.Struct)]
+public sealed class CustomTypeMarshallerAttribute : Attribute
+{
+     public CustomTypeMarshallerAttribute(Type managedType, CustomTypeMarshallerKind marshallerKind = CustomTypeMarshallerKind.Value)
+     {
+          ManagedType = managedType;
+          MarshallerKind = marshallerKind;
+     }
+
+     public Type ManagedType { get; }
+     public CustomTypeMarshallerKind MarshallerKind { get; }
+     public int BufferSize { get; set; }
+     public bool RequiresStackBuffer { get; set; }
+}
+
+public enum CustomTypeMarshallerKind
+{
+     Value
+}
 ```
 
-The `NativeMarshallingAttribute` and `MarshalUsingAttribute` attributes would require that the provided native type `TNative` is a `struct` that does not require any marshalling and has a subset of three methods with the following names and shapes (with the managed type named TManaged):
+The `NativeMarshallingAttribute` and `MarshalUsingAttribute` attributes would require that the provided native type `TNative` is a `struct` that does not require any marshalling and has the `CustomTypeMarshallerAttribute` with the first parameter being a `typeof()` of the managed type and a subset of three methods with the following names and shapes (with the managed type named TManaged):
 
 ```csharp
+[CustomTypeMarshaller(typeof(TManaged))]
 partial struct TNative
 {
      public TNative(TManaged managed) {}
@@ -57,7 +78,7 @@ The analyzer will report an error if neither the constructor nor the `ToManaged`
 
 > :question: Does this API surface and shape work for all marshalling scenarios we plan on supporting? It may have issues with the current "layout class" by-value `[Out]` parameter marshalling where the runtime updates a `class` typed object in place. We already recommend against using classes for interop for performance reasons and a struct value passed via `ref` or `out` with the same members would cover this scenario.
 
-If the native type `TNative` also has a public `Value` property, then the value of the `Value` property will be passed to native code instead of the `TNative` value itself. As a result, the type `TNative` will be allowed to require marshalling and the type of the `Value` property will be required be passable to native code without any additional marshalling. If the `Value` property is settable, then when marshalling in the native-to-managed direction, a default value of `TNative` will have its `Value` property set to the native value. If `Value` does not have a setter, then marshalling from native to managed is not supported.
+If the native type `TNative` also has a public `Value` property, then the value of the `Value` property will be passed to native code instead of the `TNative` value itself. As a result, the type `TNative` will be allowed to require marshalling and the type of the `Value` property will be required be passable to native code without any additional marshalling. When the `Value` property is provided, the `CustomTypeMarshallerAttribute` will still need to be provided on `TNative`. If the `Value` property is settable, then when marshalling in the native-to-managed direction, a default value of `TNative` will have its `Value` property set to the native value. If `Value` does not have a setter, then marshalling from native to managed is not supported.
 
 If a `Value` property is provided, the developer may also provide a ref-returning or readonly-ref-returning `GetPinnableReference` method. The `GetPinnableReference` method will be called before the `Value` property getter is called. The ref returned by `GetPinnableReference` will be pinned with a `fixed` statement, but the pinned value will not be used (it acts exclusively as a side-effect).
 
@@ -70,6 +91,7 @@ public struct TManaged
      // ...
 }
 
+[CustomTypeMarshaller(typeof(TManaged))]
 public struct TMarshaler
 {
      public TMarshaler(TManaged managed) {}
@@ -92,20 +114,17 @@ Since C# 7.3 added a feature to enable custom pinning logic for user types, we s
 
 #### Caller-allocated memory
 
-Custom marshalers of collection-like types or custom string encodings (such as UTF-32) may want to use stack space for extra storage for additional performance when possible. If the `TNative` type provides additional members with the following signatures, then it will opt in to using a caller-allocated buffer:
+Custom marshalers of collection-like types or custom string encodings (such as UTF-32) may want to use stack space for extra storage for additional performance when possible. If the `TNative` type provides additional constructor with the following signature and sets the `BufferSize` field on the `CustomTypeMarshallerAttribute`, then it will opt in to using a caller-allocated buffer:
 
 ```csharp
+[CustomTypeMarshaller(typeof(TManaged), BufferSize = /* */, RequiresStackBuffer = /* */)]
 partial struct TNative
 {
      public TNative(TManaged managed, Span<byte> buffer) {}
-
-     public const int BufferSize = /* */;
-
-     public const bool RequiresStackBuffer = /* */;
 }
 ```
 
-When these members are present, the source generator will call the two-parameter constructor with a possibly stack-allocated buffer of `BufferSize` bytes when a stack-allocated buffer is usable. If a stack-allocated buffer is a requirement, the `RequiresStackBuffer` field should be set to `true` and the `buffer` will be guaranteed to be allocated on the stack. Setting the `RequiresStackBuffer` field to `false` is the same as omitting the field definition. Since a dynamically allocated buffer is not usable in all scenarios, for example Reverse P/Invoke and struct marshalling, a one-parameter constructor must also be provided for usage in those scenarios. This may also be provided by providing a two-parameter constructor with a default value for the second parameter.
+When these members are present, the source generator will call the two-parameter constructor with a possibly stack-allocated buffer of `BufferSize` bytes when a stack-allocated buffer is usable. If a stack-allocated buffer is a requirement, the `RequiresStackBuffer` field should be set to `true` and the `buffer` will be guaranteed to be allocated on the stack. Setting the `RequiresStackBuffer` field to `false` is the same as not specifying the value in the attribute. Since a dynamically allocated buffer is not usable in all scenarios, for example Reverse P/Invoke and struct marshalling, a one-parameter constructor must also be provided for usage in those scenarios. This may also be provided by providing a two-parameter constructor with a default value for the second parameter.
 
 Type authors can pass down the `buffer` pointer to native code by defining a `Value` property that returns a pointer to the first element, generally through code using `MemoryMarshal.GetReference()` and `Unsafe.AsPointer`. If `RequiresStackBuffer` is not provided or set to `false`, the `buffer` span must be pinned to be used safely. The `buffer` span can be pinned by defining a `GetPinnableReference()` method on the native type that returns a reference to the first element of the span.
 
@@ -251,6 +270,7 @@ struct HResult
      public readonly int Result;
 }
 
+[CustomTypeMarshaller(typeof(HResult))]
 struct HRESULT
 {
      public HRESULT(HResult hr)
@@ -276,21 +296,20 @@ Building on this Transparent Structures support, we can also support ComWrappers
 class Foo
 {}
 
-struct ComWrappersMarshaler<TClass, TComWrappers>
-     where TComWrappers : ComWrappers, new()
+struct FooComWrappersMarshaler
 {
-     private static readonly TComWrappers ComWrappers = new TComWrappers();
+     private static readonly FooComWrappers ComWrappers = new FooComWrappers();
 
      private IntPtr nativeObj;
 
-     public ComWrappersMarshaler(TClass obj)
+     public ComWrappersMarshaler(Foo obj)
      {
           nativeObj = ComWrappers.GetOrCreateComInterfaceForObject(obj, CreateComInterfaceFlags.None);
      }
 
      public IntPtr Value { get => nativeObj; set => nativeObj = value; }
 
-     public TClass ToManaged() => (TClass)ComWrappers.GetOrCreateObjectForComInstance(nativeObj, CreateObjectFlags.None);
+     public Foo ToManaged() => (Foo)ComWrappers.GetOrCreateObjectForComInstance(nativeObj, CreateObjectFlags.None);
 
      public unsafe void FreeNative()
      {

eng/generators.targets

@@ -44,6 +44,9 @@
   <ItemGroup Condition="'@(EnabledGenerators)' != ''
                         and @(EnabledGenerators->AnyHaveMetadataValue('Identity', 'LibraryImportGenerator'))
                         and '$(IncludeLibraryImportGeneratorSources)' == 'true'">
+    <Compile Include="$(LibrariesProjectRoot)Common\src\System\Runtime\InteropServices\CustomTypeMarshallerKind.cs" />
+    <Compile Include="$(LibrariesProjectRoot)Common\src\System\Runtime\InteropServices\CustomTypeMarshallerDirection.cs" />
+    <Compile Include="$(LibrariesProjectRoot)Common\src\System\Runtime\InteropServices\CustomTypeMarshallerFeatures.cs" />
 
     <!-- Only add the following files if we are not on the latest TFM. -->
     <Compile Condition="'$(NetCoreAppCurrentTargetFrameworkMoniker)' != '$(TargetFrameworkMoniker)'"

src/coreclr/nativeaot/Test.CoreLib/src/Test.CoreLib.csproj

@@ -4,6 +4,7 @@
     <GenerateTargetFrameworkAttribute>false</GenerateTargetFrameworkAttribute>
     <TargetFramework>netstandard2.0</TargetFramework>
     <EnableLibraryImportGenerator>true</EnableLibraryImportGenerator>
+    <DefineConstants>$(DefineConstants);TEST_CORELIB</DefineConstants>
     <!--
       SYSLIB1053: LibraryImportGenerator Target Framework Not Supported.
      -->

src/libraries/Common/src/Interop/Interop.Ldap.cs

@@ -44,6 +44,9 @@ internal struct SEC_WINNT_AUTH_IDENTITY_EX
         public string packageList;
         public int packageListLength;
 
+#if NET7_0_OR_GREATER
+        [CustomTypeMarshaller(typeof(SEC_WINNT_AUTH_IDENTITY_EX), Direction = CustomTypeMarshallerDirection.In, Features = CustomTypeMarshallerFeatures.UnmanagedResources)]
+#endif
         [StructLayout(LayoutKind.Sequential)]
         internal struct Native
         {
@@ -173,6 +176,7 @@ internal sealed class BerVal
         public IntPtr bv_val = IntPtr.Zero;
 
 #if NET7_0_OR_GREATER
+        [CustomTypeMarshaller(typeof(BerVal), Direction = CustomTypeMarshallerDirection.In, Features = CustomTypeMarshallerFeatures.TwoStageMarshalling)]
         internal unsafe struct PinningMarshaller
         {
             private readonly BerVal _managed;
@@ -183,7 +187,7 @@ public PinningMarshaller(BerVal managed)
 
             public ref int GetPinnableReference() => ref (_managed is null ? ref Unsafe.NullRef<int>() : ref _managed.bv_len);
 
-            public void* Value => Unsafe.AsPointer(ref GetPinnableReference());
+            public void* ToNativeValue() => Unsafe.AsPointer(ref GetPinnableReference());
         }
 #endif
     }
@@ -211,6 +215,7 @@ internal struct LdapReferralCallback
 #if NET7_0_OR_GREATER
         public static readonly unsafe int Size = sizeof(Marshaller.Native);
 
+        [CustomTypeMarshaller(typeof(LdapReferralCallback), Features = CustomTypeMarshallerFeatures.UnmanagedResources | CustomTypeMarshallerFeatures.TwoStageMarshalling)]
         public unsafe struct Marshaller
         {
             public unsafe struct Native
@@ -234,11 +239,9 @@ public Marshaller(LdapReferralCallback managed)
                 _native.dereference = managed.dereference is not null ? Marshal.GetFunctionPointerForDelegate(managed.dereference) : IntPtr.Zero;
             }
 
-            public Native Value
-            {
-                get => _native;
-                set => _native = value;
-            }
+            public Native ToNativeValue() => _native;
+
+            public void FromNativeValue(Native value) => _native = value;
 
             public LdapReferralCallback ToManaged()
             {

src/libraries/Common/src/Interop/Windows/Advapi32/Interop.LsaLookupNames2.cs

@@ -26,6 +26,7 @@ internal struct MARSHALLED_UNICODE_STRING
             internal ushort MaximumLength;
             internal string Buffer;
 
+            [CustomTypeMarshaller(typeof(MARSHALLED_UNICODE_STRING), Direction = CustomTypeMarshallerDirection.In, Features = CustomTypeMarshallerFeatures.UnmanagedResources)]
             public struct Native
             {
                 internal ushort Length;

src/libraries/Common/src/Interop/Windows/CryptUI/Interop.CryptUIDlgCertificate.cs

@@ -37,6 +37,7 @@ internal struct CRYPTUI_VIEWCERTIFICATE_STRUCTW
             internal uint nStartPage;
 
 #if NET7_0_OR_GREATER
+            [CustomTypeMarshaller(typeof(CRYPTUI_VIEWCERTIFICATE_STRUCTW), Features = CustomTypeMarshallerFeatures.UnmanagedResources)]
             internal unsafe struct Native
             {
                 private uint dwSize;
@@ -139,6 +140,7 @@ internal struct CRYPTUI_SELECTCERTIFICATE_STRUCTW
             internal IntPtr hSelectedCertStore;
 
 #if NET7_0_OR_GREATER
+            [CustomTypeMarshaller(typeof(CRYPTUI_SELECTCERTIFICATE_STRUCTW), Features = CustomTypeMarshallerFeatures.UnmanagedResources)]
             internal unsafe struct Native
             {
                 private uint dwSize;