Load Assembly referenced from within a lambda closure

azure-pipelines.yml

@@ -55,14 +55,14 @@ jobs:
     inputs:
       command: test
       projects: '**/*UnitTest/*.csproj'
-      arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+      arguments: '--configuration $(buildConfiguration)'
 
   - task: DotNetCoreCLI@2
     displayName: 'E2E tests for Spark 2.3.0'
     inputs:
       command: test
       projects: '**/Microsoft.Spark.E2ETest/*.csproj'
-      arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+      arguments: '--configuration $(buildConfiguration)'
     env:
       SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7
       HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -73,7 +73,7 @@ jobs:
     inputs:
       command: test
       projects: '**/Microsoft.Spark.E2ETest/*.csproj'
-      arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+      arguments: '--configuration $(buildConfiguration)'
     env:
       SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7
       HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -84,7 +84,7 @@ jobs:
     inputs:
       command: test
       projects: '**/Microsoft.Spark.E2ETest/*.csproj'
-      arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+      arguments: '--configuration $(buildConfiguration)'
     env:
       SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7
       HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -95,7 +95,7 @@ jobs:
     inputs:
       command: test
       projects: '**/Microsoft.Spark.E2ETest/*.csproj'
-      arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+      arguments: '--configuration $(buildConfiguration)'
     env:
       SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7
       HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -106,7 +106,7 @@ jobs:
     inputs:
       command: test
       projects: '**/Microsoft.Spark.E2ETest/*.csproj'
-      arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+      arguments: '--configuration $(buildConfiguration)'
     env:
       SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7
       HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -117,7 +117,7 @@ jobs:
     inputs:
       command: test
       projects: '**/Microsoft.Spark.E2ETest/*.csproj'
-      arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+      arguments: '--configuration $(buildConfiguration)'
     env:
       SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7
       HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -128,7 +128,7 @@ jobs:
     inputs:
       command: test
       projects: '**/Microsoft.Spark.E2ETest/*.csproj'
-      arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+      arguments: '--configuration $(buildConfiguration)'
     env:
       SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7
       HADOOP_HOME: $(Build.BinariesDirectory)\hadoop

src/csharp/Microsoft.Spark/Utils/CommandSerDe.cs

@@ -212,7 +212,7 @@ private static void SerializeUdfs(
 
             foreach (UdfSerDe.FieldData field in fields)
             {
-                SerializeUdfs((Delegate)field.Value, curNode, udfWrapperNodes, udfs);
+                SerializeUdfs((Delegate)field.ValueData.Value, curNode, udfWrapperNodes, udfs);
             }
         }
 

src/csharp/Microsoft.Spark/Utils/UdfSerDe.cs

@@ -4,10 +4,13 @@
 
 using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Linq.Expressions;
 using System.Reflection;
+using System.Runtime.Serialization;
+using System.Runtime.Serialization.Formatters.Binary;
 
 namespace Microsoft.Spark.Utils
 {
@@ -68,7 +71,51 @@ internal sealed class FieldData
         {
             public TypeData TypeData { get; set; }
             public string Name { get; set; }
+            public ValueData ValueData { get; set; }
+        }
+
+        [Serializable]
+        internal sealed class ValueData : ISerializable
+        {
+            public ValueData() { }
+
+            public TypeData TypeData { get; set; }
+
             public object Value { get; set; }
+
+            public void GetObjectData(SerializationInfo info, StreamingContext context)
+            {
+                info.AddValue("TypeData", TypeData, typeof(TypeData));
+
+                var bf = new BinaryFormatter();
+                using (var ms = new MemoryStream())
+                {
+                    bf.Serialize(ms, Value);
+                    info.AddValue("ValueSerialized", ms.ToArray(), typeof(byte[]));
+                }
+            }
+
+            public ValueData(SerializationInfo info, StreamingContext context)
+            {
+                TypeData = (TypeData)info.GetValue("TypeData", typeof(TypeData));
+
+                var valueSerialized = (byte[])info.GetValue("ValueSerialized", typeof(byte[]));
+
+                var bf = new BinaryFormatter();
+                using (var ms = new MemoryStream(valueSerialized, false))
+                {
+                    try
+                    {
+                        Value = bf.Deserialize(ms);
+                    }
+                    catch (SerializationException)
+                    {
+                        ms.Seek(0, SeekOrigin.Begin);
+                        DeserializeType(TypeData);
+                        Value = bf.Deserialize(ms);
+                    }
+                }
+            }
         }
 
         internal static UdfData Serialize(Delegate udf)
@@ -125,17 +172,27 @@ private static TargetData SerializeTarget(object target)
             Type targetType = target.GetType();
             TypeData targetTypeData = SerializeType(targetType);
 
-            System.Collections.Generic.IEnumerable<FieldData> fields = targetType.GetFields(
+            var fields = new List<FieldData>();
+            foreach (var field in targetType.GetFields(
                 BindingFlags.Instance |
                 BindingFlags.Static |
                 BindingFlags.Public |
-                BindingFlags.NonPublic).
-                    Select((field) => new FieldData()
+                BindingFlags.NonPublic))
+            {
+                object value = field.GetValue(target);
+                var fieldData = new FieldData()
+                {
+                    TypeData = SerializeType(field.FieldType),
+                    Name = field.Name,
+                    ValueData = new ValueData()
                     {
-                        TypeData = SerializeType(field.FieldType),
-                        Name = field.Name,
-                        Value = field.GetValue(target)
-                    });
+                        TypeData = (value != null) ? SerializeType(value.GetType()) : default,
+                        Value = value
+                    }
+                };
+
+                fields.Add(fieldData);
+            }
 
             // Even when an UDF does not have any closure, GetFields() returns some fields
             // which include Func<> of the udf specified.
@@ -166,7 +223,7 @@ private static object DeserializeTargetData(TargetData targetData)
                     field.Name,
                     BindingFlags.Instance |
                     BindingFlags.Public |
-                    BindingFlags.NonPublic).SetValue(target, field.Value);
+                    BindingFlags.NonPublic).SetValue(target, field.ValueData.Value);
             }
 
             return target;