From 96ba8c12b7b4b02185cfa81070dbc8968b833dbb Mon Sep 17 00:00:00 2001 From: Michael Sharp <51342856+michaelgsharp@users.noreply.github.com> Date: Fri, 15 Sep 2023 18:17:57 -0600 Subject: [PATCH] 6 more naive methods for Tensor Primitives. (#92142) * 6 more naive methods * updates from pr comments --- .../ref/System.Numerics.Tensors.cs | 48 +++-- .../src/Resources/Strings.resx | 5 +- .../Numerics/Tensors/TensorPrimitives.cs | 157 +++++++++++++++ .../src/System/ThrowHelper.cs | 4 + .../tests/TensorPrimitivesTests.cs | 184 ++++++++++++++++++ 5 files changed, 376 insertions(+), 22 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs b/src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs index c657a43a054dd..c0c15cd6bfa19 100644 --- a/src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs +++ b/src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs @@ -6,27 +6,33 @@ namespace System.Numerics.Tensors { - public static class TensorPrimitives + public static partial class TensorPrimitives { - public static void Add(System.ReadOnlySpan x, float y, System.Span destination) { throw null; } - public static void Add(System.ReadOnlySpan x, System.ReadOnlySpan y, System.Span destination) { throw null; } - public static void AddMultiply(System.ReadOnlySpan x, float y, System.ReadOnlySpan multiplier, System.Span destination) { throw null; } - public static void AddMultiply(System.ReadOnlySpan x, System.ReadOnlySpan y, float multiplier, System.Span destination) { throw null; } - public static void AddMultiply(System.ReadOnlySpan x, System.ReadOnlySpan y, System.ReadOnlySpan multiplier, System.Span destination) { throw null; } - public static void Cosh(System.ReadOnlySpan x, System.Span destination) { throw null; } - public static void Divide(System.ReadOnlySpan x, float y, System.Span destination) { throw null; } - public static void Divide(System.ReadOnlySpan x, System.ReadOnlySpan y, System.Span destination) { throw null; } - public static void Exp(System.ReadOnlySpan x, System.Span destination) { throw null; } - public static void Log(System.ReadOnlySpan x, System.Span destination) { throw null; } - public static void Multiply(System.ReadOnlySpan x, float y, System.Span destination) { throw null; } - public static void Multiply(System.ReadOnlySpan x, System.ReadOnlySpan y, System.Span destination) { throw null; } - public static void MultiplyAdd(System.ReadOnlySpan x, float y, System.ReadOnlySpan addend, System.Span destination) { throw null; } - public static void MultiplyAdd(System.ReadOnlySpan x, System.ReadOnlySpan y, float addend, System.Span destination) { throw null; } - public static void MultiplyAdd(System.ReadOnlySpan x, System.ReadOnlySpan y, System.ReadOnlySpan addend, System.Span destination) { throw null; } - public static void Negate(System.ReadOnlySpan x, System.Span destination) { throw null; } - public static void Subtract(System.ReadOnlySpan x, float y, System.Span destination) { throw null; } - public static void Subtract(System.ReadOnlySpan x, System.ReadOnlySpan y, System.Span destination) { throw null; } - public static void Sinh(System.ReadOnlySpan x, System.Span destination) { throw null; } - public static void Tanh(System.ReadOnlySpan x, System.Span destination) { throw null; } + public static void Add(System.ReadOnlySpan x, System.ReadOnlySpan y, System.Span destination) { } + public static void Add(System.ReadOnlySpan x, float y, System.Span destination) { } + public static void AddMultiply(System.ReadOnlySpan x, System.ReadOnlySpan y, System.ReadOnlySpan multiplier, System.Span destination) { } + public static void AddMultiply(System.ReadOnlySpan x, System.ReadOnlySpan y, float multiplier, System.Span destination) { } + public static void AddMultiply(System.ReadOnlySpan x, float y, System.ReadOnlySpan multiplier, System.Span destination) { } + public static void Cosh(System.ReadOnlySpan x, System.Span destination) { } + public static float CosineSimilarity(System.ReadOnlySpan x, System.ReadOnlySpan y) { throw null; } + public static float Distance(System.ReadOnlySpan x, System.ReadOnlySpan y) { throw null; } + public static void Divide(System.ReadOnlySpan x, System.ReadOnlySpan y, System.Span destination) { } + public static void Divide(System.ReadOnlySpan x, float y, System.Span destination) { } + public static float Dot(System.ReadOnlySpan x, System.ReadOnlySpan y) { throw null; } + public static void Exp(System.ReadOnlySpan x, System.Span destination) { } + public static float L2Normalize(System.ReadOnlySpan x) { throw null; } + public static void Log(System.ReadOnlySpan x, System.Span destination) { } + public static void Multiply(System.ReadOnlySpan x, System.ReadOnlySpan y, System.Span destination) { } + public static void Multiply(System.ReadOnlySpan x, float y, System.Span destination) { } + public static void MultiplyAdd(System.ReadOnlySpan x, System.ReadOnlySpan y, System.ReadOnlySpan addend, System.Span destination) { } + public static void MultiplyAdd(System.ReadOnlySpan x, System.ReadOnlySpan y, float addend, System.Span destination) { } + public static void MultiplyAdd(System.ReadOnlySpan x, float y, System.ReadOnlySpan addend, System.Span destination) { } + public static void Negate(System.ReadOnlySpan x, System.Span destination) { } + public static void Sigmoid(System.ReadOnlySpan x, System.Span destination) { } + public static void Sinh(System.ReadOnlySpan x, System.Span destination) { } + public static void SoftMax(System.ReadOnlySpan x, System.Span destination) { } + public static void Subtract(System.ReadOnlySpan x, System.ReadOnlySpan y, System.Span destination) { } + public static void Subtract(System.ReadOnlySpan x, float y, System.Span destination) { } + public static void Tanh(System.ReadOnlySpan x, System.Span destination) { } } } diff --git a/src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx b/src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx index d70054a241126..45f0d8fa17893 100644 --- a/src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx +++ b/src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx @@ -120,7 +120,10 @@ Destination is too short. + + Input span arguments must not be empty. + Input span arguments must all have the same length. - + \ No newline at end of file diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs index 08bd9d362217e..87d41d1f9c0a5 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs @@ -253,5 +253,162 @@ public static void Tanh(ReadOnlySpan x, Span destination) destination[i] = MathF.Tanh(x[i]); } } + + /// Computes the cosine similarity between two non-zero vectors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The cosine similarity between the two vectors. + /// Length of '' must be same as length of ''. + /// '' and '' must not be empty. + public static float CosineSimilarity(ReadOnlySpan x, ReadOnlySpan y) + { + if (x.Length != y.Length) + { + ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); + } + if (x.Length == 0 || y.Length == 0) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + float dotprod = 0f; + float magx = 0f; + float magy = 0f; + + for (int i = 0; i < x.Length; i++) + { + dotprod += x[i] * y[i]; + magx += x[i] * x[i]; + magy += y[i] * y[i]; + } + + return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy)); + } + + /// + /// Compute the distance between two points in Euclidean space. + /// + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The Euclidean distance. + /// Length of '' must be same as length of ''. + /// '' and '' must not be empty. + public static float Distance(ReadOnlySpan x, ReadOnlySpan y) + { + if (x.Length != y.Length) + { + ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); + } + if (x.Length == 0 || y.Length == 0) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + float distance = 0f; + + for (int i = 0; i < x.Length; i++) + { + float dist = x[i] - y[i]; + distance += dist * dist; + } + + return MathF.Sqrt(distance); + } + + /// + /// A mathematical operation that takes two vectors and returns a scalar. + /// + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The dot product. + /// Length of '' must be same as length of ''. + public static float Dot(ReadOnlySpan x, ReadOnlySpan y) // BLAS1: dot + { + if (x.Length != y.Length) + { + ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); + } + + float dotprod = 0f; + + for (int i = 0; i < x.Length; i++) + { + dotprod += x[i] * y[i]; + } + + return dotprod; + } + + /// + /// A mathematical operation that takes a vector and returns the L2 norm. + /// + /// The first tensor, represented as a span. + /// The L2 norm. + public static float L2Normalize(ReadOnlySpan x) // BLAS1: nrm2 + { + float magx = 0f; + + for (int i = 0; i < x.Length; i++) + { + magx += x[i] * x[i]; + } + + return MathF.Sqrt(magx); + } + + /// + /// A function that takes a collection of real numbers and returns a probability distribution. + /// + /// The first tensor, represented as a span. + /// The destination tensor. + /// Destination is too short. + /// '' must not be empty. + public static void SoftMax(ReadOnlySpan x, Span destination) + { + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + if (x.Length == 0) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + float expSum = 0f; + + for (int i = 0; i < x.Length; i++) + { + expSum += MathF.Pow((float)Math.E, x[i]); + } + + for (int i = 0; i < x.Length; i++) + { + destination[i] = MathF.Exp(x[i]) / expSum; + } + } + + /// + /// A function that takes a real number and returns a value between 0 and 1. + /// + /// The first tensor, represented as a span. + /// The destination tensor. + /// Destination is too short. + /// '' must not be empty. + public static void Sigmoid(ReadOnlySpan x, Span destination) + { + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + if (x.Length == 0) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + for (int i = 0; i < x.Length; i++) + { + destination[i] = 1f / (1 + MathF.Exp(-x[i])); + } + } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs b/src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs index cc8d423f5a4d0..902b27787e856 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs @@ -14,5 +14,9 @@ public static void ThrowArgument_DestinationTooShort() => [DoesNotReturn] public static void ThrowArgument_SpansMustHaveSameLength() => throw new ArgumentException(SR.Argument_SpansMustHaveSameLength); + + [DoesNotReturn] + public static void ThrowArgument_SpansMustBeNonEmpty() => + throw new ArgumentException(SR.Argument_SpansMustBeNonEmpty); } } diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs index 5a9912542a8c2..e8bb48a50f6b2 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs @@ -702,5 +702,189 @@ public static void TanhTensor_ThrowsForTooShortDestination(int tensorLength) AssertExtensions.Throws("destination", () => TensorPrimitives.Tanh(x, destination)); } + + [Theory] + [MemberData(nameof(TensorLengths))] + public static void CosineSimilarity_ThrowsForMismatchedLengths_x_y(int tensorLength) + { + float[] x = CreateAndFillTensor(tensorLength); + float[] y = CreateAndFillTensor(tensorLength - 1); + + Assert.Throws(() => TensorPrimitives.CosineSimilarity(x, y)); + } + + [Fact] + public static void CosineSimilarity_ThrowsForEmpty_x_y() + { + float[] x = []; + float[] y = []; + + Assert.Throws(() => TensorPrimitives.CosineSimilarity(x, y)); + } + + [Theory] + [InlineData(new float[] { 3, 2, 0, 5 }, new float[] { 1, 0, 0, 0 }, 0.49f)] + [InlineData(new float[] { 1, 1, 1, 1, 1, 0 }, new float[] { 1, 1, 1, 1, 0, 1 }, 0.80f)] + public static void CosineSimilarity(float[] x, float[] y, float expectedResult) + { + Assert.Equal(expectedResult, TensorPrimitives.CosineSimilarity(x, y), .01f); + } + + [Fact] + public static void Distance_ThrowsForEmpty_x_y() + { + float[] x = []; + float[] y = []; + + Assert.Throws(() => TensorPrimitives.Distance(x, y)); + } + + [Theory] + [MemberData(nameof(TensorLengths))] + public static void Distance_ThrowsForMismatchedLengths_x_y(int tensorLength) + { + float[] x = CreateAndFillTensor(tensorLength); + float[] y = CreateAndFillTensor(tensorLength - 1); + + Assert.Throws(() => TensorPrimitives.Distance(x, y)); + } + + [Theory] + [InlineData(new float[] { 3, 2 }, new float[] { 4, 1 }, 1.4142f)] + [InlineData(new float[] { 0, 4 }, new float[] { 6, 2 }, 6.3245f)] + [InlineData(new float[] { 1, 2, 3 }, new float[] { 4, 5, 6 }, 5.1961f)] + [InlineData(new float[] { 5, 1, 6, 10 }, new float[] { 7, 2, 8, 4 }, 6.7082f)] + public static void Distance(float[] x, float[] y, float expectedResult) + { + Assert.Equal(expectedResult, TensorPrimitives.Distance(x, y), .001f); + } + + [Theory] + [MemberData(nameof(TensorLengths))] + public static void Dot_ThrowsForMismatchedLengths_x_y(int tensorLength) + { + float[] x = CreateAndFillTensor(tensorLength); + float[] y = CreateAndFillTensor(tensorLength - 1); + + Assert.Throws(() => TensorPrimitives.Dot(x, y)); + } + + [Theory] + [InlineData(new float[] { 1, 3, -5 }, new float[] { 4, -2, -1 }, 3)] + [InlineData(new float[] { 1, 2, 3 }, new float[] { 4, 5, 6 }, 32)] + [InlineData(new float[] { 1, 2, 3, 10, 8 }, new float[] { 4, 5, 6, -2, 7 }, 68)] + [InlineData(new float[] { }, new float[] { }, 0)] + public static void Dot(float[] x, float[] y, float expectedResult) + { + Assert.Equal(expectedResult, TensorPrimitives.Dot(x, y), .001f); + } + + [Theory] + [InlineData(new float[] { 1, 2, 3 }, 3.7416)] + [InlineData(new float[] { 3, 4 }, 5)] + [InlineData(new float[] { 3 }, 3)] + [InlineData(new float[] { 3, 4, 1, 2 }, 5.477)] + [InlineData(new float[] { }, 0f)] + public static void L2Normalize(float[] x, float expectedResult) + { + Assert.Equal(expectedResult, TensorPrimitives.L2Normalize(x), .001f); + } + + [Theory] + [MemberData(nameof(TensorLengths))] + public static void SoftMax_ThrowsForTooShortDestination(int tensorLength) + { + float[] x = CreateAndFillTensor(tensorLength); + float[] destination = CreateTensor(tensorLength - 1); + + AssertExtensions.Throws("destination", () => TensorPrimitives.SoftMax(x, destination)); + } + + [Theory] + [InlineData(new float[] { 3, 1, .2f }, new float[] { 0.8360188f, 0.11314284f, 0.05083836f })] + [InlineData(new float[] { 3, 4, 1 }, new float[] { 0.2594f, 0.7052f, 0.0351f })] + [InlineData(new float[] { 5, 3 }, new float[] { 0.8807f, 0.1192f })] + [InlineData(new float[] { 4, 2, 1, 9 }, new float[] { 0.0066f, 9.04658e-4f, 3.32805e-4f, 0.9920f})] + public static void SoftMax(float[] x, float[] expectedResult) + { + var dest = new float[x.Length]; + TensorPrimitives.SoftMax(x, dest); + + for (int i = 0; i < x.Length; i++) + { + Assert.Equal(expectedResult[i], dest[i], .001f); + } + } + + [Fact] + public static void SoftMax_DestinationLongerThanSource() + { + var x = new float[] { 3, 1, .2f }; + var expectedResult = new float[] { 0.8360188f, 0.11314284f, 0.05083836f }; + var dest = new float[x.Length + 1]; + TensorPrimitives.SoftMax(x, dest); + + for (int i = 0; i < x.Length; i++) + { + Assert.Equal(expectedResult[i], dest[i], .001f); + } + } + + [Fact] + public static void SoftMax_ThrowsForEmpty_x_y() + { + var x = new float[] { }; + var dest = new float[x.Length]; + + AssertExtensions.Throws(() => TensorPrimitives.SoftMax(x, dest)); + } + + [Theory] + [MemberData(nameof(TensorLengths))] + public static void Sigmoid_ThrowsForTooShortDestination(int tensorLength) + { + float[] x = CreateAndFillTensor(tensorLength); + float[] destination = CreateTensor(tensorLength - 1); + + AssertExtensions.Throws("destination", () => TensorPrimitives.Sigmoid(x, destination)); + } + + [Theory] + [InlineData(new float[] { -5, -4.5f, -4 }, new float[] { 0.0066f, 0.0109f, 0.0179f })] + [InlineData(new float[] { 4.5f, 5 }, new float[] { 0.9890f, 0.9933f })] + [InlineData(new float[] { 0, -3, 3, .5f }, new float[] { 0.5f, 0.0474f, 0.9525f, 0.6224f })] + public static void Sigmoid(float[] x, float[] expectedResult) + { + var dest = new float[x.Length]; + TensorPrimitives.Sigmoid(x, dest); + + for (int i = 0; i < x.Length; i++) + { + Assert.Equal(expectedResult[i], dest[i], .001f); + } + } + + [Fact] + public static void Sigmoid_DestinationLongerThanSource() + { + var x = new float[] { -5, -4.5f, -4 }; + var expectedResult = new float[] { 0.0066f, 0.0109f, 0.0179f }; + var dest = new float[x.Length + 1]; + TensorPrimitives.Sigmoid(x, dest); + + for (int i = 0; i < x.Length; i++) + { + Assert.Equal(expectedResult[i], dest[i], .001f); + } + } + + [Fact] + public static void Sigmoid_ThrowsForEmpty_x_y() + { + var x = new float[] { }; + var dest = new float[x.Length]; + + AssertExtensions.Throws(() => TensorPrimitives.Sigmoid(x, dest)); + } } }