Fixes #106142.

src/coreclr/jit/simd.h

@@ -802,17 +802,27 @@ TBase EvaluateBinaryScalarSpecialized(genTreeOps oper, TBase arg0, TBase arg1)
         case GT_ROL:
         {
             // Normalize the "rotate by" value
-            arg1 %= (sizeof(TBase) * BITS_PER_BYTE);
+            // EvaluateBinaryScalarRSZ allows overshifting and treats
+            // it as zeroing.
+            // But ROL ensures the rotateAmount is masked
+            // to be within range, so we pre-calculates this.
+            unsigned rotateCountMask = (sizeof(TBase) * BITS_PER_BYTE) - 1;
+            arg1 &= rotateCountMask;
             return EvaluateBinaryScalarSpecialized<TBase>(GT_LSH, arg0, arg1) |
-                   EvaluateBinaryScalarRSZ<TBase>(arg0, (sizeof(TBase) * 8) - arg1);
+                   EvaluateBinaryScalarRSZ<TBase>(arg0, (sizeof(TBase) * BITS_PER_BYTE) - arg1);
         }
 
         case GT_ROR:
         {
             // Normalize the "rotate by" value
-            arg1 %= (sizeof(TBase) * BITS_PER_BYTE);
+            // EvaluateBinaryScalarRSZ allows overshifting and treats
+            // it as zeroing.
+            // But ROR ensures the rotateAmount is masked
+            // to be within range, so we pre-calculates this.
+            unsigned rotateCountMask = (sizeof(TBase) * BITS_PER_BYTE) - 1;
+            arg1 &= rotateCountMask;
             return EvaluateBinaryScalarRSZ<TBase>(arg0, arg1) |
-                   EvaluateBinaryScalarSpecialized<TBase>(GT_LSH, arg0, (sizeof(TBase) * 8) - arg1);
+                   EvaluateBinaryScalarSpecialized<TBase>(GT_LSH, arg0, (sizeof(TBase) * BITS_PER_BYTE) - arg1);
         }
 
         case GT_RSH:

src/tests/JIT/Regression/JitBlue/Runtime_106142/Runtime_106142.cs

@@ -0,0 +1,68 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// Avx512FRotateLeftTest()
+// Generated by Fuzzlyn v2.2 on 2024-08-06 15:26:23
+// Run on X86 Windows
+// Seed: 17786759086488605032-vectort,vector128,vector256,x86aes,x86avx,x86avx2,x86avx512bw,x86avx512bwvl,x86avx512cd,x86avx512cdvl,x86avx512dq,x86avx512dqvl,x86avx512f,x86avx512fvl,x86bmi1,x86bmi2,x86fma,x86lzcnt,x86pclmulqdq,x86popcnt,x86sse,x86sse2,x86sse3,x86sse41,x86sse42,x86ssse3,x86x86base
+// Reduced from 186.1 KiB to 0.4 KiB in 00:13:36
+// Debug: Outputs <4611686018427387904, 0, 0, 0>
+// Release: Outputs <0, 0, 0, 0>
+
+
+// Avx512FRotateRightTest()
+// Generated by Fuzzlyn v2.2 on 2024-08-06 15:23:25
+// Run on X86 Windows
+// Seed: 4121896490095390139-vectort,vector128,vector256,x86aes,x86avx,x86avx2,x86avx512bw,x86avx512bwvl,x86avx512cd,x86avx512cdvl,x86avx512dq,x86avx512dqvl,x86avx512f,x86avx512fvl,x86bmi1,x86bmi2,x86fma,x86lzcnt,x86pclmulqdq,x86popcnt,x86sse,x86sse2,x86sse3,x86sse41,x86sse42,x86ssse3,x86x86base
+// Reduced from 31.4 KiB to 0.4 KiB in 00:01:15
+// Debug: Outputs <9007199254740992, 1, 1, 1>
+// Release: Outputs <0, 1, 1, 1>
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using Xunit;
+
+public class Runtime_106142
+{
+    [Fact]
+    public static void TestEntryPoint()
+    {
+        if (Avx512F.IsSupported)
+        {
+            Avx512FRotateLeftTest();
+            Avx512FRotateRightTest();
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void ShiftRightLogicalTest()
+    {
+        var vr17 = Vector128.CreateScalar(2558356441U);
+        var vr18 = Vector128.Create(0, 3113514718U, 0, 0);
+        var vr19 = Sse2.ShiftRightLogical(vr17, vr18);
+        if (Sse2.ConvertToUInt32(vr19) != 0)
+            throw new InvalidOperationException();
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void Avx512FRotateLeftTest()
+    {
+        var vr7 = Vector256.Create(1, 0, 0, 0);
+        var vr8 = Vector256.CreateScalar(18446744073709551614UL);
+        Vector256<long> vr9 = Avx512F.VL.RotateLeftVariable(vr7, vr8);
+        if (vr9.GetElement(0) != 4611686018427387904)
+            throw new InvalidOperationException();
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void Avx512FRotateRightTest()
+    {
+        var vr2 = Vector256.Create<long>(1);
+        var vr3 = Vector256.CreateScalar(9945469575827037067UL);
+        Vector256<long> vr4 = Avx512F.VL.RotateRightVariable(vr2, vr3);
+        if (vr4.GetElement(0) != 9007199254740992)
+            throw new InvalidOperationException();
+    }
+}

src/tests/JIT/Regression/JitBlue/Runtime_106142/Runtime_106142.csproj

@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <Optimize>True</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+</Project>