From 62004f89cc4328a77556b5e7e60a3f96f1d49100 Mon Sep 17 00:00:00 2001 From: Martin Finkel Date: Thu, 15 Sep 2022 17:02:52 +0700 Subject: [PATCH 01/10] Add option for truncated stream detection fix https://github.com/dotnet/runtime/issues/47563 --- .../CompressionStreamUnitTestBase.cs | 3 +- .../System/IO/Compression/ZipTestHelper.cs | 41 +++++++++++++++++++ .../src/Resources/Strings.resx | 3 ++ .../dec/BrotliStream.Decompress.cs | 21 ++++++++++ .../tests/ZipFile.Create.cs | 24 +++++++++++ .../src/Resources/Strings.resx | 3 ++ .../Compression/DeflateZLib/DeflateStream.cs | 40 ++++++++++++++++++ .../IO/Compression/DeflateZLib/Inflater.cs | 5 +++ .../tests/CompressionStreamUnitTests.Gzip.cs | 1 - .../tests/CompressionStreamUnitTests.ZLib.cs | 1 - 10 files changed, 139 insertions(+), 3 deletions(-) diff --git a/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs b/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs index 684af671e9ac4..d54f71c12f184 100644 --- a/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs +++ b/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs @@ -469,7 +469,6 @@ async Task GetLengthAsync(CompressionLevel compressionLevel) Assert.True(optimalLength >= smallestLength); } - [ActiveIssue("https://github.com/dotnet/runtime/issues/47563")] [Theory] [InlineData(TestScenario.ReadAsync)] [InlineData(TestScenario.Read)] @@ -479,6 +478,8 @@ async Task GetLengthAsync(CompressionLevel compressionLevel) [InlineData(TestScenario.ReadByteAsync)] public async Task StreamTruncation_IsDetected(TestScenario scenario) { + AppContext.SetSwitch("System.IO.Compression.UseStrictValidation", true); + var buffer = new byte[16]; byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray(); byte[] compressedData; diff --git a/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs b/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs index 31a3e7878266c..f55fe1ed5a1b5 100644 --- a/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs +++ b/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs @@ -110,6 +110,11 @@ public static void StreamsEqual(Stream ast, Stream bst) StreamsEqual(ast, bst, -1); } + public static async Task StreamsEqualAsync(Stream ast, Stream bst) + { + await StreamsEqualAsync(ast, bst, -1); + } + public static void StreamsEqual(Stream ast, Stream bst, int blocksToRead) { if (ast.CanSeek) @@ -146,6 +151,42 @@ public static void StreamsEqual(Stream ast, Stream bst, int blocksToRead) } while (ac == bufSize); } + public static async Task StreamsEqualAsync(Stream ast, Stream bst, int blocksToRead) + { + if (ast.CanSeek) + ast.Seek(0, SeekOrigin.Begin); + if (bst.CanSeek) + bst.Seek(0, SeekOrigin.Begin); + + const int bufSize = 4096; + byte[] ad = new byte[bufSize]; + byte[] bd = new byte[bufSize]; + + int ac = 0; + int bc = 0; + + int blocksRead = 0; + + //assume read doesn't do weird things + do + { + if (blocksToRead != -1 && blocksRead >= blocksToRead) + break; + + ac = await ReadAllBytesAsync(ast, ad, 0, 4096); + bc = await ReadAllBytesAsync(bst, bd, 0, 4096); + + if (ac != bc) + { + bd = NormalizeLineEndings(bd); + } + + Assert.True(ArraysEqual(ad, bd, ac), "Stream contents not equal: " + ast.ToString() + ", " + bst.ToString()); + + blocksRead++; + } while (ac == bufSize); + } + public static async Task IsZipSameAsDirAsync(string archiveFile, string directory, ZipArchiveMode mode) { await IsZipSameAsDirAsync(archiveFile, directory, mode, requireExplicit: false, checkTimes: false); diff --git a/src/libraries/System.IO.Compression.Brotli/src/Resources/Strings.resx b/src/libraries/System.IO.Compression.Brotli/src/Resources/Strings.resx index 355df1fd20412..4f892bf574f2c 100644 --- a/src/libraries/System.IO.Compression.Brotli/src/Resources/Strings.resx +++ b/src/libraries/System.IO.Compression.Brotli/src/Resources/Strings.resx @@ -175,6 +175,9 @@ BrotliStream.BaseStream returned more bytes than requested in Read. + + Decoder ran into truncated data. + System.IO.Compression.Brotli is not supported on this platform. diff --git a/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs b/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs index d7217bbfa0a2f..fdb0258690f14 100644 --- a/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs +++ b/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs @@ -15,6 +15,7 @@ public sealed partial class BrotliStream : Stream private BrotliDecoder _decoder; private int _bufferOffset; private int _bufferCount; + private bool _nonEmptyInput; /// Reads a number of decompressed bytes into the specified byte array. /// The array used to store decompressed bytes. @@ -65,9 +66,12 @@ public override int Read(Span buffer) int bytesRead = _stream.Read(_buffer, _bufferCount, _buffer.Length - _bufferCount); if (bytesRead <= 0) { + if (GetStrictValidationSetting() && _nonEmptyInput && !buffer.IsEmpty) + ThrowTruncatedInvalidData(); break; } + _nonEmptyInput = true; _bufferCount += bytesRead; if (_bufferCount > _buffer.Length) @@ -150,10 +154,13 @@ async ValueTask Core(Memory buffer, CancellationToken cancellationTok int bytesRead = await _stream.ReadAsync(_buffer.AsMemory(_bufferCount), cancellationToken).ConfigureAwait(false); if (bytesRead <= 0) { + if (GetStrictValidationSetting() && _nonEmptyInput && !buffer.IsEmpty) + ThrowTruncatedInvalidData(); break; } _bufferCount += bytesRead; + _nonEmptyInput = true; if (_bufferCount > _buffer.Length) { @@ -227,9 +234,23 @@ private bool TryDecompress(Span destination, out int bytesWritten) return false; } + private static bool GetStrictValidationSetting() + { + if (AppContext.TryGetSwitch("System.IO.Compression.UseStrictValidation", out bool strictValidation)) + { + return strictValidation; + } + + string? envVar = Environment.GetEnvironmentVariable("DOTNET_SYSTEM_COMPRESSION_STRICT_VALIDATION"); + return envVar is not null && (envVar == "1" || envVar.Equals("true", StringComparison.OrdinalIgnoreCase)); + } + private static void ThrowInvalidStream() => // The stream is either malicious or poorly implemented and returned a number of // bytes larger than the buffer supplied to it. throw new InvalidDataException(SR.BrotliStream_Decompress_InvalidStream); + + private static void ThrowTruncatedInvalidData() => + throw new InvalidDataException(SR.BrotliStream_Decompress_TruncatedData); } } diff --git a/src/libraries/System.IO.Compression.ZipFile/tests/ZipFile.Create.cs b/src/libraries/System.IO.Compression.ZipFile/tests/ZipFile.Create.cs index 25bd7c65ec061..7f450e73acf5c 100644 --- a/src/libraries/System.IO.Compression.ZipFile/tests/ZipFile.Create.cs +++ b/src/libraries/System.IO.Compression.ZipFile/tests/ZipFile.Create.cs @@ -45,6 +45,30 @@ public void CreateFromDirectory_IncludeBaseDirectory() } } + [Fact] + public async Task CreateFromDirectory_IncludeBaseDirectoryAsync() + { + string folderName = zfolder("normal"); + string withBaseDir = GetTestFilePath(); + ZipFile.CreateFromDirectory(folderName, withBaseDir, CompressionLevel.Optimal, true); + + IEnumerable expected = Directory.EnumerateFiles(zfolder("normal"), "*", SearchOption.AllDirectories); + using (ZipArchive actual_withbasedir = ZipFile.Open(withBaseDir, ZipArchiveMode.Read)) + { + foreach (ZipArchiveEntry actualEntry in actual_withbasedir.Entries) + { + string expectedFile = expected.Single(i => Path.GetFileName(i).Equals(actualEntry.Name)); + Assert.StartsWith("normal", actualEntry.FullName); + Assert.Equal(new FileInfo(expectedFile).Length, actualEntry.Length); + using (Stream expectedStream = File.OpenRead(expectedFile)) + using (Stream actualStream = actualEntry.Open()) + { + await StreamsEqualAsync(expectedStream, actualStream); + } + } + } + } + [Fact] public void CreateFromDirectoryUnicode() { diff --git a/src/libraries/System.IO.Compression/src/Resources/Strings.resx b/src/libraries/System.IO.Compression/src/Resources/Strings.resx index 78c96e856d2c3..66113dec492eb 100644 --- a/src/libraries/System.IO.Compression/src/Resources/Strings.resx +++ b/src/libraries/System.IO.Compression/src/Resources/Strings.resx @@ -278,6 +278,9 @@ Split or spanned archives are not supported. + + Found truncated data while decoding. + Zip file corrupt: unexpected end of stream reached. diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs index 30d5b35f12e8c..44ea0cc2128f0 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs @@ -279,6 +279,15 @@ internal int ReadCore(Span buffer) int n = _stream.Read(_buffer, 0, _buffer.Length); if (n <= 0) { + // - Inflater didn't return any data although a non-empty output buffer was passed by the caller. + // - More input is needed but there is no more input available. + // - Inflation is not finished yet. + // - Provided input wasn't completely empty + // In such case, we are dealing with a truncated input stream. + if (GetStrictValidationSetting() && !buffer.IsEmpty && !_inflater.Finished() && _inflater.NonEmptyInput()) + { + ThrowTruncatedInvalidData(); + } break; } else if (n > _buffer.Length) @@ -347,6 +356,9 @@ private static void ThrowGenericInvalidData() => // bytes < 0 || > than the buffer supplied to it. throw new InvalidDataException(SR.GenericInvalidData); + private static void ThrowTruncatedInvalidData() => + throw new InvalidDataException(SR.TruncatedData); + public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? asyncCallback, object? asyncState) => TaskToApm.Begin(ReadAsync(buffer, offset, count, CancellationToken.None), asyncCallback, asyncState); @@ -416,6 +428,15 @@ async ValueTask Core(Memory buffer, CancellationToken cancellationTok int n = await _stream.ReadAsync(new Memory(_buffer, 0, _buffer.Length), cancellationToken).ConfigureAwait(false); if (n <= 0) { + // - Inflater didn't return any data although a non-empty output buffer was passed by the caller. + // - More input is needed but there is no more input available. + // - Inflation is not finished yet. + // - Provided input wasn't completely empty + // In such case, we are dealing with a truncated input stream. + if (GetStrictValidationSetting() && !_inflater.Finished() && _inflater.NonEmptyInput() && !buffer.IsEmpty) + { + ThrowTruncatedInvalidData(); + } break; } else if (n > _buffer.Length) @@ -893,6 +914,10 @@ public async Task CopyFromSourceToDestinationAsync() // Now, use the source stream's CopyToAsync to push directly to our inflater via this helper stream await _deflateStream._stream.CopyToAsync(this, _arrayPoolBuffer.Length, _cancellationToken).ConfigureAwait(false); + if (!_deflateStream._inflater.Finished() && GetStrictValidationSetting()) + { + ThrowTruncatedInvalidData(); + } } finally { @@ -925,6 +950,10 @@ public void CopyFromSourceToDestination() // Now, use the source stream's CopyToAsync to push directly to our inflater via this helper stream _deflateStream._stream.CopyTo(this, _arrayPoolBuffer.Length); + if (!_deflateStream._inflater.Finished() && GetStrictValidationSetting()) + { + ThrowTruncatedInvalidData(); + } } finally { @@ -1049,5 +1078,16 @@ private void AsyncOperationCompleting() => private static void ThrowInvalidBeginCall() => throw new InvalidOperationException(SR.InvalidBeginCall); + + private static bool GetStrictValidationSetting() + { + if (AppContext.TryGetSwitch("System.IO.Compression.UseStrictValidation", out bool strictValidation)) + { + return strictValidation; + } + + string? envVar = Environment.GetEnvironmentVariable("DOTNET_SYSTEM_COMPRESSION_STRICT_VALIDATION"); + return envVar is not null && (envVar == "1" || envVar.Equals("true", StringComparison.OrdinalIgnoreCase)); + } } } diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/Inflater.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/Inflater.cs index 493a6f47d8cb2..ecc997f5dc8cc 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/Inflater.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/Inflater.cs @@ -17,6 +17,7 @@ internal sealed class Inflater : IDisposable private const int MinWindowBits = -15; // WindowBits must be between -8..-15 to ignore the header, 8..15 for private const int MaxWindowBits = 47; // zlib headers, 24..31 for GZip headers, or 40..47 for either Zlib or GZip + private bool _nonEmptyInput; // Whether there is any non empty input private bool _finished; // Whether the end of the stream has been reached private bool _isDisposed; // Prevents multiple disposals private readonly int _windowBits; // The WindowBits parameter passed to Inflater construction @@ -34,6 +35,7 @@ internal Inflater(int windowBits, long uncompressedSize = -1) { Debug.Assert(windowBits >= MinWindowBits && windowBits <= MaxWindowBits); _finished = false; + _nonEmptyInput = false; _isDisposed = false; _windowBits = windowBits; InflateInit(windowBits); @@ -176,6 +178,8 @@ private unsafe bool ResetStreamForLeftoverInput() public bool NeedsInput() => _zlibStream.AvailIn == 0; + public bool NonEmptyInput() => _nonEmptyInput; + public void SetInput(byte[] inputBuffer, int startIndex, int count) { Debug.Assert(NeedsInput(), "We have something left in previous input!"); @@ -200,6 +204,7 @@ public unsafe void SetInput(ReadOnlyMemory inputBuffer) _zlibStream.NextIn = (IntPtr)_inputBufferHandle.Pointer; _zlibStream.AvailIn = (uint)inputBuffer.Length; _finished = false; + _nonEmptyInput = true; } } diff --git a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs index 29628646f1f64..6ffa72b2f69c3 100644 --- a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs +++ b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs @@ -275,7 +275,6 @@ public void DerivedStream_ReadWriteSpan_UsesReadWriteArray() } - [ActiveIssue("https://github.com/dotnet/runtime/issues/47563")] [Fact] public void StreamCorruption_IsDetected() { diff --git a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs index 26176d9396fe4..df3a315da070f 100644 --- a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs +++ b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs @@ -19,7 +19,6 @@ public class ZLibStreamUnitTests : CompressionStreamUnitTestBase public override Stream BaseStream(Stream stream) => ((ZLibStream)stream).BaseStream; protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("ZLibTestData", Path.GetFileName(uncompressedPath) + ".z"); - [ActiveIssue("https://github.com/dotnet/runtime/issues/47563")] [Fact] public void StreamCorruption_IsDetected() { From fa4b613a9c3fe4f13d8b2fc147676ca2d2e92236 Mon Sep 17 00:00:00 2001 From: Martin Finkel Date: Fri, 16 Sep 2022 14:34:40 +0700 Subject: [PATCH 02/10] Use RemoteExecutor move the test to concrete classes as abstracted classes are not supported by RemoteExecutor --- .../CompressionStreamUnitTestBase.cs | 84 ----------- .../CompressionStreamUnitTests.Deflate.cs | 90 ++++++++++++ .../tests/CompressionStreamUnitTests.Gzip.cs | 88 +++++++++++ .../tests/CompressionStreamUnitTests.ZLib.cs | 139 +++++++++++++++--- .../tests/System.IO.Compression.Tests.csproj | 1 + 5 files changed, 295 insertions(+), 107 deletions(-) diff --git a/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs b/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs index d54f71c12f184..31156d796a6e1 100644 --- a/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs +++ b/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs @@ -2,13 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; -using System.IO.Compression.Tests; -using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; using Xunit; -using Xunit.Sdk; namespace System.IO.Compression { @@ -468,87 +465,6 @@ async Task GetLengthAsync(CompressionLevel compressionLevel) Assert.True(fastestLength >= optimalLength); Assert.True(optimalLength >= smallestLength); } - - [Theory] - [InlineData(TestScenario.ReadAsync)] - [InlineData(TestScenario.Read)] - [InlineData(TestScenario.Copy)] - [InlineData(TestScenario.CopyAsync)] - [InlineData(TestScenario.ReadByte)] - [InlineData(TestScenario.ReadByteAsync)] - public async Task StreamTruncation_IsDetected(TestScenario scenario) - { - AppContext.SetSwitch("System.IO.Compression.UseStrictValidation", true); - - var buffer = new byte[16]; - byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray(); - byte[] compressedData; - using (var compressed = new MemoryStream()) - using (Stream compressor = CreateStream(compressed, CompressionMode.Compress)) - { - foreach (byte b in source) - { - compressor.WriteByte(b); - } - - compressor.Dispose(); - compressedData = compressed.ToArray(); - } - - for (var i = 1; i <= compressedData.Length; i += 1) - { - bool expectException = i < compressedData.Length; - using (var compressedStream = new MemoryStream(compressedData.Take(i).ToArray())) - { - using (Stream decompressor = CreateStream(compressedStream, CompressionMode.Decompress)) - { - var decompressedStream = new MemoryStream(); - - try - { - switch (scenario) - { - case TestScenario.Copy: - decompressor.CopyTo(decompressedStream); - break; - - case TestScenario.CopyAsync: - await decompressor.CopyToAsync(decompressedStream); - break; - - case TestScenario.Read: - while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0) { }; - break; - - case TestScenario.ReadAsync: - while (await ZipFileTestBase.ReadAllBytesAsync(decompressor, buffer, 0, buffer.Length) != 0) { }; - break; - - case TestScenario.ReadByte: - while (decompressor.ReadByte() != -1) { } - break; - - case TestScenario.ReadByteAsync: - while (await decompressor.ReadByteAsync() != -1) { } - break; - } - } - catch (InvalidDataException e) - { - if (expectException) - continue; - - throw new XunitException($"An unexpected error occurred while decompressing data:{e}"); - } - - if (expectException) - { - throw new XunitException($"Truncated stream was decompressed successfully but exception was expected: length={i}/{compressedData.Length}"); - } - } - } - } - } } public enum TestScenario diff --git a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Deflate.cs b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Deflate.cs index 0dbf8a4b7dd5c..e295fd64895cb 100644 --- a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Deflate.cs +++ b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Deflate.cs @@ -3,10 +3,14 @@ using System.Collections; using System.Collections.Generic; +using System.IO.Compression.Tests; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.DotNet.RemoteExecutor; using Xunit; +using Xunit.Sdk; namespace System.IO.Compression { @@ -100,6 +104,92 @@ public void CompressorNotClosed_DecompressorStillSuccessful(bool closeCompressor } } + [InlineData(TestScenario.ReadAsync)] + [InlineData(TestScenario.Read)] + [InlineData(TestScenario.Copy)] + [InlineData(TestScenario.CopyAsync)] + [InlineData(TestScenario.ReadByte)] + [InlineData(TestScenario.ReadByteAsync)] + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public void StreamTruncation_IsDetected(TestScenario testScenario) + { + RemoteExecutor.Invoke(async (testScenario) => + { + TestScenario scenario = Enum.Parse(testScenario); + + AppContext.SetSwitch("System.IO.Compression.UseStrictValidation", true); + + var buffer = new byte[16]; + byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray(); + byte[] compressedData; + using (var compressed = new MemoryStream()) + using (Stream compressor = CreateStream(compressed, CompressionMode.Compress)) + { + foreach (byte b in source) + { + compressor.WriteByte(b); + } + + compressor.Dispose(); + compressedData = compressed.ToArray(); + } + + for (var i = 1; i <= compressedData.Length; i += 1) + { + bool expectException = i < compressedData.Length; + using (var compressedStream = new MemoryStream(compressedData.Take(i).ToArray())) + { + using (Stream decompressor = CreateStream(compressedStream, CompressionMode.Decompress)) + { + var decompressedStream = new MemoryStream(); + + try + { + switch (scenario) + { + case TestScenario.Copy: + decompressor.CopyTo(decompressedStream); + break; + + case TestScenario.CopyAsync: + await decompressor.CopyToAsync(decompressedStream); + break; + + case TestScenario.Read: + while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0) { }; + break; + + case TestScenario.ReadAsync: + while (await ZipFileTestBase.ReadAllBytesAsync(decompressor, buffer, 0, buffer.Length) != 0) { }; + break; + + case TestScenario.ReadByte: + while (decompressor.ReadByte() != -1) { } + break; + + case TestScenario.ReadByteAsync: + while (await decompressor.ReadByteAsync() != -1) { } + break; + } + } + catch (InvalidDataException e) + { + if (expectException) + continue; + + throw new XunitException($"An unexpected error occurred while decompressing data:{e}"); + } + + if (expectException) + { + throw new XunitException($"Truncated stream was decompressed successfully but exception was expected: length={i}/{compressedData.Length}"); + } + } + } + } + }, testScenario.ToString()).Dispose(); + } + private sealed class DerivedDeflateStream : DeflateStream { public bool ReadArrayInvoked = false, WriteArrayInvoked = false; diff --git a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs index 6ffa72b2f69c3..c48e3550c68a1 100644 --- a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs +++ b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs @@ -6,7 +6,9 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.DotNet.RemoteExecutor; using Xunit; +using Xunit.Sdk; namespace System.IO.Compression { @@ -321,6 +323,92 @@ public void StreamCorruption_IsDetected() } } + [InlineData(TestScenario.ReadAsync)] + [InlineData(TestScenario.Read)] + [InlineData(TestScenario.Copy)] + [InlineData(TestScenario.CopyAsync)] + [InlineData(TestScenario.ReadByte)] + [InlineData(TestScenario.ReadByteAsync)] + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public void StreamTruncation_IsDetected(TestScenario testScenario) + { + RemoteExecutor.Invoke(async (testScenario) => + { + TestScenario scenario = Enum.Parse(testScenario); + + AppContext.SetSwitch("System.IO.Compression.UseStrictValidation", true); + + var buffer = new byte[16]; + byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray(); + byte[] compressedData; + using (var compressed = new MemoryStream()) + using (Stream compressor = CreateStream(compressed, CompressionMode.Compress)) + { + foreach (byte b in source) + { + compressor.WriteByte(b); + } + + compressor.Dispose(); + compressedData = compressed.ToArray(); + } + + for (var i = 1; i <= compressedData.Length; i += 1) + { + bool expectException = i < compressedData.Length; + using (var compressedStream = new MemoryStream(compressedData.Take(i).ToArray())) + { + using (Stream decompressor = CreateStream(compressedStream, CompressionMode.Decompress)) + { + var decompressedStream = new MemoryStream(); + + try + { + switch (scenario) + { + case TestScenario.Copy: + decompressor.CopyTo(decompressedStream); + break; + + case TestScenario.CopyAsync: + await decompressor.CopyToAsync(decompressedStream); + break; + + case TestScenario.Read: + while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0) { }; + break; + + case TestScenario.ReadAsync: + while (await ZipFileTestBase.ReadAllBytesAsync(decompressor, buffer, 0, buffer.Length) != 0) { }; + break; + + case TestScenario.ReadByte: + while (decompressor.ReadByte() != -1) { } + break; + + case TestScenario.ReadByteAsync: + while (await decompressor.ReadByteAsync() != -1) { } + break; + } + } + catch (InvalidDataException e) + { + if (expectException) + continue; + + throw new XunitException($"An unexpected error occurred while decompressing data:{e}"); + } + + if (expectException) + { + throw new XunitException($"Truncated stream was decompressed successfully but exception was expected: length={i}/{compressedData.Length}"); + } + } + } + } + }, testScenario.ToString()).Dispose(); + } + private sealed class DerivedGZipStream : GZipStream { public bool ReadArrayInvoked = false, WriteArrayInvoked = false; diff --git a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs index df3a315da070f..3ff29773c9e2c 100644 --- a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs +++ b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs @@ -2,11 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers; +using System.Diagnostics; using System.IO.Compression.Tests; using System.Linq; -using System.Threading; using System.Threading.Tasks; +using Microsoft.DotNet.RemoteExecutor; using Xunit; +using Xunit.Sdk; namespace System.IO.Compression { @@ -22,40 +24,131 @@ public class ZLibStreamUnitTests : CompressionStreamUnitTestBase [Fact] public void StreamCorruption_IsDetected() { - byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray(); - var buffer = new byte[64]; - byte[] compressedData; - using (var compressed = new MemoryStream()) - using (Stream compressor = CreateStream(compressed, CompressionMode.Compress)) + RemoteExecutor.Invoke(() => { - foreach (byte b in source) + AppContext.SetSwitch("System.IO.Compression.UseStrictValidation", true); + + byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray(); + var buffer = new byte[64]; + byte[] compressedData; + using (var compressed = new MemoryStream()) + using (Stream compressor = CreateStream(compressed, CompressionMode.Compress)) { - compressor.WriteByte(b); + foreach (byte b in source) + { + compressor.WriteByte(b); + } + + compressor.Dispose(); + compressedData = compressed.ToArray(); } - compressor.Dispose(); - compressedData = compressed.ToArray(); - } + for (int byteToCorrupt = 0; byteToCorrupt < compressedData.Length; byteToCorrupt++) + { + // corrupt the data + compressedData[byteToCorrupt]++; - for (int byteToCorrupt = 0; byteToCorrupt < compressedData.Length; byteToCorrupt++) + using (var decompressedStream = new MemoryStream(compressedData)) + { + using (Stream decompressor = CreateStream(decompressedStream, CompressionMode.Decompress)) + { + Assert.Throws(() => + { + while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0); + }); + } + } + + // restore the data + compressedData[byteToCorrupt]--; + } + }).Dispose(); + } + + [InlineData(TestScenario.ReadAsync)] + [InlineData(TestScenario.Read)] + [InlineData(TestScenario.Copy)] + [InlineData(TestScenario.CopyAsync)] + [InlineData(TestScenario.ReadByte)] + [InlineData(TestScenario.ReadByteAsync)] + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public void StreamTruncation_IsDetected(TestScenario testScenario) + { + RemoteExecutor.Invoke(async (testScenario) => { - // corrupt the data - compressedData[byteToCorrupt]++; + TestScenario scenario = Enum.Parse(testScenario); + + AppContext.SetSwitch("System.IO.Compression.UseStrictValidation", true); - using (var decompressedStream = new MemoryStream(compressedData)) + var buffer = new byte[16]; + byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray(); + byte[] compressedData; + using (var compressed = new MemoryStream()) + using (Stream compressor = CreateStream(compressed, CompressionMode.Compress)) { - using (Stream decompressor = CreateStream(decompressedStream, CompressionMode.Decompress)) + foreach (byte b in source) { - Assert.Throws(() => - { - while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0); - }); + compressor.WriteByte(b); } + + compressor.Dispose(); + compressedData = compressed.ToArray(); } - // restore the data - compressedData[byteToCorrupt]--; - } + for (var i = 1; i <= compressedData.Length; i += 1) + { + bool expectException = i < compressedData.Length; + using (var compressedStream = new MemoryStream(compressedData.Take(i).ToArray())) + { + using (Stream decompressor = CreateStream(compressedStream, CompressionMode.Decompress)) + { + var decompressedStream = new MemoryStream(); + + try + { + switch (scenario) + { + case TestScenario.Copy: + decompressor.CopyTo(decompressedStream); + break; + + case TestScenario.CopyAsync: + await decompressor.CopyToAsync(decompressedStream); + break; + + case TestScenario.Read: + while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0) { }; + break; + + case TestScenario.ReadAsync: + while (await ZipFileTestBase.ReadAllBytesAsync(decompressor, buffer, 0, buffer.Length) != 0) { }; + break; + + case TestScenario.ReadByte: + while (decompressor.ReadByte() != -1) { } + break; + + case TestScenario.ReadByteAsync: + while (await decompressor.ReadByteAsync() != -1) { } + break; + } + } + catch (InvalidDataException e) + { + if (expectException) + continue; + + throw new XunitException($"An unexpected error occurred while decompressing data:{e}"); + } + + if (expectException) + { + throw new XunitException($"Truncated stream was decompressed successfully but exception was expected: length={i}/{compressedData.Length}"); + } + } + } + } + }, testScenario.ToString()).Dispose(); } } } diff --git a/src/libraries/System.IO.Compression/tests/System.IO.Compression.Tests.csproj b/src/libraries/System.IO.Compression/tests/System.IO.Compression.Tests.csproj index 229119e3aa37c..5b4d37cf93c55 100644 --- a/src/libraries/System.IO.Compression/tests/System.IO.Compression.Tests.csproj +++ b/src/libraries/System.IO.Compression/tests/System.IO.Compression.Tests.csproj @@ -2,6 +2,7 @@ $(NetCoreAppCurrent)-windows;$(NetCoreAppCurrent)-Unix;$(NetCoreAppCurrent)-Browser true + true From c7be15da77408c2b0801cf13162bf02eb07533e6 Mon Sep 17 00:00:00 2001 From: Martin Finkel Date: Fri, 16 Sep 2022 15:48:22 +0700 Subject: [PATCH 03/10] review feedback --- .../Common/tests/System/IO/Compression/ZipTestHelper.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs b/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs index f55fe1ed5a1b5..ce4ffc568b291 100644 --- a/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs +++ b/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs @@ -173,15 +173,15 @@ public static async Task StreamsEqualAsync(Stream ast, Stream bst, int blocksToR if (blocksToRead != -1 && blocksRead >= blocksToRead) break; - ac = await ReadAllBytesAsync(ast, ad, 0, 4096); - bc = await ReadAllBytesAsync(bst, bd, 0, 4096); + ac = await ast.ReadAtLeastAsync(ad, 4096); + bc = await bst.ReadAtLeastAsync(bd, 4096); if (ac != bc) { bd = NormalizeLineEndings(bd); } - Assert.True(ArraysEqual(ad, bd, ac), "Stream contents not equal: " + ast.ToString() + ", " + bst.ToString()); + AssertExtensions.SequenceEqual(ad, bd); blocksRead++; } while (ac == bufSize); From ab41e46ca521564aac3515750c44bad74737d2e8 Mon Sep 17 00:00:00 2001 From: Martin Finkel Date: Fri, 16 Sep 2022 15:58:46 +0700 Subject: [PATCH 04/10] use same error text message --- .../System.IO.Compression.Brotli/src/Resources/Strings.resx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.IO.Compression.Brotli/src/Resources/Strings.resx b/src/libraries/System.IO.Compression.Brotli/src/Resources/Strings.resx index 4f892bf574f2c..f415d5e74bc57 100644 --- a/src/libraries/System.IO.Compression.Brotli/src/Resources/Strings.resx +++ b/src/libraries/System.IO.Compression.Brotli/src/Resources/Strings.resx @@ -176,7 +176,7 @@ BrotliStream.BaseStream returned more bytes than requested in Read. - Decoder ran into truncated data. + Found truncated data while decoding. System.IO.Compression.Brotli is not supported on this platform. From 1d5bc2a8959abbba49600fb995ba41dc34ecdd6d Mon Sep 17 00:00:00 2001 From: Martin Finkel Date: Fri, 16 Sep 2022 16:08:29 +0700 Subject: [PATCH 05/10] cache appcontext getswitch --- .../dec/BrotliStream.Decompress.cs | 16 ++++----------- .../Compression/DeflateZLib/DeflateStream.cs | 20 ++++++------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs b/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs index fdb0258690f14..a6a48c6de3a46 100644 --- a/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs +++ b/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs @@ -66,7 +66,7 @@ public override int Read(Span buffer) int bytesRead = _stream.Read(_buffer, _bufferCount, _buffer.Length - _bufferCount); if (bytesRead <= 0) { - if (GetStrictValidationSetting() && _nonEmptyInput && !buffer.IsEmpty) + if (UseStrictValidation && _nonEmptyInput && !buffer.IsEmpty) ThrowTruncatedInvalidData(); break; } @@ -154,7 +154,7 @@ async ValueTask Core(Memory buffer, CancellationToken cancellationTok int bytesRead = await _stream.ReadAsync(_buffer.AsMemory(_bufferCount), cancellationToken).ConfigureAwait(false); if (bytesRead <= 0) { - if (GetStrictValidationSetting() && _nonEmptyInput && !buffer.IsEmpty) + if (UseStrictValidation && _nonEmptyInput && !buffer.IsEmpty) ThrowTruncatedInvalidData(); break; } @@ -234,16 +234,8 @@ private bool TryDecompress(Span destination, out int bytesWritten) return false; } - private static bool GetStrictValidationSetting() - { - if (AppContext.TryGetSwitch("System.IO.Compression.UseStrictValidation", out bool strictValidation)) - { - return strictValidation; - } - - string? envVar = Environment.GetEnvironmentVariable("DOTNET_SYSTEM_COMPRESSION_STRICT_VALIDATION"); - return envVar is not null && (envVar == "1" || envVar.Equals("true", StringComparison.OrdinalIgnoreCase)); - } + private static readonly bool UseStrictValidation = + AppContext.TryGetSwitch("System.IO.Compression.UseStrictValidation", out bool strictValidation) ? strictValidation : false; private static void ThrowInvalidStream() => // The stream is either malicious or poorly implemented and returned a number of diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs index 44ea0cc2128f0..4b12a84787777 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs @@ -284,7 +284,7 @@ internal int ReadCore(Span buffer) // - Inflation is not finished yet. // - Provided input wasn't completely empty // In such case, we are dealing with a truncated input stream. - if (GetStrictValidationSetting() && !buffer.IsEmpty && !_inflater.Finished() && _inflater.NonEmptyInput()) + if (UseStrictValidation && !buffer.IsEmpty && !_inflater.Finished() && _inflater.NonEmptyInput()) { ThrowTruncatedInvalidData(); } @@ -433,7 +433,7 @@ async ValueTask Core(Memory buffer, CancellationToken cancellationTok // - Inflation is not finished yet. // - Provided input wasn't completely empty // In such case, we are dealing with a truncated input stream. - if (GetStrictValidationSetting() && !_inflater.Finished() && _inflater.NonEmptyInput() && !buffer.IsEmpty) + if (UseStrictValidation && !_inflater.Finished() && _inflater.NonEmptyInput() && !buffer.IsEmpty) { ThrowTruncatedInvalidData(); } @@ -914,7 +914,7 @@ public async Task CopyFromSourceToDestinationAsync() // Now, use the source stream's CopyToAsync to push directly to our inflater via this helper stream await _deflateStream._stream.CopyToAsync(this, _arrayPoolBuffer.Length, _cancellationToken).ConfigureAwait(false); - if (!_deflateStream._inflater.Finished() && GetStrictValidationSetting()) + if (!_deflateStream._inflater.Finished() && UseStrictValidation) { ThrowTruncatedInvalidData(); } @@ -950,7 +950,7 @@ public void CopyFromSourceToDestination() // Now, use the source stream's CopyToAsync to push directly to our inflater via this helper stream _deflateStream._stream.CopyTo(this, _arrayPoolBuffer.Length); - if (!_deflateStream._inflater.Finished() && GetStrictValidationSetting()) + if (!_deflateStream._inflater.Finished() && UseStrictValidation) { ThrowTruncatedInvalidData(); } @@ -1079,15 +1079,7 @@ private void AsyncOperationCompleting() => private static void ThrowInvalidBeginCall() => throw new InvalidOperationException(SR.InvalidBeginCall); - private static bool GetStrictValidationSetting() - { - if (AppContext.TryGetSwitch("System.IO.Compression.UseStrictValidation", out bool strictValidation)) - { - return strictValidation; - } - - string? envVar = Environment.GetEnvironmentVariable("DOTNET_SYSTEM_COMPRESSION_STRICT_VALIDATION"); - return envVar is not null && (envVar == "1" || envVar.Equals("true", StringComparison.OrdinalIgnoreCase)); - } + private static readonly bool UseStrictValidation = + AppContext.TryGetSwitch("System.IO.Compression.UseStrictValidation", out bool strictValidation) ? strictValidation : false; } } From ff417336c507cd7f65c6a2fe3f6103b9f488e080 Mon Sep 17 00:00:00 2001 From: Martin Finkel Date: Thu, 22 Sep 2022 14:52:03 +0700 Subject: [PATCH 06/10] fix failing test --- .../Common/tests/System/IO/Compression/ZipTestHelper.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs b/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs index ce4ffc568b291..f8fe686b817ef 100644 --- a/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs +++ b/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs @@ -173,8 +173,8 @@ public static async Task StreamsEqualAsync(Stream ast, Stream bst, int blocksToR if (blocksToRead != -1 && blocksRead >= blocksToRead) break; - ac = await ast.ReadAtLeastAsync(ad, 4096); - bc = await bst.ReadAtLeastAsync(bd, 4096); + ac = await ast.ReadAtLeastAsync(ad, 4096, throwOnEndOfStream: false); + bc = await bst.ReadAtLeastAsync(bd, 4096, throwOnEndOfStream: false); if (ac != bc) { From ff8d01c2a92278ea2d3b60cdbc59ac106cabcbc7 Mon Sep 17 00:00:00 2001 From: Martin Finkel Date: Thu, 22 Sep 2022 14:53:26 +0700 Subject: [PATCH 07/10] slice byte array for assertion --- .../Common/tests/System/IO/Compression/ZipTestHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs b/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs index f8fe686b817ef..f4a7d0cb46a7e 100644 --- a/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs +++ b/src/libraries/Common/tests/System/IO/Compression/ZipTestHelper.cs @@ -181,7 +181,7 @@ public static async Task StreamsEqualAsync(Stream ast, Stream bst, int blocksToR bd = NormalizeLineEndings(bd); } - AssertExtensions.SequenceEqual(ad, bd); + AssertExtensions.SequenceEqual(ad.AsSpan(0, ac), bd.AsSpan(0, bc)); blocksRead++; } while (ac == bufSize); From a9eb6ed7fa15600c64efd159cc6e04b96ffac271 Mon Sep 17 00:00:00 2001 From: Martin Finkel Date: Thu, 22 Sep 2022 15:30:04 +0700 Subject: [PATCH 08/10] renaming --- .../IO/Compression/dec/BrotliStream.Decompress.cs | 6 +++--- .../System/IO/Compression/DeflateZLib/DeflateStream.cs | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs b/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs index a6a48c6de3a46..be4fe9d4e0927 100644 --- a/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs +++ b/src/libraries/System.IO.Compression.Brotli/src/System/IO/Compression/dec/BrotliStream.Decompress.cs @@ -66,7 +66,7 @@ public override int Read(Span buffer) int bytesRead = _stream.Read(_buffer, _bufferCount, _buffer.Length - _bufferCount); if (bytesRead <= 0) { - if (UseStrictValidation && _nonEmptyInput && !buffer.IsEmpty) + if (s_useStrictValidation && _nonEmptyInput && !buffer.IsEmpty) ThrowTruncatedInvalidData(); break; } @@ -154,7 +154,7 @@ async ValueTask Core(Memory buffer, CancellationToken cancellationTok int bytesRead = await _stream.ReadAsync(_buffer.AsMemory(_bufferCount), cancellationToken).ConfigureAwait(false); if (bytesRead <= 0) { - if (UseStrictValidation && _nonEmptyInput && !buffer.IsEmpty) + if (s_useStrictValidation && _nonEmptyInput && !buffer.IsEmpty) ThrowTruncatedInvalidData(); break; } @@ -234,7 +234,7 @@ private bool TryDecompress(Span destination, out int bytesWritten) return false; } - private static readonly bool UseStrictValidation = + private static readonly bool s_useStrictValidation = AppContext.TryGetSwitch("System.IO.Compression.UseStrictValidation", out bool strictValidation) ? strictValidation : false; private static void ThrowInvalidStream() => diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs index 4b12a84787777..266b68960aca5 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs @@ -284,7 +284,7 @@ internal int ReadCore(Span buffer) // - Inflation is not finished yet. // - Provided input wasn't completely empty // In such case, we are dealing with a truncated input stream. - if (UseStrictValidation && !buffer.IsEmpty && !_inflater.Finished() && _inflater.NonEmptyInput()) + if (s_useStrictValidation && !buffer.IsEmpty && !_inflater.Finished() && _inflater.NonEmptyInput()) { ThrowTruncatedInvalidData(); } @@ -433,7 +433,7 @@ async ValueTask Core(Memory buffer, CancellationToken cancellationTok // - Inflation is not finished yet. // - Provided input wasn't completely empty // In such case, we are dealing with a truncated input stream. - if (UseStrictValidation && !_inflater.Finished() && _inflater.NonEmptyInput() && !buffer.IsEmpty) + if (s_useStrictValidation && !_inflater.Finished() && _inflater.NonEmptyInput() && !buffer.IsEmpty) { ThrowTruncatedInvalidData(); } @@ -914,7 +914,7 @@ public async Task CopyFromSourceToDestinationAsync() // Now, use the source stream's CopyToAsync to push directly to our inflater via this helper stream await _deflateStream._stream.CopyToAsync(this, _arrayPoolBuffer.Length, _cancellationToken).ConfigureAwait(false); - if (!_deflateStream._inflater.Finished() && UseStrictValidation) + if (!_deflateStream._inflater.Finished() && s_useStrictValidation) { ThrowTruncatedInvalidData(); } @@ -950,7 +950,7 @@ public void CopyFromSourceToDestination() // Now, use the source stream's CopyToAsync to push directly to our inflater via this helper stream _deflateStream._stream.CopyTo(this, _arrayPoolBuffer.Length); - if (!_deflateStream._inflater.Finished() && UseStrictValidation) + if (!_deflateStream._inflater.Finished() && s_useStrictValidation) { ThrowTruncatedInvalidData(); } @@ -1079,7 +1079,7 @@ private void AsyncOperationCompleting() => private static void ThrowInvalidBeginCall() => throw new InvalidOperationException(SR.InvalidBeginCall); - private static readonly bool UseStrictValidation = + private static readonly bool s_useStrictValidation = AppContext.TryGetSwitch("System.IO.Compression.UseStrictValidation", out bool strictValidation) ? strictValidation : false; } } From 6ce9b9b0664a137433f92fa954af8d0b1d732e31 Mon Sep 17 00:00:00 2001 From: Martin Finkel Date: Thu, 29 Sep 2022 14:17:36 +0700 Subject: [PATCH 09/10] add missing RemoteExecutor.IsSupported --- .../tests/CompressionStreamUnitTests.ZLib.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs index 3ff29773c9e2c..ac030e2c1d2a0 100644 --- a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs +++ b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs @@ -21,7 +21,7 @@ public class ZLibStreamUnitTests : CompressionStreamUnitTestBase public override Stream BaseStream(Stream stream) => ((ZLibStream)stream).BaseStream; protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("ZLibTestData", Path.GetFileName(uncompressedPath) + ".z"); - [Fact] + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] public void StreamCorruption_IsDetected() { RemoteExecutor.Invoke(() => From d757d9171e0d2fa19b58f1d232f712d728e76eda Mon Sep 17 00:00:00 2001 From: Martin Finkel Date: Thu, 13 Oct 2022 15:27:31 +0700 Subject: [PATCH 10/10] fast check first --- .../src/System/IO/Compression/DeflateZLib/DeflateStream.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs index 266b68960aca5..02664b09451a8 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs @@ -914,7 +914,7 @@ public async Task CopyFromSourceToDestinationAsync() // Now, use the source stream's CopyToAsync to push directly to our inflater via this helper stream await _deflateStream._stream.CopyToAsync(this, _arrayPoolBuffer.Length, _cancellationToken).ConfigureAwait(false); - if (!_deflateStream._inflater.Finished() && s_useStrictValidation) + if (s_useStrictValidation && !_deflateStream._inflater.Finished()) { ThrowTruncatedInvalidData(); } @@ -950,7 +950,7 @@ public void CopyFromSourceToDestination() // Now, use the source stream's CopyToAsync to push directly to our inflater via this helper stream _deflateStream._stream.CopyTo(this, _arrayPoolBuffer.Length); - if (!_deflateStream._inflater.Finished() && s_useStrictValidation) + if (s_useStrictValidation && !_deflateStream._inflater.Finished()) { ThrowTruncatedInvalidData(); }