From 800d14f40b8ad99ebaf0efd29a191fc86642083b Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Wed, 2 Mar 2022 11:08:30 -0700 Subject: [PATCH 01/24] Add more assertion options --- src/LibHac/Diag/Assert.cs | 147 ++++++++++++++++++++++++++++- src/LibHac/Diag/Impl/AssertImpl.cs | 14 ++- 2 files changed, 158 insertions(+), 3 deletions(-) diff --git a/src/LibHac/Diag/Assert.cs b/src/LibHac/Diag/Assert.cs index 02b4353d..5b8a9bf0 100644 --- a/src/LibHac/Diag/Assert.cs +++ b/src/LibHac/Diag/Assert.cs @@ -572,7 +572,99 @@ internal static void SdkRequiresNull(ref T value, } // --------------------------------------------------------------------- - // In range + // Null UniqueRef + // --------------------------------------------------------------------- + + private static void NullImpl(AssertionType assertionType, in UniqueRef value, + string valueText, string functionName, string fileName, int lineNumber) where T : class, IDisposable + { + if (AssertImpl.Null(in value)) + return; + + AssertImpl.InvokeAssertionNotNull(assertionType, valueText, functionName, fileName, lineNumber); + } + + [Conditional(AssertCondition)] + public static void Null(in UniqueRef value, + [CallerArgumentExpression("value")] string valueText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + where T : class, IDisposable + { + NullImpl(AssertionType.UserAssert, in value, valueText, functionName, fileName, lineNumber); + } + + [Conditional(AssertCondition)] + internal static void SdkNull(in UniqueRef value, + [CallerArgumentExpression("value")] string valueText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + where T : class, IDisposable + { + NullImpl(AssertionType.SdkAssert, in value, valueText, functionName, fileName, lineNumber); + } + + [Conditional(AssertCondition)] + internal static void SdkRequiresNull(in UniqueRef value, + [CallerArgumentExpression("value")] string valueText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + where T : class, IDisposable + { + NullImpl(AssertionType.SdkRequires, in value, valueText, functionName, fileName, lineNumber); + } + + // --------------------------------------------------------------------- + // Null SharedRef + // --------------------------------------------------------------------- + + private static void NullImpl(AssertionType assertionType, in SharedRef value, + string valueText, string functionName, string fileName, int lineNumber) where T : class, IDisposable + { + if (AssertImpl.Null(in value)) + return; + + AssertImpl.InvokeAssertionNotNull(assertionType, valueText, functionName, fileName, lineNumber); + } + + [Conditional(AssertCondition)] + public static void Null(in SharedRef value, + [CallerArgumentExpression("value")] string valueText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + where T : class, IDisposable + { + NullImpl(AssertionType.UserAssert, in value, valueText, functionName, fileName, lineNumber); + } + + [Conditional(AssertCondition)] + internal static void SdkNull(in SharedRef value, + [CallerArgumentExpression("value")] string valueText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + where T : class, IDisposable + { + NullImpl(AssertionType.SdkAssert, in value, valueText, functionName, fileName, lineNumber); + } + + [Conditional(AssertCondition)] + internal static void SdkRequiresNull(in SharedRef value, + [CallerArgumentExpression("value")] string valueText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + where T : class, IDisposable + { + NullImpl(AssertionType.SdkRequires, in value, valueText, functionName, fileName, lineNumber); + } + + // --------------------------------------------------------------------- + // In range int // --------------------------------------------------------------------- private static void InRangeImpl(AssertionType assertionType, int value, int lower, int upper, string valueText, @@ -624,6 +716,59 @@ internal static void SdkRequiresInRange(int value, int lowerInclusive, int upper upperExclusiveText, functionName, fileName, lineNumber); } + // --------------------------------------------------------------------- + // In range long + // --------------------------------------------------------------------- + + private static void InRangeImpl(AssertionType assertionType, long value, long lower, long upper, string valueText, + string lowerText, string upperText, string functionName, string fileName, int lineNumber) + { + if (AssertImpl.WithinRange(value, lower, upper)) + return; + + AssertImpl.InvokeAssertionInRange(assertionType, value, lower, upper, valueText, lowerText, upperText, functionName, + fileName, lineNumber); + } + + [Conditional(AssertCondition)] + public static void InRange(long value, long lowerInclusive, long upperExclusive, + [CallerArgumentExpression("value")] string valueText = "", + [CallerArgumentExpression("lowerInclusive")] string lowerInclusiveText = "", + [CallerArgumentExpression("upperExclusive")] string upperExclusiveText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + { + InRangeImpl(AssertionType.UserAssert, value, lowerInclusive, upperExclusive, valueText, lowerInclusiveText, + upperExclusiveText, functionName, fileName, lineNumber); + } + + [Conditional(AssertCondition)] + internal static void SdkInRange(long value, long lowerInclusive, long upperExclusive, + [CallerArgumentExpression("value")] string valueText = "", + [CallerArgumentExpression("lowerInclusive")] string lowerInclusiveText = "", + [CallerArgumentExpression("upperExclusive")] string upperExclusiveText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + { + InRangeImpl(AssertionType.SdkAssert, value, lowerInclusive, upperExclusive, valueText, lowerInclusiveText, + upperExclusiveText, functionName, fileName, lineNumber); + } + + [Conditional(AssertCondition)] + internal static void SdkRequiresInRange(long value, long lowerInclusive, long upperExclusive, + [CallerArgumentExpression("value")] string valueText = "", + [CallerArgumentExpression("lowerInclusive")] string lowerInclusiveText = "", + [CallerArgumentExpression("upperExclusive")] string upperExclusiveText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + { + InRangeImpl(AssertionType.SdkRequires, value, lowerInclusive, upperExclusive, valueText, lowerInclusiveText, + upperExclusiveText, functionName, fileName, lineNumber); + } + // --------------------------------------------------------------------- // Within min-max int // --------------------------------------------------------------------- diff --git a/src/LibHac/Diag/Impl/AssertImpl.cs b/src/LibHac/Diag/Impl/AssertImpl.cs index cc16becc..cc2e262e 100644 --- a/src/LibHac/Diag/Impl/AssertImpl.cs +++ b/src/LibHac/Diag/Impl/AssertImpl.cs @@ -22,7 +22,7 @@ internal static void InvokeAssertionNull(AssertionType assertionType, string val $"{valueText} must be nullptr."); } - internal static void InvokeAssertionInRange(AssertionType assertionType, int value, int lower, int upper, + internal static void InvokeAssertionInRange(AssertionType assertionType, long value, long lower, long upper, string valueText, string lowerText, string upperText, string functionName, string fileName, int lineNumber) { string message = @@ -126,6 +126,16 @@ public static bool Null(ref T item) return Unsafe.IsNullRef(ref item); } + public static bool Null(in UniqueRef item) where T : class, IDisposable + { + return !item.HasValue; + } + + public static bool Null(in SharedRef item) where T : class, IDisposable + { + return !item.HasValue; + } + public static bool NotNull(T item) where T : class { return item is not null; @@ -240,4 +250,4 @@ public static bool IsAligned(ulong value, int alignment) { return Alignment.IsAlignedPow2(value, (uint)alignment); } -} +} \ No newline at end of file From cd2b8edfa8a92fbbb444ca18200d519ef6494319 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Wed, 2 Mar 2022 11:09:17 -0700 Subject: [PATCH 02/24] Skeleton AesXtsStorage --- src/LibHac/FsSystem/AesXtsStorage.cs | 115 +++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 src/LibHac/FsSystem/AesXtsStorage.cs diff --git a/src/LibHac/FsSystem/AesXtsStorage.cs b/src/LibHac/FsSystem/AesXtsStorage.cs new file mode 100644 index 00000000..229f2161 --- /dev/null +++ b/src/LibHac/FsSystem/AesXtsStorage.cs @@ -0,0 +1,115 @@ +// ReSharper disable NotAccessedField.Local +using System; +using System.Buffers.Binary; +using LibHac.Common; +using LibHac.Common.FixedArrays; +using LibHac.Crypto; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.Os; + +namespace LibHac.FsSystem; + +/// +/// Reads and writes to an that's encrypted with AES-XTS-128. +/// +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public class AesXtsStorage : IStorage +{ + public static readonly int AesBlockSize = Aes.BlockSize; + public static readonly int KeySize = Aes.KeySize128; + public static readonly int IvSize = Aes.KeySize128; + + private IStorage _baseStorage; + private Array16 _key1; + private Array16 _key2; + private Array16 _iv; + private int _blockSize; + private SdkMutexType _mutex; + + // LibHac addition: This field goes unused if initialized with a plain IStorage. + // The original class uses a template for both the shared and non-shared IStorage which avoids needing this field. + private SharedRef _baseStorageShared; + + public static void MakeAesXtsIv(Span outIv, long offset, int blockSize) + { + Assert.Equal(outIv.Length, IvSize); + Assert.SdkRequiresGreaterEqual(offset, 0); + Assert.SdkRequiresAligned((ulong)blockSize, AesBlockSize); + + BinaryPrimitives.WriteInt64BigEndian(outIv.Slice(sizeof(long)), offset / blockSize); + } + + public AesXtsStorage(IStorage baseStorage, ReadOnlySpan key1, ReadOnlySpan key2, ReadOnlySpan iv, + int blockSize) + { + _baseStorage = baseStorage; + _blockSize = blockSize; + _mutex = new SdkMutexType(); + + Assert.SdkRequiresEqual(KeySize, key1.Length); + Assert.SdkRequiresEqual(KeySize, key2.Length); + Assert.SdkRequiresEqual(IvSize, iv.Length); + Assert.SdkRequiresAligned((ulong)blockSize, AesBlockSize); + + key1.CopyTo(_key1.Items); + key2.CopyTo(_key2.Items); + iv.CopyTo(_iv.Items); + } + + public AesXtsStorage(ref SharedRef baseStorage, ReadOnlySpan key1, ReadOnlySpan key2, + ReadOnlySpan iv, int blockSize) + { + _baseStorageShared = SharedRef.CreateMove(ref baseStorage); + _baseStorage = _baseStorageShared.Get; + _blockSize = blockSize; + _mutex = new SdkMutexType(); + + Assert.SdkRequiresEqual(KeySize, key1.Length); + Assert.SdkRequiresEqual(KeySize, key2.Length); + Assert.SdkRequiresEqual(IvSize, iv.Length); + Assert.SdkRequiresAligned((ulong)blockSize, AesBlockSize); + + key1.CopyTo(_key1.Items); + key2.CopyTo(_key2.Items); + iv.CopyTo(_iv.Items); + } + + public override void Dispose() + { + _baseStorageShared.Destroy(); + + base.Dispose(); + } + + public override Result Read(long offset, Span destination) + { + throw new NotImplementedException(); + } + + public override Result Write(long offset, ReadOnlySpan source) + { + throw new NotImplementedException(); + } + + public override Result Flush() + { + throw new NotImplementedException(); + } + + public override Result GetSize(out long size) + { + throw new NotImplementedException(); + } + + public override Result SetSize(long size) + { + throw new NotImplementedException(); + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + throw new NotImplementedException(); + } +} \ No newline at end of file From b9e2e0863b0e3ba5dd65e9a0a4db72c20764cdd0 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Wed, 2 Mar 2022 11:13:37 -0700 Subject: [PATCH 03/24] Add NcaReader and some related NCA classes --- src/LibHac/Common/FixedArrays/Array96.cs | 31 + src/LibHac/Crypto/Rsa.cs | 3 + src/LibHac/Fs/Common.cs | 21 + .../FsSrv/FsCreator/StorageOnNcaCreator.cs | 1 + src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs | 1 + ...oFilterReader.cs => SaveDataInfoFilter.cs} | 0 src/LibHac/FsSystem/CompressionCommon.cs | 7 +- src/LibHac/FsSystem/NcaFileSystemDriver.cs | 63 ++ src/LibHac/FsSystem/NcaHeader.cs | 238 ++++++++ src/LibHac/FsSystem/NcaReader.cs | 561 ++++++++++++++++++ src/LibHac/FsSystem/NcaStructs.cs | 49 +- src/LibHac/Tools/Fs/SwitchFs.cs | 1 + src/LibHac/Tools/FsSystem/NcaUtils/Nca.cs | 7 +- .../Tools/FsSystem/NcaUtils/NcaFsHeader.cs | 2 +- src/hactoolnet/ProcessNca.cs | 1 + tests/LibHac.Tests/Fs/TypeLayoutTests.cs | 19 + .../LibHac.Tests/FsSystem/TypeLayoutTests.cs | 233 +++++++- 17 files changed, 1184 insertions(+), 54 deletions(-) create mode 100644 src/LibHac/Common/FixedArrays/Array96.cs create mode 100644 src/LibHac/Fs/Common.cs rename src/LibHac/FsSrv/{SaveDataInfoFilterReader.cs => SaveDataInfoFilter.cs} (100%) create mode 100644 src/LibHac/FsSystem/NcaFileSystemDriver.cs create mode 100644 src/LibHac/FsSystem/NcaHeader.cs create mode 100644 src/LibHac/FsSystem/NcaReader.cs diff --git a/src/LibHac/Common/FixedArrays/Array96.cs b/src/LibHac/Common/FixedArrays/Array96.cs new file mode 100644 index 00000000..ca4fbad4 --- /dev/null +++ b/src/LibHac/Common/FixedArrays/Array96.cs @@ -0,0 +1,31 @@ +#pragma warning disable CS0169, CS0649, IDE0051 // Field is never used, Field is never assigned to, Remove unused private members +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace LibHac.Common.FixedArrays; + +public struct Array96 +{ + public const int Length = 96; + + private Array80 _0; + private Array16 _80; + + public ref T this[int i] => ref Items[i]; + + public Span Items + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => SpanHelpers.CreateSpan(ref MemoryMarshal.GetReference(_0.Items), Length); + } + + public readonly ReadOnlySpan ItemsRo + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => SpanHelpers.CreateSpan(ref MemoryMarshal.GetReference(_0.ItemsRo), Length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static implicit operator ReadOnlySpan(in Array96 value) => value.ItemsRo; +} \ No newline at end of file diff --git a/src/LibHac/Crypto/Rsa.cs b/src/LibHac/Crypto/Rsa.cs index 672a5463..c2af9a5e 100644 --- a/src/LibHac/Crypto/Rsa.cs +++ b/src/LibHac/Crypto/Rsa.cs @@ -6,6 +6,9 @@ namespace LibHac.Crypto; public static class Rsa { + public static readonly int ModulusSize2048Pss = 256; + public static readonly int MaximumExponentSize2048Pss = 3; + public static bool VerifyRsa2048PssSha256(ReadOnlySpan signature, ReadOnlySpan modulus, ReadOnlySpan exponent, ReadOnlySpan message) => VerifyRsa2048Sha256(signature, modulus, exponent, message, RSASignaturePadding.Pss); diff --git a/src/LibHac/Fs/Common.cs b/src/LibHac/Fs/Common.cs new file mode 100644 index 00000000..97f28384 --- /dev/null +++ b/src/LibHac/Fs/Common.cs @@ -0,0 +1,21 @@ +using System.Runtime.InteropServices; + +namespace LibHac.Fs; + +[StructLayout(LayoutKind.Sequential, Pack = 4)] +public struct Int64 +{ + private long _value; + + public void Set(long value) + { + _value = value; + } + + public readonly long Get() + { + return _value; + } + + public static implicit operator long(in Int64 value) => value.Get(); +} \ No newline at end of file diff --git a/src/LibHac/FsSrv/FsCreator/StorageOnNcaCreator.cs b/src/LibHac/FsSrv/FsCreator/StorageOnNcaCreator.cs index bf2ec6da..31f1eeb1 100644 --- a/src/LibHac/FsSrv/FsCreator/StorageOnNcaCreator.cs +++ b/src/LibHac/FsSrv/FsCreator/StorageOnNcaCreator.cs @@ -7,6 +7,7 @@ using LibHac.FsSystem.Impl; using LibHac.Tools.FsSystem; using LibHac.Tools.FsSystem.NcaUtils; +using NcaFsHeader = LibHac.Tools.FsSystem.NcaUtils.NcaFsHeader; namespace LibHac.FsSrv.FsCreator; diff --git a/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs b/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs index db7be714..0e9a4362 100644 --- a/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs +++ b/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs @@ -13,6 +13,7 @@ using LibHac.Spl; using LibHac.Tools.FsSystem.NcaUtils; using LibHac.Util; +using NcaFsHeader = LibHac.Tools.FsSystem.NcaUtils.NcaFsHeader; using RightsId = LibHac.Fs.RightsId; using Utility = LibHac.FsSystem.Utility; diff --git a/src/LibHac/FsSrv/SaveDataInfoFilterReader.cs b/src/LibHac/FsSrv/SaveDataInfoFilter.cs similarity index 100% rename from src/LibHac/FsSrv/SaveDataInfoFilterReader.cs rename to src/LibHac/FsSrv/SaveDataInfoFilter.cs diff --git a/src/LibHac/FsSystem/CompressionCommon.cs b/src/LibHac/FsSystem/CompressionCommon.cs index 8670e46c..e0d77183 100644 --- a/src/LibHac/FsSystem/CompressionCommon.cs +++ b/src/LibHac/FsSystem/CompressionCommon.cs @@ -1,4 +1,6 @@ -namespace LibHac.FsSystem; +using System; + +namespace LibHac.FsSystem; public enum CompressionType : byte { @@ -8,6 +10,9 @@ public enum CompressionType : byte Unknown = 4 } +public delegate Result DecompressorFunction(Span destination, ReadOnlySpan source); +public delegate DecompressorFunction GetDecompressorFunction(CompressionType compressionType); + public static class CompressionTypeUtility { public static bool IsBlockAlignmentRequired(CompressionType type) diff --git a/src/LibHac/FsSystem/NcaFileSystemDriver.cs b/src/LibHac/FsSystem/NcaFileSystemDriver.cs new file mode 100644 index 00000000..adda8dae --- /dev/null +++ b/src/LibHac/FsSystem/NcaFileSystemDriver.cs @@ -0,0 +1,63 @@ +using LibHac.Common.FixedArrays; +using LibHac.Crypto; + +namespace LibHac.FsSystem; + +public struct NcaCryptoConfiguration +{ + public static readonly int Rsa2048KeyModulusSize = Rsa.ModulusSize2048Pss; + public static readonly int Rsa2048KeyPublicExponentSize = Rsa.MaximumExponentSize2048Pss; + public static readonly int Rsa2048KeyPrivateExponentSize = Rsa2048KeyModulusSize; + + public static readonly int Aes128KeySize = Aes.KeySize128; + + public static readonly int Header1SignatureKeyGenerationMax = 1; + + public static readonly int KeyAreaEncryptionKeyIndexCount = 3; + public static readonly int HeaderEncryptionKeyCount = 2; + + public static readonly int KeyGenerationMax = 32; + public static readonly int KeyAreaEncryptionKeyCount = KeyAreaEncryptionKeyIndexCount * KeyGenerationMax; + + public Array2> Header1SignKeyModuli; + public Array3 Header1SignKeyPublicExponent; + public Array3> KeyAreaEncryptionKeySources; + public Array16 HeaderEncryptionKeySource; + public Array2> HeaderEncryptedEncryptionKeys; + public GenerateKeyFunction GenerateKey; + public DecryptAesCtrFunction DecryptAesCtr; + public DecryptAesCtrFunction DecryptAesCtrForExternalKey; + public bool IsDev; +} + +public struct NcaCompressionConfiguration +{ + public GetDecompressorFunction GetDecompressorFunc; +} + +public static class NcaKeyFunctions +{ + public static bool IsInvalidKeyTypeValue(int keyType) + { + return keyType < 0; + } + + public static int GetKeyTypeValue(byte keyIndex, byte keyGeneration) + { + const int invalidKeyTypeValue = -1; + + if (keyIndex >= NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexCount) + return invalidKeyTypeValue; + + return NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexCount * keyGeneration + keyIndex; + } +} + +public enum KeyType +{ + NcaHeaderKey = 0x60, + NcaExternalKey = 0x61, + SaveDataDeviceUniqueMac = 0x62, + SaveDataSeedUniqueMac = 0x63, + SaveDataTransferMac = 0x64 +} \ No newline at end of file diff --git a/src/LibHac/FsSystem/NcaHeader.cs b/src/LibHac/FsSystem/NcaHeader.cs new file mode 100644 index 00000000..6de0e074 --- /dev/null +++ b/src/LibHac/FsSystem/NcaHeader.cs @@ -0,0 +1,238 @@ +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using LibHac.Common.FixedArrays; + +namespace LibHac.FsSystem; + +public struct NcaHeader +{ + public enum ContentType : byte + { + Program = 0, + Meta = 1, + Control = 2, + Manual = 3, + Data = 4, + PublicData = 5 + } + + public enum DistributionType : byte + { + Download = 0, + GameCard = 1 + } + + public enum EncryptionType : byte + { + Auto = 0, + None = 1 + } + + public enum DecryptionKey : byte + { + AesXts = 0, + AesXts1 = 0, + AesXts2 = 1, + AesCtr = 2, + AesCtrEx = 3, + AesCtrHw = 4, + Count + } + + public struct FsInfo + { + public uint StartSector; + public uint EndSector; + public uint HashSectors; + public uint Reserved; + } + + public static readonly uint Magic0 = 0x3041434E; // NCA0 + public static readonly uint Magic1 = 0x3141434E; // NCA1 + public static readonly uint Magic2 = 0x3241434E; // NCA2 + public static readonly uint Magic3 = 0x3341434E; // NCA3 + + public static readonly uint CurrentMagic = Magic3; + + public static readonly int Size = 0x400; + public static readonly int FsCountMax = 4; + public static readonly int HeaderSignCount = 2; + public static readonly int HeaderSignSize = 0x100; + public static readonly int EncryptedKeyAreaSize = 0x100; + public static readonly int SectorSize = 0x200; + public static readonly int SectorShift = 9; + public static readonly int RightsIdSize = 0x10; + public static readonly int XtsBlockSize = 0x200; + public static readonly int CtrBlockSize = 0x10; + + public Array256 Signature1; + public Array256 Signature2; + public uint Magic; + public DistributionType DistributionTypeValue; + public ContentType ContentTypeValue; + public byte KeyGeneration1; + public byte KeyAreaEncryptionKeyIndex; + public ulong ContentSize; + public ulong ProgramId; + public uint ContentIndex; + public uint SdkAddonVersion; + public byte KeyGeneration2; + public byte Header1SignatureKeyGeneration; + public Array2 Reserved222; + public Array3 Reserved224; + public Array16 RightsId; + public Array4 FsInfos; + public Array4 FsHeaderHashes; + public Array256 EncryptedKeys; + + public static ulong SectorToByte(uint sectorIndex) => sectorIndex << SectorShift; + public static uint ByteToSector(ulong byteIndex) => (uint)(byteIndex >> SectorShift); + + public readonly byte GetProperKeyGeneration() => Math.Max(KeyGeneration1, KeyGeneration2); +} + +public struct NcaPatchInfo +{ + public long IndirectOffset; + public long IndirectSize; + public Array16 IndirectHeader; + public long AesCtrExOffset; + public long AesCtrExSize; + public Array16 AesCtrExHeader; + + public readonly bool HasIndirectTable() => IndirectSize != 0; + public readonly bool HasAesCtrExTable() => AesCtrExSize != 0; +} + +public struct NcaSparseInfo +{ + public long MetaOffset; + public long MetaSize; + public Array16 MetaHeader; + public long PhysicalOffset; + public ushort Generation; + public Array6 Reserved; + + public readonly uint GetGeneration() => (uint)(Generation << 16); + public readonly long GetPhysicalSize() => MetaOffset + MetaSize; + + public readonly NcaAesCtrUpperIv MakeAesCtrUpperIv(NcaAesCtrUpperIv upperIv) + { + NcaAesCtrUpperIv sparseUpperIv = upperIv; + sparseUpperIv.Generation = GetGeneration(); + return sparseUpperIv; + } +} + +public struct NcaCompressionInfo +{ + public long TableOffset; + public long TableSize; + public Array16 TableHeader; + public ulong Reserved; +} + +[StructLayout(LayoutKind.Explicit)] +public struct NcaAesCtrUpperIv +{ + [FieldOffset(0)] public ulong Value; + + [FieldOffset(0)] public uint Generation; + [FieldOffset(4)] public uint SecureValue; + + internal NcaAesCtrUpperIv(ulong value) + { + Unsafe.SkipInit(out Generation); + Unsafe.SkipInit(out SecureValue); + Value = value; + } +} + +public struct NcaFsHeader +{ + public ushort Version; + public FsType FsTypeValue; + public HashType HashTypeValue; + public EncryptionType EncryptionTypeValue; + public Array3 Reserved; + public HashData HashDataValue; + public NcaPatchInfo PatchInfo; + public NcaAesCtrUpperIv AesCtrUpperIv; + public NcaSparseInfo SparseInfo; + public NcaCompressionInfo CompressionInfo; + public Array96 Padding; + + public enum FsType : byte + { + RomFs = 0, + PartitionFs = 1 + } + + public enum EncryptionType : byte + { + Auto = 0, + None = 1, + AesXts = 2, + AesCtr = 3, + AesCtrEx = 4 + } + + public enum HashType : byte + { + Auto = 0, + None = 1, + HierarchicalSha256Hash = 2, + HierarchicalIntegrityHash = 3 + } + + public struct Region + { + public long Offset; + public long Size; + } + + [StructLayout(LayoutKind.Explicit, Size = 0xF8)] + public struct HashData + { + [FieldOffset(0)] public HierarchicalSha256Data HierarchicalSha256; + [FieldOffset(0)] public IntegrityMetaInfo IntegrityMeta; + + public struct HierarchicalSha256Data + { + public Hash MasterHash; + public int BlockSize; + public int LayerCount; + public Array5 LayerRegions; + } + + public struct IntegrityMetaInfo + { + public uint Magic; + public uint Version; + public uint MasterHashSize; + public InfoLevelHash LevelHashInfo; + public Hash MasterHash; + + public struct InfoLevelHash + { + public int MaxLayers; + public Array6 Layers; + public SignatureSalt Salt; + + public struct HierarchicalIntegrityVerificationLevelInformation + { + public Fs.Int64 Offset; + public Fs.Int64 Size; + public int OrderBlock; + public Array4 Reserved; + } + + public struct SignatureSalt + { + public Array32 Value; + } + } + } + } +} \ No newline at end of file diff --git a/src/LibHac/FsSystem/NcaReader.cs b/src/LibHac/FsSystem/NcaReader.cs new file mode 100644 index 00000000..a8a081f0 --- /dev/null +++ b/src/LibHac/FsSystem/NcaReader.cs @@ -0,0 +1,561 @@ +using System; +using System.Runtime.CompilerServices; +using LibHac.Common; +using LibHac.Common.FixedArrays; +using LibHac.Crypto; +using LibHac.Diag; +using LibHac.Fs; + +namespace LibHac.FsSystem; + +public delegate Result GenerateKeyFunction(Span destKey, ReadOnlySpan sourceKey, int keyType, in NcaCryptoConfiguration config); +public delegate Result DecryptAesCtrFunction(Span dest, int keyType, ReadOnlySpan encryptedKey, ReadOnlySpan iv, ReadOnlySpan source); + +/// +/// Handles reading information from an NCA file's header. +/// +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public class NcaReader : IDisposable +{ + private const uint SdkAddonVersionMin = 0xB0000; + + private NcaHeader _header; + private Array5> _decryptionKeys; + private SharedRef _bodyStorage; + private UniqueRef _headerStorage; + private Array16 _externalDataDecryptionKey; + private DecryptAesCtrFunction _decryptAesCtr; + private DecryptAesCtrFunction _decryptAesCtrForExternalKey; + private bool _isSoftwareAesPrioritized; + private NcaHeader.EncryptionType _headerEncryptionType; + private GetDecompressorFunction _getDecompressorFunc; + private IHash256GeneratorFactory _hashGeneratorFactory; + + public void Dispose() + { + _headerStorage.Destroy(); + _bodyStorage.Destroy(); + } + + public Result Initialize(ref SharedRef baseStorage, in NcaCryptoConfiguration cryptoConfig, + in NcaCompressionConfiguration compressionConfig, IHash256GeneratorFactorySelector hashGeneratorFactorySelector) + { + Assert.SdkRequiresNotNull(in baseStorage); + Assert.SdkRequiresNotNull(hashGeneratorFactorySelector); + Assert.SdkRequiresNull(in _bodyStorage); + + if (cryptoConfig.GenerateKey is null) + return ResultFs.InvalidArgument.Log(); + + using var headerStorage = new UniqueRef(); + + // Generate the keys for decrypting the NCA header. + Unsafe.SkipInit(out Array2> commonDecryptionKeys); + for (int i = 0; i < NcaCryptoConfiguration.HeaderEncryptionKeyCount; i++) + { + cryptoConfig.GenerateKey(commonDecryptionKeys[i].Items, cryptoConfig.HeaderEncryptedEncryptionKeys[i], 0x60, + in cryptoConfig); + } + + // Create an XTS storage to read the encrypted header. + Array16 headerIv = default; + headerStorage.Reset(new AesXtsStorage(baseStorage.Get, commonDecryptionKeys[0], commonDecryptionKeys[1], + headerIv, NcaHeader.XtsBlockSize)); + + if (!headerStorage.HasValue) + return ResultFs.AllocationMemoryFailedInNcaReaderA.Log(); + + // Read the decrypted header. + Result rc = headerStorage.Get.Read(0, SpanHelpers.AsByteSpan(ref _header)); + if (rc.IsFailure()) return rc.Miss(); + + // Check if the NCA magic value is correct. + Result signatureResult = CheckSignature(in _header); + if (signatureResult.IsFailure()) + { + // If the magic value is not correct the header might not be encrypted. + if (cryptoConfig.IsDev) + { + // Read the header without decrypting it and check the magic value again. + rc = baseStorage.Get.Read(0, SpanHelpers.AsByteSpan(ref _header)); + if (rc.IsFailure()) return rc.Miss(); + + rc = CheckSignature(in _header); + if (rc.IsFailure()) + return signatureResult.Miss(); + + // We have a plaintext header. Get an IStorage of just the header. + rc = baseStorage.Get.GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + headerStorage.Reset(new SubStorage(in baseStorage, 0, baseStorageSize)); + + if (!headerStorage.HasValue) + return ResultFs.AllocationMemoryFailedInNcaReaderA.Log(); + + _headerEncryptionType = NcaHeader.EncryptionType.None; + } + else + { + return signatureResult.Miss(); + } + } + + // Validate the fixed key signature. + if (_header.Header1SignatureKeyGeneration > NcaCryptoConfiguration.Header1SignatureKeyGenerationMax) + return ResultFs.InvalidNcaHeader1SignatureKeyGeneration.Log(); + + int signMessageOffset = NcaHeader.HeaderSignSize * NcaHeader.HeaderSignCount; + int signMessageSize = NcaHeader.Size - signMessageOffset; + ReadOnlySpan signature = _header.Signature1; + ReadOnlySpan modulus = cryptoConfig.Header1SignKeyModuli[_header.Header1SignatureKeyGeneration]; + ReadOnlySpan exponent = cryptoConfig.Header1SignKeyPublicExponent; + ReadOnlySpan message = SpanHelpers.AsReadOnlyByteSpan(in _header).Slice(signMessageOffset, signMessageSize); + + if (!Rsa.VerifyRsa2048PssSha256(signature, modulus, exponent, message)) + return ResultFs.NcaHeaderSignature1VerificationFailed.Log(); + + // Validate the sdk version. + if (_header.SdkAddonVersion < SdkAddonVersionMin) + return ResultFs.UnsupportedSdkVersion.Log(); + + // Validate the key index. + if (_header.KeyAreaEncryptionKeyIndex >= NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexCount) + return ResultFs.InvalidNcaKeyIndex.Log(); + + // Get keys from the key area if the NCA doesn't have a rights ID. + Array16 zeroRightsId = default; + if (CryptoUtil.IsSameBytes(zeroRightsId, _header.RightsId, NcaHeader.RightsIdSize)) + { + // If we don't have a rights ID we need to generate decryption keys. + int keyType = NcaKeyFunctions.GetKeyTypeValue(_header.KeyAreaEncryptionKeyIndex, _header.GetProperKeyGeneration()); + ReadOnlySpan encryptedKeyCtr = _header.EncryptedKeys.ItemsRo.Slice((int)NcaHeader.DecryptionKey.AesCtr * Aes.KeySize128, Aes.KeySize128); + ReadOnlySpan keyCtrHw = _header.EncryptedKeys.ItemsRo.Slice((int)NcaHeader.DecryptionKey.AesCtrHw * Aes.KeySize128, Aes.KeySize128); + + cryptoConfig.GenerateKey(_decryptionKeys[(int)NcaHeader.DecryptionKey.AesCtr].Items, encryptedKeyCtr, keyType, in cryptoConfig); + + // Copy the plaintext hardware key. + keyCtrHw.CopyTo(_decryptionKeys[(int)NcaHeader.DecryptionKey.AesCtrHw].Items); + } + + _externalDataDecryptionKey.Items.Clear(); + + // Copy the configuration to the NcaReader. + _decryptAesCtr = cryptoConfig.DecryptAesCtr; + _decryptAesCtrForExternalKey = cryptoConfig.DecryptAesCtrForExternalKey; + _getDecompressorFunc = compressionConfig.GetDecompressorFunc; + _hashGeneratorFactory = hashGeneratorFactorySelector.GetFactory(); + Assert.SdkRequiresNotNull(_hashGeneratorFactory); + + _bodyStorage.SetByMove(ref baseStorage); + _headerStorage.Set(ref headerStorage.Ref()); + + return Result.Success; + + static Result CheckSignature(in NcaHeader header) + { + if (header.Magic == NcaHeader.Magic0 || + header.Magic == NcaHeader.Magic1 || + header.Magic == NcaHeader.Magic2) + { + return ResultFs.UnsupportedSdkVersion.Log(); + } + + if (header.Magic != NcaHeader.CurrentMagic) + return ResultFs.InvalidNcaSignature.Log(); + + return Result.Success; + } + } + + public Result ReadHeader(out NcaFsHeader outHeader, int index) + { + UnsafeHelpers.SkipParamInit(out outHeader); + + Assert.SdkRequiresInRange(index, 0, NcaHeader.FsCountMax); + + long offset = Unsafe.SizeOf() + Unsafe.SizeOf() * index; + return _headerStorage.Get.Read(offset, SpanHelpers.AsByteSpan(ref outHeader)); + } + + public void GetHeaderSign2(Span outBuffer) + { + Assert.SdkRequiresEqual(NcaHeader.HeaderSignSize, outBuffer.Length); + + _header.Signature2.ItemsRo.CopyTo(outBuffer); + } + + public void GetHeaderSign2TargetHash(Span outBuffer) + { + Assert.SdkRequiresNotNull(_hashGeneratorFactory); + Assert.SdkRequiresEqual(IHash256Generator.HashSize, outBuffer.Length); + + int signTargetOffset = NcaHeader.HeaderSignSize * NcaHeader.HeaderSignCount; + int signTargetSize = NcaHeader.Size - signTargetOffset; + ReadOnlySpan signTarget = + SpanHelpers.AsReadOnlyByteSpan(in _header).Slice(signTargetOffset, signTargetSize); + + _hashGeneratorFactory.GenerateHash(outBuffer, signTarget); + } + + public SharedRef GetSharedBodyStorage() + { + Assert.SdkRequiresNotNull(_bodyStorage); + + return SharedRef.CreateCopy(in _bodyStorage); + } + + public uint GetSignature() + { + Assert.SdkRequiresNotNull(_bodyStorage); + return _header.Magic; + } + + public NcaHeader.DistributionType GetDistributionType() + { + Assert.SdkRequiresNotNull(_bodyStorage); + return _header.DistributionTypeValue; + } + + public NcaHeader.ContentType GetContentType() + { + Assert.SdkRequiresNotNull(_bodyStorage); + return _header.ContentTypeValue; + } + + public byte GetKeyGeneration() + { + Assert.SdkRequiresNotNull(_bodyStorage); + return _header.GetProperKeyGeneration(); + } + + public byte GetKeyIndex() + { + Assert.SdkRequiresNotNull(_bodyStorage); + return _header.KeyAreaEncryptionKeyIndex; + } + + public ulong GetContentSize() + { + Assert.SdkRequiresNotNull(_bodyStorage); + return _header.ContentSize; + } + + public ulong GetProgramId() + { + Assert.SdkRequiresNotNull(_bodyStorage); + return _header.ProgramId; + } + + public uint GetContentIndex() + { + Assert.SdkRequiresNotNull(_bodyStorage); + return _header.ContentIndex; + } + + public uint GetSdkAddonVersion() + { + Assert.SdkRequiresNotNull(_bodyStorage); + return _header.SdkAddonVersion; + } + + public void GetRightsId(Span outBuffer) + { + Assert.SdkRequiresGreaterEqual(outBuffer.Length, NcaHeader.RightsIdSize); + + _header.RightsId.ItemsRo.CopyTo(outBuffer); + } + + public bool HasFsInfo(int index) + { + Assert.SdkRequiresInRange(index, 0, NcaHeader.FsCountMax); + + return _header.FsInfos[index].StartSector != 0 || _header.FsInfos[index].EndSector != 0; + } + + public int GetFsCount() + { + Assert.SdkRequiresNotNull(_bodyStorage); + + for (int i = 0; i < NcaHeader.FsCountMax; i++) + { + if (!HasFsInfo(i)) + { + return i; + } + } + + return NcaHeader.FsCountMax; + } + + public NcaHeader.EncryptionType GetEncryptionType() + { + return _headerEncryptionType; + } + + public ref readonly Hash GetFsHeaderHash(int index) + { + Assert.SdkRequiresNotNull(_bodyStorage); + Assert.SdkRequiresInRange(index, 0, NcaHeader.FsCountMax); + + return ref _header.FsHeaderHashes[index]; + } + + public void GetFsHeaderHash(out Hash outHash, int index) + { + Assert.SdkRequiresNotNull(_bodyStorage); + Assert.SdkRequiresInRange(index, 0, NcaHeader.FsCountMax); + + outHash = _header.FsHeaderHashes[index]; + } + + public void GetFsInfo(out NcaHeader.FsInfo outFsInfo, int index) + { + Assert.SdkRequiresNotNull(_bodyStorage); + Assert.SdkRequiresInRange(index, 0, NcaHeader.FsCountMax); + + outFsInfo = _header.FsInfos[index]; + } + + public ulong GetFsOffset(int index) + { + Assert.SdkRequiresNotNull(_bodyStorage); + Assert.SdkRequiresInRange(index, 0, NcaHeader.FsCountMax); + + return NcaHeader.SectorToByte(_header.FsInfos[index].StartSector); + } + + public ulong GetFsEndOffset(int index) + { + Assert.SdkRequiresNotNull(_bodyStorage); + Assert.SdkRequiresInRange(index, 0, NcaHeader.FsCountMax); + + return NcaHeader.SectorToByte(_header.FsInfos[index].EndSector); + } + + public ulong GetFsSize(int index) + { + Assert.SdkRequiresNotNull(_bodyStorage); + Assert.SdkRequiresInRange(index, 0, NcaHeader.FsCountMax); + + return NcaHeader.SectorToByte(_header.FsInfos[index].EndSector - _header.FsInfos[index].StartSector); + } + + public void GetEncryptedKey(Span outBuffer) + { + Assert.SdkRequiresNotNull(_bodyStorage); + Assert.SdkRequiresGreaterEqual(outBuffer.Length, NcaHeader.EncryptedKeyAreaSize); + + _header.EncryptedKeys.ItemsRo.CopyTo(outBuffer); + } + + public ReadOnlySpan GetDecryptionKey(int index) + { + Assert.SdkRequiresNotNull(_bodyStorage); + Assert.SdkRequiresInRange(index, 0, (int)NcaHeader.DecryptionKey.Count); + + return _decryptionKeys[index]; + } + + public bool HasValidInternalKey() + { + Array16 zeroKey = default; + + for (int i = 0; i < (int)NcaHeader.DecryptionKey.Count; i++) + { + if (!CryptoUtil.IsSameBytes(zeroKey, + _header.EncryptedKeys.ItemsRo.Slice(i * Aes.KeySize128, Aes.KeySize128), Aes.KeySize128)) + { + return true; + } + } + + return false; + } + + public bool HasInternalDecryptionKeyForAesHw() + { + Array16 zeroKey = default; + return !CryptoUtil.IsSameBytes(zeroKey, GetDecryptionKey((int)NcaHeader.DecryptionKey.AesCtrHw), + Array16.Length); + } + + public bool IsSwAesPrioritized() + { + return _isSoftwareAesPrioritized; + } + + public void PrioritizeSwAes() + { + _isSoftwareAesPrioritized = true; + } + + public void SetExternalDecryptionKey(ReadOnlySpan key) + { + Assert.SdkRequiresEqual(_externalDataDecryptionKey.ItemsRo.Length, key.Length); + + key.CopyTo(_externalDataDecryptionKey.Items); + } + + public ReadOnlySpan GetExternalDecryptionKey() + { + return _externalDataDecryptionKey.ItemsRo; + } + + public bool HasExternalDecryptionKey() + { + Array16 zeroKey = default; + return !CryptoUtil.IsSameBytes(zeroKey, GetExternalDecryptionKey(), Array16.Length); + } + + public void GetRawData(Span outBuffer) + { + Assert.SdkRequires(_bodyStorage.HasValue); + Assert.SdkRequiresLessEqual(Unsafe.SizeOf(), outBuffer.Length); + + SpanHelpers.AsReadOnlyByteSpan(_header).CopyTo(outBuffer); + } + + public DecryptAesCtrFunction GetExternalDecryptAesCtrFunction() + { + Assert.SdkRequiresNotNull(_decryptAesCtr); + return _decryptAesCtr; + } + + public DecryptAesCtrFunction GetExternalDecryptAesCtrFunctionForExternalKey() + { + Assert.SdkRequiresNotNull(_decryptAesCtrForExternalKey); + return _decryptAesCtrForExternalKey; + } + + public GetDecompressorFunction GetDecompressor() + { + Assert.SdkRequiresNotNull(_getDecompressorFunc); + return _getDecompressorFunc; + } + + public IHash256GeneratorFactory GetHashGeneratorFactory() + { + Assert.SdkRequiresNotNull(_hashGeneratorFactory); + return _hashGeneratorFactory; + } +} + +/// +/// Handles reading information from the of a file system inside an NCA file. +/// +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public class NcaFsHeaderReader +{ + private NcaFsHeader _header; + private int _fsIndex; + + public NcaFsHeaderReader() + { + _fsIndex = -1; + } + + public bool IsInitialized() + { + return _fsIndex >= 0; + } + + public Result Initialize(NcaReader reader, int index) + { + _fsIndex = -1; + + Result rc = reader.ReadHeader(out _header, index); + if (rc.IsFailure()) return rc.Miss(); + + Unsafe.SkipInit(out Hash hash); + reader.GetHashGeneratorFactory().GenerateHash(hash.Value.Items, SpanHelpers.AsReadOnlyByteSpan(in _header)); + + if (!CryptoUtil.IsSameBytes(reader.GetFsHeaderHash(index).Value, hash.Value, Unsafe.SizeOf())) + { + return ResultFs.NcaFsHeaderHashVerificationFailed.Log(); + } + + _fsIndex = index; + return Result.Success; + } + + public ref readonly NcaFsHeader.HashData GetHashData() + { + Assert.SdkRequires(IsInitialized()); + return ref _header.HashDataValue; + } + + public ushort GetVersion() + { + Assert.SdkRequires(IsInitialized()); + return _header.Version; + } + + public int GetFsIndex() + { + Assert.SdkRequires(IsInitialized()); + return _fsIndex; + } + + public NcaFsHeader.FsType GetFsType() + { + Assert.SdkRequires(IsInitialized()); + return _header.FsTypeValue; + } + + public NcaFsHeader.HashType GetHashType() + { + Assert.SdkRequires(IsInitialized()); + return _header.HashTypeValue; + } + + public NcaFsHeader.EncryptionType GetEncryptionType() + { + Assert.SdkRequires(IsInitialized()); + return _header.EncryptionTypeValue; + } + + public ref readonly NcaPatchInfo GetPatchInfo() + { + Assert.SdkRequires(IsInitialized()); + return ref _header.PatchInfo; + } + + public NcaAesCtrUpperIv GetAesCtrUpperIv() + { + Assert.SdkRequires(IsInitialized()); + return _header.AesCtrUpperIv; + } + + public bool ExistsSparseLayer() + { + Assert.SdkRequires(IsInitialized()); + return _header.SparseInfo.Generation != 0; + } + + public ref readonly NcaSparseInfo GetSparseInfo() + { + Assert.SdkRequires(IsInitialized()); + return ref _header.SparseInfo; + } + + public bool ExistsCompressionLayer() + { + Assert.SdkRequires(IsInitialized()); + return _header.CompressionInfo.TableOffset != 0 && _header.CompressionInfo.TableSize != 0; + } + + public ref readonly NcaCompressionInfo GetCompressionInfo() + { + Assert.SdkRequires(IsInitialized()); + return ref _header.CompressionInfo; + } + + public void GetRawData(Span outBuffer) + { + Assert.SdkRequires(IsInitialized()); + Assert.SdkRequiresLessEqual(Unsafe.SizeOf(), outBuffer.Length); + + SpanHelpers.AsReadOnlyByteSpan(in _header).CopyTo(outBuffer); + } +} \ No newline at end of file diff --git a/src/LibHac/FsSystem/NcaStructs.cs b/src/LibHac/FsSystem/NcaStructs.cs index a5bcd0c0..206295ee 100644 --- a/src/LibHac/FsSystem/NcaStructs.cs +++ b/src/LibHac/FsSystem/NcaStructs.cs @@ -1,51 +1,4 @@ -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using LibHac.Common.FixedArrays; - -namespace LibHac.FsSystem; - -public struct NcaSparseInfo -{ - public long MetaOffset; - public long MetaSize; - public Array16 MetaHeader; - public long PhysicalOffset; - public ushort Generation; - public Array6 Reserved; - - public readonly uint GetGeneration() => (uint)(Generation << 16); - public readonly long GetPhysicalSize() => MetaOffset + MetaSize; - - public readonly NcaAesCtrUpperIv MakeAesCtrUpperIv(NcaAesCtrUpperIv upperIv) - { - NcaAesCtrUpperIv sparseUpperIv = upperIv; - sparseUpperIv.Generation = GetGeneration(); - return sparseUpperIv; - } -} - -public struct NcaCompressionInfo -{ - public long MetaOffset; - public long MetaSize; - public Array16 MetaHeader; -} - -[StructLayout(LayoutKind.Explicit)] -public struct NcaAesCtrUpperIv -{ - [FieldOffset(0)] public ulong Value; - - [FieldOffset(0)] public uint Generation; - [FieldOffset(4)] public uint SecureValue; - - internal NcaAesCtrUpperIv(ulong value) - { - Unsafe.SkipInit(out Generation); - Unsafe.SkipInit(out SecureValue); - Value = value; - } -} +namespace LibHac.FsSystem; public enum NcaSectionType { diff --git a/src/LibHac/Tools/Fs/SwitchFs.cs b/src/LibHac/Tools/Fs/SwitchFs.cs index be307a04..0be82524 100644 --- a/src/LibHac/Tools/Fs/SwitchFs.cs +++ b/src/LibHac/Tools/Fs/SwitchFs.cs @@ -15,6 +15,7 @@ using LibHac.Tools.FsSystem.Save; using LibHac.Tools.Ncm; using LibHac.Util; +using KeyType = LibHac.Common.Keys.KeyType; namespace LibHac.Tools.Fs; diff --git a/src/LibHac/Tools/FsSystem/NcaUtils/Nca.cs b/src/LibHac/Tools/FsSystem/NcaUtils/Nca.cs index a7532784..b4765e8d 100644 --- a/src/LibHac/Tools/FsSystem/NcaUtils/Nca.cs +++ b/src/LibHac/Tools/FsSystem/NcaUtils/Nca.cs @@ -13,6 +13,7 @@ using LibHac.Spl; using LibHac.Tools.Crypto; using LibHac.Tools.FsSystem.RomFs; +using KeyType = LibHac.Common.Keys.KeyType; namespace LibHac.Tools.FsSystem.NcaUtils; @@ -416,13 +417,13 @@ private static IStorage OpenCompressedStorage(NcaFsHeader header, IStorage baseS ref NcaCompressionInfo compressionInfo = ref header.GetCompressionInfo(); Unsafe.SkipInit(out BucketTree.Header bucketTreeHeader); - compressionInfo.MetaHeader.ItemsRo.CopyTo(SpanHelpers.AsByteSpan(ref bucketTreeHeader)); + compressionInfo.TableHeader.ItemsRo.CopyTo(SpanHelpers.AsByteSpan(ref bucketTreeHeader)); bucketTreeHeader.Verify().ThrowIfFailure(); long nodeStorageSize = CompressedStorage.QueryNodeStorageSize(bucketTreeHeader.EntryCount); long entryStorageSize = CompressedStorage.QueryEntryStorageSize(bucketTreeHeader.EntryCount); - long tableOffset = compressionInfo.MetaOffset; - long tableSize = compressionInfo.MetaSize; + long tableOffset = compressionInfo.TableOffset; + long tableSize = compressionInfo.TableSize; if (entryStorageSize + nodeStorageSize > tableSize) throw new HorizonResultException(ResultFs.NcaInvalidCompressionInfo.Value); diff --git a/src/LibHac/Tools/FsSystem/NcaUtils/NcaFsHeader.cs b/src/LibHac/Tools/FsSystem/NcaUtils/NcaFsHeader.cs index 2ba5f2ae..a2d452ed 100644 --- a/src/LibHac/Tools/FsSystem/NcaUtils/NcaFsHeader.cs +++ b/src/LibHac/Tools/FsSystem/NcaUtils/NcaFsHeader.cs @@ -81,7 +81,7 @@ public ref NcaCompressionInfo GetCompressionInfo() public bool ExistsCompressionLayer() { - return GetCompressionInfo().MetaOffset != 0 && GetCompressionInfo().MetaSize != 0; + return GetCompressionInfo().TableOffset != 0 && GetCompressionInfo().TableSize != 0; } public ulong Counter diff --git a/src/hactoolnet/ProcessNca.cs b/src/hactoolnet/ProcessNca.cs index 55dc4e85..cf1a913b 100644 --- a/src/hactoolnet/ProcessNca.cs +++ b/src/hactoolnet/ProcessNca.cs @@ -11,6 +11,7 @@ using LibHac.Tools.FsSystem.NcaUtils; using LibHac.Tools.Npdm; using static hactoolnet.Print; +using NcaFsHeader = LibHac.Tools.FsSystem.NcaUtils.NcaFsHeader; namespace hactoolnet; diff --git a/tests/LibHac.Tests/Fs/TypeLayoutTests.cs b/tests/LibHac.Tests/Fs/TypeLayoutTests.cs index e7bb54cc..ad328e4e 100644 --- a/tests/LibHac.Tests/Fs/TypeLayoutTests.cs +++ b/tests/LibHac.Tests/Fs/TypeLayoutTests.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using LibHac.Fs; using LibHac.Fs.Impl; using Xunit; @@ -471,4 +472,22 @@ public static void GameCardHandle_Layout() Assert.Equal(0, GetOffset(in s, in s.Value)); } + + [StructLayout(LayoutKind.Sequential)] + private struct Int64AlignmentTest + { + public int A; + public Int64 B; + } + + [Fact] + public static void Int64Test_Layout() + { + var s = new Int64AlignmentTest(); + + Assert.Equal(12, Unsafe.SizeOf()); + + Assert.Equal(0, GetOffset(in s, in s.A)); + Assert.Equal(4, GetOffset(in s, in s.B)); + } } \ No newline at end of file diff --git a/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs b/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs index 2a31de87..5d0746ea 100644 --- a/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs +++ b/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs @@ -10,10 +10,241 @@ public class TypeLayoutTests [Fact] public static void Hash_Layout() { - var s = new Hash(); + Hash s = default; Assert.Equal(0x20, Unsafe.SizeOf()); Assert.Equal(0x0, GetOffset(in s, in s.Value)); } + + [Fact] + public static void NcaFsHeader_Layout() + { + NcaFsHeader s = default; + + Assert.Equal(0x200, Unsafe.SizeOf()); + + Assert.Equal(0x000, GetOffset(in s, in s.Version)); + Assert.Equal(0x002, GetOffset(in s, in s.FsTypeValue)); + Assert.Equal(0x003, GetOffset(in s, in s.HashTypeValue)); + Assert.Equal(0x004, GetOffset(in s, in s.EncryptionTypeValue)); + Assert.Equal(0x005, GetOffset(in s, in s.Reserved)); + Assert.Equal(0x008, GetOffset(in s, in s.HashDataValue)); + Assert.Equal(0x100, GetOffset(in s, in s.PatchInfo)); + Assert.Equal(0x140, GetOffset(in s, in s.AesCtrUpperIv)); + Assert.Equal(0x148, GetOffset(in s, in s.SparseInfo)); + Assert.Equal(0x178, GetOffset(in s, in s.CompressionInfo)); + Assert.Equal(0x1A0, GetOffset(in s, in s.Padding)); + } + + [Fact] + public static void NcaFsHeaderRegion_Layout() + { + NcaFsHeader.Region s = default; + + Assert.Equal(0x10, Unsafe.SizeOf()); + + Assert.Equal(0, GetOffset(in s, in s.Offset)); + Assert.Equal(8, GetOffset(in s, in s.Size)); + } + + [Fact] + public static void HashData_Layout() + { + NcaFsHeader.HashData s = default; + + Assert.Equal(0xF8, Unsafe.SizeOf()); + + Assert.Equal(0, GetOffset(in s, in s.HierarchicalSha256)); + Assert.Equal(0, GetOffset(in s, in s.IntegrityMeta)); + } + + [Fact] + public static void HierarchicalSha256Data_Layout() + { + NcaFsHeader.HashData.HierarchicalSha256Data s = default; + + Assert.Equal(0x78, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.MasterHash)); + Assert.Equal(0x20, GetOffset(in s, in s.BlockSize)); + Assert.Equal(0x24, GetOffset(in s, in s.LayerCount)); + Assert.Equal(0x28, GetOffset(in s, in s.LayerRegions)); + } + + [Fact] + public static void IntegrityMetaInfo_Layout() + { + NcaFsHeader.HashData.IntegrityMetaInfo s = default; + + Assert.Equal(0xE0, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.Magic)); + Assert.Equal(0x04, GetOffset(in s, in s.Version)); + Assert.Equal(0x08, GetOffset(in s, in s.MasterHashSize)); + Assert.Equal(0x0C, GetOffset(in s, in s.LevelHashInfo)); + Assert.Equal(0xC0, GetOffset(in s, in s.MasterHash)); + } + + [Fact] + public static void InfoLevelHash_Layout() + { + NcaFsHeader.HashData.IntegrityMetaInfo.InfoLevelHash s = default; + + Assert.Equal(0xB4, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.MaxLayers)); + Assert.Equal(0x04, GetOffset(in s, in s.Layers)); + Assert.Equal(0x94, GetOffset(in s, in s.Salt)); + } + + [Fact] + public static void HierarchicalIntegrityVerificationLevelInformation_Layout() + { + NcaFsHeader.HashData.IntegrityMetaInfo.InfoLevelHash.HierarchicalIntegrityVerificationLevelInformation s = default; + + Assert.Equal(0x18, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.Offset)); + Assert.Equal(0x08, GetOffset(in s, in s.Size)); + Assert.Equal(0x10, GetOffset(in s, in s.OrderBlock)); + Assert.Equal(0x14, GetOffset(in s, in s.Reserved)); + } + + [Fact] + public static void SignatureSalt_Layout() + { + NcaFsHeader.HashData.IntegrityMetaInfo.InfoLevelHash.SignatureSalt s = default; + + Assert.Equal(0x20, Unsafe.SizeOf()); + + Assert.Equal(0, GetOffset(in s, in s.Value)); + } + + [Fact] + public static void NcaPatchInfo_Layout() + { + NcaPatchInfo s = default; + + Assert.Equal(0x40, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.IndirectOffset)); + Assert.Equal(0x08, GetOffset(in s, in s.IndirectSize)); + Assert.Equal(0x10, GetOffset(in s, in s.IndirectHeader)); + Assert.Equal(0x20, GetOffset(in s, in s.AesCtrExOffset)); + Assert.Equal(0x28, GetOffset(in s, in s.AesCtrExSize)); + Assert.Equal(0x30, GetOffset(in s, in s.AesCtrExHeader)); + } + + [Fact] + public static void NcaSparseInfo_Layout() + { + NcaSparseInfo s = default; + + Assert.Equal(0x30, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.MetaOffset)); + Assert.Equal(0x08, GetOffset(in s, in s.MetaSize)); + Assert.Equal(0x10, GetOffset(in s, in s.MetaHeader)); + Assert.Equal(0x20, GetOffset(in s, in s.PhysicalOffset)); + Assert.Equal(0x28, GetOffset(in s, in s.Generation)); + Assert.Equal(0x2A, GetOffset(in s, in s.Reserved)); + } + + [Fact] + public static void NcaCompressionInfo_Layout() + { + NcaCompressionInfo s = default; + + Assert.Equal(0x28, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.TableOffset)); + Assert.Equal(0x08, GetOffset(in s, in s.TableSize)); + Assert.Equal(0x10, GetOffset(in s, in s.TableHeader)); + Assert.Equal(0x20, GetOffset(in s, in s.Reserved)); + } + + [Fact] + public static void NcaAesCtrUpperIv_Layout() + { + NcaAesCtrUpperIv s = default; + + Assert.Equal(8, Unsafe.SizeOf()); + + Assert.Equal(0, GetOffset(in s, in s.Value)); + Assert.Equal(0, GetOffset(in s, in s.Generation)); + Assert.Equal(4, GetOffset(in s, in s.SecureValue)); + } + + [Fact] + public static void NcaHeader_Layout() + { + NcaHeader s = default; + + Assert.Equal(0x400, Unsafe.SizeOf()); + + Assert.Equal(0x000, GetOffset(in s, in s.Signature1)); + Assert.Equal(0x100, GetOffset(in s, in s.Signature2)); + Assert.Equal(0x200, GetOffset(in s, in s.Magic)); + Assert.Equal(0x204, GetOffset(in s, in s.DistributionTypeValue)); + Assert.Equal(0x205, GetOffset(in s, in s.ContentTypeValue)); + Assert.Equal(0x206, GetOffset(in s, in s.KeyGeneration1)); + Assert.Equal(0x207, GetOffset(in s, in s.KeyAreaEncryptionKeyIndex)); + Assert.Equal(0x208, GetOffset(in s, in s.ContentSize)); + Assert.Equal(0x210, GetOffset(in s, in s.ProgramId)); + Assert.Equal(0x218, GetOffset(in s, in s.ContentIndex)); + Assert.Equal(0x21C, GetOffset(in s, in s.SdkAddonVersion)); + Assert.Equal(0x220, GetOffset(in s, in s.KeyGeneration2)); + Assert.Equal(0x221, GetOffset(in s, in s.Header1SignatureKeyGeneration)); + Assert.Equal(0x222, GetOffset(in s, in s.Reserved222)); + Assert.Equal(0x224, GetOffset(in s, in s.Reserved224)); + Assert.Equal(0x230, GetOffset(in s, in s.RightsId)); + Assert.Equal(0x240, GetOffset(in s, in s.FsInfos)); + Assert.Equal(0x280, GetOffset(in s, in s.FsHeaderHashes)); + Assert.Equal(0x300, GetOffset(in s, in s.EncryptedKeys)); + + Assert.Equal(NcaHeader.Size, Unsafe.SizeOf()); + Assert.Equal(NcaHeader.SectorSize, 1 << NcaHeader.SectorShift); + + Assert.Equal(NcaHeader.HeaderSignSize, s.Signature1.ItemsRo.Length); + Assert.Equal(NcaHeader.HeaderSignSize, s.Signature2.ItemsRo.Length); + Assert.Equal(NcaHeader.RightsIdSize, s.RightsId.ItemsRo.Length); + Assert.Equal(NcaHeader.FsCountMax, s.FsInfos.ItemsRo.Length); + Assert.Equal(NcaHeader.FsCountMax, s.FsHeaderHashes.ItemsRo.Length); + Assert.Equal(NcaHeader.EncryptedKeyAreaSize, s.EncryptedKeys.ItemsRo.Length); + } + + [Fact] + public static void NcaHeader_FsInfo_Layout() + { + NcaHeader.FsInfo s = default; + + Assert.Equal(0x10, Unsafe.SizeOf()); + + Assert.Equal(0x0, GetOffset(in s, in s.StartSector)); + Assert.Equal(0x4, GetOffset(in s, in s.EndSector)); + Assert.Equal(0x8, GetOffset(in s, in s.HashSectors)); + Assert.Equal(0xC, GetOffset(in s, in s.Reserved)); + } + + [Fact] + public static void KeyType_Layout() + { + NcaCryptoConfiguration s = default; + + Assert.Equal(NcaCryptoConfiguration.Header1SignatureKeyGenerationMax + 1, s.Header1SignKeyModuli.ItemsRo.Length); + Assert.Equal(NcaCryptoConfiguration.Rsa2048KeyModulusSize, s.Header1SignKeyModuli.ItemsRo[0].ItemsRo.Length); + Assert.Equal(NcaCryptoConfiguration.Rsa2048KeyPublicExponentSize, s.Header1SignKeyPublicExponent.ItemsRo.Length); + Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexCount, s.KeyAreaEncryptionKeySources.ItemsRo.Length); + Assert.Equal(NcaCryptoConfiguration.Aes128KeySize, s.KeyAreaEncryptionKeySources.ItemsRo[0].ItemsRo.Length); + Assert.Equal(NcaCryptoConfiguration.Aes128KeySize, s.HeaderEncryptionKeySource.ItemsRo.Length); + Assert.Equal(NcaCryptoConfiguration.HeaderEncryptionKeyCount, s.HeaderEncryptedEncryptionKeys.ItemsRo.Length); + Assert.Equal(NcaCryptoConfiguration.Aes128KeySize, s.HeaderEncryptedEncryptionKeys.ItemsRo[0].ItemsRo.Length); + + Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 0, (int)KeyType.NcaHeaderKey); + Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 1, (int)KeyType.NcaExternalKey); + Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 2, (int)KeyType.SaveDataDeviceUniqueMac); + Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 3, (int)KeyType.SaveDataSeedUniqueMac); + Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 4, (int)KeyType.SaveDataTransferMac); + } } \ No newline at end of file From c9352fcb5ada43dbc838f8381dbeeefa50e07b06 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Thu, 3 Mar 2022 16:16:08 -0700 Subject: [PATCH 04/24] Make AES crypto return the number of bytes written --- src/LibHac/Boot/Package1.cs | 2 +- src/LibHac/Crypto/Aes.cs | 58 +++++++++++--------------- src/LibHac/Crypto/AesCbcCipher.cs | 10 ++--- src/LibHac/Crypto/AesCbcCipherNi.cs | 10 ++--- src/LibHac/Crypto/AesCtrCipher.cs | 6 +-- src/LibHac/Crypto/AesCtrCipherNi.cs | 6 +-- src/LibHac/Crypto/AesEcbCipher.cs | 10 ++--- src/LibHac/Crypto/AesEcbCipherNi.cs | 10 ++--- src/LibHac/Crypto/AesXtsCipher.cs | 10 ++--- src/LibHac/Crypto/AesXtsCipherNi.cs | 10 ++--- src/LibHac/Crypto/ICipher.cs | 4 +- src/LibHac/Crypto/Impl/AesCbcMode.cs | 10 ++--- src/LibHac/Crypto/Impl/AesCbcModeNi.cs | 8 +++- src/LibHac/Crypto/Impl/AesCore.cs | 28 +++++++------ src/LibHac/Crypto/Impl/AesCoreNi.cs | 12 ++++-- src/LibHac/Crypto/Impl/AesCtrMode.cs | 6 ++- src/LibHac/Crypto/Impl/AesCtrModeNi.cs | 7 +++- src/LibHac/Crypto/Impl/AesEcbMode.cs | 10 ++--- src/LibHac/Crypto/Impl/AesEcbModeNi.cs | 10 ++--- src/LibHac/Crypto/Impl/AesXtsMode.cs | 8 +++- src/LibHac/Crypto/Impl/AesXtsModeNi.cs | 10 +++-- 21 files changed, 131 insertions(+), 114 deletions(-) diff --git a/src/LibHac/Boot/Package1.cs b/src/LibHac/Boot/Package1.cs index 03b16e05..eae84d17 100644 --- a/src/LibHac/Boot/Package1.cs +++ b/src/LibHac/Boot/Package1.cs @@ -300,7 +300,7 @@ private Result SetPk11Storage() return Result.Success; } - private delegate void Decryptor(ReadOnlySpan input, Span output, ReadOnlySpan key, + private delegate int Decryptor(ReadOnlySpan input, Span output, ReadOnlySpan key, ReadOnlySpan iv, bool preferDotNetCrypto = false); private bool TryFindEristaKeyRevision() diff --git a/src/LibHac/Crypto/Aes.cs b/src/LibHac/Crypto/Aes.cs index eb2bcd08..28ae12c7 100644 --- a/src/LibHac/Crypto/Aes.cs +++ b/src/LibHac/Crypto/Aes.cs @@ -102,7 +102,7 @@ public static ICipherWithIv CreateXtsEncryptor(ReadOnlySpan key1, ReadOnly return new AesXtsEncryptor(key1, key2, iv); } - public static void EncryptEcb128(ReadOnlySpan input, Span output, ReadOnlySpan key, + public static int EncryptEcb128(ReadOnlySpan input, Span output, ReadOnlySpan key, bool preferDotNetCrypto = false) { if (IsAesNiSupported() && !preferDotNetCrypto) @@ -110,16 +110,15 @@ public static void EncryptEcb128(ReadOnlySpan input, Span output, Re Unsafe.SkipInit(out AesEcbModeNi cipherNi); cipherNi.Initialize(key, false); - cipherNi.Encrypt(input, output); - return; + return cipherNi.Encrypt(input, output); } ICipher cipher = CreateEcbEncryptor(key, preferDotNetCrypto); - cipher.Transform(input, output); + return cipher.Transform(input, output); } - public static void DecryptEcb128(ReadOnlySpan input, Span output, ReadOnlySpan key, + public static int DecryptEcb128(ReadOnlySpan input, Span output, ReadOnlySpan key, bool preferDotNetCrypto = false) { if (IsAesNiSupported() && !preferDotNetCrypto) @@ -127,16 +126,15 @@ public static void DecryptEcb128(ReadOnlySpan input, Span output, Re Unsafe.SkipInit(out AesEcbModeNi cipherNi); cipherNi.Initialize(key, true); - cipherNi.Decrypt(input, output); - return; + return cipherNi.Decrypt(input, output); } ICipher cipher = CreateEcbDecryptor(key, preferDotNetCrypto); - cipher.Transform(input, output); + return cipher.Transform(input, output); } - public static void EncryptCbc128(ReadOnlySpan input, Span output, ReadOnlySpan key, + public static int EncryptCbc128(ReadOnlySpan input, Span output, ReadOnlySpan key, ReadOnlySpan iv, bool preferDotNetCrypto = false) { if (IsAesNiSupported() && !preferDotNetCrypto) @@ -144,16 +142,15 @@ public static void EncryptCbc128(ReadOnlySpan input, Span output, Re Unsafe.SkipInit(out AesCbcModeNi cipherNi); cipherNi.Initialize(key, iv, false); - cipherNi.Encrypt(input, output); - return; + return cipherNi.Encrypt(input, output); } ICipher cipher = CreateCbcEncryptor(key, iv, preferDotNetCrypto); - cipher.Transform(input, output); + return cipher.Transform(input, output); } - public static void DecryptCbc128(ReadOnlySpan input, Span output, ReadOnlySpan key, + public static int DecryptCbc128(ReadOnlySpan input, Span output, ReadOnlySpan key, ReadOnlySpan iv, bool preferDotNetCrypto = false) { if (IsAesNiSupported() && !preferDotNetCrypto) @@ -161,16 +158,15 @@ public static void DecryptCbc128(ReadOnlySpan input, Span output, Re Unsafe.SkipInit(out AesCbcModeNi cipherNi); cipherNi.Initialize(key, iv, true); - cipherNi.Decrypt(input, output); - return; + return cipherNi.Decrypt(input, output); } ICipher cipher = CreateCbcDecryptor(key, iv, preferDotNetCrypto); - cipher.Transform(input, output); + return cipher.Transform(input, output); } - public static void EncryptCtr128(ReadOnlySpan input, Span output, ReadOnlySpan key, + public static int EncryptCtr128(ReadOnlySpan input, Span output, ReadOnlySpan key, ReadOnlySpan iv, bool preferDotNetCrypto = false) { if (IsAesNiSupported() && !preferDotNetCrypto) @@ -178,16 +174,15 @@ public static void EncryptCtr128(ReadOnlySpan input, Span output, Re Unsafe.SkipInit(out AesCtrModeNi cipherNi); cipherNi.Initialize(key, iv); - cipherNi.Transform(input, output); - return; + return cipherNi.Transform(input, output); } ICipher cipher = CreateCtrEncryptor(key, iv, preferDotNetCrypto); - cipher.Transform(input, output); + return cipher.Transform(input, output); } - public static void DecryptCtr128(ReadOnlySpan input, Span output, ReadOnlySpan key, + public static int DecryptCtr128(ReadOnlySpan input, Span output, ReadOnlySpan key, ReadOnlySpan iv, bool preferDotNetCrypto = false) { if (IsAesNiSupported() && !preferDotNetCrypto) @@ -195,16 +190,15 @@ public static void DecryptCtr128(ReadOnlySpan input, Span output, Re Unsafe.SkipInit(out AesCtrModeNi cipherNi); cipherNi.Initialize(key, iv); - cipherNi.Transform(input, output); - return; + return cipherNi.Transform(input, output); } ICipher cipher = CreateCtrDecryptor(key, iv, preferDotNetCrypto); - cipher.Transform(input, output); + return cipher.Transform(input, output); } - public static void EncryptXts128(ReadOnlySpan input, Span output, ReadOnlySpan key1, + public static int EncryptXts128(ReadOnlySpan input, Span output, ReadOnlySpan key1, ReadOnlySpan key2, ReadOnlySpan iv, bool preferDotNetCrypto = false) { if (IsAesNiSupported() && !preferDotNetCrypto) @@ -212,16 +206,15 @@ public static void EncryptXts128(ReadOnlySpan input, Span output, Re Unsafe.SkipInit(out AesXtsModeNi cipherNi); cipherNi.Initialize(key1, key2, iv, false); - cipherNi.Encrypt(input, output); - return; + return cipherNi.Encrypt(input, output); } ICipher cipher = CreateXtsEncryptor(key1, key2, iv, preferDotNetCrypto); - cipher.Transform(input, output); + return cipher.Transform(input, output); } - public static void DecryptXts128(ReadOnlySpan input, Span output, ReadOnlySpan key1, + public static int DecryptXts128(ReadOnlySpan input, Span output, ReadOnlySpan key1, ReadOnlySpan key2, ReadOnlySpan iv, bool preferDotNetCrypto = false) { if (IsAesNiSupported() && !preferDotNetCrypto) @@ -229,13 +222,12 @@ public static void DecryptXts128(ReadOnlySpan input, Span output, Re Unsafe.SkipInit(out AesXtsModeNi cipherNi); cipherNi.Initialize(key1, key2, iv, true); - cipherNi.Decrypt(input, output); - return; + return cipherNi.Decrypt(input, output); } ICipher cipher = CreateXtsDecryptor(key1, key2, iv, preferDotNetCrypto); - cipher.Transform(input, output); + return cipher.Transform(input, output); } /// @@ -307,4 +299,4 @@ private static void LeftShiftBytes(ReadOnlySpan input, Span output) carry = (byte)((b & 0xff00) >> 8); } } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/AesCbcCipher.cs b/src/LibHac/Crypto/AesCbcCipher.cs index 4502d6de..1b8138b9 100644 --- a/src/LibHac/Crypto/AesCbcCipher.cs +++ b/src/LibHac/Crypto/AesCbcCipher.cs @@ -13,9 +13,9 @@ public AesCbcEncryptor(ReadOnlySpan key, ReadOnlySpan iv) _baseCipher.Initialize(key, iv, false); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Encrypt(input, output); + return _baseCipher.Encrypt(input, output); } } @@ -29,8 +29,8 @@ public AesCbcDecryptor(ReadOnlySpan key, ReadOnlySpan iv) _baseCipher.Initialize(key, iv, true); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Decrypt(input, output); + return _baseCipher.Decrypt(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/AesCbcCipherNi.cs b/src/LibHac/Crypto/AesCbcCipherNi.cs index 418bb3e0..96cc098c 100644 --- a/src/LibHac/Crypto/AesCbcCipherNi.cs +++ b/src/LibHac/Crypto/AesCbcCipherNi.cs @@ -18,9 +18,9 @@ public AesCbcEncryptorNi(ReadOnlySpan key, ReadOnlySpan iv) _baseCipher.Initialize(key, iv, false); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Encrypt(input, output); + return _baseCipher.Encrypt(input, output); } } @@ -36,8 +36,8 @@ public AesCbcDecryptorNi(ReadOnlySpan key, ReadOnlySpan iv) _baseCipher.Initialize(key, iv, true); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Decrypt(input, output); + return _baseCipher.Decrypt(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/AesCtrCipher.cs b/src/LibHac/Crypto/AesCtrCipher.cs index 6661b53e..45a3e6f8 100644 --- a/src/LibHac/Crypto/AesCtrCipher.cs +++ b/src/LibHac/Crypto/AesCtrCipher.cs @@ -16,8 +16,8 @@ public AesCtrCipher(ReadOnlySpan key, ReadOnlySpan iv) _baseCipher.Initialize(key, iv); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Transform(input, output); + return _baseCipher.Transform(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/AesCtrCipherNi.cs b/src/LibHac/Crypto/AesCtrCipherNi.cs index 6474b533..30a1b106 100644 --- a/src/LibHac/Crypto/AesCtrCipherNi.cs +++ b/src/LibHac/Crypto/AesCtrCipherNi.cs @@ -18,8 +18,8 @@ public AesCtrCipherNi(ReadOnlySpan key, ReadOnlySpan iv) _baseCipher.Initialize(key, iv); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Transform(input, output); + return _baseCipher.Transform(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/AesEcbCipher.cs b/src/LibHac/Crypto/AesEcbCipher.cs index 887acf27..0a35e7bb 100644 --- a/src/LibHac/Crypto/AesEcbCipher.cs +++ b/src/LibHac/Crypto/AesEcbCipher.cs @@ -13,9 +13,9 @@ public AesEcbEncryptor(ReadOnlySpan key) _baseCipher.Initialize(key, false); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Encrypt(input, output); + return _baseCipher.Encrypt(input, output); } } @@ -29,8 +29,8 @@ public AesEcbDecryptor(ReadOnlySpan key) _baseCipher.Initialize(key, true); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Decrypt(input, output); + return _baseCipher.Decrypt(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/AesEcbCipherNi.cs b/src/LibHac/Crypto/AesEcbCipherNi.cs index 17a9f43e..0002f8bd 100644 --- a/src/LibHac/Crypto/AesEcbCipherNi.cs +++ b/src/LibHac/Crypto/AesEcbCipherNi.cs @@ -13,9 +13,9 @@ public AesEcbEncryptorNi(ReadOnlySpan key) _baseCipher.Initialize(key, false); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Encrypt(input, output); + return _baseCipher.Encrypt(input, output); } } @@ -29,8 +29,8 @@ public AesEcbDecryptorNi(ReadOnlySpan key) _baseCipher.Initialize(key, true); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Decrypt(input, output); + return _baseCipher.Decrypt(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/AesXtsCipher.cs b/src/LibHac/Crypto/AesXtsCipher.cs index 57ddb18d..04c65419 100644 --- a/src/LibHac/Crypto/AesXtsCipher.cs +++ b/src/LibHac/Crypto/AesXtsCipher.cs @@ -16,9 +16,9 @@ public AesXtsEncryptor(ReadOnlySpan key1, ReadOnlySpan key2, ReadOnl _baseCipher.Initialize(key1, key2, iv, false); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Encrypt(input, output); + return _baseCipher.Encrypt(input, output); } } @@ -34,8 +34,8 @@ public AesXtsDecryptor(ReadOnlySpan key1, ReadOnlySpan key2, ReadOnl _baseCipher.Initialize(key1, key2, iv, true); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Decrypt(input, output); + return _baseCipher.Decrypt(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/AesXtsCipherNi.cs b/src/LibHac/Crypto/AesXtsCipherNi.cs index 6799378f..06b06dec 100644 --- a/src/LibHac/Crypto/AesXtsCipherNi.cs +++ b/src/LibHac/Crypto/AesXtsCipherNi.cs @@ -18,9 +18,9 @@ public AesXtsEncryptorNi(ReadOnlySpan key1, ReadOnlySpan key2, ReadO _baseCipher.Initialize(key1, key2, iv, false); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Encrypt(input, output); + return _baseCipher.Encrypt(input, output); } } @@ -36,8 +36,8 @@ public AesXtsDecryptorNi(ReadOnlySpan key1, ReadOnlySpan key2, ReadO _baseCipher.Initialize(key1, key2, iv, true); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - _baseCipher.Decrypt(input, output); + return _baseCipher.Decrypt(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/ICipher.cs b/src/LibHac/Crypto/ICipher.cs index 1ee75e81..5d2b6f76 100644 --- a/src/LibHac/Crypto/ICipher.cs +++ b/src/LibHac/Crypto/ICipher.cs @@ -5,10 +5,10 @@ namespace LibHac.Crypto; public interface ICipher { - void Transform(ReadOnlySpan input, Span output); + int Transform(ReadOnlySpan input, Span output); } public interface ICipherWithIv : ICipher { ref Buffer16 Iv { get; } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/Impl/AesCbcMode.cs b/src/LibHac/Crypto/Impl/AesCbcMode.cs index 09c7e1ea..009b453a 100644 --- a/src/LibHac/Crypto/Impl/AesCbcMode.cs +++ b/src/LibHac/Crypto/Impl/AesCbcMode.cs @@ -13,13 +13,13 @@ public void Initialize(ReadOnlySpan key, ReadOnlySpan iv, bool isDec _aesCore.Initialize(key, iv, CipherMode.CBC, isDecrypting); } - public void Encrypt(ReadOnlySpan input, Span output) + public int Encrypt(ReadOnlySpan input, Span output) { - _aesCore.Encrypt(input, output); + return _aesCore.Encrypt(input, output); } - public void Decrypt(ReadOnlySpan input, Span output) + public int Decrypt(ReadOnlySpan input, Span output) { - _aesCore.Decrypt(input, output); + return _aesCore.Decrypt(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/Impl/AesCbcModeNi.cs b/src/LibHac/Crypto/Impl/AesCbcModeNi.cs index 81983c18..0f8efc67 100644 --- a/src/LibHac/Crypto/Impl/AesCbcModeNi.cs +++ b/src/LibHac/Crypto/Impl/AesCbcModeNi.cs @@ -22,7 +22,7 @@ public void Initialize(ReadOnlySpan key, ReadOnlySpan iv, bool isDec Iv = Unsafe.ReadUnaligned>(ref MemoryMarshal.GetReference(iv)); } - public void Encrypt(ReadOnlySpan input, Span output) + public int Encrypt(ReadOnlySpan input, Span output) { int blockCount = Math.Min(input.Length, output.Length) >> 4; @@ -42,9 +42,11 @@ public void Encrypt(ReadOnlySpan input, Span output) } Iv = iv; + + return Math.Min(input.Length, output.Length) & ~0xF; } - public void Decrypt(ReadOnlySpan input, Span output) + public int Decrypt(ReadOnlySpan input, Span output) { int remainingBlocks = Math.Min(input.Length, output.Length) >> 4; @@ -104,5 +106,7 @@ public void Decrypt(ReadOnlySpan input, Span output) } Iv = iv; + + return Math.Min(input.Length, output.Length) & ~0xF; } } \ No newline at end of file diff --git a/src/LibHac/Crypto/Impl/AesCore.cs b/src/LibHac/Crypto/Impl/AesCore.cs index 548c5153..a4111997 100644 --- a/src/LibHac/Crypto/Impl/AesCore.cs +++ b/src/LibHac/Crypto/Impl/AesCore.cs @@ -31,43 +31,45 @@ public void Initialize(ReadOnlySpan key, ReadOnlySpan iv, CipherMode _isDecrypting = isDecrypting; } - public void Encrypt(ReadOnlySpan input, Span output) + public int Encrypt(ReadOnlySpan input, Span output) { Debug.Assert(!_isDecrypting); - Transform(input, output); + return Transform(input, output); } - public void Decrypt(ReadOnlySpan input, Span output) + public int Decrypt(ReadOnlySpan input, Span output) { Debug.Assert(_isDecrypting); - Transform(input, output); + return Transform(input, output); } - public void Encrypt(byte[] input, byte[] output, int length) + public int Encrypt(byte[] input, byte[] output, int length) { Debug.Assert(!_isDecrypting); - Transform(input, output, length); + return Transform(input, output, length); } - public void Decrypt(byte[] input, byte[] output, int length) + public int Decrypt(byte[] input, byte[] output, int length) { Debug.Assert(_isDecrypting); - Transform(input, output, length); + return Transform(input, output, length); } - private void Transform(ReadOnlySpan input, Span output) + private int Transform(ReadOnlySpan input, Span output) { using var rented = new RentedArray(input.Length); input.CopyTo(rented.Array); - Transform(rented.Array, rented.Array, input.Length); + int bytesWritten = Transform(rented.Array, rented.Array, input.Length); rented.Array.CopyTo(output); + + return bytesWritten; } - private void Transform(byte[] input, byte[] output, int length) + private int Transform(byte[] input, byte[] output, int length) { - _transform.TransformBlock(input, 0, length, output, 0); + return _transform.TransformBlock(input, 0, length, output, 0); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/Impl/AesCoreNi.cs b/src/LibHac/Crypto/Impl/AesCoreNi.cs index 24b34e9f..9b928495 100644 --- a/src/LibHac/Crypto/Impl/AesCoreNi.cs +++ b/src/LibHac/Crypto/Impl/AesCoreNi.cs @@ -98,9 +98,10 @@ public readonly Vector128 DecryptBlock(Vector128 input) return AesNi.DecryptLast(b, keys[0]); } - public readonly void EncryptInterleaved8(ReadOnlySpan input, Span output) + public readonly int EncryptInterleaved8(ReadOnlySpan input, Span output) { int remainingBlocks = Math.Min(input.Length, output.Length) >> 4; + int length = remainingBlocks << 4; ref Vector128 inBlock = ref Unsafe.As>(ref MemoryMarshal.GetReference(input)); ref Vector128 outBlock = ref Unsafe.As>(ref MemoryMarshal.GetReference(output)); @@ -138,11 +139,14 @@ out Unsafe.Add(ref outBlock, 6), outBlock = ref Unsafe.Add(ref outBlock, 1); remainingBlocks -= 1; } + + return length; } - public readonly void DecryptInterleaved8(ReadOnlySpan input, Span output) + public readonly int DecryptInterleaved8(ReadOnlySpan input, Span output) { int remainingBlocks = Math.Min(input.Length, output.Length) >> 4; + int length = remainingBlocks << 4; ref Vector128 inBlock = ref Unsafe.As>(ref MemoryMarshal.GetReference(input)); ref Vector128 outBlock = ref Unsafe.As>(ref MemoryMarshal.GetReference(output)); @@ -180,6 +184,8 @@ out Unsafe.Add(ref outBlock, 6), outBlock = ref Unsafe.Add(ref outBlock, 1); remainingBlocks -= 1; } + + return length; } // When inlining this function, RyuJIT will almost make the @@ -574,4 +580,4 @@ private static Vector128 KeyExpansion(Vector128 s, Vector128 t return Sse2.Xor(s, t); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/Impl/AesCtrMode.cs b/src/LibHac/Crypto/Impl/AesCtrMode.cs index b5ac9859..72cf111b 100644 --- a/src/LibHac/Crypto/Impl/AesCtrMode.cs +++ b/src/LibHac/Crypto/Impl/AesCtrMode.cs @@ -24,7 +24,7 @@ public void Initialize(ReadOnlySpan key, ReadOnlySpan iv) Iv = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(iv)); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { int blockCount = BitUtil.DivideUp(input.Length, Aes.BlockSize); int length = blockCount * Aes.BlockSize; @@ -34,6 +34,8 @@ public void Transform(ReadOnlySpan input, Span output) _aesCore.Encrypt(counterBuffer.Array, counterBuffer.Array, length); Utilities.XorArrays(output, input, counterBuffer.Span); + + return Math.Min(input.Length, output.Length); } private static void FillDecryptedCounter(Span counter, Span buffer) @@ -53,4 +55,4 @@ private static void FillDecryptedCounter(Span counter, Span buffer) counterL[1] = BinaryPrimitives.ReverseEndianness(lo); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/Impl/AesCtrModeNi.cs b/src/LibHac/Crypto/Impl/AesCtrModeNi.cs index 492fd230..547aaf69 100644 --- a/src/LibHac/Crypto/Impl/AesCtrModeNi.cs +++ b/src/LibHac/Crypto/Impl/AesCtrModeNi.cs @@ -23,9 +23,10 @@ public void Initialize(ReadOnlySpan key, ReadOnlySpan iv) Iv = Unsafe.ReadUnaligned>(ref MemoryMarshal.GetReference(iv)); } - public void Transform(ReadOnlySpan input, Span output) + public int Transform(ReadOnlySpan input, Span output) { - int remaining = Math.Min(input.Length, output.Length); + int length = Math.Min(input.Length, output.Length); + int remaining = length; int blockCount = remaining >> 4; ref Vector128 inBlock = ref Unsafe.As>(ref MemoryMarshal.GetReference(input)); @@ -103,6 +104,8 @@ public void Transform(ReadOnlySpan input, Span output) { EncryptCtrPartialBlock(input.Slice(blockCount * 0x10), output.Slice(blockCount * 0x10)); } + + return length; } private void EncryptCtrPartialBlock(ReadOnlySpan input, Span output) diff --git a/src/LibHac/Crypto/Impl/AesEcbMode.cs b/src/LibHac/Crypto/Impl/AesEcbMode.cs index 2d76d5fb..42ff39fd 100644 --- a/src/LibHac/Crypto/Impl/AesEcbMode.cs +++ b/src/LibHac/Crypto/Impl/AesEcbMode.cs @@ -13,13 +13,13 @@ public void Initialize(ReadOnlySpan key, bool isDecrypting) _aesCore.Initialize(key, ReadOnlySpan.Empty, CipherMode.ECB, isDecrypting); } - public void Encrypt(ReadOnlySpan input, Span output) + public int Encrypt(ReadOnlySpan input, Span output) { - _aesCore.Encrypt(input, output); + return _aesCore.Encrypt(input, output); } - public void Decrypt(ReadOnlySpan input, Span output) + public int Decrypt(ReadOnlySpan input, Span output) { - _aesCore.Decrypt(input, output); + return _aesCore.Decrypt(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/Impl/AesEcbModeNi.cs b/src/LibHac/Crypto/Impl/AesEcbModeNi.cs index 88b5826f..b2f8085f 100644 --- a/src/LibHac/Crypto/Impl/AesEcbModeNi.cs +++ b/src/LibHac/Crypto/Impl/AesEcbModeNi.cs @@ -11,13 +11,13 @@ public void Initialize(ReadOnlySpan key, bool isDecrypting) _aesCore.Initialize(key, isDecrypting); } - public void Encrypt(ReadOnlySpan input, Span output) + public int Encrypt(ReadOnlySpan input, Span output) { - _aesCore.EncryptInterleaved8(input, output); + return _aesCore.EncryptInterleaved8(input, output); } - public void Decrypt(ReadOnlySpan input, Span output) + public int Decrypt(ReadOnlySpan input, Span output) { - _aesCore.DecryptInterleaved8(input, output); + return _aesCore.DecryptInterleaved8(input, output); } -} +} \ No newline at end of file diff --git a/src/LibHac/Crypto/Impl/AesXtsMode.cs b/src/LibHac/Crypto/Impl/AesXtsMode.cs index 2fafd67e..cf70ace3 100644 --- a/src/LibHac/Crypto/Impl/AesXtsMode.cs +++ b/src/LibHac/Crypto/Impl/AesXtsMode.cs @@ -26,7 +26,7 @@ public void Initialize(ReadOnlySpan key1, ReadOnlySpan key2, ReadOnl Iv = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(iv)); } - public void Encrypt(ReadOnlySpan input, Span output) + public int Encrypt(ReadOnlySpan input, Span output) { int length = Math.Min(input.Length, output.Length); int blockCount = length >> 4; @@ -74,9 +74,11 @@ public void Encrypt(ReadOnlySpan input, Span output) _dataAesCore.Encrypt(tmp, tmp); XorBuffer(ref prevOutBlock, ref tmp, ref tweak); } + + return length; } - public void Decrypt(ReadOnlySpan input, Span output) + public int Decrypt(ReadOnlySpan input, Span output) { int length = Math.Min(input.Length, output.Length); int blockCount = length >> 4; @@ -139,6 +141,8 @@ public void Decrypt(ReadOnlySpan input, Span output) _dataAesCore.Decrypt(tmp, tmp); XorBuffer(ref outBlock, ref tmp, ref tweak); } + + return length; } private static Buffer16 FillTweakBuffer(Buffer16 initialTweak, Span tweakBuffer) diff --git a/src/LibHac/Crypto/Impl/AesXtsModeNi.cs b/src/LibHac/Crypto/Impl/AesXtsModeNi.cs index aa308070..ebb8c409 100644 --- a/src/LibHac/Crypto/Impl/AesXtsModeNi.cs +++ b/src/LibHac/Crypto/Impl/AesXtsModeNi.cs @@ -25,7 +25,7 @@ public void Initialize(ReadOnlySpan key1, ReadOnlySpan key2, ReadOnl Iv = Unsafe.ReadUnaligned>(ref MemoryMarshal.GetReference(iv)); } - public void Encrypt(ReadOnlySpan input, Span output) + public int Encrypt(ReadOnlySpan input, Span output) { int length = Math.Min(input.Length, output.Length); int remainingBlocks = length >> 4; @@ -101,9 +101,11 @@ public void Encrypt(ReadOnlySpan input, Span output) { EncryptPartialFinalBlock(ref inBlock, ref outBlock, tweak, leftover); } + + return length; } - public void Decrypt(ReadOnlySpan input, Span output) + public int Decrypt(ReadOnlySpan input, Span output) { int length = Math.Min(input.Length, output.Length); int remainingBlocks = length >> 4; @@ -181,6 +183,8 @@ public void Decrypt(ReadOnlySpan input, Span output) { DecryptPartialFinalBlock(ref inBlock, ref outBlock, tweak, mask, leftover); } + + return length; } // ReSharper disable once RedundantAssignment @@ -251,4 +255,4 @@ private static Vector128 Gf128Mul(Vector128 iv, Vector128 mask return Sse2.Xor(tmp1, tmp2); } -} +} \ No newline at end of file From e8d54159e371b52bc469432920887b02534e4fcc Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Fri, 4 Mar 2022 17:35:03 -0700 Subject: [PATCH 05/24] Add AesCtrStorage --- src/LibHac/FsSystem/AesCtrStorage.cs | 235 +++++++++++++++++++++++++++ src/LibHac/FsSystem/PooledBuffer.cs | 26 ++- 2 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 src/LibHac/FsSystem/AesCtrStorage.cs diff --git a/src/LibHac/FsSystem/AesCtrStorage.cs b/src/LibHac/FsSystem/AesCtrStorage.cs new file mode 100644 index 00000000..750e4477 --- /dev/null +++ b/src/LibHac/FsSystem/AesCtrStorage.cs @@ -0,0 +1,235 @@ +using System; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using LibHac.Common; +using LibHac.Common.FixedArrays; +using LibHac.Crypto; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.Util; + +namespace LibHac.FsSystem; + +/// +/// Reads and writes to an that's encrypted with AES-CTR-128. +/// +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public class AesCtrStorage : IStorage +{ + public static readonly int BlockSize = Aes.BlockSize; + public static readonly int KeySize = Aes.KeySize128; + public static readonly int IvSize = Aes.KeySize128; + + private IStorage _baseStorage; + private Array16 _key; + private Array16 _iv; + + // LibHac addition: This field goes unused if initialized with a plain IStorage. + // The original class uses a template for both the shared and non-shared IStorage which avoids needing this field. + private SharedRef _baseStorageShared; + + public static void MakeIv(Span outIv, ulong upperIv, long offset) + { + Assert.SdkRequiresEqual(outIv.Length, IvSize); + Assert.SdkRequiresGreaterEqual(offset, 0); + + BinaryPrimitives.WriteUInt64BigEndian(outIv, upperIv); + BinaryPrimitives.WriteInt64BigEndian(outIv.Slice(sizeof(long)), offset / BlockSize); + } + + public AesCtrStorage(IStorage baseStorage, ReadOnlySpan key, ReadOnlySpan iv) + { + Assert.SdkRequiresNotNull(baseStorage); + Assert.SdkRequiresEqual(key.Length, KeySize); + Assert.SdkRequiresEqual(iv.Length, IvSize); + + _baseStorage = baseStorage; + + key.CopyTo(_key.Items); + iv.CopyTo(_iv.Items); + } + + public AesCtrStorage(in SharedRef baseStorage, ReadOnlySpan key, ReadOnlySpan iv) + { + Assert.SdkRequiresNotNull(in baseStorage); + Assert.SdkRequiresEqual(key.Length, KeySize); + Assert.SdkRequiresEqual(iv.Length, IvSize); + + _baseStorage = baseStorage.Get; + _baseStorageShared = SharedRef.CreateCopy(in baseStorage); + + key.CopyTo(_key.Items); + iv.CopyTo(_iv.Items); + } + + public override void Dispose() + { + _baseStorageShared.Destroy(); + + base.Dispose(); + } + + public override Result Read(long offset, Span destination) + { + if (destination.Length == 0) + return Result.Success; + + // Reads cannot contain any partial blocks. + if (!Alignment.IsAlignedPow2(offset, (uint)BlockSize)) + return ResultFs.InvalidArgument.Log(); + + if (!Alignment.IsAlignedPow2(destination.Length, (uint)BlockSize)) + return ResultFs.InvalidArgument.Log(); + + Result rc = _baseStorage.Read(offset, destination); + if (rc.IsFailure()) return rc.Miss(); + + using var changePriority = new ScopedThreadPriorityChanger(1, ScopedThreadPriorityChanger.Mode.Relative); + + Array16 counter = _iv; + Utility.AddCounter(counter.Items, (ulong)offset / (uint)BlockSize); + + int decSize = Aes.DecryptCtr128(destination, destination, _key, counter); + if (decSize != destination.Length) + return ResultFs.UnexpectedInAesCtrStorageA.Log(); + + return Result.Success; + } + + public override Result Write(long offset, ReadOnlySpan source) + { + if (source.Length == 0) + return Result.Success; + + // We can only write full, aligned blocks. + if (!Alignment.IsAlignedPow2(offset, (uint)BlockSize)) + return ResultFs.InvalidArgument.Log(); + + if (!Alignment.IsAlignedPow2(source.Length, (uint)BlockSize)) + return ResultFs.InvalidArgument.Log(); + + // Get a pooled buffer. + // Note: The original code will const_cast the input buffer and encrypt the data in-place if the provided + // buffer is from the pooled buffer heap. This seems very error-prone since the data in buffers you pass + // as const might unexpectedly be modified. We make IsDeviceAddress() always return false + // so this won't happen, but the code that does the encryption in-place will be left in as a reference. + using var pooledBuffer = new PooledBuffer(); + bool useWorkBuffer = PooledBufferGlobalMethods.IsDeviceAddress(source); + if (useWorkBuffer) + pooledBuffer.Allocate(source.Length, BlockSize); + + // Setup the counter. + var counter = new Array16(); + Utility.AddCounter(counter.Items, (ulong)offset / (uint)BlockSize); + + // Loop until all data is written. + int remaining = source.Length; + int currentOffset = 0; + + while (remaining > 0) + { + // Determine data we're writing and where. + int writeSize = useWorkBuffer ? Math.Max(pooledBuffer.GetSize(), remaining) : remaining; + Span writeBuffer = useWorkBuffer + ? pooledBuffer.GetBuffer().Slice(0, writeSize) + : MemoryMarshal.CreateSpan(ref MemoryMarshal.GetReference(source), source.Length).Slice(0, writeSize); + + // Encrypt the data, with temporarily increased priority. + using (new ScopedThreadPriorityChanger(1, ScopedThreadPriorityChanger.Mode.Relative)) + { + int encSize = Aes.EncryptCtr128(source.Slice(currentOffset, writeSize), writeBuffer, _key, _iv); + if (encSize != writeSize) + return ResultFs.UnexpectedInAesCtrStorageA.Log(); + } + + // Write the encrypted data. + Result rc = _baseStorage.Write(offset + currentOffset, writeBuffer); + if (rc.IsFailure()) return rc.Miss(); + + // Advance. + currentOffset += writeSize; + remaining -= writeSize; + if (remaining > 0) + { + Utility.AddCounter(counter.Items, (uint)writeSize / (uint)BlockSize); + } + } + + return Result.Success; + } + + public override Result Flush() + { + return _baseStorage.Flush(); + } + + public override Result SetSize(long size) + { + return ResultFs.UnsupportedSetSizeForAesCtrStorage.Log(); + } + + public override Result GetSize(out long size) + { + return _baseStorage.GetSize(out size); + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + if (operationId != OperationId.InvalidateCache) + { + if (size == 0) + { + if (operationId == OperationId.QueryRange) + { + if (outBuffer.Length != Unsafe.SizeOf()) + return ResultFs.InvalidSize.Log(); + + Unsafe.As(ref MemoryMarshal.GetReference(outBuffer)).Clear(); + } + + return Result.Success; + } + + if (!Alignment.IsAlignedPow2(offset, (uint)BlockSize)) + return ResultFs.InvalidArgument.Log(); + + if (!Alignment.IsAlignedPow2(size, (uint)BlockSize)) + return ResultFs.InvalidArgument.Log(); + } + + switch (operationId) + { + case OperationId.QueryRange: + { + if (outBuffer.Length != Unsafe.SizeOf()) + return ResultFs.InvalidSize.Log(); + + ref QueryRangeInfo outInfo = + ref Unsafe.As(ref MemoryMarshal.GetReference(outBuffer)); + + // Get the QueryRangeInfo of the underlying base storage. + Result rc = _baseStorage.OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + Unsafe.SkipInit(out QueryRangeInfo info); + info.Clear(); + info.AesCtrKeyType = (int)QueryRangeInfo.AesCtrKeyTypeFlag.InternalKeyForSoftwareAes; + + outInfo.Merge(in info); + + break; + } + default: + { + Result rc = _baseStorage.OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + break; + } + } + + return Result.Success; + } +} \ No newline at end of file diff --git a/src/LibHac/FsSystem/PooledBuffer.cs b/src/LibHac/FsSystem/PooledBuffer.cs index 1c0437f1..154db9c9 100644 --- a/src/LibHac/FsSystem/PooledBuffer.cs +++ b/src/LibHac/FsSystem/PooledBuffer.cs @@ -8,6 +8,12 @@ namespace LibHac.FsSystem; public static class PooledBufferGlobalMethods { + // ReSharper disable once UnusedParameter.Global + public static bool IsPooledBuffer(ReadOnlySpan buffer) + { + return false; + } + public static long GetPooledBufferRetriedCount(this FileSystemServer fsSrv) { return fsSrv.Globals.PooledBuffer.CountRetried; @@ -42,6 +48,24 @@ public static void ClearPooledBufferPeak(this FileSystemServer fsSrv) g.CountReduceAllocation = 0; g.CountFailedIdealAllocationOnAsyncAccess = 0; } + + public static bool IsAdditionalDeviceAddress(ReadOnlySpan buffer) + { + return false; + } + + // ReSharper disable once UnusedParameter.Global + /// + /// Checks if the provided buffer is located at a "device address". + /// + /// The buffer to check. + /// if this is a device address; otherwise . + /// A device address is one that is either located in the pooled buffer heap + /// or in any of the registered additional device address ranges. + public static bool IsDeviceAddress(ReadOnlySpan buffer) + { + return IsPooledBuffer(buffer) || IsAdditionalDeviceAddress(buffer); + } } internal struct PooledBufferGlobals @@ -168,4 +192,4 @@ public void Dispose() { Deallocate(); } -} +} \ No newline at end of file From 00819731ae5e67232380ca1bdf611b3f14499967 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Fri, 4 Mar 2022 21:01:35 -0700 Subject: [PATCH 06/24] Add AesCtrCounterExtendedStorage --- .../FsSystem/AesCtrCounterExtendedStorage.cs | 476 ++++++++++++++++++ .../LibHac.Tests/FsSystem/TypeLayoutTests.cs | 12 + 2 files changed, 488 insertions(+) create mode 100644 src/LibHac/FsSystem/AesCtrCounterExtendedStorage.cs diff --git a/src/LibHac/FsSystem/AesCtrCounterExtendedStorage.cs b/src/LibHac/FsSystem/AesCtrCounterExtendedStorage.cs new file mode 100644 index 00000000..578ddd35 --- /dev/null +++ b/src/LibHac/FsSystem/AesCtrCounterExtendedStorage.cs @@ -0,0 +1,476 @@ +using System; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using LibHac.Common; +using LibHac.Common.FixedArrays; +using LibHac.Crypto; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.Util; + +namespace LibHac.FsSystem; + +/// +/// Reads from an encrypted with AES-CTR-128 using a table of counters. +/// +/// The base data used for this storage comes with a table of ranges and counter values that are used +/// to decrypt each range. This encryption scheme is used for encrypting content updates so that no counter values +/// are ever reused. +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public class AesCtrCounterExtendedStorage : IStorage +{ + public delegate Result DecryptFunction(Span destination, int index, ReadOnlySpan encryptedKey, + ReadOnlySpan iv, ReadOnlySpan source); + + public interface IDecryptor : IDisposable + { + Result Decrypt(Span destination, ReadOnlySpan encryptedKey, ReadOnlySpan iv); + bool HasExternalDecryptionKey(); + } + + public struct Entry + { + public Array8 Offset; + public int Reserved; + public int Generation; + + public void SetOffset(long value) + { + BinaryPrimitives.WriteInt64LittleEndian(Offset.Items, value); + } + + public readonly long GetOffset() + { + return BinaryPrimitives.ReadInt64LittleEndian(Offset.ItemsRo); + } + } + + public static readonly int BlockSize = Aes.BlockSize; + public static readonly int KeySize = Aes.KeySize128; + public static readonly int IvSize = Aes.BlockSize; + public static readonly int NodeSize = 1024 * 16; + + private BucketTree _table; + private ValueSubStorage _dataStorage; + private Array16 _key; + private uint _secureValue; + private long _counterOffset; + private UniqueRef _decryptor; + + public static long QueryHeaderStorageSize() + { + return BucketTree.QueryHeaderStorageSize(); + } + + public static long QueryNodeStorageSize(int entryCount) + { + return BucketTree.QueryNodeStorageSize(NodeSize, Unsafe.SizeOf(), entryCount); + } + + public static long QueryEntryStorageSize(int entryCount) + { + return BucketTree.QueryEntryStorageSize(NodeSize, Unsafe.SizeOf(), entryCount); + } + + public static Result CreateExternalDecryptor(ref UniqueRef outDecryptor, + DecryptFunction decryptFunction, int keyIndex) + { + using var decryptor = new UniqueRef(new ExternalDecryptor(decryptFunction, keyIndex)); + + if (!decryptor.HasValue) + return ResultFs.AllocationMemoryFailedInAesCtrCounterExtendedStorageA.Log(); + + outDecryptor.Set(ref decryptor.Ref()); + return Result.Success; + } + + public static Result CreateSoftwareDecryptor(ref UniqueRef outDecryptor) + { + using var decryptor = new UniqueRef(new SoftwareDecryptor()); + + if (!decryptor.HasValue) + return ResultFs.AllocationMemoryFailedInAesCtrCounterExtendedStorageA.Log(); + + outDecryptor.Set(ref decryptor.Ref()); + return Result.Success; + } + + public AesCtrCounterExtendedStorage() + { + _table = new BucketTree(); + } + + public override void Dispose() + { + FinalizeObject(); + + _decryptor.Destroy(); + _dataStorage.Dispose(); + _table.Dispose(); + + base.Dispose(); + } + + public bool IsInitialized() + { + return _table.IsInitialized(); + } + + // ReSharper disable once UnusedMember.Local + private Result Initialize(MemoryResource allocator, ReadOnlySpan key, uint secureValue, + in ValueSubStorage dataStorage, in ValueSubStorage tableStorage) + { + Unsafe.SkipInit(out BucketTree.Header header); + + Result rc = tableStorage.Read(0, SpanHelpers.AsByteSpan(ref header)); + if (rc.IsFailure()) return rc.Miss(); + + rc = header.Verify(); + if (rc.IsFailure()) return rc.Miss(); + + long nodeStorageSize = QueryNodeStorageSize(header.EntryCount); + long entryStorageSize = QueryEntryStorageSize(header.EntryCount); + long nodeStorageOffset = QueryHeaderStorageSize(); + long entryStorageOffset = nodeStorageOffset + nodeStorageSize; + + using var decryptor = new UniqueRef(); + rc = CreateSoftwareDecryptor(ref decryptor.Ref()); + if (rc.IsFailure()) return rc.Miss(); + + rc = tableStorage.GetSize(out long storageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (nodeStorageOffset + nodeStorageSize + entryStorageSize > storageSize) + return ResultFs.InvalidAesCtrCounterExtendedMetaStorageSize.Log(); + + using var entryStorage = new ValueSubStorage(in tableStorage, entryStorageOffset, entryStorageSize); + using var nodeStorage = new ValueSubStorage(in tableStorage, nodeStorageOffset, nodeStorageSize); + + return Initialize(allocator, key, secureValue, counterOffset: 0, in dataStorage, in nodeStorage, + in entryStorage, header.EntryCount, ref decryptor.Ref()); + } + + public Result Initialize(MemoryResource allocator, ReadOnlySpan key, uint secureValue, long counterOffset, + in ValueSubStorage dataStorage, in ValueSubStorage nodeStorage, in ValueSubStorage entryStorage, int entryCount, + ref UniqueRef decryptor) + { + Assert.SdkRequiresEqual(key.Length, KeySize); + Assert.SdkRequiresGreaterEqual(counterOffset, 0); + Assert.SdkRequiresNotNull(in decryptor); + + Result rc = _table.Initialize(allocator, in nodeStorage, in entryStorage, NodeSize, Unsafe.SizeOf(), + entryCount); + if (rc.IsFailure()) return rc.Miss(); + + rc = dataStorage.GetSize(out long dataStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + rc = _table.GetOffsets(out BucketTree.Offsets offsets); + if (rc.IsFailure()) return rc.Miss(); + + if (offsets.EndOffset > dataStorageSize) + return ResultFs.InvalidAesCtrCounterExtendedDataStorageSize.Log(); + + _dataStorage.Set(in dataStorage); + key.CopyTo(_key.Items); + _secureValue = secureValue; + _counterOffset = counterOffset; + _decryptor.Set(ref decryptor); + + return Result.Success; + } + + public void FinalizeObject() + { + if (IsInitialized()) + { + _table.FinalizeObject(); + + using var emptyStorage = new ValueSubStorage(); + _dataStorage.Set(in emptyStorage); + } + } + + public override Result Read(long offset, Span destination) + { + Assert.SdkRequiresLessEqual(0, offset); + Assert.SdkRequires(IsInitialized()); + + if (destination.Length == 0) + return Result.Success; + + // Reads cannot contain any partial blocks. + if (!Alignment.IsAlignedPow2(offset, (uint)BlockSize)) + return ResultFs.InvalidOffset.Log(); + + if (!Alignment.IsAlignedPow2(destination.Length, (uint)BlockSize)) + return ResultFs.InvalidSize.Log(); + + // Ensure the the requested range is within the bounds of the table. + Result rc = _table.GetOffsets(out BucketTree.Offsets offsets); + if (rc.IsFailure()) return rc.Miss(); + + if (!offsets.IsInclude(offset, destination.Length)) + return ResultFs.OutOfRange.Log(); + + // Fill the destination buffer with the encrypted data. + rc = _dataStorage.Read(offset, destination); + if (rc.IsFailure()) return rc.Miss(); + + // Temporarily increase our thread priority. + using var changePriority = new ScopedThreadPriorityChanger(1, ScopedThreadPriorityChanger.Mode.Relative); + + // Find the entry in the table that contains our current offset. + using var visitor = new BucketTree.Visitor(); + rc = _table.Find(ref visitor.Ref, offset); + if (rc.IsFailure()) return rc.Miss(); + + // Verify that the entry's offset is aligned to an AES block and within the bounds of the table. + long entryOffset = visitor.Get().GetOffset(); + if (!Alignment.IsAlignedPow2(entryOffset, (uint)BlockSize) || entryOffset < 0 || + !offsets.IsInclude(entryOffset)) + { + return ResultFs.InvalidAesCtrCounterExtendedEntryOffset.Log(); + } + + Span currentData = destination; + long currentOffset = offset; + long endOffset = offset + destination.Length; + + while (currentOffset < endOffset) + { + // Get the current entry and validate its offset. + // No need to check its alignment since it was already checked elsewhere. + var entry = visitor.Get(); + + long entryStartOffset = entry.GetOffset(); + if (entryStartOffset > currentOffset) + return ResultFs.InvalidAesCtrCounterExtendedEntryOffset.Log(); + + // Get current entry's end offset. + long entryEndOffset; + if (visitor.CanMoveNext()) + { + // Advance to the next entry so we know where our current entry ends. + // The current entry's end offset is the next entry's start offset. + rc = visitor.MoveNext(); + if (rc.IsFailure()) return rc.Miss(); + + entryEndOffset = visitor.Get().GetOffset(); + if (!offsets.IsInclude(entryEndOffset)) + return ResultFs.InvalidAesCtrCounterExtendedEntryOffset.Log(); + } + else + { + // If this is the last entry its end offset is the table's end offset. + entryEndOffset = offsets.EndOffset; + } + + if (!Alignment.IsAlignedPow2((ulong)entryEndOffset, (uint)BlockSize) || currentOffset >= entryEndOffset) + return ResultFs.InvalidAesCtrCounterExtendedEntryOffset.Log(); + + // Get the part of the entry that contains the data we read. + long dataOffset = currentOffset - entryStartOffset; + long dataSize = entryEndOffset - currentOffset; + Assert.SdkLess(0, dataSize); + + long remainingSize = endOffset - currentOffset; + long readSize = Math.Min(remainingSize, dataSize); + Assert.SdkLessEqual(readSize, destination.Length); + + // Create the counter for the first data block we're decrypting. + long counterOffset = _counterOffset + entryStartOffset + dataOffset; + var upperIv = new NcaAesCtrUpperIv + { + Generation = (uint)entry.Generation, + SecureValue = _secureValue + }; + + Unsafe.SkipInit(out Array16 counter); + AesCtrStorage.MakeIv(counter.Items, upperIv.Value, counterOffset); + + // Decrypt the data from the current entry. + rc = _decryptor.Get.Decrypt(currentData.Slice(0, (int)dataSize), _key, counter); + if (rc.IsFailure()) return rc.Miss(); + + // Advance the current offsets. + currentData = currentData.Slice((int)dataSize); + currentOffset -= dataSize; + } + + return Result.Success; + } + + public override Result Write(long offset, ReadOnlySpan source) + { + return ResultFs.UnsupportedWriteForAesCtrCounterExtendedStorage.Log(); + } + + public override Result Flush() + { + return Result.Success; + } + + public override Result GetSize(out long size) + { + UnsafeHelpers.SkipParamInit(out size); + + Result rc = _table.GetOffsets(out BucketTree.Offsets offsets); + if (rc.IsFailure()) return rc.Miss(); + + size = offsets.EndOffset; + return Result.Success; + } + + public override Result SetSize(long size) + { + return ResultFs.UnsupportedSetSizeForAesCtrCounterExtendedStorage.Log(); + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + switch (operationId) + { + case OperationId.InvalidateCache: + { + Assert.SdkRequires(IsInitialized()); + + // Invalidate the table's cache. + Result rc = _table.InvalidateCache(); + if (rc.IsFailure()) return rc.Miss(); + + // Invalidate the data storage's cache. + rc = _dataStorage.OperateRange(OperationId.InvalidateCache, offset: 0, size: long.MaxValue); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + case OperationId.QueryRange: + { + Assert.SdkRequiresLessEqual(0, offset); + Assert.SdkRequires(IsInitialized()); + + if (outBuffer.Length != Unsafe.SizeOf()) + return ResultFs.InvalidSize.Log(); + + ref QueryRangeInfo outInfo = + ref Unsafe.As(ref MemoryMarshal.GetReference(outBuffer)); + + if (size == 0) + { + outInfo.Clear(); + return Result.Success; + } + + if (!Alignment.IsAlignedPow2(offset, (uint)BlockSize)) + return ResultFs.InvalidOffset.Log(); + + if (!Alignment.IsAlignedPow2(size, (uint)BlockSize)) + return ResultFs.InvalidSize.Log(); + + // Ensure the storage contains the provided offset and size. + Result rc = _table.GetOffsets(out BucketTree.Offsets offsets); + if (rc.IsFailure()) return rc.Miss(); + + if (!offsets.IsInclude(offset, size)) + return ResultFs.OutOfRange.Log(); + + // Get the QueryRangeInfo of the underlying data storage. + rc = _dataStorage.OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + // Set the key type in the info and merge it with the output info. + Unsafe.SkipInit(out QueryRangeInfo info); + info.Clear(); + info.AesCtrKeyType = (int)(_decryptor.Get.HasExternalDecryptionKey() + ? QueryRangeInfo.AesCtrKeyTypeFlag.ExternalKeyForHardwareAes + : QueryRangeInfo.AesCtrKeyTypeFlag.InternalKeyForHardwareAes); + + outInfo.Merge(in info); + + return Result.Success; + } + + default: + return ResultFs.UnsupportedOperateRangeForAesCtrCounterExtendedStorage.Log(); + } + } + + private class ExternalDecryptor : IDecryptor + { + private DecryptFunction _decryptFunction; + private int _keyIndex; + + public ExternalDecryptor(DecryptFunction decryptFunction, int keyIndex) + { + Assert.SdkRequiresNotNull(decryptFunction); + + _decryptFunction = decryptFunction; + _keyIndex = keyIndex; + } + + public void Dispose() { } + + public Result Decrypt(Span destination, ReadOnlySpan encryptedKey, ReadOnlySpan iv) + { + Assert.SdkRequiresEqual(encryptedKey.Length, KeySize); + Assert.SdkRequiresEqual(iv.Length, IvSize); + + Unsafe.SkipInit(out Array16 counter); + iv.CopyTo(counter.Items); + + int remainingSize = destination.Length; + int currentOffset = 0; + + // Todo: Align the buffer to the block size + using var pooledBuffer = new PooledBuffer(); + pooledBuffer.AllocateParticularlyLarge(destination.Length, BlockSize); + Assert.SdkAssert(pooledBuffer.GetSize() > 0); + + while (remainingSize > 0) + { + int currentSize = Math.Min(pooledBuffer.GetSize(), remainingSize); + Span dstBuffer = destination.Slice(currentOffset, currentSize); + Span workBuffer = pooledBuffer.GetBuffer().Slice(0, currentSize); + + Result rc = _decryptFunction(workBuffer, _keyIndex, encryptedKey, counter, dstBuffer); + if (rc.IsFailure()) return rc.Miss(); + + workBuffer.CopyTo(dstBuffer); + + currentOffset += currentSize; + remainingSize -= currentSize; + + if (remainingSize > 0) + { + Utility.AddCounter(counter.Items, (uint)currentSize / (uint)BlockSize); + } + } + + return Result.Success; + } + + public bool HasExternalDecryptionKey() + { + return _keyIndex < 0; + } + } + + private class SoftwareDecryptor : IDecryptor + { + public void Dispose() { } + + public Result Decrypt(Span destination, ReadOnlySpan key, ReadOnlySpan iv) + { + Aes.DecryptCtr128(destination, destination, key, iv); + return Result.Success; + } + + public bool HasExternalDecryptionKey() + { + return false; + } + } +} \ No newline at end of file diff --git a/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs b/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs index 5d0746ea..092c1986 100644 --- a/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs +++ b/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs @@ -247,4 +247,16 @@ public static void KeyType_Layout() Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 3, (int)KeyType.SaveDataSeedUniqueMac); Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 4, (int)KeyType.SaveDataTransferMac); } + + [Fact] + public static void AesCtrCounterExtendedStorage_Entry_Layout() + { + AesCtrCounterExtendedStorage.Entry s = default; + + Assert.Equal(0x10, Unsafe.SizeOf()); + + Assert.Equal(0x0, GetOffset(in s, in s.Offset)); + Assert.Equal(0x8, GetOffset(in s, in s.Reserved)); + Assert.Equal(0xC, GetOffset(in s, in s.Generation)); + } } \ No newline at end of file From 3e64f9ef0322a02852640db82cbb5e0871d89209 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 7 Mar 2022 14:08:05 -0700 Subject: [PATCH 07/24] Skeleton NcaFileSystemDriver --- src/LibHac/FsSystem/CompressedStorage.cs | 6 +- src/LibHac/FsSystem/NcaFileSystemDriver.cs | 194 ++++++++++++++++++++- 2 files changed, 195 insertions(+), 5 deletions(-) diff --git a/src/LibHac/FsSystem/CompressedStorage.cs b/src/LibHac/FsSystem/CompressedStorage.cs index ec0b819b..e8e16bd8 100644 --- a/src/LibHac/FsSystem/CompressedStorage.cs +++ b/src/LibHac/FsSystem/CompressedStorage.cs @@ -1,4 +1,5 @@ -using System; +// ReSharper disable UnusedMember.Local NotAccessedField.Local +using System; using System.Runtime.CompilerServices; using LibHac.Diag; using LibHac.Fs; @@ -12,9 +13,6 @@ namespace LibHac.FsSystem; public class CompressedStorage : IStorage, IAsynchronousAccessSplitter { - public delegate Result DecompressorFunction(Span destination, ReadOnlySpan source); - public delegate DecompressorFunction GetDecompressorFunction(CompressionType type); - public class CompressedStorageCore : IDisposable { private long _blockSizeMax; diff --git a/src/LibHac/FsSystem/NcaFileSystemDriver.cs b/src/LibHac/FsSystem/NcaFileSystemDriver.cs index adda8dae..0f651f37 100644 --- a/src/LibHac/FsSystem/NcaFileSystemDriver.cs +++ b/src/LibHac/FsSystem/NcaFileSystemDriver.cs @@ -1,5 +1,10 @@ -using LibHac.Common.FixedArrays; +// ReSharper disable UnusedMember.Local +using System; +using LibHac.Common; +using LibHac.Common.FixedArrays; using LibHac.Crypto; +using LibHac.Fs; +using LibHac.FsSrv; namespace LibHac.FsSystem; @@ -60,4 +65,191 @@ public enum KeyType SaveDataDeviceUniqueMac = 0x62, SaveDataSeedUniqueMac = 0x63, SaveDataTransferMac = 0x64 +} + +public class NcaFileSystemDriver : IDisposable +{ + public struct StorageContext : IDisposable + { + public bool OpenRawStorage; + public SharedRef BodySubStorage; + public SharedRef CurrentSparseStorage; + public SharedRef SparseStorageMetaStorage; + public SharedRef OriginalSparseStorage; + // Todo: externalCurrentSparseStorage, externalOriginalSparseStorage + public SharedRef AesCtrExStorageMetaStorage; + public SharedRef AesCtrExStorageDataStorage; + public SharedRef AesCtrExStorage; + public SharedRef IndirectStorageMetaStorage; + public SharedRef IndirectStorage; + public SharedRef FsDataStorage; + public SharedRef CompressedStorageMetaStorage; + public SharedRef CompressedStorage; + + public void Dispose() + { + BodySubStorage.Destroy(); + CurrentSparseStorage.Destroy(); + SparseStorageMetaStorage.Destroy(); + OriginalSparseStorage.Destroy(); + AesCtrExStorageMetaStorage.Destroy(); + AesCtrExStorageDataStorage.Destroy(); + AesCtrExStorage.Destroy(); + IndirectStorageMetaStorage.Destroy(); + IndirectStorage.Destroy(); + FsDataStorage.Destroy(); + CompressedStorageMetaStorage.Destroy(); + CompressedStorage.Destroy(); + } + } + + private enum AlignmentStorageRequirement + { + CacheBlockSize = 0, + None = 1 + } + + public NcaFileSystemDriver(ref SharedRef ncaReader, MemoryResource allocator, + IBufferManager bufferManager, IHash256GeneratorFactorySelector hashGeneratorFactorySelector) + { + throw new NotImplementedException(); + } + + public NcaFileSystemDriver(ref SharedRef originalNcaReader, ref SharedRef currentNcaReader, + MemoryResource allocator, IBufferManager bufferManager, + IHash256GeneratorFactorySelector hashGeneratorFactorySelector) + { + throw new NotImplementedException(); + } + + public void Dispose() + { + throw new NotImplementedException(); + } + + private bool IsUsingHwAesCtrForSpeedEmulation(FileSystemServer fs) + { + throw new NotImplementedException(); + } + + private long GetFsOffset(NcaReader reader, int index) + { + return (long)reader.GetFsOffset(index); + } + + private long GetFsEndOffset(NcaReader reader, int index) + { + return (long)reader.GetFsEndOffset(index); + } + + public Result OpenStorage(ref SharedRef outStorage, + ref SharedRef outStorageAccessSplitter, out NcaFsHeaderReader outHeaderReader, + int fsIndex) + { + throw new NotImplementedException(); + } + + private Result OpenStorageImpl(ref SharedRef outStorage, out NcaFsHeaderReader outHeaderReader, + int fsIndex, ref StorageContext storageContext) + { + throw new NotImplementedException(); + } + + private Result OpenIndirectableStorageAsOriginal(ref SharedRef outStorage, + in NcaFsHeaderReader headerReader, ref StorageContext storageContext) + { + throw new NotImplementedException(); + } + + private Result CreateBodySubStorage(ref SharedRef outStorage, long offset, long size) + { + throw new NotImplementedException(); + } + + private Result CreateAesCtrStorage(ref SharedRef outStorage, ref SharedRef baseStorage, + long offset, in NcaAesCtrUpperIv upperIv, AlignmentStorageRequirement alignmentRequirement) + { + throw new NotImplementedException(); + } + + private Result CreateAesXtsStorage(ref SharedRef outStorage, ref SharedRef baseStorage, + long offset) + { + throw new NotImplementedException(); + } + + private Result CreateSparseStorageMetaStorage(ref SharedRef outStorage, + ref SharedRef baseStorage, long offset, in NcaAesCtrUpperIv upperIv, in NcaSparseInfo sparseInfo) + { + throw new NotImplementedException(); + } + + private Result CreateSparseStorageCore(ref SharedRef outStorage, ref SharedRef baseStorage, + long baseStorageSize, ref SharedRef sparseStorageMetaStorage, in NcaSparseInfo sparseInfo, + bool hasExternalInfo) + { + throw new NotImplementedException(); + } + + private Result CreateSparseStorage(ref SharedRef outStorage, out long outFsDataOffset, + ref SharedRef outSparseStorage, ref SharedRef outSparseStorageMetaStorage, int index, + in NcaAesCtrUpperIv upperIv, in NcaSparseInfo sparseInfo) + { + throw new NotImplementedException(); + } + + private Result CreateAesCtrExStorageMetaStorage(ref SharedRef outStorage, + ref SharedRef baseStorage, long offset, in NcaAesCtrUpperIv upperIv, in NcaPatchInfo patchInfo) + { + throw new NotImplementedException(); + } + + private Result CreateAesCtrExStorage(ref SharedRef outStorage, + ref SharedRef outAesCtrExStorage, ref SharedRef baseStorage, + ref SharedRef aesCtrExMetaStorage, long counterOffset, in NcaAesCtrUpperIv upperIv, + in NcaPatchInfo patchInfo) + { + throw new NotImplementedException(); + } + + private Result CreateIndirectStorageMetaStorage(ref SharedRef outStorage, + ref SharedRef baseStorage, in NcaPatchInfo patchInfo) + { + throw new NotImplementedException(); + } + + private Result CreateIndirectStorage(ref SharedRef outStorage, + ref SharedRef outIndirectStorage, ref SharedRef baseStorage, + ref SharedRef originalDataStorage, ref SharedRef indirectStorageMetaStorage, + in NcaPatchInfo patchInfo) + { + throw new NotImplementedException(); + } + + private Result CreateSha256Storage(ref SharedRef outStorage, ref SharedRef baseStorage, + in NcaFsHeader.HashData.HierarchicalSha256Data sha256Data) + { + throw new NotImplementedException(); + } + + private Result HierarchicalSha256Data(ref SharedRef outStorage, ref SharedRef baseStorage, + in NcaFsHeader.HashData.IntegrityMetaInfo metaInfo) + { + throw new NotImplementedException(); + } + + public static Result CreateCompressedStorage(ref SharedRef outStorage, + ref SharedRef outCompressedStorage, ref SharedRef outMetaStorage, + ref SharedRef baseStorage, in NcaCompressionInfo compressionInfo, + GetDecompressorFunction getDecompressor, MemoryResource allocator, IBufferManager bufferManager) + { + throw new NotImplementedException(); + } + + public Result CreateCompressedStorage(ref SharedRef outStorage, + ref SharedRef outCompressedStorage, ref SharedRef outMetaStorage, + ref SharedRef baseStorage, in NcaCompressionInfo compressionInfo) + { + throw new NotImplementedException(); + } } \ No newline at end of file From d28c38a9d0b682a065e64d184604076fdda7d802 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Tue, 8 Mar 2022 17:20:41 -0700 Subject: [PATCH 08/24] Add MemoryResourceBufferHoldStorage --- .../MemoryResourceBufferHoldStorage.cs | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/LibHac/FsSystem/MemoryResourceBufferHoldStorage.cs diff --git a/src/LibHac/FsSystem/MemoryResourceBufferHoldStorage.cs b/src/LibHac/FsSystem/MemoryResourceBufferHoldStorage.cs new file mode 100644 index 00000000..00f77c1c --- /dev/null +++ b/src/LibHac/FsSystem/MemoryResourceBufferHoldStorage.cs @@ -0,0 +1,91 @@ +using System; +using LibHac.Common; +using LibHac.Diag; +using LibHac.Fs; + +namespace LibHac.FsSystem; + +/// +/// Allocates a buffer from the provided that is deallocated +/// when the is disposed. +/// +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public class MemoryResourceBufferHoldStorage : IStorage +{ + private SharedRef _storage; + private MemoryResource _memoryResource; + private Mem.Buffer _buffer; + + public MemoryResourceBufferHoldStorage(ref SharedRef baseStorage, MemoryResource memoryResource, + int bufferSize) + { + _storage = SharedRef.CreateMove(ref baseStorage); + _memoryResource = memoryResource; + _buffer = memoryResource.Allocate(bufferSize); + } + + public override void Dispose() + { + if (!_buffer.IsNull) + { + _memoryResource.Deallocate(ref _buffer); + } + + _storage.Destroy(); + + base.Dispose(); + } + + public bool IsValid() + { + return !_buffer.IsNull; + } + + public Mem.Buffer GetBuffer() + { + return _buffer; + } + + public override Result Read(long offset, Span destination) + { + Assert.SdkRequiresNotNull(_storage); + + return _storage.Get.Read(offset, destination); + } + + public override Result Write(long offset, ReadOnlySpan source) + { + Assert.SdkRequiresNotNull(_storage); + + return _storage.Get.Write(offset, source); + } + + public override Result Flush() + { + Assert.SdkRequiresNotNull(_storage); + + return _storage.Get.Flush(); + } + + public override Result SetSize(long size) + { + Assert.SdkRequiresNotNull(_storage); + + return _storage.Get.SetSize(size); + } + + public override Result GetSize(out long size) + { + Assert.SdkRequiresNotNull(_storage); + + return _storage.Get.GetSize(out size); + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + Assert.SdkRequiresNotNull(_storage); + + return _storage.Get.OperateRange(outBuffer, operationId, offset, size, inBuffer); + } +} \ No newline at end of file From c765ab999e483c85f7964a94fe3a5e414577c901 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Thu, 10 Mar 2022 14:08:20 -0700 Subject: [PATCH 09/24] Rename StorageType to StorageLayoutType --- src/LibHac/FsSrv/BaseFileSystemService.cs | 4 +- src/LibHac/FsSrv/BaseStorageService.cs | 2 +- src/LibHac/FsSrv/FileSystemProxyImpl.cs | 4 +- src/LibHac/FsSrv/NcaFileSystemService.cs | 14 ++--- src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs | 6 +- src/LibHac/FsSrv/SaveDataFileSystemService.cs | 56 +++++++++---------- .../FsSystem/StorageLayoutTypeSetter.cs | 34 +++++------ 7 files changed, 60 insertions(+), 60 deletions(-) diff --git a/src/LibHac/FsSrv/BaseFileSystemService.cs b/src/LibHac/FsSrv/BaseFileSystemService.cs index 4c18be83..7d322c23 100644 --- a/src/LibHac/FsSrv/BaseFileSystemService.cs +++ b/src/LibHac/FsSrv/BaseFileSystemService.cs @@ -112,7 +112,7 @@ public Result OpenBisFileSystem(ref SharedRef outFileSystem, in F if (!accessibility.CanRead || !accessibility.CanWrite) return ResultFs.PermissionDenied.Log(); - const StorageType storageFlag = StorageType.Bis; + const StorageLayoutType storageFlag = StorageLayoutType.Bis; using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); // Normalize the path @@ -218,7 +218,7 @@ public Result OpenSdCardFileSystem(ref SharedRef outFileSystem) if (!accessibility.CanRead || !accessibility.CanWrite) return ResultFs.PermissionDenied.Log(); - const StorageType storageFlag = StorageType.Bis; + const StorageLayoutType storageFlag = StorageLayoutType.Bis; using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); using var fileSystem = new SharedRef(); diff --git a/src/LibHac/FsSrv/BaseStorageService.cs b/src/LibHac/FsSrv/BaseStorageService.cs index 6300c9e2..8f34721b 100644 --- a/src/LibHac/FsSrv/BaseStorageService.cs +++ b/src/LibHac/FsSrv/BaseStorageService.cs @@ -61,7 +61,7 @@ private static Result GetAccessibilityForOpenBisPartition(out Accessibility acce public Result OpenBisStorage(ref SharedRef outStorage, BisPartitionId id) { - var storageFlag = StorageType.Bis; + var storageFlag = StorageLayoutType.Bis; using var scopedLayoutType = new ScopedStorageLayoutTypeSetter(storageFlag); Result rc = GetProgramInfo(out ProgramInfo programInfo); diff --git a/src/LibHac/FsSrv/FileSystemProxyImpl.cs b/src/LibHac/FsSrv/FileSystemProxyImpl.cs index 72d5f91c..60e783a4 100644 --- a/src/LibHac/FsSrv/FileSystemProxyImpl.cs +++ b/src/LibHac/FsSrv/FileSystemProxyImpl.cs @@ -803,7 +803,7 @@ public Result OpenContentStorageFileSystem(ref SharedRef outFileS public Result OpenCloudBackupWorkStorageFileSystem(ref SharedRef outFileSystem, CloudBackupWorkStorageId storageId) { - var storageFlag = StorageType.NonGameCard; + var storageFlag = StorageLayoutType.NonGameCard; using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); Result rc = GetProgramInfo(out ProgramInfo programInfo); @@ -836,7 +836,7 @@ public Result OpenCloudBackupWorkStorageFileSystem(ref SharedRef public Result OpenCustomStorageFileSystem(ref SharedRef outFileSystem, CustomStorageId storageId) { - var storageFlag = StorageType.NonGameCard; + var storageFlag = StorageLayoutType.NonGameCard; using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); Result rc = GetProgramInfo(out ProgramInfo programInfo); diff --git a/src/LibHac/FsSrv/NcaFileSystemService.cs b/src/LibHac/FsSrv/NcaFileSystemService.cs index c0f025db..152cba2f 100644 --- a/src/LibHac/FsSrv/NcaFileSystemService.cs +++ b/src/LibHac/FsSrv/NcaFileSystemService.cs @@ -73,7 +73,7 @@ private Result GetProgramInfoByProgramId(out ProgramInfo programInfo, ulong prog public Result OpenFileSystemWithPatch(ref SharedRef outFileSystem, ProgramId programId, FileSystemProxyType fsType) { - const StorageType storageFlag = StorageType.All; + const StorageLayoutType storageFlag = StorageLayoutType.All; using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); // Get the program info for the caller and verify permissions @@ -228,7 +228,7 @@ public Result OpenDataStorageByProgramId(ref SharedRef outStorage, P public Result OpenFileSystemWithId(ref SharedRef outFileSystem, in FspPath path, ulong id, FileSystemProxyType fsType) { - const StorageType storageFlag = StorageType.All; + const StorageLayoutType storageFlag = StorageLayoutType.All; using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); Result rc = GetProgramInfo(out ProgramInfo programInfo); @@ -361,7 +361,7 @@ public Result GetRightsId(out RightsId outRightsId, ProgramId programId, Storage { UnsafeHelpers.SkipParamInit(out outRightsId); - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.All); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.All); Result rc = GetProgramInfo(out ProgramInfo programInfo); if (rc.IsFailure()) return rc; @@ -389,7 +389,7 @@ public Result GetRightsIdAndKeyGenerationByPath(out RightsId outRightsId, out by const ulong checkThroughProgramId = ulong.MaxValue; UnsafeHelpers.SkipParamInit(out outRightsId, out outKeyGeneration); - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.All); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.All); Result rc = GetProgramInfo(out ProgramInfo programInfo); if (rc.IsFailure()) return rc; @@ -426,7 +426,7 @@ private Result OpenDataFileSystemCore(ref SharedRef outFileSystem, { UnsafeHelpers.SkipParamInit(out isHostFs); - StorageType storageFlag = _serviceImpl.GetStorageFlag(programId); + StorageLayoutType storageFlag = _serviceImpl.GetStorageFlag(programId); using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); using var programPath = new Path(); @@ -452,7 +452,7 @@ private Result OpenDataFileSystemCore(ref SharedRef outFileSystem, public Result OpenContentStorageFileSystem(ref SharedRef outFileSystem, ContentStorageId contentStorageId) { - StorageType storageFlag = contentStorageId == ContentStorageId.System ? StorageType.Bis : StorageType.All; + StorageLayoutType storageFlag = contentStorageId == ContentStorageId.System ? StorageLayoutType.Bis : StorageLayoutType.All; using var scopedLayoutType = new ScopedStorageLayoutTypeSetter(storageFlag); Result rc = GetProgramInfo(out ProgramInfo programInfo); @@ -535,7 +535,7 @@ public Result RegisterUpdatePartition() public Result OpenRegisteredUpdatePartition(ref SharedRef outFileSystem) { - var storageFlag = StorageType.All; + var storageFlag = StorageLayoutType.All; using var scopedLayoutType = new ScopedStorageLayoutTypeSetter(storageFlag); Result rc = GetProgramInfo(out ProgramInfo programInfo); diff --git a/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs b/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs index 0e9a4362..ba2131e3 100644 --- a/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs +++ b/src/LibHac/FsSrv/NcaFileSystemServiceImpl.cs @@ -815,13 +815,13 @@ public Result ResolveRegisteredHtmlDocumentPath(ref Path path, ulong id) return _locationResolverSet.ResolveRegisteredHtmlDocumentPath(ref path, id); } - internal StorageType GetStorageFlag(ulong programId) + internal StorageLayoutType GetStorageFlag(ulong programId) { if (programId >= _config.SpeedEmulationRange.ProgramIdMin && programId <= _config.SpeedEmulationRange.ProgramIdMax) - return StorageType.Bis; + return StorageLayoutType.Bis; else - return StorageType.All; + return StorageLayoutType.All; } public Result HandleResolubleAccessFailure(out bool wasDeferred, Result resultForNoFailureDetected, diff --git a/src/LibHac/FsSrv/SaveDataFileSystemService.cs b/src/LibHac/FsSrv/SaveDataFileSystemService.cs index 2038fea9..bfa316c2 100644 --- a/src/LibHac/FsSrv/SaveDataFileSystemService.cs +++ b/src/LibHac/FsSrv/SaveDataFileSystemService.cs @@ -450,7 +450,7 @@ public Result RegisterSaveDataFileSystemAtomicDeletion(InBuffer saveDataIds) public Result DeleteSaveDataFileSystem(ulong saveDataId) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.Bis); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.Bis); return DeleteSaveDataFileSystemCommon(SaveDataSpaceId.System, saveDataId); } @@ -473,7 +473,7 @@ private Result DeleteSaveDataFileSystemCore(SaveDataSpaceId spaceId, ulong saveD public Result DeleteSaveDataFileSystemBySaveDataSpaceId(SaveDataSpaceId spaceId, ulong saveDataId) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); return DeleteSaveDataFileSystemBySaveDataSpaceIdCore(spaceId, saveDataId); } @@ -499,7 +499,7 @@ private Result DeleteSaveDataFileSystemBySaveDataSpaceIdCore(SaveDataSpaceId spa public Result DeleteSaveDataFileSystemBySaveDataAttribute(SaveDataSpaceId spaceId, in SaveDataAttribute attribute) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); Result rs = GetSaveDataInfo(out SaveDataInfo info, spaceId, in attribute); if (rs.IsFailure()) return rs; @@ -691,7 +691,7 @@ private Result CreateSaveDataFileSystemCore(in SaveDataAttribute attribute, bool accessorInitialized = false; Result rc; - StorageType storageFlag = DecidePossibleStorageFlag(attribute.Type, creationInfo.SpaceId); + StorageLayoutType storageFlag = DecidePossibleStorageFlag(attribute.Type, creationInfo.SpaceId); using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); using var accessor = new UniqueRef(); @@ -863,7 +863,7 @@ public Result GetSaveDataInfo(out SaveDataInfo info, SaveDataSpaceId spaceId, in { UnsafeHelpers.SkipParamInit(out info); - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); using var accessor = new UniqueRef(); Result rc = OpenSaveDataIndexerAccessor(ref accessor.Ref(), spaceId); @@ -906,7 +906,7 @@ public Result CreateSaveDataFileSystemWithHashSalt(in SaveDataAttribute attribut private Result CreateSaveDataFileSystemWithHashSaltImpl(in SaveDataAttribute attribute, in SaveDataCreationInfo creationInfo, in SaveDataMetaInfo metaInfo, in Optional hashSalt) { - StorageType storageFlag = DecidePossibleStorageFlag(attribute.Type, creationInfo.SpaceId); + StorageLayoutType storageFlag = DecidePossibleStorageFlag(attribute.Type, creationInfo.SpaceId); using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); Result rc = GetProgramInfo(out ProgramInfo programInfo); @@ -943,7 +943,7 @@ private Result CreateSaveDataFileSystemWithHashSaltImpl(in SaveDataAttribute att public Result CreateSaveDataFileSystemBySystemSaveDataId(in SaveDataAttribute attribute, in SaveDataCreationInfo creationInfo) { - StorageType storageFlag = DecidePossibleStorageFlag(attribute.Type, creationInfo.SpaceId); + StorageLayoutType storageFlag = DecidePossibleStorageFlag(attribute.Type, creationInfo.SpaceId); using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); Result rc = GetProgramInfo(out ProgramInfo programInfo); @@ -1083,7 +1083,7 @@ Result RemoveSaveIndexerEntry() private Result OpenUserSaveDataFileSystemCore(ref SharedRef outFileSystem, SaveDataSpaceId spaceId, in SaveDataAttribute attribute, ProgramInfo programInfo, bool openReadOnly) { - StorageType storageFlag = DecidePossibleStorageFlag(attribute.Type, spaceId); + StorageLayoutType storageFlag = DecidePossibleStorageFlag(attribute.Type, spaceId); using var scopedContext = new ScopedStorageLayoutTypeSetter(storageFlag); // Try grabbing the mount count semaphore @@ -1199,7 +1199,7 @@ public Result OpenSaveDataFileSystemBySystemSaveDataId(ref SharedRef(); Result rc = OpenSaveDataIndexerAccessor(ref accessor.Ref(), spaceId); @@ -1291,7 +1291,7 @@ private Result ReadSaveDataFileSystemExtraDataCore(out SaveDataExtraData extraDa { UnsafeHelpers.SkipParamInit(out extraData); - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); Result rc = GetProgramInfo(out ProgramInfo programInfo); if (rc.IsFailure()) return rc; @@ -1445,7 +1445,7 @@ public Result ReadSaveDataFileSystemExtraDataWithMaskBySaveDataAttribute(OutBuff private Result WriteSaveDataFileSystemExtraDataCore(SaveDataSpaceId spaceId, ulong saveDataId, in SaveDataExtraData extraData, SaveDataType saveType, bool updateTimeStamp) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); using Path saveDataRootPath = _saveDataRootPath.DangerousGetPath(); return _serviceImpl.WriteSaveDataFileSystemExtraData(spaceId, saveDataId, in extraData, in saveDataRootPath, @@ -1455,7 +1455,7 @@ private Result WriteSaveDataFileSystemExtraDataCore(SaveDataSpaceId spaceId, ulo private Result WriteSaveDataFileSystemExtraDataWithMaskCore(ulong saveDataId, SaveDataSpaceId spaceId, in SaveDataExtraData extraData, in SaveDataExtraData extraDataMask) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); Result rc = GetProgramInfo(out ProgramInfo programInfo); if (rc.IsFailure()) return rc; @@ -1539,7 +1539,7 @@ public Result WriteSaveDataFileSystemExtraDataWithMask(ulong saveDataId, SaveDat public Result OpenSaveDataInfoReader(ref SharedRef outInfoReader) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.Bis); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.Bis); Result rc = GetProgramInfo(out ProgramInfo programInfo); if (rc.IsFailure()) return rc; @@ -1569,7 +1569,7 @@ public Result OpenSaveDataInfoReader(ref SharedRef outInfoR public Result OpenSaveDataInfoReaderBySaveDataSpaceId( ref SharedRef outInfoReader, SaveDataSpaceId spaceId) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); Result rc = GetProgramInfo(out ProgramInfo programInfo); if (rc.IsFailure()) return rc; @@ -1603,7 +1603,7 @@ public Result OpenSaveDataInfoReaderBySaveDataSpaceId( public Result OpenSaveDataInfoReaderWithFilter(ref SharedRef outInfoReader, SaveDataSpaceId spaceId, in SaveDataFilter filter) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); Result rc = GetProgramInfo(out ProgramInfo programInfo); if (rc.IsFailure()) return rc; @@ -1664,7 +1664,7 @@ public Result FindSaveDataWithFilter(out long count, OutBuffer saveDataInfoBuffe if (saveDataInfoBuffer.Size != Unsafe.SizeOf()) return ResultFs.InvalidArgument.Log(); - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); Result rc = GetProgramInfo(out ProgramInfo programInfo); if (rc.IsFailure()) return rc; @@ -1730,7 +1730,7 @@ public Result GetSaveDataCommitId(out long commitId, SaveDataSpaceId spaceId, ul public Result OpenSaveDataInfoReaderOnlyCacheStorage(ref SharedRef outInfoReader) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); // Find where the current program's cache storage is located Result rc = GetCacheStorageSpaceId(out SaveDataSpaceId spaceId); @@ -1749,7 +1749,7 @@ public Result OpenSaveDataInfoReaderOnlyCacheStorage(ref SharedRef outInfoReader, SaveDataSpaceId spaceId) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); Result rc = GetProgramInfo(out ProgramInfo programInfo); if (rc.IsFailure()) return rc; @@ -1876,7 +1876,7 @@ private Result FindCacheStorage(out SaveDataInfo saveInfo, out SaveDataSpaceId s public Result DeleteCacheStorage(ushort index) { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); Result rc = FindCacheStorage(out SaveDataInfo saveInfo, out SaveDataSpaceId spaceId, index); if (rc.IsFailure()) return rc; @@ -1891,7 +1891,7 @@ public Result GetCacheStorageSize(out long usableDataSize, out long journalSize, { UnsafeHelpers.SkipParamInit(out usableDataSize, out journalSize); - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.NonGameCard); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.NonGameCard); Result rc = FindCacheStorage(out SaveDataInfo saveInfo, out SaveDataSpaceId spaceId, index); if (rc.IsFailure()) return rc; @@ -1972,7 +1972,7 @@ public Result CorruptSaveDataFileSystemByOffset(SaveDataSpaceId spaceId, ulong s public Result CleanUpSaveData() { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.Bis); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.Bis); using var accessor = new UniqueRef(); Result rc = OpenSaveDataIndexerAccessor(ref accessor.Ref(), SaveDataSpaceId.System); @@ -1989,7 +1989,7 @@ private Result CleanUpSaveData(SaveDataIndexerAccessor accessor) public Result CompleteSaveDataExtension() { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.Bis); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.Bis); using var accessor = new UniqueRef(); Result rc = OpenSaveDataIndexerAccessor(ref accessor.Ref(), SaveDataSpaceId.System); @@ -2006,7 +2006,7 @@ private Result CompleteSaveDataExtension(SaveDataIndexerAccessor accessor) public Result CleanUpTemporaryStorage() { - using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageType.Bis); + using var scopedContext = new ScopedStorageLayoutTypeSetter(StorageLayoutType.Bis); using var fileSystem = new SharedRef(); Result rc = _serviceImpl.OpenSaveDataDirectoryFileSystem(ref fileSystem.Ref(), SaveDataSpaceId.Temporary); @@ -2197,16 +2197,16 @@ private void MaskExtraData(ref SaveDataExtraData extraData, in SaveDataExtraData } } - private StorageType DecidePossibleStorageFlag(SaveDataType type, SaveDataSpaceId spaceId) + private StorageLayoutType DecidePossibleStorageFlag(SaveDataType type, SaveDataSpaceId spaceId) { if (type == SaveDataType.Cache || type == SaveDataType.Bcat) - return StorageType.Bis | StorageType.SdCard | StorageType.Usb; + return StorageLayoutType.Bis | StorageLayoutType.SdCard | StorageLayoutType.Usb; if (type == SaveDataType.System || spaceId != SaveDataSpaceId.SdSystem && spaceId != SaveDataSpaceId.SdUser) - return StorageType.Bis; + return StorageLayoutType.Bis; - return StorageType.SdCard | StorageType.Usb; + return StorageLayoutType.SdCard | StorageLayoutType.Usb; } Result ISaveDataTransferCoreInterface.CreateSaveDataFileSystemCore(in SaveDataAttribute attribute, diff --git a/src/LibHac/FsSystem/StorageLayoutTypeSetter.cs b/src/LibHac/FsSystem/StorageLayoutTypeSetter.cs index 3450513e..59a7eb96 100644 --- a/src/LibHac/FsSystem/StorageLayoutTypeSetter.cs +++ b/src/LibHac/FsSystem/StorageLayoutTypeSetter.cs @@ -7,7 +7,7 @@ namespace LibHac.FsSystem; [Flags] -internal enum StorageType +internal enum StorageLayoutType { Bis = 1 << 0, SdCard = 1 << 1, @@ -22,9 +22,9 @@ internal enum StorageType /// Contains functions for validating the storage layout type flag. /// /// Based on FS 13.1.0 (nnSdk 13.4.0) -internal static class StorageLayoutType +internal static class StorageLayoutTypeFunctions { - public static bool IsStorageFlagValid(StorageType storageFlag) + public static bool IsStorageFlagValid(StorageLayoutType storageFlag) { return storageFlag != 0; } @@ -33,7 +33,7 @@ public static bool IsStorageFlagValid(StorageType storageFlag) internal struct ScopedStorageLayoutTypeSetter : IDisposable { // ReSharper disable once UnusedParameter.Local - public ScopedStorageLayoutTypeSetter(StorageType storageFlag) + public ScopedStorageLayoutTypeSetter(StorageLayoutType storageFlag) { // Todo: Implement } @@ -52,14 +52,14 @@ public void Dispose() internal class StorageLayoutTypeSetStorage : IStorage { private SharedRef _baseStorage; - private StorageType _storageFlag; + private StorageLayoutType _storageFlag; - public StorageLayoutTypeSetStorage(ref SharedRef baseStorage, StorageType storageFlag) + public StorageLayoutTypeSetStorage(ref SharedRef baseStorage, StorageLayoutType storageFlag) { _baseStorage = SharedRef.CreateMove(ref baseStorage); _storageFlag = storageFlag; - Assert.SdkAssert(StorageLayoutType.IsStorageFlagValid(storageFlag)); + Assert.SdkAssert(StorageLayoutTypeFunctions.IsStorageFlagValid(storageFlag)); } public override void Dispose() @@ -120,24 +120,24 @@ internal class StorageLayoutTypeSetFile : IFile private IFile _baseFile; private UniqueRef _baseFileUnique; private SharedRef _baseFileShared; - private StorageType _storageFlag; + private StorageLayoutType _storageFlag; - public StorageLayoutTypeSetFile(ref UniqueRef baseFile, StorageType storageFlag) + public StorageLayoutTypeSetFile(ref UniqueRef baseFile, StorageLayoutType storageFlag) { _baseFile = baseFile.Get; _baseFileUnique = new UniqueRef(ref baseFile); _storageFlag = storageFlag; - Assert.SdkAssert(StorageLayoutType.IsStorageFlagValid(storageFlag)); + Assert.SdkAssert(StorageLayoutTypeFunctions.IsStorageFlagValid(storageFlag)); } - public StorageLayoutTypeSetFile(ref SharedRef baseFile, StorageType storageFlag) + public StorageLayoutTypeSetFile(ref SharedRef baseFile, StorageLayoutType storageFlag) { _baseFile = baseFile.Get; _baseFileShared = SharedRef.CreateMove(ref baseFile); _storageFlag = storageFlag; - Assert.SdkAssert(StorageLayoutType.IsStorageFlagValid(storageFlag)); + Assert.SdkAssert(StorageLayoutTypeFunctions.IsStorageFlagValid(storageFlag)); } public override void Dispose() @@ -198,9 +198,9 @@ protected override Result DoOperateRange(Span outBuffer, OperationId opera internal class StorageLayoutTypeSetDirectory : IDirectory { private UniqueRef _baseDirectory; - private StorageType _storageFlag; + private StorageLayoutType _storageFlag; - public StorageLayoutTypeSetDirectory(ref UniqueRef baseDirectory, StorageType storageFlag) + public StorageLayoutTypeSetDirectory(ref UniqueRef baseDirectory, StorageLayoutType storageFlag) { _baseDirectory = new UniqueRef(ref baseDirectory); _storageFlag = storageFlag; @@ -236,14 +236,14 @@ protected override Result DoGetEntryCount(out long entryCount) internal class StorageLayoutTypeSetFileSystem : IFileSystem { private SharedRef _baseFileSystem; - private StorageType _storageFlag; + private StorageLayoutType _storageFlag; - public StorageLayoutTypeSetFileSystem(ref SharedRef baseFileSystem, StorageType storageFlag) + public StorageLayoutTypeSetFileSystem(ref SharedRef baseFileSystem, StorageLayoutType storageFlag) { _baseFileSystem = SharedRef.CreateMove(ref baseFileSystem); _storageFlag = storageFlag; - Assert.SdkAssert(StorageLayoutType.IsStorageFlagValid(storageFlag)); + Assert.SdkAssert(StorageLayoutTypeFunctions.IsStorageFlagValid(storageFlag)); } public override void Dispose() From eaff8059ba1badbb06ed0b7ab021d7fb85c243f7 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Thu, 10 Mar 2022 18:39:59 -0700 Subject: [PATCH 10/24] Skeleton BlockCacheBufferedStorage --- src/LibHac/Fs/FsEnums.cs | 7 + src/LibHac/FsSystem/BitmapUtils.cs | 14 + .../FsSystem/BlockCacheBufferedStorage.cs | 249 ++++++++++++++++++ src/LibHac/FsSystem/CompressedStorage.cs | 4 +- src/LibHac/FsSystem/Impl/BlockCacheManager.cs | 7 + src/LibHac/Util/BitUtil.cs | 8 +- 6 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 src/LibHac/FsSystem/BitmapUtils.cs create mode 100644 src/LibHac/FsSystem/BlockCacheBufferedStorage.cs diff --git a/src/LibHac/Fs/FsEnums.cs b/src/LibHac/Fs/FsEnums.cs index f5c3ceb4..dc10a21e 100644 --- a/src/LibHac/Fs/FsEnums.cs +++ b/src/LibHac/Fs/FsEnums.cs @@ -287,4 +287,11 @@ public enum SdmmcPort Mmc = 0, SdCard = 1, GcAsic = 2 +} + +public enum StorageType +{ + SaveData = 0, + RomFs = 1, + Authoring = 2 } \ No newline at end of file diff --git a/src/LibHac/FsSystem/BitmapUtils.cs b/src/LibHac/FsSystem/BitmapUtils.cs new file mode 100644 index 00000000..c425ec86 --- /dev/null +++ b/src/LibHac/FsSystem/BitmapUtils.cs @@ -0,0 +1,14 @@ +using LibHac.Diag; + +namespace LibHac.FsSystem; + +public static class BitmapUtils +{ + public static uint ILog2(uint value) + { + Assert.SdkRequiresGreater(value, 0u); + + const uint intBitCount = 32; + return intBitCount - 1 - (uint)Util.BitUtil.CountLeadingZeros(value); + } +} \ No newline at end of file diff --git a/src/LibHac/FsSystem/BlockCacheBufferedStorage.cs b/src/LibHac/FsSystem/BlockCacheBufferedStorage.cs new file mode 100644 index 00000000..1c339870 --- /dev/null +++ b/src/LibHac/FsSystem/BlockCacheBufferedStorage.cs @@ -0,0 +1,249 @@ +using System; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.FsSystem.Impl; +using LibHac.Os; +using Buffer = LibHac.Mem.Buffer; + +// Todo: Remove warning suppressions after implementing +// ReSharper disable All +#pragma warning disable CS0414 + +namespace LibHac.FsSystem; + +public class BlockCacheBufferedStorage : IStorage +{ + public struct CacheEntry : IBlockCacheManagerEntry + { + public AccessRange Range { get; } + public bool IsValid { get; set; } + public bool IsWriteBack { get; set; } + public bool IsCached { get; set; } + public bool IsFlushing { get; set; } + public short Age { get; set; } + public ulong Handle { get; set; } + public Buffer Buffer { get; set; } + + public void Invalidate() + { + throw new NotImplementedException(); + } + + public readonly bool IsAllocated() + { + throw new NotImplementedException(); + } + } + + public struct AccessRange : IBlockCacheManagerRange + { + public long Offset { get; set; } + public long Size { get; set; } + + public readonly long GetEndOffset() + { + throw new NotImplementedException(); + } + + public readonly long IsIncluded(long offset) + { + throw new NotImplementedException(); + } + } + + private SdkRecursiveMutex _mutex; + private IStorage _storageData; + private Result _lastResult; + private long _sizeBytesData; + private int _sizeBytesVerificationBlock; + private int _shiftBytesVerificationBlock; + private int _flags; + private int _bufferLevel; + private StorageType _storageType; + private BlockCacheManager _cacheManager; + + public BlockCacheBufferedStorage() + { + _bufferLevel = -1; + _cacheManager = new BlockCacheManager(); + } + + public override void Dispose() + { + FinalizeObject(); + _cacheManager.Dispose(); + + base.Dispose(); + } + + public bool IsEnabledKeepBurstMode() + { + throw new NotImplementedException(); + } + + public void SetKeepBurstMode(bool isEnabled) + { + throw new NotImplementedException(); + } + + public void SetRealDataCache(bool isEnabled) + { + throw new NotImplementedException(); + } + + public Result Initialize(IBufferManager bufferManager, SdkRecursiveMutex mutex, IStorage data, long dataSize, + int sizeBytesVerificationBlock, int maxCacheEntries, bool useRealDataCache, sbyte bufferLevel, + bool useKeepBurstMode, StorageType storageType) + { + Assert.SdkNotNull(data); + Assert.SdkNotNull(mutex); + Assert.SdkNull(_mutex); + Assert.SdkNull(_storageData); + Assert.SdkGreater(maxCacheEntries, 0); + + Result rc = _cacheManager.Initialize(bufferManager, maxCacheEntries); + if (rc.IsFailure()) return rc.Miss(); + + _mutex = mutex; + _storageData = data; + _sizeBytesData = dataSize; + _sizeBytesVerificationBlock = sizeBytesVerificationBlock; + _lastResult = Result.Success; + _flags = 0; + _bufferLevel = bufferLevel; + _storageType = storageType; + _shiftBytesVerificationBlock = (int)BitmapUtils.ILog2((uint)sizeBytesVerificationBlock); + + Assert.SdkEqual(1 << _shiftBytesVerificationBlock, _sizeBytesVerificationBlock); + + SetKeepBurstMode(useKeepBurstMode); + SetRealDataCache(useRealDataCache); + + return Result.Success; + } + + public void FinalizeObject() + { + throw new NotImplementedException(); + } + + public override Result GetSize(out long size) + { + throw new NotImplementedException(); + } + + public override Result SetSize(long size) + { + throw new NotImplementedException(); + } + + public override Result Read(long offset, Span destination) + { + throw new NotImplementedException(); + } + + public override Result Write(long offset, ReadOnlySpan source) + { + throw new NotImplementedException(); + } + + public override Result Flush() + { + throw new NotImplementedException(); + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + throw new NotImplementedException(); + } + + public Result Commit() + { + throw new NotImplementedException(); + } + + public Result OnRollback() + { + throw new NotImplementedException(); + } + + private Result FillZeroImpl(long offset, long size) + { + throw new NotImplementedException(); + } + + private Result DestroySignatureImpl(long offset, long size) + { + throw new NotImplementedException(); + } + + private Result InvalidateImpl() + { + throw new NotImplementedException(); + } + + private Result QueryRangeImpl(Span outBuffer, long offset, long size) + { + throw new NotImplementedException(); + } + + private Result GetAssociateBuffer(out Buffer outRange, out CacheEntry outEntry, long offset, int idealSize, + bool isAllocateForWrite) + { + throw new NotImplementedException(); + } + + private Result StoreOrDestroyBuffer(in Buffer memoryRange, ref CacheEntry entry) + { + throw new NotImplementedException(); + } + + private Result StoreOrDestroyBuffer(out int outCacheIndex, in Buffer memoryRange, ref CacheEntry entry) + { + throw new NotImplementedException(); + } + + private Result FlushCacheEntry(int index, bool isWithInvalidate) + { + throw new NotImplementedException(); + } + + private Result FlushRangeCacheEntries(long offset, long size, bool isWithInvalidate) + { + throw new NotImplementedException(); + } + + private Result FlushAllCacheEntries() + { + throw new NotImplementedException(); + } + + private Result ControlDirtiness() + { + throw new NotImplementedException(); + } + + private Result UpdateLastResult(Result result) + { + throw new NotImplementedException(); + } + + private Result ReadHeadCache(out Buffer outMemoryRange, out CacheEntry outEntry, out bool outIsCacheNeeded, + ref long offset, ref long offsetAligned, long offsetAlignedEnd, ref Span buffer) + { + throw new NotImplementedException(); + } + + private Result ReadTailCache(out Buffer outMemoryRange, out CacheEntry outEntry, out bool outIsCacheNeeded, + long offset, long offsetAligned, long offsetAlignedEnd, Span buffer, ref long size) + { + throw new NotImplementedException(); + } + + private Result BulkRead(long offset, Span buffer, ref Buffer memoryRangeHead, ref Buffer memoryRangeTail, + ref CacheEntry entryHead, ref CacheEntry entryTail, bool isHeadCacheNeeded, bool isTailCacheNeeded) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/src/LibHac/FsSystem/CompressedStorage.cs b/src/LibHac/FsSystem/CompressedStorage.cs index e8e16bd8..7ea54a39 100644 --- a/src/LibHac/FsSystem/CompressedStorage.cs +++ b/src/LibHac/FsSystem/CompressedStorage.cs @@ -162,8 +162,8 @@ public struct Range : IBlockCacheManagerRange public long Offset { get; set; } public uint Size { get; set; } - public long GetEndOffset() => Offset + Size; - public bool IsIncluded(long offset) => Offset <= offset && offset < GetEndOffset(); + public readonly long GetEndOffset() => Offset + Size; + public readonly bool IsIncluded(long offset) => Offset <= offset && offset < GetEndOffset(); } public struct AccessRange diff --git a/src/LibHac/FsSystem/Impl/BlockCacheManager.cs b/src/LibHac/FsSystem/Impl/BlockCacheManager.cs index 51a24273..be673cbc 100644 --- a/src/LibHac/FsSystem/Impl/BlockCacheManager.cs +++ b/src/LibHac/FsSystem/Impl/BlockCacheManager.cs @@ -28,6 +28,13 @@ public interface IBlockCacheManagerRange long GetEndOffset(); } +/// +/// Handles caching buffers with the given . +/// +/// The type of the entries to be stores in the cache manager. +/// The type that uses +/// to represent the range cached by an entry. +/// Based on FS 13.1.0 (nnSdk 13.4.0) public class BlockCacheManager : IDisposable where TEntry : struct, IBlockCacheManagerEntry where TRange : struct, IBlockCacheManagerRange diff --git a/src/LibHac/Util/BitUtil.cs b/src/LibHac/Util/BitUtil.cs index 002af32d..b883167b 100644 --- a/src/LibHac/Util/BitUtil.cs +++ b/src/LibHac/Util/BitUtil.cs @@ -1,4 +1,5 @@ -using System.Runtime.CompilerServices; +using System.Numerics; +using System.Runtime.CompilerServices; namespace LibHac.Util; @@ -36,6 +37,11 @@ private static ulong ResetLeastSignificantOneBit(ulong value) return value & (value - 1); } + public static int CountLeadingZeros(uint value) + { + return BitOperations.LeadingZeroCount(value); + } + // DivideUp comes from a C++ template that always casts to unsigned types public static uint DivideUp(uint value, uint divisor) => (value + divisor - 1) / divisor; public static ulong DivideUp(ulong value, ulong divisor) => (value + divisor - 1) / divisor; From 2c154ec3ba7deedb598219ee8b67a1fae4d1dac6 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sat, 12 Mar 2022 15:02:54 -0700 Subject: [PATCH 11/24] Implement AlignmentMatchingStorage classes --- .../FsSystem/AlignmentMatchingStorage.cs | 518 ++++++++++++++++++ .../FsSystem/AlignmentMatchingStorageImpl.cs | 191 +++++++ 2 files changed, 709 insertions(+) create mode 100644 src/LibHac/FsSystem/AlignmentMatchingStorage.cs create mode 100644 src/LibHac/FsSystem/AlignmentMatchingStorageImpl.cs diff --git a/src/LibHac/FsSystem/AlignmentMatchingStorage.cs b/src/LibHac/FsSystem/AlignmentMatchingStorage.cs new file mode 100644 index 00000000..a1aa15ff --- /dev/null +++ b/src/LibHac/FsSystem/AlignmentMatchingStorage.cs @@ -0,0 +1,518 @@ +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using LibHac.Common; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.Util; + +namespace LibHac.FsSystem; + +public interface IAlignmentMatchingStorageSize { } + +[StructLayout(LayoutKind.Sequential, Size = 1)] public struct AlignmentMatchingStorageSize1 : IAlignmentMatchingStorageSize { } +[StructLayout(LayoutKind.Sequential, Size = 16)] public struct AlignmentMatchingStorageSize16 : IAlignmentMatchingStorageSize { } +[StructLayout(LayoutKind.Sequential, Size = 512)] public struct AlignmentMatchingStorageSize512 : IAlignmentMatchingStorageSize { } + +/// +/// Handles accessing a base that must always be accessed via an aligned offset and size. +/// +/// The alignment of all accesses made to the base storage. +/// Must be a power of 2 that is less than or equal to 0x200. +/// The alignment of the destination buffer for the core read. Must be a power of 2. +/// This class uses a work buffer on the stack to avoid allocations. Because of this the data alignment +/// must be kept small; no larger than 0x200. The class +/// should be used for data alignment sizes larger than this. +/// Based on FS 13.1.0 (nnSdk 13.4.0) +[SkipLocalsInit] +public class AlignmentMatchingStorage : IStorage + where TDataAlignment : struct, IAlignmentMatchingStorageSize + where TBufferAlignment : struct, IAlignmentMatchingStorageSize +{ + public static uint DataAlign => (uint)Unsafe.SizeOf(); + public static uint BufferAlign => (uint)Unsafe.SizeOf(); + + public static uint DataAlignMax => 0x200; + + private static void VerifyTypeParameters() + { + Abort.DoAbortUnless(DataAlign <= DataAlignMax); + Abort.DoAbortUnless(BitUtil.IsPowerOfTwo(DataAlign)); + Abort.DoAbortUnless(BitUtil.IsPowerOfTwo(BufferAlign)); + } + + private IStorage _baseStorage; + private long _baseStorageSize; + private bool _isBaseStorageSizeDirty; + private SharedRef _sharedBaseStorage; + + public AlignmentMatchingStorage(ref SharedRef baseStorage) + { + VerifyTypeParameters(); + + _baseStorage = baseStorage.Get; + _isBaseStorageSizeDirty = true; + _sharedBaseStorage = SharedRef.CreateMove(ref baseStorage); + } + + public AlignmentMatchingStorage(IStorage baseStorage) + { + VerifyTypeParameters(); + + _baseStorage = baseStorage; + _isBaseStorageSizeDirty = true; + } + + public override void Dispose() + { + _sharedBaseStorage.Destroy(); + + base.Dispose(); + } + + public override Result Read(long offset, Span destination) + { + Span workBuffer = stackalloc byte[(int)DataAlign]; + + if (destination.Length == 0) + return Result.Success; + + Result rc = GetSize(out long totalSize); + if (rc.IsFailure()) return rc.Miss(); + + if (!CheckAccessRange(offset, destination.Length, totalSize)) + return ResultFs.OutOfRange.Log(); + + return AlignmentMatchingStorageImpl.Read(_baseStorage, workBuffer, DataAlign, BufferAlign, offset, destination); + } + + public override Result Write(long offset, ReadOnlySpan source) + { + Span workBuffer = stackalloc byte[(int)DataAlign]; + + if (source.Length == 0) + return Result.Success; + + Result rc = GetSize(out long totalSize); + if (rc.IsFailure()) return rc.Miss(); + + if (!CheckAccessRange(offset, source.Length, totalSize)) + return ResultFs.OutOfRange.Log(); + + return AlignmentMatchingStorageImpl.Write(_baseStorage, workBuffer, DataAlign, BufferAlign, offset, source); + } + + public override Result Flush() + { + return _baseStorage.Flush(); + } + + public override Result SetSize(long size) + { + Result rc = _baseStorage.SetSize(Alignment.AlignUpPow2(size, DataAlign)); + _isBaseStorageSizeDirty = true; + + return rc; + } + + public override Result GetSize(out long size) + { + UnsafeHelpers.SkipParamInit(out size); + + if (_isBaseStorageSizeDirty) + { + Result rc = _baseStorage.GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + _baseStorageSize = baseStorageSize; + _isBaseStorageSizeDirty = false; + } + + size = _baseStorageSize; + return Result.Success; + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + if (operationId == OperationId.InvalidateCache) + { + return _baseStorage.OperateRange(OperationId.InvalidateCache, offset, size); + } + + if (size == 0) + return Result.Success; + + Result rc = GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (!CheckOffsetAndSize(offset, size)) + return ResultFs.OutOfRange.Log(); + + long validSize = Math.Min(size, baseStorageSize - offset); + long alignedOffset = Alignment.AlignDownPow2(offset, DataAlign); + long alignedOffsetEnd = Alignment.AlignUpPow2(offset + validSize, DataAlign); + long alignedSize = alignedOffsetEnd - alignedOffset; + + return _baseStorage.OperateRange(outBuffer, operationId, alignedOffset, alignedSize, inBuffer); + } +} + +/// +/// Handles accessing a base that must always be accessed via an aligned offset and size. +/// +/// The alignment of the destination buffer for the core read. Must be a power of 2. +/// On every access this class allocates a work buffer that is used for handling any partial blocks at +/// the beginning or end of the requested range. For data alignment sizes of 0x200 or smaller +/// should be used instead +/// to avoid these allocations. +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public class AlignmentMatchingStoragePooledBuffer : IStorage + where TBufferAlignment : struct, IAlignmentMatchingStorageSize +{ + public static uint BufferAlign => (uint)Unsafe.SizeOf(); + + private IStorage _baseStorage; + private long _baseStorageSize; + private uint _dataAlignment; + private bool _isBaseStorageSizeDirty; + + // LibHac addition: This field goes unused if initialized with a plain IStorage. + // The original class uses a template for both the shared and non-shared IStorage which avoids needing this field. + private SharedRef _sharedBaseStorage; + + public AlignmentMatchingStoragePooledBuffer(IStorage baseStorage, int dataAlign) + { + Abort.DoAbortUnless(BitUtil.IsPowerOfTwo(BufferAlign)); + + _baseStorage = baseStorage; + _dataAlignment = (uint)dataAlign; + _isBaseStorageSizeDirty = true; + + Assert.SdkRequires(BitUtil.IsPowerOfTwo(dataAlign), "DataAlign must be a power of 2."); + } + + public AlignmentMatchingStoragePooledBuffer(in SharedRef baseStorage, int dataAlign) + { + Abort.DoAbortUnless(BitUtil.IsPowerOfTwo(BufferAlign)); + + _baseStorage = baseStorage.Get; + _dataAlignment = (uint)dataAlign; + _isBaseStorageSizeDirty = true; + + Assert.SdkRequires(BitUtil.IsPowerOfTwo(dataAlign), "DataAlign must be a power of 2."); + + _sharedBaseStorage = SharedRef.CreateCopy(in baseStorage); + } + + public override void Dispose() + { + _sharedBaseStorage.Destroy(); + + base.Dispose(); + } + + public override Result Read(long offset, Span destination) + { + if (destination.Length == 0) + return Result.Success; + + Result rc = GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (!CheckAccessRange(offset, destination.Length, baseStorageSize)) + return ResultFs.OutOfRange.Log(); + + using var pooledBuffer = new PooledBuffer(); + pooledBuffer.AllocateParticularlyLarge((int)_dataAlignment, (int)_dataAlignment); + + return AlignmentMatchingStorageImpl.Read(_baseStorage, pooledBuffer.GetBuffer(), _dataAlignment, BufferAlign, + offset, destination); + } + + public override Result Write(long offset, ReadOnlySpan source) + { + if (source.Length == 0) + return Result.Success; + + Result rc = GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (!CheckAccessRange(offset, source.Length, baseStorageSize)) + return ResultFs.OutOfRange.Log(); + + using var pooledBuffer = new PooledBuffer(); + pooledBuffer.AllocateParticularlyLarge((int)_dataAlignment, (int)_dataAlignment); + + return AlignmentMatchingStorageImpl.Write(_baseStorage, pooledBuffer.GetBuffer(), _dataAlignment, BufferAlign, + offset, source); + } + + public override Result Flush() + { + return _baseStorage.Flush(); + } + + public override Result SetSize(long size) + { + Result rc = _baseStorage.SetSize(Alignment.AlignUpPow2(size, _dataAlignment)); + _isBaseStorageSizeDirty = true; + + return rc; + } + + public override Result GetSize(out long size) + { + UnsafeHelpers.SkipParamInit(out size); + + if (_isBaseStorageSizeDirty) + { + Result rc = _baseStorage.GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + _isBaseStorageSizeDirty = false; + _baseStorageSize = baseStorageSize; + } + + size = _baseStorageSize; + return Result.Success; + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + if (operationId == OperationId.InvalidateCache) + { + return _baseStorage.OperateRange(OperationId.InvalidateCache, offset, size); + } + + if (size == 0) + return Result.Success; + + Result rc = GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (!CheckOffsetAndSize(offset, size)) + return ResultFs.OutOfRange.Log(); + + long validSize = Math.Min(size, baseStorageSize - offset); + long alignedOffset = Alignment.AlignDownPow2(offset, _dataAlignment); + long alignedOffsetEnd = Alignment.AlignUpPow2(offset + validSize, _dataAlignment); + long alignedSize = alignedOffsetEnd - alignedOffset; + + return _baseStorage.OperateRange(outBuffer, operationId, alignedOffset, alignedSize, inBuffer); + } +} + +/// +/// Handles accessing a base that must always be accessed via an aligned offset and size. +/// +/// The alignment of the destination buffer for the core read. Must be a power of 2. +/// This class is basically the same as except +/// it doesn't allocate a work buffer for reads that are already aligned, and it ignores the buffer alignment for reads. +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public class AlignmentMatchingStorageInBulkRead : IStorage + where TBufferAlignment : struct, IAlignmentMatchingStorageSize +{ + public static uint BufferAlign => (uint)Unsafe.SizeOf(); + + private IStorage _baseStorage; + private SharedRef _sharedBaseStorage; + private long _baseStorageSize; + private uint _dataAlignment; + + public AlignmentMatchingStorageInBulkRead(IStorage baseStorage, int dataAlignment) + { + Abort.DoAbortUnless(BitUtil.IsPowerOfTwo(BufferAlign)); + + _baseStorage = baseStorage; + _baseStorageSize = -1; + _dataAlignment = (uint)dataAlignment; + + Assert.SdkRequires(BitUtil.IsPowerOfTwo(dataAlignment)); + } + + public AlignmentMatchingStorageInBulkRead(in SharedRef baseStorage, int dataAlignment) + { + Abort.DoAbortUnless(BitUtil.IsPowerOfTwo(BufferAlign)); + + _baseStorage = baseStorage.Get; + _baseStorageSize = -1; + _dataAlignment = (uint)dataAlignment; + + Assert.SdkRequires(BitUtil.IsPowerOfTwo(dataAlignment)); + + _sharedBaseStorage = SharedRef.CreateCopy(in baseStorage); + } + + public override void Dispose() + { + _sharedBaseStorage.Destroy(); + + base.Dispose(); + } + + // The original template doesn't define this function, requiring a specialized function for each TBufferAlignment used. + // The only buffer alignment used by that template is 1, so we use that specialization for our Read method. + public override Result Read(long offset, Span destination) + { + if (destination.Length == 0) + return Result.Success; + + Result rc = GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (!CheckAccessRange(offset, destination.Length, baseStorageSize)) + return ResultFs.OutOfRange.Log(); + + // Calculate the aligned offsets of the requested region. + long offsetEnd = offset + destination.Length; + long alignedOffset = Alignment.AlignDownPow2(offset, _dataAlignment); + long alignedOffsetEnd = Alignment.AlignUpPow2(offsetEnd, _dataAlignment); + long alignedSize = alignedOffsetEnd - alignedOffset; + + using var pooledBuffer = new PooledBuffer(); + + // If we aren't aligned we need to allocate a buffer. + if (alignedOffset != offset || alignedSize != destination.Length) + { + if (alignedSize <= PooledBuffer.GetAllocatableSizeMax()) + { + // Try to allocate a buffer that will fit the entire aligned read. + pooledBuffer.Allocate((int)alignedSize, (int)_dataAlignment); + + // If we were able to get a buffer that fits the entire aligned read then read it + // into the buffer and copy the unaligned portion to the destination buffer. + if (alignedSize <= pooledBuffer.GetSize()) + { + rc = _baseStorage.Read(alignedOffset, pooledBuffer.GetBuffer().Slice(0, (int)alignedSize)); + if (rc.IsFailure()) return rc.Miss(); + + pooledBuffer.GetBuffer().Slice((int)(offset - alignedOffset), destination.Length) + .CopyTo(destination); + + return Result.Success; + } + + // We couldn't get as large a buffer as we wanted. + // Shrink the buffer since we only need a single block. + pooledBuffer.Shrink((int)_dataAlignment); + } + else + { + // The requested read is larger than we can allocate, so only allocate a single block. + pooledBuffer.Allocate((int)_dataAlignment, (int)_dataAlignment); + } + } + + // Determine read extents for the aligned portion. + long coreOffset = Alignment.AlignUpPow2(offset, _dataAlignment); + long coreOffsetEnd = Alignment.AlignDownPow2(offsetEnd, _dataAlignment); + + // Handle any data before the aligned portion. + if (offset < coreOffset) + { + int headSize = (int)(coreOffset - offset); + Assert.SdkLess(headSize, destination.Length); + + rc = _baseStorage.Read(alignedOffset, pooledBuffer.GetBuffer().Slice(0, (int)_dataAlignment)); + if (rc.IsFailure()) return rc.Miss(); + + pooledBuffer.GetBuffer().Slice((int)(offset - alignedOffset), headSize).CopyTo(destination); + } + + // Handle the aligned portion. + if (coreOffset < coreOffsetEnd) + { + int coreSize = (int)(coreOffsetEnd - coreOffset); + Span coreBuffer = destination.Slice((int)(coreOffset - offset), coreSize); + + rc = _baseStorage.Read(coreOffset, coreBuffer); + if (rc.IsFailure()) return rc.Miss(); + } + + // Handle any data after the aligned portion. + if (coreOffsetEnd < offsetEnd) + { + int tailSize = (int)(offsetEnd - coreOffsetEnd); + + rc = _baseStorage.Read(coreOffsetEnd, pooledBuffer.GetBuffer().Slice(0, (int)_dataAlignment)); + if (rc.IsFailure()) return rc.Miss(); + + pooledBuffer.GetBuffer().Slice(0, tailSize).CopyTo(destination.Slice((int)(coreOffsetEnd - offset))); + } + + return Result.Success; + } + + public override Result Write(long offset, ReadOnlySpan source) + { + if (source.Length == 0) + return Result.Success; + + Result rc = GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (!CheckAccessRange(offset, source.Length, baseStorageSize)) + return ResultFs.OutOfRange.Log(); + + using var pooledBuffer = new PooledBuffer((int)_dataAlignment, (int)_dataAlignment); + + return AlignmentMatchingStorageImpl.Write(_baseStorage, pooledBuffer.GetBuffer(), _dataAlignment, BufferAlign, + offset, source); + } + + public override Result Flush() + { + return _baseStorage.Flush(); + } + + public override Result SetSize(long size) + { + Result rc = _baseStorage.SetSize(Alignment.AlignUpPow2(size, _dataAlignment)); + _baseStorageSize = -1; + + return rc; + } + + public override Result GetSize(out long size) + { + UnsafeHelpers.SkipParamInit(out size); + + if (_baseStorageSize < 0) + { + Result rc = _baseStorage.GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + _baseStorageSize = baseStorageSize; + } + + size = _baseStorageSize; + return Result.Success; + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + if (operationId == OperationId.InvalidateCache) + { + return _baseStorage.OperateRange(OperationId.InvalidateCache, offset, size); + } + + if (size == 0) + return Result.Success; + + Result rc = GetSize(out long baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (!CheckOffsetAndSize(offset, size)) + return ResultFs.OutOfRange.Log(); + + long validSize = Math.Min(size, baseStorageSize - offset); + long alignedOffset = Alignment.AlignDownPow2(offset, _dataAlignment); + long alignedOffsetEnd = Alignment.AlignUpPow2(offset + validSize, _dataAlignment); + long alignedSize = alignedOffsetEnd - alignedOffset; + + return _baseStorage.OperateRange(outBuffer, operationId, alignedOffset, alignedSize, inBuffer); + } +} \ No newline at end of file diff --git a/src/LibHac/FsSystem/AlignmentMatchingStorageImpl.cs b/src/LibHac/FsSystem/AlignmentMatchingStorageImpl.cs new file mode 100644 index 00000000..33b1f868 --- /dev/null +++ b/src/LibHac/FsSystem/AlignmentMatchingStorageImpl.cs @@ -0,0 +1,191 @@ +using System; +using LibHac.Common; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.Util; + +namespace LibHac.FsSystem; + +/// +/// Contains the functions used by classes like for +/// accessing an aligned . +/// +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public static class AlignmentMatchingStorageImpl +{ + public static uint GetRoundDownDifference(int value, uint alignment) + { + return (uint)(value - Alignment.AlignDownPow2(value, alignment)); + } + + public static uint GetRoundDownDifference(long value, uint alignment) + { + return (uint)(value - Alignment.AlignDownPow2(value, alignment)); + } + + public static uint GetRoundUpDifference(int value, uint alignment) + { + return (uint)(Alignment.AlignUpPow2(value, alignment) - value); + } + + private static uint GetRoundUpDifference(long value, uint alignment) + { + return (uint)(Alignment.AlignUpPow2(value, alignment) - value); + } + + public static Result Read(in SharedRef storage, Span workBuffer, uint dataAlignment, + uint bufferAlignment, long offset, Span destination) + { + return Read(storage.Get, workBuffer, dataAlignment, bufferAlignment, offset, destination); + } + + public static Result Write(in SharedRef storage, Span subBuffer, uint dataAlignment, + uint bufferAlignment, long offset, ReadOnlySpan source) + { + return Write(storage.Get, subBuffer, dataAlignment, bufferAlignment, offset, source); + } + + public static Result Read(IStorage storage, Span workBuffer, uint dataAlignment, uint bufferAlignment, + long offset, Span destination) + { + // We don't support buffer alignment because Nintendo never uses any alignment other than 1, and because + // we'd have to mess with pinning the buffer. + Abort.DoAbortUnless(bufferAlignment == 1); + + Assert.SdkRequiresGreaterEqual((uint)workBuffer.Length, dataAlignment); + + if (destination.Length == 0) + return Result.Success; + + // Calculate the range that contains only full data blocks. + uint offsetRoundUpDifference = GetRoundUpDifference(offset, dataAlignment); + + long coreOffset = Alignment.AlignUpPow2(offset, dataAlignment); + long coreSize = destination.Length < offsetRoundUpDifference + ? 0 + : Alignment.AlignDownPow2(destination.Length - offsetRoundUpDifference, dataAlignment); + + long coveredOffset = coreSize > 0 ? coreOffset : offset; + + // Read the core portion that doesn't contain any partial blocks. + if (coreSize > 0) + { + Result rc = storage.Read(coreOffset, destination.Slice((int)offsetRoundUpDifference, (int)coreSize)); + if (rc.IsFailure()) return rc.Miss(); + } + + // Read any partial block at the head of the requested range + if (offset < coveredOffset) + { + long headOffset = Alignment.AlignDownPow2(offset, dataAlignment); + int headSize = (int)(coveredOffset - offset); + + Assert.SdkAssert(GetRoundDownDifference(offset, dataAlignment) + headSize <= workBuffer.Length); + + Result rc = storage.Read(headOffset, workBuffer.Slice(0, (int)dataAlignment)); + if (rc.IsFailure()) return rc.Miss(); + + workBuffer.Slice((int)GetRoundDownDifference(offset, dataAlignment), headSize).CopyTo(destination); + } + + long tailOffset = coveredOffset + coreSize; + long remainingTailSize = offset + destination.Length - tailOffset; + + // Read any partial block at the tail of the requested range + while (remainingTailSize > 0) + { + long alignedTailOffset = Alignment.AlignDownPow2(tailOffset, dataAlignment); + long copySize = Math.Min(alignedTailOffset + dataAlignment - tailOffset, remainingTailSize); + + Result rc = storage.Read(alignedTailOffset, workBuffer.Slice(0, (int)dataAlignment)); + if (rc.IsFailure()) return rc.Miss(); + + Assert.SdkAssert(tailOffset - offset + copySize <= destination.Length); + Assert.SdkAssert(tailOffset - alignedTailOffset + copySize <= dataAlignment); + workBuffer.Slice((int)(tailOffset - alignedTailOffset), (int)copySize) + .CopyTo(destination.Slice((int)(tailOffset - offset))); + + remainingTailSize -= copySize; + tailOffset += copySize; + } + + return Result.Success; + } + + public static Result Write(IStorage storage, Span workBuffer, uint dataAlignment, uint bufferAlignment, + long offset, ReadOnlySpan source) + { + // We don't support buffer alignment because Nintendo never uses any alignment other than 1, and because + // we'd have to mess with pinning the buffer. + Abort.DoAbortUnless(bufferAlignment == 1); + + Assert.SdkRequiresGreaterEqual((uint)workBuffer.Length, dataAlignment); + + if (source.Length == 0) + return Result.Success; + + // Calculate the range that contains only full data blocks. + uint offsetRoundUpDifference = GetRoundUpDifference(offset, dataAlignment); + + long coreOffset = Alignment.AlignUpPow2(offset, dataAlignment); + long coreSize = source.Length < offsetRoundUpDifference + ? 0 + : Alignment.AlignDownPow2(source.Length - offsetRoundUpDifference, dataAlignment); + + long coveredOffset = coreSize > 0 ? coreOffset : offset; + + // Write the core portion that doesn't contain any partial blocks. + if (coreSize > 0) + { + Result rc = storage.Write(coreOffset, source.Slice((int)offsetRoundUpDifference, (int)coreSize)); + if (rc.IsFailure()) return rc.Miss(); + } + + // Write any partial block at the head of the specified range + if (offset < coveredOffset) + { + long headOffset = Alignment.AlignDownPow2(offset, dataAlignment); + int headSize = (int)(coveredOffset - offset); + + Assert.SdkAssert((offset - headOffset) + headSize <= workBuffer.Length); + + // Read the existing block, copy the partial block to the appropriate portion, + // and write the modified block back to the base storage. + Result rc = storage.Read(headOffset, workBuffer.Slice(0, (int)dataAlignment)); + if (rc.IsFailure()) return rc.Miss(); + + source.Slice(0, headSize).CopyTo(workBuffer.Slice((int)(offset - headOffset))); + + rc = storage.Write(headOffset, workBuffer.Slice(0, (int)dataAlignment)); + if (rc.IsFailure()) return rc.Miss(); + } + + long tailOffset = coveredOffset + coreSize; + long remainingTailSize = offset + source.Length - tailOffset; + + // Write any partial block at the tail of the specified range + while (remainingTailSize > 0) + { + Assert.SdkAssert(tailOffset - offset < source.Length); + + long alignedTailOffset = Alignment.AlignDownPow2(tailOffset, dataAlignment); + long copySize = Math.Min(alignedTailOffset + dataAlignment - tailOffset, remainingTailSize); + + // Read the existing block, copy the partial block to the appropriate portion, + // and write the modified block back to the base storage. + Result rc = storage.Read(alignedTailOffset, workBuffer.Slice(0, (int)dataAlignment)); + if (rc.IsFailure()) return rc.Miss(); + + source.Slice((int)(tailOffset - offset), (int)copySize) + .CopyTo(workBuffer.Slice((int)GetRoundDownDifference(tailOffset, dataAlignment))); + + rc = storage.Write(alignedTailOffset, workBuffer.Slice(0, (int)dataAlignment)); + if (rc.IsFailure()) return rc.Miss(); + + remainingTailSize -= copySize; + tailOffset += copySize; + } + + return Result.Success; + } +} \ No newline at end of file From 3a05e779f944975da49500f8504ee23b6365a17a Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 14 Mar 2022 13:34:52 -0700 Subject: [PATCH 12/24] Add ReadOnlyBlockCacheStorage --- src/LibHac/Diag/Assert.cs | 52 ++++- src/LibHac/Diag/Impl/AssertImpl.cs | 7 +- src/LibHac/FsSystem/BitmapUtils.cs | 1 + src/LibHac/FsSystem/LruListCache.cs | 87 ++++++++ .../FsSystem/ReadOnlyBlockCacheStorage.cs | 141 ++++++++++++ .../ReadOnlyBlockCacheStorageTests.cs | 200 ++++++++++++++++++ 6 files changed, 486 insertions(+), 2 deletions(-) create mode 100644 src/LibHac/FsSystem/LruListCache.cs create mode 100644 src/LibHac/FsSystem/ReadOnlyBlockCacheStorage.cs create mode 100644 tests/LibHac.Tests/FsSystem/ReadOnlyBlockCacheStorageTests.cs diff --git a/src/LibHac/Diag/Assert.cs b/src/LibHac/Diag/Assert.cs index 5b8a9bf0..8f87b05b 100644 --- a/src/LibHac/Diag/Assert.cs +++ b/src/LibHac/Diag/Assert.cs @@ -1272,7 +1272,57 @@ internal static void SdkRequiresGreaterEqual(T lhs, T rhs, } // --------------------------------------------------------------------- - // Aligned + // Aligned long + // --------------------------------------------------------------------- + + private static void AlignedImpl(AssertionType assertionType, long value, int alignment, string valueText, + string alignmentText, string functionName, string fileName, int lineNumber) + { + if (AssertImpl.IsAligned(value, alignment)) + return; + + AssertImpl.InvokeAssertionAligned(assertionType, value, alignment, valueText, alignmentText, functionName, fileName, + lineNumber); + } + + [Conditional(AssertCondition)] + public static void Aligned(long value, int alignment, + [CallerArgumentExpression("value")] string valueText = "", + [CallerArgumentExpression("alignment")] string alignmentText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + { + AlignedImpl(AssertionType.UserAssert, value, alignment, valueText, alignmentText, functionName, fileName, + lineNumber); + } + + [Conditional(AssertCondition)] + internal static void SdkAligned(long value, int alignment, + [CallerArgumentExpression("value")] string valueText = "", + [CallerArgumentExpression("alignment")] string alignmentText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + { + AlignedImpl(AssertionType.SdkAssert, value, alignment, valueText, alignmentText, functionName, fileName, + lineNumber); + } + + [Conditional(AssertCondition)] + internal static void SdkRequiresAligned(long value, int alignment, + [CallerArgumentExpression("value")] string valueText = "", + [CallerArgumentExpression("alignment")] string alignmentText = "", + [CallerMemberName] string functionName = "", + [CallerFilePath] string fileName = "", + [CallerLineNumber] int lineNumber = 0) + { + AlignedImpl(AssertionType.SdkRequires, value, alignment, valueText, alignmentText, functionName, fileName, + lineNumber); + } + + // --------------------------------------------------------------------- + // Aligned ulong // --------------------------------------------------------------------- private static void AlignedImpl(AssertionType assertionType, ulong value, int alignment, string valueText, diff --git a/src/LibHac/Diag/Impl/AssertImpl.cs b/src/LibHac/Diag/Impl/AssertImpl.cs index cc2e262e..e27cc9bd 100644 --- a/src/LibHac/Diag/Impl/AssertImpl.cs +++ b/src/LibHac/Diag/Impl/AssertImpl.cs @@ -106,7 +106,7 @@ internal static void InvokeAssertionGreaterEqual(AssertionType assertionType, Assert.OnAssertionFailure(assertionType, "GreaterEqual", functionName, fileName, lineNumber, message); } - internal static void InvokeAssertionAligned(AssertionType assertionType, ulong value, int alignment, + internal static void InvokeAssertionAligned(AssertionType assertionType, T value, int alignment, string valueText, string alignmentText, string functionName, string fileName, int lineNumber) { string message = @@ -246,6 +246,11 @@ public static bool GreaterEqual(ref T lhs, ref T rhs) where T : IComparable= 0; } + public static bool IsAligned(long value, int alignment) + { + return Alignment.IsAlignedPow2(value, (uint)alignment); + } + public static bool IsAligned(ulong value, int alignment) { return Alignment.IsAlignedPow2(value, (uint)alignment); diff --git a/src/LibHac/FsSystem/BitmapUtils.cs b/src/LibHac/FsSystem/BitmapUtils.cs index c425ec86..0cdb4bfc 100644 --- a/src/LibHac/FsSystem/BitmapUtils.cs +++ b/src/LibHac/FsSystem/BitmapUtils.cs @@ -4,6 +4,7 @@ namespace LibHac.FsSystem; public static class BitmapUtils { + // ReSharper disable once InconsistentNaming public static uint ILog2(uint value) { Assert.SdkRequiresGreater(value, 0u); diff --git a/src/LibHac/FsSystem/LruListCache.cs b/src/LibHac/FsSystem/LruListCache.cs new file mode 100644 index 00000000..9ffb20c0 --- /dev/null +++ b/src/LibHac/FsSystem/LruListCache.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using LibHac.Diag; + +namespace LibHac.FsSystem; + +/// +/// Represents a list of key/value pairs that are ordered by when they were last accessed. +/// +/// The type of the keys in the list. +/// The type of the values in the list. +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public class LruListCache where TKey : IEquatable +{ + public struct Node + { + public TKey Key; + public TValue Value; + + public Node(TValue value) + { + Key = default; + Value = value; + } + } + + private LinkedList _list; + + public LruListCache() + { + _list = new LinkedList(); + } + + public bool FindValueAndUpdateMru(out TValue value, TKey key) + { + LinkedListNode currentNode = _list.First; + + while (currentNode is not null) + { + if (currentNode.ValueRef.Key.Equals(key)) + { + value = currentNode.ValueRef.Value; + + _list.Remove(currentNode); + _list.AddFirst(currentNode); + + return true; + } + + currentNode = currentNode.Next; + } + + value = default; + return false; + } + + public LinkedListNode PopLruNode() + { + Abort.DoAbortUnless(_list.Count != 0); + + LinkedListNode lru = _list.Last; + _list.RemoveLast(); + + return lru; + } + + public void PushMruNode(LinkedListNode node, TKey key) + { + node.ValueRef.Key = key; + _list.AddFirst(node); + } + + public void DeleteAllNodes() + { + _list.Clear(); + } + + public int GetSize() + { + return _list.Count; + } + + public bool IsEmpty() + { + return _list.Count == 0; + } +} \ No newline at end of file diff --git a/src/LibHac/FsSystem/ReadOnlyBlockCacheStorage.cs b/src/LibHac/FsSystem/ReadOnlyBlockCacheStorage.cs new file mode 100644 index 00000000..64f10ecf --- /dev/null +++ b/src/LibHac/FsSystem/ReadOnlyBlockCacheStorage.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using LibHac.Common; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.Os; +using LibHac.Util; + +using BlockCache = LibHac.FsSystem.LruListCache>; + +namespace LibHac.FsSystem; + +/// +/// Caches reads to a base using a least-recently-used cache of data blocks. +/// The offset and size read from the storage must be aligned to multiples of the block size. +/// Only reads that access a single block will use the cache. Reads that access multiple blocks will +/// be passed down to the base to be handled without caching. +/// +/// Based on FS 13.1.0 (nnSdk 13.4.0) +public class ReadOnlyBlockCacheStorage : IStorage +{ + private SdkMutexType _mutex; + private BlockCache _blockCache; + private SharedRef _baseStorage; + private int _blockSize; + + public ReadOnlyBlockCacheStorage(ref SharedRef baseStorage, int blockSize, Memory buffer, + int cacheBlockCount) + { + _baseStorage = SharedRef.CreateMove(ref baseStorage); + _blockSize = blockSize; + _blockCache = new BlockCache(); + _mutex = new SdkMutexType(); + + Assert.SdkRequiresGreaterEqual(buffer.Length, _blockSize); + Assert.SdkRequires(BitUtil.IsPowerOfTwo(blockSize), $"{nameof(blockSize)} must be power of 2."); + Assert.SdkRequiresGreater(cacheBlockCount, 0); + Assert.SdkRequiresGreaterEqual(buffer.Length, blockSize * cacheBlockCount); + + for (int i = 0; i < cacheBlockCount; i++) + { + Memory nodeBuffer = buffer.Slice(i * blockSize, blockSize); + var node = new LinkedListNode(new BlockCache.Node(nodeBuffer)); + Assert.SdkNotNull(node); + + _blockCache.PushMruNode(node, -1); + } + } + + public override void Dispose() + { + _blockCache.DeleteAllNodes(); + _baseStorage.Destroy(); + + base.Dispose(); + } + + public override Result Read(long offset, Span destination) + { + Assert.SdkRequiresAligned(offset, _blockSize); + Assert.SdkRequiresAligned(destination.Length, _blockSize); + + if (destination.Length == _blockSize) + { + // Search the cache for the requested block. + using (new ScopedLock(ref _mutex)) + { + bool found = _blockCache.FindValueAndUpdateMru(out Memory cachedBuffer, offset / _blockSize); + if (found) + { + cachedBuffer.Span.CopyTo(destination); + return Result.Success; + } + } + + // The block wasn't in the cache. Read from the base storage. + Result rc = _baseStorage.Get.Read(offset, destination); + if (rc.IsFailure()) return rc.Miss(); + + // Add the block to the cache. + using (new ScopedLock(ref _mutex)) + { + LinkedListNode lru = _blockCache.PopLruNode(); + destination.CopyTo(lru.ValueRef.Value.Span); + _blockCache.PushMruNode(lru, offset / _blockSize); + } + + return Result.Success; + } + else + { + return _baseStorage.Get.Read(offset, destination); + } + } + + public override Result Write(long offset, ReadOnlySpan source) + { + // Missing: Log output + return ResultFs.UnsupportedWriteForReadOnlyBlockCacheStorage.Log(); + } + + public override Result Flush() + { + return Result.Success; + } + + public override Result SetSize(long size) + { + return ResultFs.UnsupportedSetSizeForReadOnlyBlockCacheStorage.Log(); + } + + public override Result GetSize(out long size) + { + return _baseStorage.Get.GetSize(out size); + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + if (operationId == OperationId.InvalidateCache) + { + // Invalidate all the blocks in our cache. + using var scopedLock = new ScopedLock(ref _mutex); + + int cacheBlockCount = _blockCache.GetSize(); + for (int i = 0; i < cacheBlockCount; i++) + { + LinkedListNode lru = _blockCache.PopLruNode(); + _blockCache.PushMruNode(lru, -1); + } + } + else + { + Assert.SdkRequiresAligned(offset, _blockSize); + Assert.SdkRequiresAligned(size, _blockSize); + } + + // Pass the request to the base storage. + return _baseStorage.Get.OperateRange(outBuffer, operationId, offset, size, inBuffer); + } +} \ No newline at end of file diff --git a/tests/LibHac.Tests/FsSystem/ReadOnlyBlockCacheStorageTests.cs b/tests/LibHac.Tests/FsSystem/ReadOnlyBlockCacheStorageTests.cs new file mode 100644 index 00000000..15ec81d3 --- /dev/null +++ b/tests/LibHac.Tests/FsSystem/ReadOnlyBlockCacheStorageTests.cs @@ -0,0 +1,200 @@ +using System; +using System.Buffers.Binary; +using LibHac.Common; +using LibHac.Fs; +using LibHac.FsSystem; +using Xunit; + +namespace LibHac.Tests.FsSystem; + +public class ReadOnlyBlockCacheStorageTests +{ + private class TestContext + { + private int _blockSize; + private int _cacheBlockCount; + + public byte[] BaseData; + public byte[] ModifiedBaseData; + public byte[] CacheBuffer; + + public ReadOnlyBlockCacheStorage CacheStorage; + + public TestContext(int blockSize, int cacheBlockCount, int storageBlockCount, ulong rngSeed) + { + _blockSize = blockSize; + _cacheBlockCount = cacheBlockCount; + + BaseData = new byte[_blockSize * storageBlockCount]; + ModifiedBaseData = new byte[_blockSize * storageBlockCount]; + CacheBuffer = new byte[_blockSize * _cacheBlockCount]; + + new Random(rngSeed).NextBytes(BaseData); + BaseData.AsSpan().CopyTo(ModifiedBaseData); + + for (int i = 0; i < storageBlockCount; i++) + { + ModifyBlock(GetModifiedBaseDataBlock(i)); + } + + using var baseStorage = new SharedRef(new MemoryStorage(BaseData)); + CacheStorage = new ReadOnlyBlockCacheStorage(ref baseStorage.Ref(), _blockSize, CacheBuffer, _cacheBlockCount); + } + + public Span GetBaseDataBlock(int index) => BaseData.AsSpan(_blockSize * index, _blockSize); + public Span GetModifiedBaseDataBlock(int index) => ModifiedBaseData.AsSpan(_blockSize * index, _blockSize); + public Span GetCacheDataBlock(int index) => CacheBuffer.AsSpan(_blockSize * index, _blockSize); + + private void ModifyBlock(Span block) + { + BinaryPrimitives.WriteUInt64LittleEndian(block, ulong.MaxValue); + } + + public void ModifyAllCacheBlocks() + { + for (int i = 0; i < _cacheBlockCount; i++) + { + ModifyBlock(GetCacheDataBlock(i)); + } + } + + public Span ReadCachedStorage(int blockIndex) + { + byte[] buffer = new byte[_blockSize]; + Assert.Success(CacheStorage.Read(_blockSize * blockIndex, buffer)); + return buffer; + } + + public Span ReadCachedStorage(long offset, int size) + { + byte[] buffer = new byte[size]; + Assert.Success(CacheStorage.Read(offset, buffer)); + return buffer; + } + + public void InvalidateCache() + { + Assert.Success(CacheStorage.OperateRange(OperationId.InvalidateCache, 0, long.MaxValue)); + } + } + + private const int BlockSize = 0x4000; + private const int CacheBlockCount = 4; + private const int StorageBlockCount = 16; + + [Fact] + public void Read_CompleteBlocks_ReadsCorrectData() + { + var context = new TestContext(BlockSize, CacheBlockCount, StorageBlockCount, 21341); + + for (int i = 0; i < StorageBlockCount; i++) + { + Assert.True(context.GetBaseDataBlock(i).SequenceEqual(context.ReadCachedStorage(i))); + Assert.True(context.GetBaseDataBlock(i).SequenceEqual(context.ReadCachedStorage(i))); + } + } + + [Fact] + public void Read_PreviouslyCachedBlock_ReturnsDataFromCache() + { + const int index = 4; + var context = new TestContext(BlockSize, CacheBlockCount, StorageBlockCount, 21341); + + // Cache the block + context.ReadCachedStorage(index); + + // Directly modify the cache buffer + context.ModifyAllCacheBlocks(); + + // Next read should return the modified data from the cache buffer + Assert.True(context.GetModifiedBaseDataBlock(index).SequenceEqual(context.ReadCachedStorage(index))); + } + + [Fact] + public void Read_BlockEvictedFromCache_ReturnsDataFromBaseStorage() + { + const int index = 4; + var context = new TestContext(BlockSize, CacheBlockCount, StorageBlockCount, 21341); + + context.ReadCachedStorage(index); + context.ModifyAllCacheBlocks(); + + // Read enough additional blocks to push the initial block out of the cache + context.ReadCachedStorage(6); + context.ReadCachedStorage(7); + context.ReadCachedStorage(8); + context.ReadCachedStorage(9); + + // Reading the initial block should now return the original data + Assert.True(context.GetBaseDataBlock(index).SequenceEqual(context.ReadCachedStorage(index))); + } + + [Fact] + public void Read_ReadMultipleBlocks_BlocksAreEvictedAtTheRightTime() + { + const int index = 4; + var context = new TestContext(BlockSize, CacheBlockCount, StorageBlockCount, 21341); + + context.ReadCachedStorage(index); + context.ModifyAllCacheBlocks(); + + context.ReadCachedStorage(6); + context.ReadCachedStorage(7); + context.ReadCachedStorage(8); + + // Reading the initial block should return the cached data + Assert.True(context.GetModifiedBaseDataBlock(index).SequenceEqual(context.ReadCachedStorage(index))); + + for (int i = 0; i < 3; i++) + context.ReadCachedStorage(9 + i); + + context.ModifyAllCacheBlocks(); + + // The initial block should have been moved to the top of the cache when it was last accessed + Assert.True(context.GetModifiedBaseDataBlock(index).SequenceEqual(context.ReadCachedStorage(index))); + + // Access all the other blocks in the cache so the initial block is the least recently accessed + for (int i = 0; i < 3; i++) + Assert.True(context.GetModifiedBaseDataBlock(9 + i).SequenceEqual(context.ReadCachedStorage(9 + i))); + + // Add a new block to the cache + Assert.True(context.GetBaseDataBlock(2).SequenceEqual(context.ReadCachedStorage(2))); + + // The initial block should have been removed from the cache + Assert.True(context.GetBaseDataBlock(index).SequenceEqual(context.ReadCachedStorage(index))); + } + + [Fact] + public void Read_UnalignedBlock_ReturnsOriginalData() + { + const int index = 4; + var context = new TestContext(BlockSize, CacheBlockCount, StorageBlockCount, 21341); + + context.ReadCachedStorage(index); + context.ModifyAllCacheBlocks(); + + // Read two blocks at once + int offset = index * BlockSize; + int size = BlockSize * 2; + + // The cache should be bypassed, returning the original data + Assert.True(context.BaseData.AsSpan(offset, size).SequenceEqual(context.ReadCachedStorage(offset, size))); + } + + [Fact] + public void OperateRange_InvalidateCache_PreviouslyCachedBlockReturnsDataFromBaseStorage() + { + const int index = 4; + var context = new TestContext(BlockSize, CacheBlockCount, StorageBlockCount, 21341); + + context.ReadCachedStorage(index); + context.ModifyAllCacheBlocks(); + + // Next read should return the modified data from the cache buffer + Assert.True(context.GetModifiedBaseDataBlock(index).SequenceEqual(context.ReadCachedStorage(index))); + + // Reading after invalidating the cache should return the original data + context.InvalidateCache(); + Assert.True(context.GetBaseDataBlock(index).SequenceEqual(context.ReadCachedStorage(index))); + } +} \ No newline at end of file From 4e5e9a4627b0156b9af997b60f6c95750428d6eb Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 11 Apr 2022 13:54:44 -0700 Subject: [PATCH 13/24] Add memory fence functions --- src/LibHac/Os/Impl/MemoryFence.cs | 128 ++++++++++++++++++++++++++++++ src/LibHac/Os/MemoryFenceApi.cs | 19 +++++ 2 files changed, 147 insertions(+) create mode 100644 src/LibHac/Os/Impl/MemoryFence.cs create mode 100644 src/LibHac/Os/MemoryFenceApi.cs diff --git a/src/LibHac/Os/Impl/MemoryFence.cs b/src/LibHac/Os/Impl/MemoryFence.cs new file mode 100644 index 00000000..725d188d --- /dev/null +++ b/src/LibHac/Os/Impl/MemoryFence.cs @@ -0,0 +1,128 @@ +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.X86; + +namespace LibHac.Os.Impl; + +public static class MemoryFence +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void FenceMemoryStoreStore() + { + if (Sse.IsSupported) + { + Sse.StoreFence(); + } + else + { + // This only needs to be a store barrier on aarch64 + System.Threading.Thread.MemoryBarrier(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void FenceMemoryStoreLoad() + { + if (Sse2.IsSupported) + { + Sse2.MemoryFence(); + } + else + { + System.Threading.Thread.MemoryBarrier(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void FenceMemoryStoreAny() + { + if (Sse2.IsSupported) + { + Sse2.MemoryFence(); + } + else + { + System.Threading.Thread.MemoryBarrier(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void FenceMemoryLoadStore() + { + if (Sse2.IsSupported) + { + Sse2.MemoryFence(); + } + else + { + // This only needs to be a load barrier on aarch64 + System.Threading.Thread.MemoryBarrier(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void FenceMemoryLoadLoad() + { + if (Sse2.IsSupported) + { + Sse2.LoadFence(); + } + else + { + // This only needs to be a load barrier on aarch64 + System.Threading.Thread.MemoryBarrier(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void FenceMemoryLoadAny() + { + if (Sse2.IsSupported) + { + Sse2.MemoryFence(); + } + else + { + // This only needs to be a load barrier on aarch64 + System.Threading.Thread.MemoryBarrier(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void FenceMemoryAnyStore() + { + if (Sse2.IsSupported) + { + Sse2.MemoryFence(); + } + else + { + System.Threading.Thread.MemoryBarrier(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void FenceMemoryAnyLoad() + { + if (Sse2.IsSupported) + { + Sse2.MemoryFence(); + } + else + { + System.Threading.Thread.MemoryBarrier(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void FenceMemoryAnyAny() + { + if (Sse2.IsSupported) + { + Sse2.MemoryFence(); + } + else + { + System.Threading.Thread.MemoryBarrier(); + } + } +} \ No newline at end of file diff --git a/src/LibHac/Os/MemoryFenceApi.cs b/src/LibHac/Os/MemoryFenceApi.cs new file mode 100644 index 00000000..35949014 --- /dev/null +++ b/src/LibHac/Os/MemoryFenceApi.cs @@ -0,0 +1,19 @@ +using System.Runtime.CompilerServices; +using LibHac.Os.Impl; + +namespace LibHac.Os; + +public static class MemoryFenceApi +{ + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void FenceMemoryStoreStore() => MemoryFence.FenceMemoryStoreStore(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void FenceMemoryStoreLoad() => MemoryFence.FenceMemoryStoreLoad(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void FenceMemoryStoreAny() => MemoryFence.FenceMemoryStoreAny(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void FenceMemoryLoadStore() => MemoryFence.FenceMemoryLoadStore(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void FenceMemoryLoadLoad() => MemoryFence.FenceMemoryLoadLoad(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void FenceMemoryLoadAny() => MemoryFence.FenceMemoryLoadAny(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void FenceMemoryAnyStore() => MemoryFence.FenceMemoryAnyStore(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void FenceMemoryAnyLoad() => MemoryFence.FenceMemoryAnyLoad(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void FenceMemoryAnyAny() => MemoryFence.FenceMemoryAnyAny(); +} \ No newline at end of file From e46c1f0231528417c6a6a0199dd7c55190c628a7 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 11 Apr 2022 14:22:00 -0700 Subject: [PATCH 14/24] Add multiple wait APIs This adds the base work that will be needed for waiting on multiple objects. Some tweaks will still probably be necessary to make it work nicely with the .NET APIs. --- src/LibHac/Os/CommonTypes.cs | 8 + .../Os/Impl/InternalCriticalSection-os.net.cs | 7 +- src/LibHac/Os/Impl/InternalCriticalSection.cs | 11 +- src/LibHac/Os/Impl/MultiWaitHolderImpl.cs | 14 + src/LibHac/Os/Impl/MultipleWaitHolderBase.cs | 59 +++ .../Impl/MultipleWaitHolderOfNativeHandle.cs | 22 ++ src/LibHac/Os/Impl/MultipleWaitImpl.cs | 349 ++++++++++++++++++ src/LibHac/Os/Impl/MultipleWaitObjectList.cs | 47 +++ .../Os/Impl/MultipleWaitTargetImpl-os.net.cs | 116 ++++++ src/LibHac/Os/MultipleWait.cs | 179 +++++++++ src/LibHac/Os/MultipleWaitTypes.cs | 22 ++ src/LibHac/Os/OsTypes.cs | 8 + 12 files changed, 838 insertions(+), 4 deletions(-) create mode 100644 src/LibHac/Os/CommonTypes.cs create mode 100644 src/LibHac/Os/Impl/MultiWaitHolderImpl.cs create mode 100644 src/LibHac/Os/Impl/MultipleWaitHolderBase.cs create mode 100644 src/LibHac/Os/Impl/MultipleWaitHolderOfNativeHandle.cs create mode 100644 src/LibHac/Os/Impl/MultipleWaitImpl.cs create mode 100644 src/LibHac/Os/Impl/MultipleWaitObjectList.cs create mode 100644 src/LibHac/Os/Impl/MultipleWaitTargetImpl-os.net.cs create mode 100644 src/LibHac/Os/MultipleWait.cs create mode 100644 src/LibHac/Os/MultipleWaitTypes.cs create mode 100644 src/LibHac/Os/OsTypes.cs diff --git a/src/LibHac/Os/CommonTypes.cs b/src/LibHac/Os/CommonTypes.cs new file mode 100644 index 00000000..49f5b653 --- /dev/null +++ b/src/LibHac/Os/CommonTypes.cs @@ -0,0 +1,8 @@ +namespace LibHac.Os; + +public enum TriBool +{ + False = 0, + True = 1, + Undefined = 2 +} \ No newline at end of file diff --git a/src/LibHac/Os/Impl/InternalCriticalSection-os.net.cs b/src/LibHac/Os/Impl/InternalCriticalSection-os.net.cs index fcb852e3..c6de32d0 100644 --- a/src/LibHac/Os/Impl/InternalCriticalSection-os.net.cs +++ b/src/LibHac/Os/Impl/InternalCriticalSection-os.net.cs @@ -1,8 +1,9 @@ -using System.Threading; +using System; +using System.Threading; namespace LibHac.Os.Impl; -internal struct InternalCriticalSectionImpl +internal struct InternalCriticalSectionImpl : IDisposable { private object _obj; @@ -11,6 +12,8 @@ public InternalCriticalSectionImpl() _obj = new object(); } + public void Dispose() { } + public void Initialize() { _obj = new object(); diff --git a/src/LibHac/Os/Impl/InternalCriticalSection.cs b/src/LibHac/Os/Impl/InternalCriticalSection.cs index 2a937ce1..4777dee0 100644 --- a/src/LibHac/Os/Impl/InternalCriticalSection.cs +++ b/src/LibHac/Os/Impl/InternalCriticalSection.cs @@ -1,6 +1,8 @@ -namespace LibHac.Os.Impl; +using System; -public struct InternalCriticalSection : ILockable +namespace LibHac.Os.Impl; + +public struct InternalCriticalSection : ILockable, IDisposable { private InternalCriticalSectionImpl _impl; @@ -9,6 +11,11 @@ public InternalCriticalSection() _impl = new InternalCriticalSectionImpl(); } + public void Dispose() + { + _impl.Dispose(); + } + public void Initialize() => _impl.Initialize(); public void FinalizeObject() => _impl.FinalizeObject(); diff --git a/src/LibHac/Os/Impl/MultiWaitHolderImpl.cs b/src/LibHac/Os/Impl/MultiWaitHolderImpl.cs new file mode 100644 index 00000000..1564e186 --- /dev/null +++ b/src/LibHac/Os/Impl/MultiWaitHolderImpl.cs @@ -0,0 +1,14 @@ +namespace LibHac.Os.Impl; + +public class MultiWaitHolderImpl +{ + private MultiWaitHolderBase _holder; + + public MultiWaitHolderBase HolderBase => _holder; + public MultiWaitHolderOfNativeHandle HolderOfNativeHandle => (MultiWaitHolderOfNativeHandle)_holder; + + public MultiWaitHolderImpl(MultiWaitHolderOfNativeHandle holder) + { + _holder = holder; + } +} \ No newline at end of file diff --git a/src/LibHac/Os/Impl/MultipleWaitHolderBase.cs b/src/LibHac/Os/Impl/MultipleWaitHolderBase.cs new file mode 100644 index 00000000..5c7969b9 --- /dev/null +++ b/src/LibHac/Os/Impl/MultipleWaitHolderBase.cs @@ -0,0 +1,59 @@ +namespace LibHac.Os.Impl; + +public abstract class MultiWaitHolderBase +{ + private MultiWaitImpl _multiWait; + + // LibHac addition because we can't reinterpret_cast a MultiWaitHolderBase + // to a MultiWaitHolderType like the original does in c++ + public MultiWaitHolderType Holder { get; protected set; } + + public abstract TriBool IsSignaled(); + public abstract TriBool AddToObjectList(); + public abstract void RemoveFromObjectList(); + public abstract bool GetNativeHandle(out OsNativeHandle handle); + + public virtual TimeSpan GetAbsoluteTimeToWakeup() + { + return TimeSpan.FromNanoSeconds(long.MaxValue); + } + + public void SetMultiWait(MultiWaitImpl multiWait) + { + _multiWait = multiWait; + } + + public MultiWaitImpl GetMultiWait() + { + return _multiWait; + } + + public bool IsLinked() + { + return _multiWait is not null; + } + + public bool IsNotLinked() + { + return _multiWait is null; + } +} + +public abstract class MultiWaitHolderOfUserWaitObject : MultiWaitHolderBase +{ + public override bool GetNativeHandle(out OsNativeHandle handle) + { + handle = default; + return false; + } +} + +public abstract class MultiWaitHolderOfNativeWaitObject : MultiWaitHolderBase +{ + public override TriBool AddToObjectList() + { + return TriBool.Undefined; + } + + public override void RemoveFromObjectList() { /* ... */ } +} \ No newline at end of file diff --git a/src/LibHac/Os/Impl/MultipleWaitHolderOfNativeHandle.cs b/src/LibHac/Os/Impl/MultipleWaitHolderOfNativeHandle.cs new file mode 100644 index 00000000..7c8cbd17 --- /dev/null +++ b/src/LibHac/Os/Impl/MultipleWaitHolderOfNativeHandle.cs @@ -0,0 +1,22 @@ +namespace LibHac.Os.Impl; + +public class MultiWaitHolderOfNativeHandle : MultiWaitHolderOfNativeWaitObject +{ + private OsNativeHandle _handle; + + internal MultiWaitHolderOfNativeHandle(OsNativeHandle handle) + { + _handle = handle; + } + + public override TriBool IsSignaled() + { + return TriBool.Undefined; + } + + public override bool GetNativeHandle(out OsNativeHandle handle) + { + handle = _handle; + return false; + } +} \ No newline at end of file diff --git a/src/LibHac/Os/Impl/MultipleWaitImpl.cs b/src/LibHac/Os/Impl/MultipleWaitImpl.cs new file mode 100644 index 00000000..d8cdd536 --- /dev/null +++ b/src/LibHac/Os/Impl/MultipleWaitImpl.cs @@ -0,0 +1,349 @@ +using System; +using System.Collections.Generic; +using LibHac.Diag; + +namespace LibHac.Os.Impl; + +public class MultiWaitImpl : IDisposable +{ + public const int MaximumHandleCount = 64; + public const int WaitInvalid = -3; + public const int WaitCancelled = -2; + public const int WaitTimedOut = -1; + + private LinkedList _multiWaitList; + private MultiWaitHolderBase _signaledHolder; + private TimeSpan _currentTime; + private InternalCriticalSection _csWait; + private MultiWaitTargetImpl _targetImpl; + + // LibHac additions + private OsState _os; + private MultiWaitType _parent; + public MultiWaitType GetMultiWaitType() => _parent; + + public MultiWaitImpl(OsState os, MultiWaitType parent) + { + _multiWaitList = new LinkedList(); + _currentTime = new TimeSpan(0); + _csWait = new InternalCriticalSection(); + _targetImpl = new MultiWaitTargetImpl(os); + + _os = os; + _parent = parent; + } + + public void Dispose() + { + _csWait.Dispose(); + _targetImpl.Dispose(); + } + + public MultiWaitHolderBase WaitAny() + { + return WaitAnyImpl(infinite: true, TimeSpan.FromNanoSeconds(long.MaxValue)); + } + + public MultiWaitHolderBase TryWaitAny() + { + return WaitAnyImpl(infinite: false, new TimeSpan(0)); + } + + public MultiWaitHolderBase TimedWaitAny(TimeSpan timeout) + { + return WaitAnyImpl(infinite: false, timeout); + } + + public Result ReplyAndReceive(out MultiWaitHolderBase outHolder, OsNativeHandle replyTarget) + { + return WaitAnyImpl(out outHolder, infinite: true, TimeSpan.FromNanoSeconds(long.MaxValue), reply: true, + replyTarget); + } + + public bool IsListEmpty() + { + return _multiWaitList.Count == 0; + } + + public bool IsListNotEmpty() + { + return _multiWaitList.Count != 0; + } + + public void PushBackToList(MultiWaitHolderBase holder) + { + _multiWaitList.AddLast(holder); + } + + public void EraseFromList(MultiWaitHolderBase holder) + { + bool wasInList = _multiWaitList.Remove(holder); + Assert.SdkAssert(wasInList); + } + + public void EraseAllFromList() + { + _multiWaitList.Clear(); + } + + public void MoveAllFromOther(MultiWaitImpl other) + { + // Set ourselves as multi wait for all of the other's holders. + foreach (MultiWaitHolderBase holder in other._multiWaitList) + { + holder.SetMultiWait(this); + } + + LinkedListNode node = other._multiWaitList.First; + + while (node is not null) + { + other._multiWaitList.Remove(node); + _multiWaitList.AddLast(node); + + node = other._multiWaitList.First; + } + } + + public TimeSpan GetCurrentTime() + { + return _currentTime; + } + + public void NotifyAndWakeupThread(MultiWaitHolderBase holder) + { + using ScopedLock lk = ScopedLock.Lock(ref _csWait); + + if (_signaledHolder is null) + { + _signaledHolder = holder; + _targetImpl.CancelWait(); + } + } + + private MultiWaitHolderBase WaitAnyImpl(bool infinite, TimeSpan timeout) + { + Result waitResult = WaitAnyImpl(out MultiWaitHolderBase holder, infinite, timeout, false, OsTypes.InvalidNativeHandle); + + Assert.SdkAssert(waitResult.IsSuccess()); + + return holder; + } + + private Result WaitAnyImpl(out MultiWaitHolderBase outHolder, bool infinite, TimeSpan timeout, bool reply, + OsNativeHandle replyTarget) + { + // Prepare for processing. + _signaledHolder = null; + _targetImpl.SetCurrentThreadHandleForCancelWait(); + MultiWaitHolderBase holder = AddToEachObjectListAndCheckObjectState(); + + // Check if we've been signaled. + using (ScopedLock.Lock(ref _csWait)) + { + if (_signaledHolder is not null) + holder = _signaledHolder; + } + + // Process object array. + Result waitResult = Result.Success; + if (holder is null) + { + waitResult = InternalWaitAnyImpl(out holder, infinite, timeout, reply, replyTarget); + } + else if (reply && replyTarget != OsTypes.InvalidNativeHandle) + { + waitResult = _targetImpl.TimedReplyAndReceive(out int _, null, num: 0, replyTarget, + new TimeSpan(0)); + + if (waitResult.IsFailure()) + holder = null; + } + + // Unlink holders from the current object list. + RemoveFromEachObjectList(); + + _targetImpl.ClearCurrentThreadHandleForCancelWait(); + + outHolder = holder; + return waitResult; + } + + private Result InternalWaitAnyImpl(out MultiWaitHolderBase outHolder, bool infinite, TimeSpan timeout, bool reply, + OsNativeHandle replyTarget) + { + var objectsArray = new OsNativeHandle[MaximumHandleCount]; + var objectsArrayToHolder = new MultiWaitHolderBase[MaximumHandleCount]; + + int objectCount = ConstructObjectsArray(objectsArray, objectsArrayToHolder, MaximumHandleCount); + + TimeSpan absoluteEndTime = infinite + ? TimeSpan.FromNanoSeconds(long.MaxValue) + : _os.GetCurrentTick().ToTimeSpan(_os) + timeout; + + while (true) + { + _currentTime = _os.GetCurrentTick().ToTimeSpan(_os); + + MultiWaitHolderBase minTimeoutObject = RecalcMultiWaitTimeout(out TimeSpan timeoutMin, absoluteEndTime); + + int index; + Result waitResult = Result.Success; + + if (reply) + { + if (infinite && minTimeoutObject is null) + { + waitResult = _targetImpl.ReplyAndReceive(out index, objectsArray, objectCount, replyTarget); + } + else + { + waitResult = _targetImpl.TimedReplyAndReceive(out index, objectsArray, objectCount, replyTarget, timeoutMin); + } + } + else + { + if (infinite && minTimeoutObject is null) + { + waitResult = _targetImpl.WaitAny(out index, objectsArray, objectCount); + } + else + { + if (objectCount == 0 && timeoutMin == new TimeSpan(0)) + { + index = WaitTimedOut; + } + else + { + waitResult = _targetImpl.TimedWaitAny(out index, objectsArray, objectCount, timeoutMin); + } + } + } + + switch (index) + { + case WaitTimedOut: + if (minTimeoutObject is not null) + { + _currentTime = _os.GetCurrentTick().ToTimeSpan(_os); + + if (minTimeoutObject.IsSignaled() == TriBool.True) + { + using ScopedLock lk = ScopedLock.Lock(ref _csWait); + + _signaledHolder = minTimeoutObject; + outHolder = minTimeoutObject; + return waitResult; + } + } + else + { + outHolder = null; + return waitResult; + } + break; + case WaitCancelled: + { + using ScopedLock lk = ScopedLock.Lock(ref _csWait); + + if (_signaledHolder is not null) + { + outHolder = _signaledHolder; + return waitResult; + } + + break; + } + case WaitInvalid: + outHolder = null; + return waitResult; + default: + { + Assert.SdkAssert(index >= 0 && index < objectCount); + + using ScopedLock lk = ScopedLock.Lock(ref _csWait); + + _signaledHolder = objectsArrayToHolder[index]; + outHolder = _signaledHolder; + return waitResult; + } + } + + replyTarget = OsTypes.InvalidNativeHandle; + } + } + + public int ConstructObjectsArray(Span outHandles, Span outObjects, int num) + { + Assert.SdkRequiresGreaterEqual(outHandles.Length, num); + Assert.SdkRequiresGreaterEqual(outObjects.Length, num); + + int count = 0; + + foreach (MultiWaitHolderBase holderBase in _multiWaitList) + { + if (holderBase.GetNativeHandle(out OsNativeHandle handle)) + { + Abort.DoAbortUnless(count < num); + + outHandles[count] = handle; + outObjects[count] = holderBase; + count++; + } + } + + return count; + } + + private MultiWaitHolderBase AddToEachObjectListAndCheckObjectState() + { + MultiWaitHolderBase signaledHolder = null; + + foreach (MultiWaitHolderBase holderBase in _multiWaitList) + { + TriBool isSignaled = holderBase.AddToObjectList(); + + if (signaledHolder is null && isSignaled == TriBool.True) + { + signaledHolder = holderBase; + } + } + + return signaledHolder; + } + + private void RemoveFromEachObjectList() + { + foreach (MultiWaitHolderBase holderBase in _multiWaitList) + { + holderBase.RemoveFromObjectList(); + } + } + + public MultiWaitHolderBase RecalcMultiWaitTimeout(out TimeSpan outMinTimeout, TimeSpan endTime) + { + MultiWaitHolderBase minTimeoutHolder = null; + TimeSpan endTimeMin = endTime; + + foreach (MultiWaitHolderBase holderBase in _multiWaitList) + { + TimeSpan wakeupTime = holderBase.GetAbsoluteTimeToWakeup(); + if (wakeupTime < endTimeMin) + { + endTimeMin = wakeupTime; + minTimeoutHolder = holderBase; + } + } + + if (endTimeMin < _currentTime) + { + outMinTimeout = new TimeSpan(0); + } + else + { + outMinTimeout = endTimeMin - _currentTime; + } + + return minTimeoutHolder; + } +} \ No newline at end of file diff --git a/src/LibHac/Os/Impl/MultipleWaitObjectList.cs b/src/LibHac/Os/Impl/MultipleWaitObjectList.cs new file mode 100644 index 00000000..b4354ba3 --- /dev/null +++ b/src/LibHac/Os/Impl/MultipleWaitObjectList.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; +using LibHac.Diag; + +namespace LibHac.Os.Impl; + +public class MultiWaitObjectList +{ + private LinkedList _objectList; + + public MultiWaitObjectList() + { + _objectList = new LinkedList(); + } + + public void WakeupAllMultiWaitThreadsUnsafe() + { + foreach (MultiWaitHolderBase holderBase in _objectList) + { + holderBase.GetMultiWait().NotifyAndWakeupThread(holderBase); + } + } + + public void BroadcastToUpdateObjectStateUnsafe() + { + foreach (MultiWaitHolderBase holderBase in _objectList) + { + holderBase.GetMultiWait().NotifyAndWakeupThread(null); + } + } + + public bool IsEmpty() + { + return _objectList.Count == 0; + } + + public void PushBackToList(MultiWaitHolderBase holderBase) + { + _objectList.AddLast(holderBase); + } + + public void EraseFromList(MultiWaitHolderBase holderBase) + { + Assert.SdkRequires(_objectList.Contains(holderBase)); + + _objectList.Remove(holderBase); + } +} \ No newline at end of file diff --git a/src/LibHac/Os/Impl/MultipleWaitTargetImpl-os.net.cs b/src/LibHac/Os/Impl/MultipleWaitTargetImpl-os.net.cs new file mode 100644 index 00000000..003a5a96 --- /dev/null +++ b/src/LibHac/Os/Impl/MultipleWaitTargetImpl-os.net.cs @@ -0,0 +1,116 @@ +using System; +using System.Threading; +using LibHac.Common; +using LibHac.Diag; +using LibHac.Fs; + +namespace LibHac.Os.Impl; + +public class MultiWaitTargetImpl : IDisposable +{ + private EventWaitHandle _cancelEvent; + + // LibHac addition + private OsState _os; + + public MultiWaitTargetImpl(OsState os) + { + _cancelEvent = new EventWaitHandle(false, EventResetMode.AutoReset); + _os = os; + } + + public void Dispose() + { + _cancelEvent.Dispose(); + } + + public void CancelWait() + { + _cancelEvent.Set(); + } + + public Result WaitAny(out int index, Span handles, int num) + { + return WaitForAnyObjects(out index, num, handles, int.MaxValue); + } + + public Result TimedWaitAny(out int outIndex, Span handles, int num, TimeSpan timeout) + { + UnsafeHelpers.SkipParamInit(out outIndex); + + var timeoutHelper = new TimeoutHelper(_os, timeout); + + do + { + Result rc = WaitForAnyObjects(out int index, num, handles, + (int)timeoutHelper.GetTimeLeftOnTarget().GetMilliSeconds()); + if (rc.IsFailure()) return rc.Miss(); + + if (index == MultiWaitImpl.WaitTimedOut) + { + outIndex = index; + return Result.Success; + } + } while (!timeoutHelper.TimedOut()); + + outIndex = MultiWaitImpl.WaitTimedOut; + return Result.Success; + } + + public Result ReplyAndReceive(out int index, Span handles, int num, WaitHandle replyTarget) + { + return ReplyAndReceiveImpl(out index, handles, num, replyTarget, TimeSpan.FromNanoSeconds(long.MaxValue)); + } + + public Result TimedReplyAndReceive(out int index, Span handles, int num, WaitHandle replyTarget, + TimeSpan timeout) + { + return ReplyAndReceiveImpl(out index, handles, num, replyTarget, timeout); + } + + public void SetCurrentThreadHandleForCancelWait() + { + /* ... */ + } + + public void ClearCurrentThreadHandleForCancelWait() + { + /* ... */ + } + + private Result WaitForAnyObjects(out int outIndex, int num, Span handles, int timeoutMs) + { + // Check that we can add our cancel handle to the wait. + Abort.DoAbortUnless(num + 1 < handles.Length); + + handles[num] = _cancelEvent; + int index = WaitHandle.WaitAny(handles.Slice(0, num + 1).ToArray(), timeoutMs); + + if (index == WaitHandle.WaitTimeout) + { + outIndex = MultiWaitImpl.WaitTimedOut; + return Result.Success; + } + + Assert.SdkAssert(index >= 0 && index <= num); + + if (index == num) + { + outIndex = MultiWaitImpl.WaitCancelled; + return Result.Success; + } + + outIndex = index; + return Result.Success; + } + + private Result ReplyAndReceiveImpl(out int outIndex, Span handles, int num, WaitHandle replyTarget, + TimeSpan timeout) + { + UnsafeHelpers.SkipParamInit(out outIndex); + + Abort.DoAbortUnlessSuccess(ResultFs.NotImplemented.Value); + + return ResultFs.NotImplemented.Log(); + } +} \ No newline at end of file diff --git a/src/LibHac/Os/MultipleWait.cs b/src/LibHac/Os/MultipleWait.cs new file mode 100644 index 00000000..68b24922 --- /dev/null +++ b/src/LibHac/Os/MultipleWait.cs @@ -0,0 +1,179 @@ +using LibHac.Diag; +using LibHac.Os.Impl; + +namespace LibHac.Os; + +// Todo: Handling waiting in .NET in an OS-agnostic way requires using WaitHandles. +// I'm not sure if this might cause issues in the future if we need to wait on objects other than +// those supported by WaitHandle. +public static class MultipleWait +{ + private static MultiWaitImpl GetMultiWaitImpl(MultiWaitType multiWait) + { + return multiWait.Impl; + } + + private static MultiWaitHolderType CastToMultiWaitHolder(MultiWaitHolderBase holderBase) + { + return holderBase.Holder; + } + + // Note: The "IsWaiting" field is only used in develop builds + public static void InitializeMultiWait(this OsState os, MultiWaitType multiWait) + { + multiWait.Impl = new MultiWaitImpl(os, multiWait); + + multiWait.IsWaiting = false; + MemoryFenceApi.FenceMemoryStoreStore(); + + multiWait.CurrentState = MultiWaitType.State.Initialized; + } + + public static void FinalizeMultiWait(this OsState os, MultiWaitType multiWait) + { + MultiWaitImpl impl = GetMultiWaitImpl(multiWait); + + Assert.SdkRequires(multiWait.CurrentState == MultiWaitType.State.Initialized); + Assert.SdkRequires(impl.IsListEmpty()); + + multiWait.CurrentState = MultiWaitType.State.NotInitialized; + impl.Dispose(); + } + + public static MultiWaitHolderType WaitAny(this OsState os, MultiWaitType multiWait) + { + MultiWaitImpl impl = GetMultiWaitImpl(multiWait); + + Assert.SdkRequires(multiWait.CurrentState == MultiWaitType.State.Initialized); + Assert.SdkRequires(impl.IsListNotEmpty()); + + multiWait.IsWaiting = true; + MemoryFenceApi.FenceMemoryStoreAny(); + + MultiWaitHolderType holder = CastToMultiWaitHolder(impl.WaitAny()); + + MemoryFenceApi.FenceMemoryAnyStore(); + multiWait.IsWaiting = false; + + Assert.SdkAssert(holder is not null); + return holder; + } + + public static MultiWaitHolderType TryWaitAny(this OsState os, MultiWaitType multiWait) + { + MultiWaitImpl impl = GetMultiWaitImpl(multiWait); + + Assert.SdkRequires(multiWait.CurrentState == MultiWaitType.State.Initialized); + Assert.SdkRequires(impl.IsListNotEmpty()); + + multiWait.IsWaiting = true; + MemoryFenceApi.FenceMemoryStoreAny(); + + MultiWaitHolderType holder = CastToMultiWaitHolder(impl.TryWaitAny()); + + MemoryFenceApi.FenceMemoryAnyStore(); + multiWait.IsWaiting = false; + + return holder; + } + + public static MultiWaitHolderType TimedWaitAny(this OsState os, MultiWaitType multiWait, TimeSpan timeout) + { + MultiWaitImpl impl = GetMultiWaitImpl(multiWait); + + Assert.SdkRequires(multiWait.CurrentState == MultiWaitType.State.Initialized); + Assert.SdkRequires(impl.IsListNotEmpty()); + Assert.SdkRequires(timeout.GetNanoSeconds() >= 0); + + multiWait.IsWaiting = true; + MemoryFenceApi.FenceMemoryStoreAny(); + + MultiWaitHolderType holder = CastToMultiWaitHolder(impl.TimedWaitAny(timeout)); + + MemoryFenceApi.FenceMemoryAnyStore(); + multiWait.IsWaiting = false; + + return holder; + } + + public static void FinalizeMultiWaitHolder(this OsState os, MultiWaitHolderType holder) + { + MultiWaitHolderBase holderBase = holder.Impl.HolderBase; + Assert.SdkRequires(holderBase.IsNotLinked()); + } + + public static void LinkMultiWaitHolder(this OsState os, MultiWaitType multiWait, MultiWaitHolderType holder) + { + MultiWaitImpl impl = GetMultiWaitImpl(multiWait); + MultiWaitHolderBase holderBase = holder.Impl.HolderBase; + + Assert.SdkRequires(multiWait.CurrentState == MultiWaitType.State.Initialized); + Assert.SdkRequires(holderBase.IsNotLinked()); + + Assert.SdkEqual(false, multiWait.IsWaiting); + MemoryFenceApi.FenceMemoryLoadAny(); + + impl.PushBackToList(holderBase); + holderBase.SetMultiWait(impl); + } + + public static void UnlinkMultiWaitHolder(this OsState os, MultiWaitHolderType holder) + { + MultiWaitHolderBase holderBase = holder.Impl.HolderBase; + + Assert.SdkRequires(holderBase.IsLinked()); + + Assert.SdkEqual(false, holderBase.GetMultiWait().GetMultiWaitType().IsWaiting); + MemoryFenceApi.FenceMemoryLoadAny(); + + holderBase.GetMultiWait().EraseFromList(holderBase); + holderBase.SetMultiWait(null); + } + + public static void UnlinkAllMultiWaitHolder(this OsState os, MultiWaitType multiWait) + { + MultiWaitImpl impl = GetMultiWaitImpl(multiWait); + + Assert.SdkRequires(multiWait.CurrentState == MultiWaitType.State.Initialized); + + Assert.SdkEqual(false, multiWait.IsWaiting); + MemoryFenceApi.FenceMemoryLoadAny(); + + impl.EraseAllFromList(); + } + + public static void MoveAllMultiWaitHolder(this OsState os, MultiWaitType dest, MultiWaitType source) + { + MultiWaitImpl dstImpl = GetMultiWaitImpl(dest); + MultiWaitImpl srcImpl = GetMultiWaitImpl(source); + + Assert.SdkRequires(dest.CurrentState == MultiWaitType.State.Initialized); + Assert.SdkRequires(source.CurrentState == MultiWaitType.State.Initialized); + + Assert.SdkEqual(false, dest.IsWaiting); + MemoryFenceApi.FenceMemoryLoadAny(); + + Assert.SdkEqual(false, source.IsWaiting); + MemoryFenceApi.FenceMemoryLoadAny(); + + dstImpl.MoveAllFromOther(srcImpl); + } + + public static void SetMultiWaitHolderUserData(this OsState os, MultiWaitHolderType holder, object userData) + { + holder.UserData = userData; + } + + public static object GetMultiWaitHolderUserData(this OsState os, MultiWaitHolderType holder) + { + return holder.UserData; + } + + public static void InitializeMultiWaitHolder(this OsState os, MultiWaitHolderType holder, OsNativeHandle handle) + { + Assert.SdkRequires(handle != OsTypes.InvalidNativeHandle); + + holder.Impl = new MultiWaitHolderImpl(new MultiWaitHolderOfNativeHandle(handle)); + holder.UserData = null; + } +} \ No newline at end of file diff --git a/src/LibHac/Os/MultipleWaitTypes.cs b/src/LibHac/Os/MultipleWaitTypes.cs new file mode 100644 index 00000000..3e2f7625 --- /dev/null +++ b/src/LibHac/Os/MultipleWaitTypes.cs @@ -0,0 +1,22 @@ +using LibHac.Os.Impl; + +namespace LibHac.Os; + +public class MultiWaitType +{ + public enum State : byte + { + NotInitialized, + Initialized + } + + public State CurrentState; + public bool IsWaiting; + public MultiWaitImpl Impl; +} + +public class MultiWaitHolderType +{ + public MultiWaitHolderImpl Impl; + public object UserData; +} \ No newline at end of file diff --git a/src/LibHac/Os/OsTypes.cs b/src/LibHac/Os/OsTypes.cs new file mode 100644 index 00000000..8e870542 --- /dev/null +++ b/src/LibHac/Os/OsTypes.cs @@ -0,0 +1,8 @@ +global using OsNativeHandle = System.Threading.WaitHandle; + +namespace LibHac.Os; + +public static class OsTypes +{ + public static OsNativeHandle InvalidNativeHandle => default; +} \ No newline at end of file From 7dd137da6a60f13ee8f63f67145016e55354892d Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 11 Apr 2022 15:03:27 -0700 Subject: [PATCH 15/24] Add os::Semaphore --- .../Impl/InternalConditionVariable-os.net.cs | 6 +- .../Os/Impl/InternalConditionVariable.cs | 11 +- src/LibHac/Os/Impl/MultiWaitHolderImpl.cs | 7 +- .../Os/Impl/MultipleWaitHolderOfSemaphore.cs | 38 ++++ src/LibHac/Os/Semaphore.cs | 206 ++++++++++++++++++ 5 files changed, 260 insertions(+), 8 deletions(-) create mode 100644 src/LibHac/Os/Impl/MultipleWaitHolderOfSemaphore.cs create mode 100644 src/LibHac/Os/Semaphore.cs diff --git a/src/LibHac/Os/Impl/InternalConditionVariable-os.net.cs b/src/LibHac/Os/Impl/InternalConditionVariable-os.net.cs index 5a5da6d4..571d4691 100644 --- a/src/LibHac/Os/Impl/InternalConditionVariable-os.net.cs +++ b/src/LibHac/Os/Impl/InternalConditionVariable-os.net.cs @@ -1,13 +1,15 @@ -using System.Threading; +using System; +using System.Threading; using LibHac.Diag; namespace LibHac.Os.Impl; -internal struct InternalConditionVariableImpl +internal struct InternalConditionVariableImpl : IDisposable { private object _obj; public InternalConditionVariableImpl() => _obj = new object(); + public void Dispose() { } public void Initialize() => _obj = new object(); public void Signal() diff --git a/src/LibHac/Os/Impl/InternalConditionVariable.cs b/src/LibHac/Os/Impl/InternalConditionVariable.cs index 143a1db0..a5510efa 100644 --- a/src/LibHac/Os/Impl/InternalConditionVariable.cs +++ b/src/LibHac/Os/Impl/InternalConditionVariable.cs @@ -1,6 +1,8 @@ -namespace LibHac.Os.Impl; +using System; -internal struct InternalConditionVariable +namespace LibHac.Os.Impl; + +internal struct InternalConditionVariable : IDisposable { private InternalConditionVariableImpl _impl; @@ -9,6 +11,11 @@ public InternalConditionVariable() _impl = new InternalConditionVariableImpl(); } + public void Dispose() + { + _impl.Dispose(); + } + public void Initialize() => _impl.Initialize(); public void Signal() => _impl.Signal(); public void Broadcast() => _impl.Broadcast(); diff --git a/src/LibHac/Os/Impl/MultiWaitHolderImpl.cs b/src/LibHac/Os/Impl/MultiWaitHolderImpl.cs index 1564e186..3f40abf9 100644 --- a/src/LibHac/Os/Impl/MultiWaitHolderImpl.cs +++ b/src/LibHac/Os/Impl/MultiWaitHolderImpl.cs @@ -6,9 +6,8 @@ public class MultiWaitHolderImpl public MultiWaitHolderBase HolderBase => _holder; public MultiWaitHolderOfNativeHandle HolderOfNativeHandle => (MultiWaitHolderOfNativeHandle)_holder; + public MultiWaitHolderOfSemaphore HolderOfSemaphore => (MultiWaitHolderOfSemaphore)_holder; - public MultiWaitHolderImpl(MultiWaitHolderOfNativeHandle holder) - { - _holder = holder; - } + public MultiWaitHolderImpl(MultiWaitHolderOfNativeHandle holder) => _holder = holder; + public MultiWaitHolderImpl(MultiWaitHolderOfSemaphore holder) => _holder = holder; } \ No newline at end of file diff --git a/src/LibHac/Os/Impl/MultipleWaitHolderOfSemaphore.cs b/src/LibHac/Os/Impl/MultipleWaitHolderOfSemaphore.cs new file mode 100644 index 00000000..8c0642f8 --- /dev/null +++ b/src/LibHac/Os/Impl/MultipleWaitHolderOfSemaphore.cs @@ -0,0 +1,38 @@ +namespace LibHac.Os.Impl; + +public class MultiWaitHolderOfSemaphore : MultiWaitHolderOfUserWaitObject +{ + private Semaphore _semaphore; + + public MultiWaitHolderOfSemaphore(Semaphore semaphore) + { + _semaphore = semaphore; + } + + public override TriBool IsSignaled() + { + using ScopedLock lk = ScopedLock.Lock(ref _semaphore.GetBase().CsSemaphore); + + return IsSignaledUnsafe(); + } + + public override TriBool AddToObjectList() + { + using ScopedLock lk = ScopedLock.Lock(ref _semaphore.GetBase().CsSemaphore); + + _semaphore.GetBase().WaitList.PushBackToList(this); + return IsSignaledUnsafe(); + } + + public override void RemoveFromObjectList() + { + using ScopedLock lk = ScopedLock.Lock(ref _semaphore.GetBase().CsSemaphore); + + _semaphore.GetBase().WaitList.EraseFromList(this); + } + + private TriBool IsSignaledUnsafe() + { + return _semaphore.GetBase().Count > 0 ? TriBool.True : TriBool.False; + } +} \ No newline at end of file diff --git a/src/LibHac/Os/Semaphore.cs b/src/LibHac/Os/Semaphore.cs new file mode 100644 index 00000000..a38e73d2 --- /dev/null +++ b/src/LibHac/Os/Semaphore.cs @@ -0,0 +1,206 @@ +using System; +using LibHac.Diag; +using LibHac.Os.Impl; + +namespace LibHac.Os; + +public class Semaphore : IDisposable +{ + private SemaphoreType _semaphore; + + public Semaphore(OsState os, int initialCount, int maxCount) => + _semaphore = new SemaphoreType(os, initialCount, maxCount); + + public void Dispose() => _semaphore.Dispose(); + public void Acquire() => _semaphore.Acquire(); + public bool TryAcquire() => _semaphore.TryAcquire(); + public bool TimedAcquire(TimeSpan timeout) => _semaphore.TimedAcquire(timeout); + public void Release() => _semaphore.Release(); + public void Release(int count) => _semaphore.Release(count); + public int GetCurrentCount() => _semaphore.GetCurrentCount(); + public ref SemaphoreType GetBase() => ref _semaphore; +} + +public struct SemaphoreType : IDisposable +{ + public enum State : byte + { + NotInitialized = 0, + Initialized = 1 + } + + internal MultiWaitObjectList WaitList; + internal State CurrentState; + internal int Count; + internal int MaxCount; + + internal InternalCriticalSection CsSemaphore; + internal InternalConditionVariable CvNotZero; + + private readonly OsState _os; + + public SemaphoreType(OsState os, int initialCount, int maxCount) + { + Assert.SdkRequires(maxCount >= 1); + Assert.SdkRequires(initialCount >= 0 && initialCount <= maxCount); + + CsSemaphore = new InternalCriticalSection(); + CvNotZero = new InternalConditionVariable(); + + WaitList = new MultiWaitObjectList(); + + Count = initialCount; + MaxCount = maxCount; + CurrentState = State.Initialized; + _os = os; + } + + public void Dispose() + { + Assert.SdkRequires(CurrentState == State.Initialized); + Assert.SdkRequires(WaitList.IsEmpty()); + + CurrentState = State.NotInitialized; + + CsSemaphore.Dispose(); + CvNotZero.Dispose(); + } + + public void Acquire() + { + Assert.SdkRequires(CurrentState == State.Initialized); + + using ScopedLock lk = ScopedLock.Lock(ref CsSemaphore); + + while (Count == 0) + { + CvNotZero.Wait(ref CsSemaphore); + } + + Count--; + } + + public bool TryAcquire() + { + Assert.SdkRequires(CurrentState == State.Initialized); + + using ScopedLock lk = ScopedLock.Lock(ref CsSemaphore); + + if (Count == 0) + { + return false; + } + + Count--; + + return true; + } + + public bool TimedAcquire(TimeSpan timeout) + { + Assert.SdkRequires(CurrentState == State.Initialized); + Assert.SdkRequires(timeout.GetNanoSeconds() >= 0); + + var timeoutHelper = new TimeoutHelper(_os, timeout); + using ScopedLock lk = ScopedLock.Lock(ref CsSemaphore); + + while (Count == 0) + { + if (timeoutHelper.TimedOut()) + return false; + + CvNotZero.TimedWait(ref CsSemaphore, in timeoutHelper); + } + + Count--; + + return true; + } + + public void Release() + { + Assert.SdkRequires(CurrentState == State.Initialized); + + using ScopedLock lk = ScopedLock.Lock(ref CsSemaphore); + + Assert.SdkAssert(Count + 1 <= MaxCount); + + Count++; + + CvNotZero.Signal(); + WaitList.WakeupAllMultiWaitThreadsUnsafe(); + } + + public void Release(int count) + { + Assert.SdkRequires(CurrentState == State.Initialized); + Assert.SdkRequires(Count >= 1); + + using ScopedLock lk = ScopedLock.Lock(ref CsSemaphore); + + Assert.SdkAssert(Count + count <= MaxCount); + + Count += count; + + CvNotZero.Broadcast(); + WaitList.WakeupAllMultiWaitThreadsUnsafe(); + } + + public readonly int GetCurrentCount() + { + Assert.SdkRequires(CurrentState == State.Initialized); + + return Count; + } +} + +public static class SemaphoreApi +{ + public static void InitializeSemaphore(this OsState os, out SemaphoreType semaphore, int initialCount, int maxCount) + { + semaphore = new SemaphoreType(os, initialCount, maxCount); + } + + public static void FinalizeSemaphore(this OsState os, ref SemaphoreType semaphore) + { + semaphore.Dispose(); + } + + public static void AcquireSemaphore(this OsState os, ref SemaphoreType semaphore) + { + semaphore.Acquire(); + } + + public static bool TryAcquireSemaphore(this OsState os, ref SemaphoreType semaphore) + { + return semaphore.TryAcquire(); + } + + public static bool TimedAcquireSemaphore(this OsState os, ref SemaphoreType semaphore, TimeSpan timeout) + { + return semaphore.TimedAcquire(timeout); + } + + public static void ReleaseSemaphore(this OsState os, ref SemaphoreType semaphore) + { + semaphore.Release(); + } + + public static void ReleaseSemaphore(this OsState os, ref SemaphoreType semaphore, int count) + { + semaphore.Release(count); + } + + public static int GetCurrentSemaphoreCount(this OsState os, in SemaphoreType semaphore) + { + return semaphore.GetCurrentCount(); + } + + public static void InitializeMultiWaitHolder(this OsState os, MultiWaitHolderType holder, Semaphore semaphore) + { + Assert.SdkRequires(semaphore.GetBase().CurrentState == SemaphoreType.State.Initialized); + + holder.Impl = new MultiWaitHolderImpl(new MultiWaitHolderOfSemaphore(semaphore)); + holder.UserData = null; + } +} \ No newline at end of file From fc3fb188c78248fa43c143a2bbe91d52bc3b4a4a Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Thu, 14 Apr 2022 14:18:24 -0700 Subject: [PATCH 16/24] Update IStorage range check functions for 14.0.0 --- src/LibHac/Fs/Common/FileStorage.cs | 66 +++++++-------- src/LibHac/Fs/IStorage.cs | 80 +++++++++++++++---- src/LibHac/Fs/MemoryStorage.cs | 10 +-- src/LibHac/Fs/SubStorage.cs | 44 ++++++---- src/LibHac/Fs/ValueSubStorage.cs | 33 +++++--- .../FsSystem/AlignmentMatchingStorage.cs | 40 +++++----- src/LibHac/Tools/FsSystem/AesCbcStorage.cs | 6 +- src/LibHac/Tools/FsSystem/CachedStorage.cs | 8 +- .../Tools/FsSystem/ConcatenationStorage.cs | 12 +-- .../Tools/FsSystem/Save/DuplexStorage.cs | 12 +-- .../Tools/FsSystem/Save/JournalStorage.cs | 12 +-- src/LibHac/Util/IntUtil.cs | 37 +++++++++ 12 files changed, 236 insertions(+), 124 deletions(-) create mode 100644 src/LibHac/Util/IntUtil.cs diff --git a/src/LibHac/Fs/Common/FileStorage.cs b/src/LibHac/Fs/Common/FileStorage.cs index a12fce38..e6667c1c 100644 --- a/src/LibHac/Fs/Common/FileStorage.cs +++ b/src/LibHac/Fs/Common/FileStorage.cs @@ -11,7 +11,7 @@ namespace LibHac.Fs; /// /// Allows interacting with an via an interface. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class FileStorage : IStorage { private const long InvalidSize = -1; @@ -60,8 +60,8 @@ public override Result Read(long offset, Span destination) Result rc = UpdateSize(); if (rc.IsFailure()) return rc.Miss(); - if (!CheckAccessRange(offset, destination.Length, _fileSize)) - return ResultFs.OutOfRange.Log(); + rc = CheckAccessRange(offset, destination.Length, _fileSize); + if (rc.IsFailure()) return rc.Miss(); return _baseFile.Read(out _, offset, destination, ReadOption.None); } @@ -74,8 +74,8 @@ public override Result Write(long offset, ReadOnlySpan source) Result rc = UpdateSize(); if (rc.IsFailure()) return rc.Miss(); - if (!CheckAccessRange(offset, source.Length, _fileSize)) - return ResultFs.OutOfRange.Log(); + rc = CheckAccessRange(offset, source.Length, _fileSize); + if (rc.IsFailure()) return rc.Miss(); return _baseFile.Write(offset, source, WriteOption.None); } @@ -104,35 +104,37 @@ public override Result SetSize(long size) public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, ReadOnlySpan inBuffer) { - if (operationId == OperationId.InvalidateCache) - { - Result rc = _baseFile.OperateRange(OperationId.InvalidateCache, offset, size); - if (rc.IsFailure()) return rc.Miss(); - - return Result.Success; - } - - if (operationId == OperationId.QueryRange) + switch (operationId) { - if (size == 0) + case OperationId.InvalidateCache: { - if (outBuffer.Length != Unsafe.SizeOf()) - return ResultFs.InvalidSize.Log(); + Result rc = _baseFile.OperateRange(OperationId.InvalidateCache, offset, size); + if (rc.IsFailure()) return rc.Miss(); - SpanHelpers.AsStruct(outBuffer).Clear(); return Result.Success; } + case OperationId.QueryRange: + { + if (size == 0) + { + if (outBuffer.Length != Unsafe.SizeOf()) + return ResultFs.InvalidSize.Log(); - Result rc = UpdateSize(); - if (rc.IsFailure()) return rc.Miss(); + SpanHelpers.AsStruct(outBuffer).Clear(); + return Result.Success; + } - if (!CheckOffsetAndSize(offset, size)) - return ResultFs.OutOfRange.Log(); + Result rc = UpdateSize(); + if (rc.IsFailure()) return rc.Miss(); - return _baseFile.OperateRange(outBuffer, operationId, offset, size, inBuffer); - } + rc = CheckOffsetAndSize(offset, size); + if (rc.IsFailure()) return rc.Miss(); - return ResultFs.UnsupportedOperateRangeForFileStorage.Log(); + return _baseFile.OperateRange(outBuffer, operationId, offset, size, inBuffer); + } + default: + return ResultFs.UnsupportedOperateRangeForFileStorage.Log(); + } } private Result UpdateSize() @@ -153,7 +155,7 @@ private Result UpdateSize() /// interface. The opened file will automatically be closed when the /// is disposed. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class FileStorageBasedFileSystem : FileStorage { private SharedRef _baseFileSystem; @@ -196,7 +198,7 @@ public Result Initialize(ref SharedRef baseFileSystem, in Path path /// Provides an interface for interacting with an opened file from a mounted file system. /// The caller may choose whether or not the file will be closed when the is disposed. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class FileHandleStorage : IStorage { private const long InvalidSize = -1; @@ -230,7 +232,7 @@ public FileHandleStorage(FileSystemClient fsClient, FileHandle handle, bool clos _handle = handle; _closeFile = closeFile; _size = InvalidSize; - _mutex.Initialize(); + _mutex = new SdkMutexType(); } public override void Dispose() @@ -255,8 +257,8 @@ public override Result Read(long offset, Span destination) Result rc = UpdateSize(); if (rc.IsFailure()) return rc.Miss(); - if (!CheckAccessRange(offset, destination.Length, _size)) - return ResultFs.OutOfRange.Log(); + rc = CheckAccessRange(offset, destination.Length, _size); + if (rc.IsFailure()) return rc.Miss(); return _fsClient.ReadFile(_handle, offset, destination, ReadOption.None); } @@ -271,8 +273,8 @@ public override Result Write(long offset, ReadOnlySpan source) Result rc = UpdateSize(); if (rc.IsFailure()) return rc.Miss(); - if (!CheckAccessRange(offset, source.Length, _size)) - return ResultFs.OutOfRange.Log(); + rc = CheckAccessRange(offset, source.Length, _size); + if (rc.IsFailure()) return rc.Miss(); return _fsClient.WriteFile(_handle, offset, source, WriteOption.None); } diff --git a/src/LibHac/Fs/IStorage.cs b/src/LibHac/Fs/IStorage.cs index 8dd6c702..02b93c3a 100644 --- a/src/LibHac/Fs/IStorage.cs +++ b/src/LibHac/Fs/IStorage.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.CompilerServices; +using LibHac.Util; namespace LibHac.Fs; @@ -7,12 +8,7 @@ namespace LibHac.Fs; /// /// Provides an interface for reading and writing a sequence of bytes. /// -/// -/// The official IStorage makes the Read etc. methods abstract and doesn't -/// have DoRead etc. methods. We're using them here so we can make sure -/// the object isn't disposed before calling the method implementation. -/// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public abstract class IStorage : IDisposable { public virtual void Dispose() { } @@ -78,20 +74,70 @@ public Result OperateRange(OperationId operationId, long offset, long size) return OperateRange(Span.Empty, operationId, offset, size, ReadOnlySpan.Empty); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool CheckAccessRange(long offset, long size, long totalSize) + public static Result CheckAccessRange(long offset, long size, long totalSize) { - return offset >= 0 && - size >= 0 && - size <= totalSize && - offset <= totalSize - size; + if (offset < 0) + return ResultFs.InvalidOffset.Log(); + + if (size < 0) + return ResultFs.InvalidSize.Log(); + + if (!IntUtil.CanAddWithoutOverflow(offset, size)) + return ResultFs.OutOfRange.Log(); + + if (size + offset > totalSize) + return ResultFs.OutOfRange.Log(); + + return Result.Success; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool CheckOffsetAndSize(long offset, long size) + public static Result CheckAccessRange(long offset, ulong size, long totalSize) + { + Result rc = CheckAccessRange(offset, unchecked((long)size), totalSize); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public static Result CheckOffsetAndSize(long offset, long size) { - return offset >= 0 && - size >= 0 && - offset <= offset + size; + if (offset < 0) + return ResultFs.InvalidOffset.Log(); + + if (size < 0) + return ResultFs.InvalidSize.Log(); + + if (!IntUtil.CanAddWithoutOverflow(offset, size)) + return ResultFs.OutOfRange.Log(); + + return Result.Success; + } + + public static Result CheckOffsetAndSize(long offset, ulong size) + { + Result rc = CheckOffsetAndSize(offset, unchecked((long)size)); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public static Result CheckOffsetAndSizeWithResult(long offset, long size, Result resultOnFailure) + { + Result rc = CheckOffsetAndSize(offset, size); + + if (rc.IsFailure()) + return resultOnFailure.Log(); + + return Result.Success; + } + + public static Result CheckOffsetAndSizeWithResult(long offset, ulong size, Result resultOnFailure) + { + Result rc = CheckOffsetAndSize(offset, size); + + if (rc.IsFailure()) + return resultOnFailure.Log(); + + return Result.Success; } } \ No newline at end of file diff --git a/src/LibHac/Fs/MemoryStorage.cs b/src/LibHac/Fs/MemoryStorage.cs index f9cdf523..0e4bf511 100644 --- a/src/LibHac/Fs/MemoryStorage.cs +++ b/src/LibHac/Fs/MemoryStorage.cs @@ -7,7 +7,7 @@ namespace LibHac.Fs; /// /// Allows interacting with a array via the interface. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class MemoryStorage : IStorage { private byte[] _storageBuffer; @@ -22,8 +22,8 @@ public override Result Read(long offset, Span destination) if (destination.Length == 0) return Result.Success; - if (!CheckAccessRange(offset, destination.Length, _storageBuffer.Length)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, destination.Length, _storageBuffer.Length); + if (rc.IsFailure()) return rc.Miss(); _storageBuffer.AsSpan((int)offset, destination.Length).CopyTo(destination); @@ -35,8 +35,8 @@ public override Result Write(long offset, ReadOnlySpan source) if (source.Length == 0) return Result.Success; - if (!CheckAccessRange(offset, source.Length, _storageBuffer.Length)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, source.Length, _storageBuffer.Length); + if (rc.IsFailure()) return rc.Miss(); source.CopyTo(_storageBuffer.AsSpan((int)offset)); diff --git a/src/LibHac/Fs/SubStorage.cs b/src/LibHac/Fs/SubStorage.cs index 72394195..e6f23741 100644 --- a/src/LibHac/Fs/SubStorage.cs +++ b/src/LibHac/Fs/SubStorage.cs @@ -8,16 +8,17 @@ namespace LibHac.Fs; /// Presents a subsection of a base IStorage as a new IStorage. /// /// -/// A SubStorage presents a sub-range of an IStorage as a separate IStorage. +/// A SubStorage presents a sub-range of an IStorage as a separate IStorage. /// -/// The SubStorage doesn't check if the offset and size provided are actually in the base storage. +/// The SubStorage doesn't check if the offset and size provided are actually in the base storage. /// GetSize will return the size given to the SubStorage at initialization and will not query -/// the base storage's size. +/// the base storage's size. /// -/// A SubStorage is non-resizable by default. may be used to mark +/// A SubStorage is non-resizable by default. may be used to mark /// the SubStorage as resizable. The SubStorage may only be resized if the end of the SubStorage /// is located at the end of the base storage. When resizing the SubStorage, the base storage -/// will be resized to the appropriate length. +/// will be resized to the appropriate length. +/// Based on FS 14.1.0 (nnSdk 14.3.0) /// public class SubStorage : IStorage { @@ -158,10 +159,13 @@ public override Result Read(long offset, Span destination) if (!IsValid()) return ResultFs.NotInitialized.Log(); if (destination.Length == 0) return Result.Success; - if (!CheckAccessRange(offset, destination.Length, _size)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, destination.Length, _size); + if (rc.IsFailure()) return rc.Miss(); - return BaseStorage.Read(_offset + offset, destination); + rc = BaseStorage.Read(_offset + offset, destination); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; } public override Result Write(long offset, ReadOnlySpan source) @@ -169,26 +173,34 @@ public override Result Write(long offset, ReadOnlySpan source) if (!IsValid()) return ResultFs.NotInitialized.Log(); if (source.Length == 0) return Result.Success; - if (!CheckAccessRange(offset, source.Length, _size)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, source.Length, _size); + if (rc.IsFailure()) return rc.Miss(); - return BaseStorage.Write(_offset + offset, source); + rc = BaseStorage.Write(_offset + offset, source); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; } public override Result Flush() { if (!IsValid()) return ResultFs.NotInitialized.Log(); - return BaseStorage.Flush(); + Result rc = BaseStorage.Flush(); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; } public override Result SetSize(long size) { if (!IsValid()) return ResultFs.NotInitialized.Log(); if (!_isResizable) return ResultFs.UnsupportedSetSizeForNotResizableSubStorage.Log(); - if (!CheckOffsetAndSize(_offset, size)) return ResultFs.InvalidSize.Log(); - Result rc = BaseStorage.GetSize(out long currentSize); + Result rc = CheckOffsetAndSize(_offset, size); + if (rc.IsFailure()) return rc.Miss(); + + rc = BaseStorage.GetSize(out long currentSize); if (rc.IsFailure()) return rc; if (currentSize != _offset + _size) @@ -222,7 +234,9 @@ public override Result OperateRange(Span outBuffer, OperationId operationI if (operationId != OperationId.InvalidateCache) { if (size == 0) return Result.Success; - if (!CheckOffsetAndSize(_offset, size)) return ResultFs.OutOfRange.Log(); + + Result rc = CheckOffsetAndSize(_offset, size); + if (rc.IsFailure()) return rc.Miss(); } return BaseStorage.OperateRange(outBuffer, operationId, _offset + offset, size, inBuffer); diff --git a/src/LibHac/Fs/ValueSubStorage.cs b/src/LibHac/Fs/ValueSubStorage.cs index 4605acf8..7cabaf9c 100644 --- a/src/LibHac/Fs/ValueSubStorage.cs +++ b/src/LibHac/Fs/ValueSubStorage.cs @@ -112,10 +112,13 @@ public readonly Result Read(long offset, Span destination) if (!IsValid()) return ResultFs.NotInitialized.Log(); if (destination.Length == 0) return Result.Success; - if (!IStorage.CheckAccessRange(offset, destination.Length, _size)) - return ResultFs.OutOfRange.Log(); + Result rc = IStorage.CheckAccessRange(offset, destination.Length, _size); + if (rc.IsFailure()) return rc.Miss(); - return _baseStorage.Read(_offset + offset, destination); + rc = _baseStorage.Read(_offset + offset, destination); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; } public readonly Result Write(long offset, ReadOnlySpan source) @@ -123,26 +126,34 @@ public readonly Result Write(long offset, ReadOnlySpan source) if (!IsValid()) return ResultFs.NotInitialized.Log(); if (source.Length == 0) return Result.Success; - if (!IStorage.CheckAccessRange(offset, source.Length, _size)) - return ResultFs.OutOfRange.Log(); + Result rc = IStorage.CheckAccessRange(offset, source.Length, _size); + if (rc.IsFailure()) return rc.Miss(); + + rc = _baseStorage.Write(_offset + offset, source); + if (rc.IsFailure()) return rc.Miss(); - return _baseStorage.Write(_offset + offset, source); + return Result.Success; } public readonly Result Flush() { if (!IsValid()) return ResultFs.NotInitialized.Log(); - return _baseStorage.Flush(); + Result rc = _baseStorage.Flush(); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; } public Result SetSize(long size) { if (!IsValid()) return ResultFs.NotInitialized.Log(); if (!_isResizable) return ResultFs.UnsupportedSetSizeForNotResizableSubStorage.Log(); - if (!IStorage.CheckOffsetAndSize(_offset, size)) return ResultFs.InvalidSize.Log(); - Result rc = _baseStorage.GetSize(out long currentSize); + Result rc = IStorage.CheckOffsetAndSize(_offset, size); + if (rc.IsFailure()) return rc; + + rc = _baseStorage.GetSize(out long currentSize); if (rc.IsFailure()) return rc; if (currentSize != _offset + _size) @@ -181,7 +192,9 @@ public readonly Result OperateRange(Span outBuffer, OperationId operationI if (operationId != OperationId.InvalidateCache) { if (size == 0) return Result.Success; - if (!IStorage.CheckOffsetAndSize(_offset, size)) return ResultFs.OutOfRange.Log(); + + Result rc = IStorage.CheckOffsetAndSize(_offset, size); + if (rc.IsFailure()) return rc.Miss(); } return _baseStorage.OperateRange(outBuffer, operationId, _offset + offset, size, inBuffer); diff --git a/src/LibHac/FsSystem/AlignmentMatchingStorage.cs b/src/LibHac/FsSystem/AlignmentMatchingStorage.cs index a1aa15ff..bd2e239c 100644 --- a/src/LibHac/FsSystem/AlignmentMatchingStorage.cs +++ b/src/LibHac/FsSystem/AlignmentMatchingStorage.cs @@ -23,7 +23,7 @@ public interface IAlignmentMatchingStorageSize { } /// This class uses a work buffer on the stack to avoid allocations. Because of this the data alignment /// must be kept small; no larger than 0x200. The class /// should be used for data alignment sizes larger than this. -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) [SkipLocalsInit] public class AlignmentMatchingStorage : IStorage where TDataAlignment : struct, IAlignmentMatchingStorageSize @@ -80,8 +80,8 @@ public override Result Read(long offset, Span destination) Result rc = GetSize(out long totalSize); if (rc.IsFailure()) return rc.Miss(); - if (!CheckAccessRange(offset, destination.Length, totalSize)) - return ResultFs.OutOfRange.Log(); + rc = CheckAccessRange(offset, destination.Length, totalSize); + if (rc.IsFailure()) return rc.Miss(); return AlignmentMatchingStorageImpl.Read(_baseStorage, workBuffer, DataAlign, BufferAlign, offset, destination); } @@ -96,8 +96,8 @@ public override Result Write(long offset, ReadOnlySpan source) Result rc = GetSize(out long totalSize); if (rc.IsFailure()) return rc.Miss(); - if (!CheckAccessRange(offset, source.Length, totalSize)) - return ResultFs.OutOfRange.Log(); + rc = CheckAccessRange(offset, source.Length, totalSize); + if (rc.IsFailure()) return rc.Miss(); return AlignmentMatchingStorageImpl.Write(_baseStorage, workBuffer, DataAlign, BufferAlign, offset, source); } @@ -146,8 +146,8 @@ public override Result OperateRange(Span outBuffer, OperationId operationI Result rc = GetSize(out long baseStorageSize); if (rc.IsFailure()) return rc.Miss(); - if (!CheckOffsetAndSize(offset, size)) - return ResultFs.OutOfRange.Log(); + rc = CheckOffsetAndSize(offset, size); + if (rc.IsFailure()) return rc.Miss(); long validSize = Math.Min(size, baseStorageSize - offset); long alignedOffset = Alignment.AlignDownPow2(offset, DataAlign); @@ -166,7 +166,7 @@ public override Result OperateRange(Span outBuffer, OperationId operationI /// the beginning or end of the requested range. For data alignment sizes of 0x200 or smaller /// should be used instead /// to avoid these allocations. -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class AlignmentMatchingStoragePooledBuffer : IStorage where TBufferAlignment : struct, IAlignmentMatchingStorageSize { @@ -220,8 +220,8 @@ public override Result Read(long offset, Span destination) Result rc = GetSize(out long baseStorageSize); if (rc.IsFailure()) return rc.Miss(); - if (!CheckAccessRange(offset, destination.Length, baseStorageSize)) - return ResultFs.OutOfRange.Log(); + rc = CheckAccessRange(offset, destination.Length, baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); using var pooledBuffer = new PooledBuffer(); pooledBuffer.AllocateParticularlyLarge((int)_dataAlignment, (int)_dataAlignment); @@ -238,8 +238,8 @@ public override Result Write(long offset, ReadOnlySpan source) Result rc = GetSize(out long baseStorageSize); if (rc.IsFailure()) return rc.Miss(); - if (!CheckAccessRange(offset, source.Length, baseStorageSize)) - return ResultFs.OutOfRange.Log(); + rc = CheckAccessRange(offset, source.Length, baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); using var pooledBuffer = new PooledBuffer(); pooledBuffer.AllocateParticularlyLarge((int)_dataAlignment, (int)_dataAlignment); @@ -292,8 +292,8 @@ public override Result OperateRange(Span outBuffer, OperationId operationI Result rc = GetSize(out long baseStorageSize); if (rc.IsFailure()) return rc.Miss(); - if (!CheckOffsetAndSize(offset, size)) - return ResultFs.OutOfRange.Log(); + rc = CheckOffsetAndSize(offset, size); + if (rc.IsFailure()) return rc.Miss(); long validSize = Math.Min(size, baseStorageSize - offset); long alignedOffset = Alignment.AlignDownPow2(offset, _dataAlignment); @@ -362,8 +362,8 @@ public override Result Read(long offset, Span destination) Result rc = GetSize(out long baseStorageSize); if (rc.IsFailure()) return rc.Miss(); - if (!CheckAccessRange(offset, destination.Length, baseStorageSize)) - return ResultFs.OutOfRange.Log(); + rc = CheckAccessRange(offset, destination.Length, baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); // Calculate the aligned offsets of the requested region. long offsetEnd = offset + destination.Length; @@ -453,8 +453,8 @@ public override Result Write(long offset, ReadOnlySpan source) Result rc = GetSize(out long baseStorageSize); if (rc.IsFailure()) return rc.Miss(); - if (!CheckAccessRange(offset, source.Length, baseStorageSize)) - return ResultFs.OutOfRange.Log(); + rc = CheckAccessRange(offset, source.Length, baseStorageSize); + if (rc.IsFailure()) return rc.Miss(); using var pooledBuffer = new PooledBuffer((int)_dataAlignment, (int)_dataAlignment); @@ -505,8 +505,8 @@ public override Result OperateRange(Span outBuffer, OperationId operationI Result rc = GetSize(out long baseStorageSize); if (rc.IsFailure()) return rc.Miss(); - if (!CheckOffsetAndSize(offset, size)) - return ResultFs.OutOfRange.Log(); + rc = CheckOffsetAndSize(offset, size); + if (rc.IsFailure()) return rc.Miss(); long validSize = Math.Min(size, baseStorageSize - offset); long alignedOffset = Alignment.AlignDownPow2(offset, _dataAlignment); diff --git a/src/LibHac/Tools/FsSystem/AesCbcStorage.cs b/src/LibHac/Tools/FsSystem/AesCbcStorage.cs index 8045d2f6..48fbd552 100644 --- a/src/LibHac/Tools/FsSystem/AesCbcStorage.cs +++ b/src/LibHac/Tools/FsSystem/AesCbcStorage.cs @@ -28,10 +28,10 @@ public AesCbcStorage(IStorage baseStorage, ReadOnlySpan key, ReadOnlySpan< public override Result Read(long offset, Span destination) { - if (!CheckAccessRange(offset, destination.Length, _size)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, destination.Length, _size); + if (rc.IsFailure()) return rc.Miss(); - Result rc = base.Read(offset, destination); + rc = base.Read(offset, destination); if (rc.IsFailure()) return rc; rc = GetDecryptor(out ICipher cipher, offset); diff --git a/src/LibHac/Tools/FsSystem/CachedStorage.cs b/src/LibHac/Tools/FsSystem/CachedStorage.cs index 89de720d..cf5f140c 100644 --- a/src/LibHac/Tools/FsSystem/CachedStorage.cs +++ b/src/LibHac/Tools/FsSystem/CachedStorage.cs @@ -39,8 +39,8 @@ public override Result Read(long offset, Span destination) long inOffset = offset; int outOffset = 0; - if (!CheckAccessRange(offset, destination.Length, Length)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, destination.Length, Length); + if (rc.IsFailure()) return rc.Miss(); lock (Blocks) { @@ -69,8 +69,8 @@ public override Result Write(long offset, ReadOnlySpan source) long inOffset = offset; int outOffset = 0; - if (!CheckAccessRange(offset, source.Length, Length)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, source.Length, Length); + if (rc.IsFailure()) return rc.Miss(); lock (Blocks) { diff --git a/src/LibHac/Tools/FsSystem/ConcatenationStorage.cs b/src/LibHac/Tools/FsSystem/ConcatenationStorage.cs index cbb8102d..10d722fc 100644 --- a/src/LibHac/Tools/FsSystem/ConcatenationStorage.cs +++ b/src/LibHac/Tools/FsSystem/ConcatenationStorage.cs @@ -34,8 +34,8 @@ public override Result Read(long offset, Span destination) int outPos = 0; int remaining = destination.Length; - if (!CheckAccessRange(offset, destination.Length, Length)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, destination.Length, Length); + if (rc.IsFailure()) return rc.Miss(); int sourceIndex = FindSource(inPos); @@ -47,7 +47,7 @@ public override Result Read(long offset, Span destination) int bytesToRead = (int)Math.Min(entryRemain, remaining); - Result rc = entry.Storage.Read(entryPos, destination.Slice(outPos, bytesToRead)); + rc = entry.Storage.Read(entryPos, destination.Slice(outPos, bytesToRead)); if (rc.IsFailure()) return rc; outPos += bytesToRead; @@ -65,8 +65,8 @@ public override Result Write(long offset, ReadOnlySpan source) int outPos = 0; int remaining = source.Length; - if (!CheckAccessRange(offset, source.Length, Length)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, source.Length, Length); + if (rc.IsFailure()) return rc.Miss(); int sourceIndex = FindSource(inPos); @@ -78,7 +78,7 @@ public override Result Write(long offset, ReadOnlySpan source) int bytesToWrite = (int)Math.Min(entryRemain, remaining); - Result rc = entry.Storage.Write(entryPos, source.Slice(outPos, bytesToWrite)); + rc = entry.Storage.Write(entryPos, source.Slice(outPos, bytesToWrite)); if (rc.IsFailure()) return rc; outPos += bytesToWrite; diff --git a/src/LibHac/Tools/FsSystem/Save/DuplexStorage.cs b/src/LibHac/Tools/FsSystem/Save/DuplexStorage.cs index af0e4d32..6b535543 100644 --- a/src/LibHac/Tools/FsSystem/Save/DuplexStorage.cs +++ b/src/LibHac/Tools/FsSystem/Save/DuplexStorage.cs @@ -33,8 +33,8 @@ public override Result Read(long offset, Span destination) int outPos = 0; int remaining = destination.Length; - if (!CheckAccessRange(offset, destination.Length, Length)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, destination.Length, Length); + if (rc.IsFailure()) return rc.Miss(); while (remaining > 0) { @@ -45,7 +45,7 @@ public override Result Read(long offset, Span destination) IStorage data = Bitmap.Bitmap[blockNum] ? DataB : DataA; - Result rc = data.Read(inPos, destination.Slice(outPos, bytesToRead)); + rc = data.Read(inPos, destination.Slice(outPos, bytesToRead)); if (rc.IsFailure()) return rc; outPos += bytesToRead; @@ -62,8 +62,8 @@ public override Result Write(long offset, ReadOnlySpan source) int outPos = 0; int remaining = source.Length; - if (!CheckAccessRange(offset, source.Length, Length)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, source.Length, Length); + if (rc.IsFailure()) return rc.Miss(); while (remaining > 0) { @@ -74,7 +74,7 @@ public override Result Write(long offset, ReadOnlySpan source) IStorage data = Bitmap.Bitmap[blockNum] ? DataB : DataA; - Result rc = data.Write(inPos, source.Slice(outPos, bytesToWrite)); + rc = data.Write(inPos, source.Slice(outPos, bytesToWrite)); if (rc.IsFailure()) return rc; outPos += bytesToWrite; diff --git a/src/LibHac/Tools/FsSystem/Save/JournalStorage.cs b/src/LibHac/Tools/FsSystem/Save/JournalStorage.cs index 7d681963..4e3774a9 100644 --- a/src/LibHac/Tools/FsSystem/Save/JournalStorage.cs +++ b/src/LibHac/Tools/FsSystem/Save/JournalStorage.cs @@ -40,8 +40,8 @@ public override Result Read(long offset, Span destination) int outPos = 0; int remaining = destination.Length; - if (!CheckAccessRange(offset, destination.Length, Length)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, destination.Length, Length); + if (rc.IsFailure()) return rc.Miss(); while (remaining > 0) { @@ -52,7 +52,7 @@ public override Result Read(long offset, Span destination) int bytesToRead = Math.Min(remaining, BlockSize - blockPos); - Result rc = BaseStorage.Read(physicalOffset, destination.Slice(outPos, bytesToRead)); + rc = BaseStorage.Read(physicalOffset, destination.Slice(outPos, bytesToRead)); if (rc.IsFailure()) return rc; outPos += bytesToRead; @@ -69,8 +69,8 @@ public override Result Write(long offset, ReadOnlySpan source) int outPos = 0; int remaining = source.Length; - if (!CheckAccessRange(offset, source.Length, Length)) - return ResultFs.OutOfRange.Log(); + Result rc = CheckAccessRange(offset, source.Length, Length); + if (rc.IsFailure()) return rc.Miss(); while (remaining > 0) { @@ -81,7 +81,7 @@ public override Result Write(long offset, ReadOnlySpan source) int bytesToWrite = Math.Min(remaining, BlockSize - blockPos); - Result rc = BaseStorage.Write(physicalOffset, source.Slice(outPos, bytesToWrite)); + rc = BaseStorage.Write(physicalOffset, source.Slice(outPos, bytesToWrite)); if (rc.IsFailure()) return rc; outPos += bytesToWrite; diff --git a/src/LibHac/Util/IntUtil.cs b/src/LibHac/Util/IntUtil.cs new file mode 100644 index 00000000..c0582580 --- /dev/null +++ b/src/LibHac/Util/IntUtil.cs @@ -0,0 +1,37 @@ +namespace LibHac.Util; + +public static class IntUtil +{ + // Todo: Use generic math once C# 11 is out + public static bool IsIntValueRepresentableAsLong(ulong value) + { + return value <= long.MaxValue; + } + + public static bool IsIntValueRepresentableAsULong(long value) + { + return value >= 0; + } + + public static bool IsIntValueRepresentableAsUInt(long value) + { + return value >= uint.MinValue && value <= uint.MaxValue; + } + + public static bool CanAddWithoutOverflow(long x, long y) + { + if (y >= 0) + { + return x <= long.MaxValue - y; + } + else + { + return x >= unchecked(long.MinValue - y); + } + } + + public static bool CanAddWithoutOverflow(ulong x, ulong y) + { + return x <= ulong.MaxValue - y; + } +} \ No newline at end of file From 398a142b2722c8c9797e33516e9a775901ad56f2 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sat, 16 Apr 2022 20:55:46 -0700 Subject: [PATCH 17/24] Update some Nca classes for 14.0.0 --- src/LibHac/Crypto/Rsa.cs | 4 +- src/LibHac/FsSystem/AesXtsStorageExternal.cs | 304 ++++++++++++++++++ src/LibHac/FsSystem/IHash256Generator.cs | 24 +- src/LibHac/FsSystem/NcaFileSystemDriver.cs | 114 +++++-- src/LibHac/FsSystem/NcaHeader.cs | 71 +++- src/LibHac/FsSystem/NcaReader.cs | 155 +++++++-- tests/LibHac.Tests/Fs/TypeLayoutTests.cs | 2 +- .../LibHac.Tests/FsSystem/TypeLayoutTests.cs | 42 ++- 8 files changed, 634 insertions(+), 82 deletions(-) create mode 100644 src/LibHac/FsSystem/AesXtsStorageExternal.cs diff --git a/src/LibHac/Crypto/Rsa.cs b/src/LibHac/Crypto/Rsa.cs index c2af9a5e..2605f3ee 100644 --- a/src/LibHac/Crypto/Rsa.cs +++ b/src/LibHac/Crypto/Rsa.cs @@ -6,8 +6,8 @@ namespace LibHac.Crypto; public static class Rsa { - public static readonly int ModulusSize2048Pss = 256; - public static readonly int MaximumExponentSize2048Pss = 3; + public const int ModulusSize2048Pss = 256; + public const int MaximumExponentSize2048Pss = 3; public static bool VerifyRsa2048PssSha256(ReadOnlySpan signature, ReadOnlySpan modulus, ReadOnlySpan exponent, ReadOnlySpan message) => diff --git a/src/LibHac/FsSystem/AesXtsStorageExternal.cs b/src/LibHac/FsSystem/AesXtsStorageExternal.cs new file mode 100644 index 00000000..a68d24a0 --- /dev/null +++ b/src/LibHac/FsSystem/AesXtsStorageExternal.cs @@ -0,0 +1,304 @@ +using System; +using System.Runtime.InteropServices; +using LibHac.Common; +using LibHac.Common.FixedArrays; +using LibHac.Crypto; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.Util; + +namespace LibHac.FsSystem; + +/// +/// Reads and writes to an that's encrypted with AES-XTS-128. +/// All encryption or decryption will be done externally via a provided function. +/// This allows for using hardware decryption where the FS process doesn't has access to the actual keys. +/// +/// Based on FS 14.1.0 (nnSdk 14.3.0) +public class AesXtsStorageExternal : IStorage +{ + public const int AesBlockSize = Aes.BlockSize; + public const int KeySize = Aes.KeySize128; + public const int IvSize = Aes.KeySize128; + + private IStorage _baseStorage; + private Array2> _key; + private Array16 _iv; + private uint _blockSize; + private CryptAesXtsFunction _encryptFunction; + private CryptAesXtsFunction _decryptFunction; + + // The original class uses a template for both the shared and non-shared IStorage which avoids needing this field. + private SharedRef _baseStorageShared; + + public AesXtsStorageExternal(IStorage baseStorage, ReadOnlySpan key1, ReadOnlySpan key2, + ReadOnlySpan iv, uint blockSize, CryptAesXtsFunction encryptFunction, CryptAesXtsFunction decryptFunction) + { + _baseStorage = baseStorage; + _blockSize = blockSize; + _encryptFunction = encryptFunction; + _decryptFunction = decryptFunction; + + Assert.SdkRequires(key1.Length is 0 or KeySize); + Assert.SdkRequires(key2.Length is 0 or KeySize); + Assert.SdkRequiresEqual(IvSize, iv.Length); + Assert.SdkRequiresAligned(blockSize, AesBlockSize); + + if (key1.Length != 0) + key1.CopyTo(_key[0].Items); + + if (key2.Length != 0) + key2.CopyTo(_key[1].Items); + + iv.CopyTo(_iv.Items); + } + + public AesXtsStorageExternal(in SharedRef baseStorage, ReadOnlySpan key1, ReadOnlySpan key2, + ReadOnlySpan iv, uint blockSize, CryptAesXtsFunction encryptFunction, CryptAesXtsFunction decryptFunction) + : this(baseStorage.Get, key1, key2, iv, blockSize, encryptFunction, decryptFunction) + { + _baseStorageShared = SharedRef.CreateCopy(in baseStorage); + } + + public override void Dispose() + { + _baseStorageShared.Destroy(); + + base.Dispose(); + } + + public override Result Read(long offset, Span destination) + { + // Allow zero size. + if (destination.Length == 0) + return Result.Success; + + // Ensure we can decrypt. + if (_decryptFunction is null) + return ResultFs.NullptrArgument.Log(); + + // We can only read at block aligned offsets. + if (!Alignment.IsAlignedPow2(offset, AesBlockSize)) + return ResultFs.InvalidArgument.Log(); + + if (!Alignment.IsAlignedPow2(destination.Length, AesBlockSize)) + return ResultFs.InvalidArgument.Log(); + + // Read the encrypted data. + Result rc = _baseStorage.Read(offset, destination); + if (rc.IsFailure()) return rc.Miss(); + + // Temporarily increase our thread priority while decrypting. + using var changePriority = new ScopedThreadPriorityChanger(1, ScopedThreadPriorityChanger.Mode.Relative); + + // Setup the counter. + Span counter = stackalloc byte[IvSize]; + _iv.ItemsRo.CopyTo(counter); + Utility.AddCounter(counter, (ulong)offset / _blockSize); + + // Handle any unaligned data before the start. + int processedSize = 0; + // ReSharper disable once ConditionIsAlwaysTrueOrFalse + if (offset % _blockSize != 0) + { + // Determine the size of the pre-data read. + int skipSize = (int)(offset - Alignment.AlignDownPow2(offset, _blockSize)); + int dataSize = (int)Math.Min(destination.Length, _blockSize - skipSize); + + // Decrypt into a pooled buffer. + using (var tmpBuffer = new PooledBuffer((int)_blockSize, (int)_blockSize)) + { + Assert.SdkAssert(tmpBuffer.GetSize() >= _blockSize); + + tmpBuffer.GetBuffer().Slice(0, skipSize).Clear(); + destination.Slice(0, dataSize).CopyTo(tmpBuffer.GetBuffer().Slice(skipSize, dataSize)); + Span decryptionBuffer = tmpBuffer.GetBuffer().Slice(0, (int)_blockSize); + + // Decrypt and copy the partial block to the output buffer. + rc = _decryptFunction(decryptionBuffer, _key[0], _key[1], counter, decryptionBuffer); + if (rc.IsFailure()) return rc.Miss(); + + tmpBuffer.GetBuffer().Slice(skipSize, dataSize).CopyTo(destination); + } + + Utility.AddCounter(counter, 1); + processedSize += dataSize; + Assert.SdkAssert(processedSize == Math.Min(destination.Length, _blockSize - skipSize)); + } + + // Decrypt aligned chunks. + Span currentOutput = destination.Slice(processedSize); + int remainingSize = destination.Length - processedSize; + + while (remainingSize > 0) + { + Span currentBlock = currentOutput.Slice(0, Math.Min((int)_blockSize, remainingSize)); + + rc = _decryptFunction(currentBlock, _key[0], _key[1], counter, currentBlock); + if (rc.IsFailure()) return rc.Miss(); + + remainingSize -= currentBlock.Length; + currentOutput = currentBlock.Slice(currentBlock.Length); + + Utility.AddCounter(counter, 1); + } + + return Result.Success; + } + + public override Result Write(long offset, ReadOnlySpan source) + { + Result rc; + + // Allow zero-size writes. + if (source.Length == 0) + return Result.Success; + + // Ensure we can encrypt. + if (_encryptFunction is null) + return ResultFs.NullptrArgument.Log(); + + // We can only write at block aligned offsets. + if (!Alignment.IsAlignedPow2(offset, AesBlockSize)) + return ResultFs.InvalidArgument.Log(); + + if (!Alignment.IsAlignedPow2(source.Length, AesBlockSize)) + return ResultFs.InvalidArgument.Log(); + + // Get a pooled buffer. + using var pooledBuffer = new PooledBuffer(); + bool useWorkBuffer = !PooledBufferGlobalMethods.IsDeviceAddress(source); + if (useWorkBuffer) + { + pooledBuffer.Allocate(source.Length, (int)_blockSize); + } + + // Setup the counter. + Span counter = stackalloc byte[IvSize]; + _iv.ItemsRo.CopyTo(counter); + Utility.AddCounter(counter, (ulong)offset / _blockSize); + + // Handle any unaligned data before the start. + int processedSize = 0; + + // Todo: remove when fixed in Resharper + // ReSharper disable once ConditionIsAlwaysTrueOrFalse + if (offset % _blockSize != 0) + { + // Determine the size of the pre-data write. + int skipSize = (int)(offset - Alignment.AlignDownPow2(offset, _blockSize)); + int dataSize = (int)Math.Min(source.Length, _blockSize - skipSize); + + // Encrypt into a pooled buffer. + // Note: Nintendo allocates a second pooled buffer here despite having one already allocated above. + using (var tmpBuffer = new PooledBuffer((int)_blockSize, (int)_blockSize)) + { + Assert.SdkAssert(tmpBuffer.GetSize() >= _blockSize); + + tmpBuffer.GetBuffer().Slice(0, skipSize).Clear(); + source.Slice(0, dataSize).CopyTo(tmpBuffer.GetBuffer().Slice(skipSize, dataSize)); + Span encryptionBuffer = tmpBuffer.GetBuffer().Slice(0, (int)_blockSize); + + rc = _encryptFunction(encryptionBuffer, _key[0], _key[1], counter, encryptionBuffer); + if (rc.IsFailure()) return rc.Miss(); + + rc = _baseStorage.Write(offset, tmpBuffer.GetBuffer().Slice(skipSize, dataSize)); + if (rc.IsFailure()) return rc.Miss(); + } + + Utility.AddCounter(counter, 1); + processedSize += dataSize; + Assert.SdkAssert(processedSize == Math.Min(source.Length, _blockSize - skipSize)); + } + + // Encrypt aligned chunks. + int remainingSize = source.Length - processedSize; + long currentOffset = offset + processedSize; + + while (remainingSize > 0) + { + // Determine data we're writing and where. + int writeSize = useWorkBuffer ? Math.Min(pooledBuffer.GetSize(), remainingSize) : remainingSize; + + // Encrypt the data with temporarily increased priority. + using (new ScopedThreadPriorityChanger(1, ScopedThreadPriorityChanger.Mode.Relative)) + { + int remainingEncryptSize = writeSize; + int encryptOffset = 0; + + // Encrypt one block at a time. + while (remainingEncryptSize > 0) + { + int currentSize = Math.Min(remainingEncryptSize, (int)_blockSize); + ReadOnlySpan encryptSource = source.Slice(processedSize + encryptOffset, currentSize); + + // const_cast the input buffer and encrypt in-place if it's a "device buffer". + Span encryptDest = useWorkBuffer + ? pooledBuffer.GetBuffer().Slice(encryptOffset, currentSize) + : MemoryMarshal.CreateSpan(ref MemoryMarshal.GetReference(encryptSource), encryptSource.Length); + + rc = _encryptFunction(encryptDest, _key[0], _key[1], counter, encryptSource); + if (rc.IsFailure()) return rc.Miss(); + + Utility.AddCounter(counter, 1); + + encryptOffset += currentSize; + remainingEncryptSize -= currentSize; + } + } + + // Write the encrypted data. + ReadOnlySpan writeBuffer = useWorkBuffer + ? pooledBuffer.GetBuffer().Slice(0, writeSize) + : source.Slice(processedSize, writeSize); + + rc = _baseStorage.Write(currentOffset, writeBuffer); + if (rc.IsFailure()) return rc.Miss(); + + // Advance. + currentOffset += writeSize; + processedSize += writeSize; + remainingSize -= writeSize; + } + + return Result.Success; + } + + public override Result Flush() + { + return _baseStorage.Flush(); + } + + public override Result SetSize(long size) + { + return _baseStorage.SetSize(size); + } + + public override Result GetSize(out long size) + { + return _baseStorage.GetSize(out size); + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + if (operationId != OperationId.InvalidateCache) + { + // Handle the zero size case. + if (size == 0) + return Result.Success; + + // Ensure alignment. + if (!Alignment.IsAlignedPow2(offset, AesBlockSize)) + return ResultFs.InvalidArgument.Log(); + + if (!Alignment.IsAlignedPow2(size, AesBlockSize)) + return ResultFs.InvalidArgument.Log(); + } + + Result rc = _baseStorage.OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } +} \ No newline at end of file diff --git a/src/LibHac/FsSystem/IHash256Generator.cs b/src/LibHac/FsSystem/IHash256Generator.cs index 3f2e26dc..807ec7d4 100644 --- a/src/LibHac/FsSystem/IHash256Generator.cs +++ b/src/LibHac/FsSystem/IHash256Generator.cs @@ -5,11 +5,17 @@ namespace LibHac.FsSystem; +public enum HashAlgorithmType : byte +{ + Sha2 = 0, + Sha3 = 1 +} + /// /// Generates a hash for a stream of data. The data can be given to the /// as multiple, smaller sequential blocks of data. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public abstract class IHash256Generator : IDisposable { public static readonly long HashSize = 256 / 8; @@ -41,14 +47,14 @@ public void GetHash(Span hashBuffer) /// /// Creates objects and can generate a hash for a single, in-memory block of data. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public abstract class IHash256GeneratorFactory : IDisposable { public virtual void Dispose() { } - public UniqueRef Create() + public Result Create(ref UniqueRef outGenerator) { - return DoCreate(); + return DoCreate(ref outGenerator); } public void GenerateHash(Span hashBuffer, ReadOnlySpan data) @@ -58,22 +64,22 @@ public void GenerateHash(Span hashBuffer, ReadOnlySpan data) DoGenerateHash(hashBuffer, data); } - protected abstract UniqueRef DoCreate(); + protected abstract Result DoCreate(ref UniqueRef outGenerator); protected abstract void DoGenerateHash(Span hashBuffer, ReadOnlySpan data); } /// /// Creates objects. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public abstract class IHash256GeneratorFactorySelector : IDisposable { public virtual void Dispose() { } - public IHash256GeneratorFactory GetFactory() + public IHash256GeneratorFactory GetFactory(HashAlgorithmType type) { - return DoGetFactory(); + return DoGetFactory(type); } - protected abstract IHash256GeneratorFactory DoGetFactory(); + protected abstract IHash256GeneratorFactory DoGetFactory(HashAlgorithmType type); } \ No newline at end of file diff --git a/src/LibHac/FsSystem/NcaFileSystemDriver.cs b/src/LibHac/FsSystem/NcaFileSystemDriver.cs index 0f651f37..823483ad 100644 --- a/src/LibHac/FsSystem/NcaFileSystemDriver.cs +++ b/src/LibHac/FsSystem/NcaFileSystemDriver.cs @@ -8,21 +8,27 @@ namespace LibHac.FsSystem; +/// +/// Contains the configuration used for decrypting NCAs. +/// +/// Based on FS 14.1.0 (nnSdk 14.3.0) public struct NcaCryptoConfiguration { - public static readonly int Rsa2048KeyModulusSize = Rsa.ModulusSize2048Pss; - public static readonly int Rsa2048KeyPublicExponentSize = Rsa.MaximumExponentSize2048Pss; - public static readonly int Rsa2048KeyPrivateExponentSize = Rsa2048KeyModulusSize; + public const int Rsa2048KeyModulusSize = Rsa.ModulusSize2048Pss; + public const int Rsa2048KeyPublicExponentSize = Rsa.MaximumExponentSize2048Pss; + public const int Rsa2048KeyPrivateExponentSize = Rsa2048KeyModulusSize; - public static readonly int Aes128KeySize = Aes.KeySize128; + public const int Aes128KeySize = Aes.KeySize128; - public static readonly int Header1SignatureKeyGenerationMax = 1; + public const int Header1SignatureKeyGenerationMax = 1; - public static readonly int KeyAreaEncryptionKeyIndexCount = 3; - public static readonly int HeaderEncryptionKeyCount = 2; + public const int KeyAreaEncryptionKeyIndexCount = 3; + public const int HeaderEncryptionKeyCount = 2; - public static readonly int KeyGenerationMax = 32; - public static readonly int KeyAreaEncryptionKeyCount = KeyAreaEncryptionKeyIndexCount * KeyGenerationMax; + public const byte KeyAreaEncryptionKeyIndexZeroKey = 0xFF; + + public const int KeyGenerationMax = 32; + public const int KeyAreaEncryptionKeyCount = KeyAreaEncryptionKeyIndexCount * KeyGenerationMax; public Array2> Header1SignKeyModuli; public Array3 Header1SignKeyPublicExponent; @@ -30,9 +36,13 @@ public struct NcaCryptoConfiguration public Array16 HeaderEncryptionKeySource; public Array2> HeaderEncryptedEncryptionKeys; public GenerateKeyFunction GenerateKey; + public CryptAesXtsFunction EncryptAesXtsForExternalKey; + public CryptAesXtsFunction DecryptAesXtsForExternalKey; public DecryptAesCtrFunction DecryptAesCtr; public DecryptAesCtrFunction DecryptAesCtrForExternalKey; + public VerifySign1Function VerifySign1; public bool IsDev; + public bool IsAvailableSwKey; } public struct NcaCompressionConfiguration @@ -49,22 +59,30 @@ public static bool IsInvalidKeyTypeValue(int keyType) public static int GetKeyTypeValue(byte keyIndex, byte keyGeneration) { - const int invalidKeyTypeValue = -1; + if (keyIndex == NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexZeroKey) + { + return (int)KeyType.ZeroKey; + } - if (keyIndex >= NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexCount) - return invalidKeyTypeValue; + if (keyIndex < NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexCount) + { + return NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexCount * keyGeneration + keyIndex; + } - return NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexCount * keyGeneration + keyIndex; + return (int)KeyType.InvalidKey; } } public enum KeyType { - NcaHeaderKey = 0x60, - NcaExternalKey = 0x61, - SaveDataDeviceUniqueMac = 0x62, - SaveDataSeedUniqueMac = 0x63, - SaveDataTransferMac = 0x64 + ZeroKey = -2, + InvalidKey = -1, + NcaHeaderKey1 = NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 0, + NcaHeaderKey2 = NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 1, + NcaExternalKey = NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 2, + SaveDataDeviceUniqueMac = NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 3, + SaveDataSeedUniqueMac = NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 4, + SaveDataTransferMac = NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 5 } public class NcaFileSystemDriver : IDisposable @@ -85,6 +103,8 @@ public struct StorageContext : IDisposable public SharedRef FsDataStorage; public SharedRef CompressedStorageMetaStorage; public SharedRef CompressedStorage; + public SharedRef PatchLayerInfoStorage; + public SharedRef SparseLayerInfoStorage; public void Dispose() { @@ -100,6 +120,8 @@ public void Dispose() FsDataStorage.Destroy(); CompressedStorageMetaStorage.Destroy(); CompressedStorage.Destroy(); + PatchLayerInfoStorage.Destroy(); + SparseLayerInfoStorage.Destroy(); } } @@ -184,6 +206,14 @@ private Result CreateSparseStorageMetaStorage(ref SharedRef outStorage throw new NotImplementedException(); } + private Result CreateSparseStorageMetaStorageWithVerification(ref SharedRef outStorage, + ref SharedRef outLayerInfoStorage, ref SharedRef baseStorage, long offset, + in NcaAesCtrUpperIv upperIv, in NcaSparseInfo sparseInfo, in NcaMetaDataHashDataInfo metaDataHashDataInfo, + IHash256GeneratorFactory hashGeneratorFactory) + { + throw new NotImplementedException(); + } + private Result CreateSparseStorageCore(ref SharedRef outStorage, ref SharedRef baseStorage, long baseStorageSize, ref SharedRef sparseStorageMetaStorage, in NcaSparseInfo sparseInfo, bool hasExternalInfo) @@ -198,8 +228,26 @@ private Result CreateSparseStorage(ref SharedRef outStorage, out long throw new NotImplementedException(); } + private Result CreateSparseStorageWithVerification(ref SharedRef outStorage, out long outFsDataOffset, + out SharedRef outSparseStorage, ref SharedRef outSparseStorageMetaStorage, + ref SharedRef outLayerInfoStorage, int index, in NcaAesCtrUpperIv upperIv, + in NcaSparseInfo sparseInfo, in NcaMetaDataHashDataInfo metaDataHashDataInfo, + NcaFsHeader.MetaDataHashType metaDataHashType) + { + throw new NotImplementedException(); + } + + private Result CreatePatchMetaStorage(ref SharedRef outAesCtrExMetaStorage, + ref SharedRef outIndirectMetaStorage, ref SharedRef outLayerInfoStorage, + ref SharedRef baseStorage, long offset, in NcaAesCtrUpperIv upperIv, in NcaPatchInfo patchInfo, + in NcaMetaDataHashDataInfo metaDataHashDataInfo, IHash256GeneratorFactory hashGeneratorFactory) + { + throw new NotImplementedException(); + } + private Result CreateAesCtrExStorageMetaStorage(ref SharedRef outStorage, - ref SharedRef baseStorage, long offset, in NcaAesCtrUpperIv upperIv, in NcaPatchInfo patchInfo) + ref SharedRef baseStorage, long offset, NcaFsHeader.EncryptionType encryptionType, + in NcaAesCtrUpperIv upperIv, in NcaPatchInfo patchInfo) { throw new NotImplementedException(); } @@ -227,13 +275,29 @@ private Result CreateIndirectStorage(ref SharedRef outStorage, } private Result CreateSha256Storage(ref SharedRef outStorage, ref SharedRef baseStorage, - in NcaFsHeader.HashData.HierarchicalSha256Data sha256Data) + in NcaFsHeader.HashData.HierarchicalSha256Data sha256Data, IHash256GeneratorFactory hashGeneratorFactory) + { + throw new NotImplementedException(); + } + + private Result CreateIntegrityVerificationStorage(ref SharedRef outStorage, + ref SharedRef baseStorage, in NcaFsHeader.HashData.IntegrityMetaInfo metaInfo, + IHash256GeneratorFactory hashGeneratorFactory) + { + throw new NotImplementedException(); + } + + private Result CreateIntegrityVerificationStorageImpl(ref SharedRef outStorage, + ref SharedRef baseStorage, in NcaFsHeader.HashData.IntegrityMetaInfo metaInfo, long layerInfoOffset, + int maxDataCacheEntries, int maxHashCacheEntries, sbyte bufferLevel, + IHash256GeneratorFactory hashGeneratorFactory) { throw new NotImplementedException(); } - private Result HierarchicalSha256Data(ref SharedRef outStorage, ref SharedRef baseStorage, - in NcaFsHeader.HashData.IntegrityMetaInfo metaInfo) + private Result CreateIntegrityVerificationStorageForMeta(ref SharedRef outStorage, + ref SharedRef outLayerInfoStorage, ref SharedRef baseStorage, long offset, + in NcaMetaDataHashDataInfo metaDataHashDataInfo, IHash256GeneratorFactory hashGeneratorFactory) { throw new NotImplementedException(); } @@ -252,4 +316,10 @@ public Result CreateCompressedStorage(ref SharedRef outStorage, { throw new NotImplementedException(); } + + public Result CreateRegionSwitchStorage(ref SharedRef outStorage, NcaFsHeaderReader headerReader, + ref SharedRef insideRegionStorage, ref SharedRef outsideRegionStorage) + { + throw new NotImplementedException(); + } } \ No newline at end of file diff --git a/src/LibHac/FsSystem/NcaHeader.cs b/src/LibHac/FsSystem/NcaHeader.cs index 6de0e074..32c34800 100644 --- a/src/LibHac/FsSystem/NcaHeader.cs +++ b/src/LibHac/FsSystem/NcaHeader.cs @@ -1,10 +1,16 @@ using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using LibHac.Common; using LibHac.Common.FixedArrays; +using LibHac.Fs; namespace LibHac.FsSystem; +/// +/// The structure used as the header for an NCA file. +/// +/// Based on FS 14.1.0 (nnSdk 14.3.0) public struct NcaHeader { public enum ContentType : byte @@ -133,6 +139,19 @@ public struct NcaCompressionInfo public ulong Reserved; } +public struct NcaMetaDataHashDataInfo +{ + public long Offset; + public long Size; + public Hash Hash; +} + +public struct NcaMetaDataHashData +{ + public long LayerInfoOffset; + public NcaFsHeader.HashData.IntegrityMetaInfo IntegrityMetaInfo; +} + [StructLayout(LayoutKind.Explicit)] public struct NcaAesCtrUpperIv { @@ -149,19 +168,25 @@ internal NcaAesCtrUpperIv(ulong value) } } +/// +/// The structure used as the header for an NCA file system. +/// +/// Based on FS 14.1.0 (nnSdk 14.3.0) public struct NcaFsHeader { public ushort Version; public FsType FsTypeValue; public HashType HashTypeValue; public EncryptionType EncryptionTypeValue; - public Array3 Reserved; + public MetaDataHashType MetaDataHashTypeValue; + public Array2 Reserved; public HashData HashDataValue; public NcaPatchInfo PatchInfo; public NcaAesCtrUpperIv AesCtrUpperIv; public NcaSparseInfo SparseInfo; public NcaCompressionInfo CompressionInfo; - public Array96 Padding; + public NcaMetaDataHashDataInfo MetaDataHashDataInfo; + public Array48 Padding; public enum FsType : byte { @@ -175,7 +200,9 @@ public enum EncryptionType : byte None = 1, AesXts = 2, AesCtr = 3, - AesCtrEx = 4 + AesCtrEx = 4, + AesCtrSkipLayerHash = 5, + AesCtrExSkipLayerHash = 6 } public enum HashType : byte @@ -183,7 +210,16 @@ public enum HashType : byte Auto = 0, None = 1, HierarchicalSha256Hash = 2, - HierarchicalIntegrityHash = 3 + HierarchicalIntegrityHash = 3, + AutoSha3 = 4, + HierarchicalSha3256Hash = 5, + HierarchicalIntegritySha3Hash = 6 + } + + public enum MetaDataHashType : byte + { + None = 0, + HierarchicalIntegrity = 1 } public struct Region @@ -235,4 +271,31 @@ public struct SignatureSalt } } } + + public readonly Result GetHashTargetOffset(out long outOffset) + { + UnsafeHelpers.SkipParamInit(out outOffset); + + if (HashTypeValue is HashType.HierarchicalIntegrityHash or HashType.HierarchicalIntegritySha3Hash) + { + ref readonly HashData.IntegrityMetaInfo.InfoLevelHash hashInfo = ref HashDataValue.IntegrityMeta.LevelHashInfo; + outOffset = hashInfo.Layers[hashInfo.MaxLayers - 2].Offset; + } + else if (HashTypeValue is HashType.HierarchicalSha256Hash or HashType.HierarchicalSha3256Hash) + { + ref readonly HashData.HierarchicalSha256Data hashInfo = ref HashDataValue.HierarchicalSha256; + outOffset = hashInfo.LayerRegions[hashInfo.LayerCount - 1].Offset; + } + else + { + return ResultFs.InvalidNcaFsHeader.Log(); + } + + return Result.Success; + } + + public readonly bool IsSkipLayerHashEncryption() + { + return EncryptionTypeValue is EncryptionType.AesCtrSkipLayerHash or EncryptionType.AesCtrExSkipLayerHash; + } } \ No newline at end of file diff --git a/src/LibHac/FsSystem/NcaReader.cs b/src/LibHac/FsSystem/NcaReader.cs index a8a081f0..28ba2feb 100644 --- a/src/LibHac/FsSystem/NcaReader.cs +++ b/src/LibHac/FsSystem/NcaReader.cs @@ -8,13 +8,15 @@ namespace LibHac.FsSystem; -public delegate Result GenerateKeyFunction(Span destKey, ReadOnlySpan sourceKey, int keyType, in NcaCryptoConfiguration config); -public delegate Result DecryptAesCtrFunction(Span dest, int keyType, ReadOnlySpan encryptedKey, ReadOnlySpan iv, ReadOnlySpan source); +public delegate void GenerateKeyFunction(Span destKey, ReadOnlySpan sourceKey, int keyType); +public delegate Result DecryptAesCtrFunction(Span dest, int keyIndex, int keyGeneration, ReadOnlySpan encryptedKey, ReadOnlySpan iv, ReadOnlySpan source); +public delegate Result CryptAesXtsFunction(Span dest, ReadOnlySpan key1, ReadOnlySpan key2, ReadOnlySpan iv, ReadOnlySpan source); +public delegate bool VerifySign1Function(ReadOnlySpan signature, ReadOnlySpan data, bool isProd, byte generation); /// /// Handles reading information from an NCA file's header. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class NcaReader : IDisposable { private const uint SdkAddonVersionMin = 0xB0000; @@ -27,9 +29,10 @@ public class NcaReader : IDisposable private DecryptAesCtrFunction _decryptAesCtr; private DecryptAesCtrFunction _decryptAesCtrForExternalKey; private bool _isSoftwareAesPrioritized; + private bool _isAvailableSwKey; private NcaHeader.EncryptionType _headerEncryptionType; private GetDecompressorFunction _getDecompressorFunc; - private IHash256GeneratorFactory _hashGeneratorFactory; + private IHash256GeneratorFactorySelector _hashGeneratorFactorySelector; public void Dispose() { @@ -44,23 +47,44 @@ public Result Initialize(ref SharedRef baseStorage, in NcaCryptoConfig Assert.SdkRequiresNotNull(hashGeneratorFactorySelector); Assert.SdkRequiresNull(in _bodyStorage); - if (cryptoConfig.GenerateKey is null) + if (cryptoConfig.VerifySign1 is null) return ResultFs.InvalidArgument.Log(); using var headerStorage = new UniqueRef(); - // Generate the keys for decrypting the NCA header. - Unsafe.SkipInit(out Array2> commonDecryptionKeys); - for (int i = 0; i < NcaCryptoConfiguration.HeaderEncryptionKeyCount; i++) + if (cryptoConfig.IsAvailableSwKey) { - cryptoConfig.GenerateKey(commonDecryptionKeys[i].Items, cryptoConfig.HeaderEncryptedEncryptionKeys[i], 0x60, - in cryptoConfig); - } + if (cryptoConfig.GenerateKey is null) + return ResultFs.InvalidArgument.Log(); + + ReadOnlySpan headerKeyTypes = stackalloc int[NcaCryptoConfiguration.HeaderEncryptionKeyCount] + { (int)KeyType.NcaHeaderKey1, (int)KeyType.NcaHeaderKey2 }; - // Create an XTS storage to read the encrypted header. - Array16 headerIv = default; - headerStorage.Reset(new AesXtsStorage(baseStorage.Get, commonDecryptionKeys[0], commonDecryptionKeys[1], - headerIv, NcaHeader.XtsBlockSize)); + // Generate the keys for decrypting the NCA header. + Unsafe.SkipInit(out Array2> commonDecryptionKeys); + for (int i = 0; i < NcaCryptoConfiguration.HeaderEncryptionKeyCount; i++) + { + cryptoConfig.GenerateKey(commonDecryptionKeys[i].Items, cryptoConfig.HeaderEncryptedEncryptionKeys[i], + headerKeyTypes[i]); + } + + // Create an XTS storage to read the encrypted header. + Array16 headerIv = default; + headerStorage.Reset(new AesXtsStorage(baseStorage.Get, commonDecryptionKeys[0], commonDecryptionKeys[1], + headerIv, NcaHeader.XtsBlockSize)); + } + else + { + // Software key isn't available, so we need to be able to decrypt externally. + if (cryptoConfig.DecryptAesXtsForExternalKey is null) + return ResultFs.InvalidArgument.Log(); + + // Create the header storage. + Array16 headerIv = default; + headerStorage.Reset(new AesXtsStorageExternal(baseStorage.Get, ReadOnlySpan.Empty, + ReadOnlySpan.Empty, headerIv, (uint)NcaHeader.XtsBlockSize, cryptoConfig.EncryptAesXtsForExternalKey, + cryptoConfig.DecryptAesXtsForExternalKey)); + } if (!headerStorage.HasValue) return ResultFs.AllocationMemoryFailedInNcaReaderA.Log(); @@ -108,11 +132,9 @@ public Result Initialize(ref SharedRef baseStorage, in NcaCryptoConfig int signMessageOffset = NcaHeader.HeaderSignSize * NcaHeader.HeaderSignCount; int signMessageSize = NcaHeader.Size - signMessageOffset; ReadOnlySpan signature = _header.Signature1; - ReadOnlySpan modulus = cryptoConfig.Header1SignKeyModuli[_header.Header1SignatureKeyGeneration]; - ReadOnlySpan exponent = cryptoConfig.Header1SignKeyPublicExponent; ReadOnlySpan message = SpanHelpers.AsReadOnlyByteSpan(in _header).Slice(signMessageOffset, signMessageSize); - if (!Rsa.VerifyRsa2048PssSha256(signature, modulus, exponent, message)) + if (!cryptoConfig.VerifySign1(signature, message, !cryptoConfig.IsDev, _header.Header1SignatureKeyGeneration)) return ResultFs.NcaHeaderSignature1VerificationFailed.Log(); // Validate the sdk version. @@ -120,32 +142,40 @@ public Result Initialize(ref SharedRef baseStorage, in NcaCryptoConfig return ResultFs.UnsupportedSdkVersion.Log(); // Validate the key index. - if (_header.KeyAreaEncryptionKeyIndex >= NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexCount) + if (_header.KeyAreaEncryptionKeyIndex >= NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexCount && + _header.KeyAreaEncryptionKeyIndex != NcaCryptoConfiguration.KeyAreaEncryptionKeyIndexZeroKey) + { return ResultFs.InvalidNcaKeyIndex.Log(); + } + + _hashGeneratorFactorySelector = hashGeneratorFactorySelector; // Get keys from the key area if the NCA doesn't have a rights ID. Array16 zeroRightsId = default; if (CryptoUtil.IsSameBytes(zeroRightsId, _header.RightsId, NcaHeader.RightsIdSize)) { - // If we don't have a rights ID we need to generate decryption keys. - int keyType = NcaKeyFunctions.GetKeyTypeValue(_header.KeyAreaEncryptionKeyIndex, _header.GetProperKeyGeneration()); - ReadOnlySpan encryptedKeyCtr = _header.EncryptedKeys.ItemsRo.Slice((int)NcaHeader.DecryptionKey.AesCtr * Aes.KeySize128, Aes.KeySize128); - ReadOnlySpan keyCtrHw = _header.EncryptedKeys.ItemsRo.Slice((int)NcaHeader.DecryptionKey.AesCtrHw * Aes.KeySize128, Aes.KeySize128); + // If we don't have a rights ID we need to generate decryption keys if software keys are available. + if (cryptoConfig.IsAvailableSwKey) + { + int keyTypeValue = NcaKeyFunctions.GetKeyTypeValue(_header.KeyAreaEncryptionKeyIndex, _header.GetProperKeyGeneration()); + ReadOnlySpan encryptedKeyCtr = _header.EncryptedKeys.ItemsRo.Slice((int)NcaHeader.DecryptionKey.AesCtr * Aes.KeySize128, Aes.KeySize128); - cryptoConfig.GenerateKey(_decryptionKeys[(int)NcaHeader.DecryptionKey.AesCtr].Items, encryptedKeyCtr, keyType, in cryptoConfig); + cryptoConfig.GenerateKey(_decryptionKeys[(int)NcaHeader.DecryptionKey.AesCtr].Items, encryptedKeyCtr, keyTypeValue); + } // Copy the plaintext hardware key. + ReadOnlySpan keyCtrHw = _header.EncryptedKeys.ItemsRo.Slice((int)NcaHeader.DecryptionKey.AesCtrHw * Aes.KeySize128, Aes.KeySize128); keyCtrHw.CopyTo(_decryptionKeys[(int)NcaHeader.DecryptionKey.AesCtrHw].Items); } + // Clear the external decryption key. _externalDataDecryptionKey.Items.Clear(); // Copy the configuration to the NcaReader. + _isAvailableSwKey = cryptoConfig.IsAvailableSwKey; _decryptAesCtr = cryptoConfig.DecryptAesCtr; _decryptAesCtrForExternalKey = cryptoConfig.DecryptAesCtrForExternalKey; _getDecompressorFunc = compressionConfig.GetDecompressorFunc; - _hashGeneratorFactory = hashGeneratorFactorySelector.GetFactory(); - Assert.SdkRequiresNotNull(_hashGeneratorFactory); _bodyStorage.SetByMove(ref baseStorage); _headerStorage.Set(ref headerStorage.Ref()); @@ -187,7 +217,7 @@ public void GetHeaderSign2(Span outBuffer) public void GetHeaderSign2TargetHash(Span outBuffer) { - Assert.SdkRequiresNotNull(_hashGeneratorFactory); + Assert.SdkRequiresNotNull(_hashGeneratorFactorySelector); Assert.SdkRequiresEqual(IHash256Generator.HashSize, outBuffer.Length); int signTargetOffset = NcaHeader.HeaderSignSize * NcaHeader.HeaderSignCount; @@ -195,7 +225,8 @@ public void GetHeaderSign2TargetHash(Span outBuffer) ReadOnlySpan signTarget = SpanHelpers.AsReadOnlyByteSpan(in _header).Slice(signTargetOffset, signTargetSize); - _hashGeneratorFactory.GenerateHash(outBuffer, signTarget); + IHash256GeneratorFactory factory = _hashGeneratorFactorySelector.GetFactory(HashAlgorithmType.Sha2); + factory.GenerateHash(outBuffer, signTarget); } public SharedRef GetSharedBodyStorage() @@ -390,6 +421,11 @@ public void PrioritizeSwAes() _isSoftwareAesPrioritized = true; } + public bool IsAvailableSwKey() + { + return _isAvailableSwKey; + } + public void SetExternalDecryptionKey(ReadOnlySpan key) { Assert.SdkRequiresEqual(_externalDataDecryptionKey.ItemsRo.Length, key.Length); @@ -434,17 +470,17 @@ public GetDecompressorFunction GetDecompressor() return _getDecompressorFunc; } - public IHash256GeneratorFactory GetHashGeneratorFactory() + public IHash256GeneratorFactorySelector GetHashGeneratorFactorySelector() { - Assert.SdkRequiresNotNull(_hashGeneratorFactory); - return _hashGeneratorFactory; + Assert.SdkRequiresNotNull(_hashGeneratorFactorySelector); + return _hashGeneratorFactorySelector; } } /// /// Handles reading information from the of a file system inside an NCA file. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class NcaFsHeaderReader { private NcaFsHeader _header; @@ -468,7 +504,8 @@ public Result Initialize(NcaReader reader, int index) if (rc.IsFailure()) return rc.Miss(); Unsafe.SkipInit(out Hash hash); - reader.GetHashGeneratorFactory().GenerateHash(hash.Value.Items, SpanHelpers.AsReadOnlyByteSpan(in _header)); + IHash256GeneratorFactory generator = reader.GetHashGeneratorFactorySelector().GetFactory(HashAlgorithmType.Sha2); + generator.GenerateHash(hash.Value.Items, SpanHelpers.AsReadOnlyByteSpan(in _header)); if (!CryptoUtil.IsSameBytes(reader.GetFsHeaderHash(index).Value, hash.Value, Unsafe.SizeOf())) { @@ -515,6 +552,34 @@ public NcaFsHeader.EncryptionType GetEncryptionType() return _header.EncryptionTypeValue; } + public NcaFsHeader.MetaDataHashType GetPatchMetaHashType() + { + Assert.SdkRequires(IsInitialized()); + return _header.MetaDataHashTypeValue; + } + + public NcaFsHeader.MetaDataHashType GetSparseMetaHashType() + { + Assert.SdkRequires(IsInitialized()); + return _header.MetaDataHashTypeValue; + } + + public Result GetHashTargetOffset(out long outOffset) + { + Assert.SdkRequires(IsInitialized()); + + Result rc = _header.GetHashTargetOffset(out outOffset); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public bool IsSkipLayerHashEncryption() + { + Assert.SdkRequires(IsInitialized()); + return _header.IsSkipLayerHashEncryption(); + } + public ref readonly NcaPatchInfo GetPatchInfo() { Assert.SdkRequires(IsInitialized()); @@ -551,6 +616,30 @@ public ref readonly NcaCompressionInfo GetCompressionInfo() return ref _header.CompressionInfo; } + public bool ExistsPatchMetaHashLayer() + { + Assert.SdkRequires(IsInitialized()); + return _header.MetaDataHashDataInfo.Size != 0 && GetPatchInfo().HasIndirectTable(); + } + + public bool ExistsSparseMetaHashLayer() + { + Assert.SdkRequires(IsInitialized()); + return _header.MetaDataHashDataInfo.Size != 0 && ExistsSparseLayer(); + } + + public ref readonly NcaMetaDataHashDataInfo GetPatchMetaDataHashDataInfo() + { + Assert.SdkRequires(IsInitialized()); + return ref _header.MetaDataHashDataInfo; + } + + public ref readonly NcaMetaDataHashDataInfo GetSparseMetaDataHashDataInfo() + { + Assert.SdkRequires(IsInitialized()); + return ref _header.MetaDataHashDataInfo; + } + public void GetRawData(Span outBuffer) { Assert.SdkRequires(IsInitialized()); diff --git a/tests/LibHac.Tests/Fs/TypeLayoutTests.cs b/tests/LibHac.Tests/Fs/TypeLayoutTests.cs index ad328e4e..8035c0c6 100644 --- a/tests/LibHac.Tests/Fs/TypeLayoutTests.cs +++ b/tests/LibHac.Tests/Fs/TypeLayoutTests.cs @@ -476,7 +476,7 @@ public static void GameCardHandle_Layout() [StructLayout(LayoutKind.Sequential)] private struct Int64AlignmentTest { - public int A; + public byte A; public Int64 B; } diff --git a/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs b/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs index 092c1986..3e518524 100644 --- a/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs +++ b/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs @@ -28,13 +28,15 @@ public static void NcaFsHeader_Layout() Assert.Equal(0x002, GetOffset(in s, in s.FsTypeValue)); Assert.Equal(0x003, GetOffset(in s, in s.HashTypeValue)); Assert.Equal(0x004, GetOffset(in s, in s.EncryptionTypeValue)); - Assert.Equal(0x005, GetOffset(in s, in s.Reserved)); + Assert.Equal(0x005, GetOffset(in s, in s.MetaDataHashTypeValue)); + Assert.Equal(0x006, GetOffset(in s, in s.Reserved)); Assert.Equal(0x008, GetOffset(in s, in s.HashDataValue)); Assert.Equal(0x100, GetOffset(in s, in s.PatchInfo)); Assert.Equal(0x140, GetOffset(in s, in s.AesCtrUpperIv)); Assert.Equal(0x148, GetOffset(in s, in s.SparseInfo)); Assert.Equal(0x178, GetOffset(in s, in s.CompressionInfo)); - Assert.Equal(0x1A0, GetOffset(in s, in s.Padding)); + Assert.Equal(0x1A0, GetOffset(in s, in s.MetaDataHashDataInfo)); + Assert.Equal(0x1D0, GetOffset(in s, in s.Padding)); } [Fact] @@ -99,7 +101,7 @@ public static void InfoLevelHash_Layout() } [Fact] - public static void HierarchicalIntegrityVerificationLevelInformation_Layout() + public static void InfoLevelHash_HierarchicalIntegrityVerificationLevelInformation_Layout() { NcaFsHeader.HashData.IntegrityMetaInfo.InfoLevelHash.HierarchicalIntegrityVerificationLevelInformation s = default; @@ -112,7 +114,7 @@ public static void HierarchicalIntegrityVerificationLevelInformation_Layout() } [Fact] - public static void SignatureSalt_Layout() + public static void InfoLevelHash_SignatureSalt_Layout() { NcaFsHeader.HashData.IntegrityMetaInfo.InfoLevelHash.SignatureSalt s = default; @@ -164,6 +166,29 @@ public static void NcaCompressionInfo_Layout() Assert.Equal(0x20, GetOffset(in s, in s.Reserved)); } + [Fact] + public static void NcaMetaDataHashDataInfo_Layout() + { + NcaMetaDataHashDataInfo s = default; + + Assert.Equal(0x30, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.Offset)); + Assert.Equal(0x08, GetOffset(in s, in s.Size)); + Assert.Equal(0x10, GetOffset(in s, in s.Hash)); + } + + [Fact] + public static void NcaMetaDataHashData_Layout() + { + NcaMetaDataHashData s = default; + + Assert.Equal(0xE8, Unsafe.SizeOf()); + + Assert.Equal(0, GetOffset(in s, in s.LayerInfoOffset)); + Assert.Equal(8, GetOffset(in s, in s.IntegrityMetaInfo)); + } + [Fact] public static void NcaAesCtrUpperIv_Layout() { @@ -240,12 +265,6 @@ public static void KeyType_Layout() Assert.Equal(NcaCryptoConfiguration.Aes128KeySize, s.HeaderEncryptionKeySource.ItemsRo.Length); Assert.Equal(NcaCryptoConfiguration.HeaderEncryptionKeyCount, s.HeaderEncryptedEncryptionKeys.ItemsRo.Length); Assert.Equal(NcaCryptoConfiguration.Aes128KeySize, s.HeaderEncryptedEncryptionKeys.ItemsRo[0].ItemsRo.Length); - - Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 0, (int)KeyType.NcaHeaderKey); - Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 1, (int)KeyType.NcaExternalKey); - Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 2, (int)KeyType.SaveDataDeviceUniqueMac); - Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 3, (int)KeyType.SaveDataSeedUniqueMac); - Assert.Equal(NcaCryptoConfiguration.KeyAreaEncryptionKeyCount + 4, (int)KeyType.SaveDataTransferMac); } [Fact] @@ -256,7 +275,8 @@ public static void AesCtrCounterExtendedStorage_Entry_Layout() Assert.Equal(0x10, Unsafe.SizeOf()); Assert.Equal(0x0, GetOffset(in s, in s.Offset)); - Assert.Equal(0x8, GetOffset(in s, in s.Reserved)); + Assert.Equal(0x8, GetOffset(in s, in s.EncryptionValue)); + Assert.Equal(0x9, GetOffset(in s, in s.Reserved)); Assert.Equal(0xC, GetOffset(in s, in s.Generation)); } } \ No newline at end of file From 2241a7ced334a9ac4f57298d28685a60853f9fc0 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sat, 16 Apr 2022 23:37:10 -0700 Subject: [PATCH 18/24] Update AesCtrCounterExtendedStorage for 14.0.0 --- .../FsSystem/AesCtrCounterExtendedStorage.cs | 65 ++++++++++++------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/src/LibHac/FsSystem/AesCtrCounterExtendedStorage.cs b/src/LibHac/FsSystem/AesCtrCounterExtendedStorage.cs index 578ddd35..3ec61e7c 100644 --- a/src/LibHac/FsSystem/AesCtrCounterExtendedStorage.cs +++ b/src/LibHac/FsSystem/AesCtrCounterExtendedStorage.cs @@ -17,11 +17,11 @@ namespace LibHac.FsSystem; /// The base data used for this storage comes with a table of ranges and counter values that are used /// to decrypt each range. This encryption scheme is used for encrypting content updates so that no counter values /// are ever reused. -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class AesCtrCounterExtendedStorage : IStorage { - public delegate Result DecryptFunction(Span destination, int index, ReadOnlySpan encryptedKey, - ReadOnlySpan iv, ReadOnlySpan source); + public delegate Result DecryptFunction(Span destination, int index, int generation, + ReadOnlySpan encryptedKey, ReadOnlySpan iv, ReadOnlySpan source); public interface IDecryptor : IDisposable { @@ -32,9 +32,16 @@ public interface IDecryptor : IDisposable public struct Entry { public Array8 Offset; - public int Reserved; + public Encryption EncryptionValue; + public Array3 Reserved; public int Generation; + public enum Encryption : byte + { + Encrypted = 0, + NotEncrypted = 1 + } + public void SetOffset(long value) { BinaryPrimitives.WriteInt64LittleEndian(Offset.Items, value); @@ -74,9 +81,9 @@ public static long QueryEntryStorageSize(int entryCount) } public static Result CreateExternalDecryptor(ref UniqueRef outDecryptor, - DecryptFunction decryptFunction, int keyIndex) + DecryptFunction decryptFunction, int keyIndex, int keyGeneration) { - using var decryptor = new UniqueRef(new ExternalDecryptor(decryptFunction, keyIndex)); + using var decryptor = new UniqueRef(new ExternalDecryptor(decryptFunction, keyIndex, keyGeneration)); if (!decryptor.HasValue) return ResultFs.AllocationMemoryFailedInAesCtrCounterExtendedStorageA.Log(); @@ -159,9 +166,18 @@ public Result Initialize(MemoryResource allocator, ReadOnlySpan key, uint Assert.SdkRequiresGreaterEqual(counterOffset, 0); Assert.SdkRequiresNotNull(in decryptor); - Result rc = _table.Initialize(allocator, in nodeStorage, in entryStorage, NodeSize, Unsafe.SizeOf(), - entryCount); - if (rc.IsFailure()) return rc.Miss(); + Result rc; + + if (entryCount > 0) + { + rc = _table.Initialize(allocator, in nodeStorage, in entryStorage, NodeSize, Unsafe.SizeOf(), + entryCount); + if (rc.IsFailure()) return rc.Miss(); + } + else + { + _table.Initialize(NodeSize, 0); + } rc = dataStorage.GetSize(out long dataStorageSize); if (rc.IsFailure()) return rc.Miss(); @@ -279,20 +295,23 @@ public override Result Read(long offset, Span destination) long readSize = Math.Min(remainingSize, dataSize); Assert.SdkLessEqual(readSize, destination.Length); - // Create the counter for the first data block we're decrypting. - long counterOffset = _counterOffset + entryStartOffset + dataOffset; - var upperIv = new NcaAesCtrUpperIv + if (entry.EncryptionValue == Entry.Encryption.Encrypted) { - Generation = (uint)entry.Generation, - SecureValue = _secureValue - }; + // Create the counter for the first data block we're decrypting. + long counterOffset = _counterOffset + entryStartOffset + dataOffset; + var upperIv = new NcaAesCtrUpperIv + { + Generation = (uint)entry.Generation, + SecureValue = _secureValue + }; - Unsafe.SkipInit(out Array16 counter); - AesCtrStorage.MakeIv(counter.Items, upperIv.Value, counterOffset); + Unsafe.SkipInit(out Array16 counter); + AesCtrStorage.MakeIv(counter.Items, upperIv.Value, counterOffset); - // Decrypt the data from the current entry. - rc = _decryptor.Get.Decrypt(currentData.Slice(0, (int)dataSize), _key, counter); - if (rc.IsFailure()) return rc.Miss(); + // Decrypt the data from the current entry. + rc = _decryptor.Get.Decrypt(currentData.Slice(0, (int)dataSize), _key, counter); + if (rc.IsFailure()) return rc.Miss(); + } // Advance the current offsets. currentData = currentData.Slice((int)dataSize); @@ -402,13 +421,15 @@ private class ExternalDecryptor : IDecryptor { private DecryptFunction _decryptFunction; private int _keyIndex; + private int _keyGeneration; - public ExternalDecryptor(DecryptFunction decryptFunction, int keyIndex) + public ExternalDecryptor(DecryptFunction decryptFunction, int keyIndex, int keyGeneration) { Assert.SdkRequiresNotNull(decryptFunction); _decryptFunction = decryptFunction; _keyIndex = keyIndex; + _keyGeneration = keyGeneration; } public void Dispose() { } @@ -435,7 +456,7 @@ public Result Decrypt(Span destination, ReadOnlySpan encryptedKey, R Span dstBuffer = destination.Slice(currentOffset, currentSize); Span workBuffer = pooledBuffer.GetBuffer().Slice(0, currentSize); - Result rc = _decryptFunction(workBuffer, _keyIndex, encryptedKey, counter, dstBuffer); + Result rc = _decryptFunction(workBuffer, _keyIndex, _keyGeneration, encryptedKey, counter, dstBuffer); if (rc.IsFailure()) return rc.Miss(); workBuffer.CopyTo(dstBuffer); From b54f5d17fa92db0dd5d1c4f090745fe1a056e591 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sun, 17 Apr 2022 21:25:26 -0700 Subject: [PATCH 19/24] Add IntegrityVerificationStorage --- build/CodeGen/results.csv | 4 +- src/LibHac/Fs/ResultFs.cs | 4 +- src/LibHac/FsSrv/Delegates.cs | 4 +- .../SaveDataTransferCryptoConfiguration.cs | 1 + src/LibHac/FsSystem/Delegates.cs | 5 + .../FsSystem/DirectorySaveDataFileSystem.cs | 1 - .../FsSystem/IntegrityVerificationStorage.cs | 623 ++++++++++++++++++ src/LibHac/FsSystem/PooledBuffer.cs | 7 + .../DirectorySaveDataFileSystemTests.cs | 1 - 9 files changed, 641 insertions(+), 9 deletions(-) create mode 100644 src/LibHac/FsSystem/Delegates.cs create mode 100644 src/LibHac/FsSystem/IntegrityVerificationStorage.cs diff --git a/build/CodeGen/results.csv b/build/CodeGen/results.csv index 49f9eaf2..e390c181 100644 --- a/build/CodeGen/results.csv +++ b/build/CodeGen/results.csv @@ -996,10 +996,10 @@ Module,DescriptionStart,DescriptionEnd,Flags,Namespace,Name,Summary 2,6316,,,,UnsupportedSetSizeForHierarchicalIntegrityVerificationStorage, 2,6317,,,,UnsupportedOperateRangeForHierarchicalIntegrityVerificationStorage, 2,6318,,,,UnsupportedSetSizeForIntegrityVerificationStorage, -2,6319,,,,UnsupportedOperateRangeForNonSaveDataIntegrityVerificationStorage, +2,6319,,,,UnsupportedOperateRangeForWritableIntegrityVerificationStorage, 2,6320,,,,UnsupportedOperateRangeForIntegrityVerificationStorage, 2,6321,,,,UnsupportedSetSizeForBlockCacheBufferedStorage, -2,6322,,,,UnsupportedOperateRangeForNonSaveDataBlockCacheBufferedStorage, +2,6322,,,,UnsupportedOperateRangeForWritableBlockCacheBufferedStorage, 2,6323,,,,UnsupportedOperateRangeForBlockCacheBufferedStorage, 2,6324,,,,UnsupportedWriteForIndirectStorage, 2,6325,,,,UnsupportedSetSizeForIndirectStorage, diff --git a/src/LibHac/Fs/ResultFs.cs b/src/LibHac/Fs/ResultFs.cs index b81011f9..50943f44 100644 --- a/src/LibHac/Fs/ResultFs.cs +++ b/src/LibHac/Fs/ResultFs.cs @@ -1817,13 +1817,13 @@ public static class ResultFs /// Error code: 2002-6318; Inner value: 0x315c02 public static Result.Base UnsupportedSetSizeForIntegrityVerificationStorage => new Result.Base(ModuleFs, 6318); /// Error code: 2002-6319; Inner value: 0x315e02 - public static Result.Base UnsupportedOperateRangeForNonSaveDataIntegrityVerificationStorage => new Result.Base(ModuleFs, 6319); + public static Result.Base UnsupportedOperateRangeForWritableIntegrityVerificationStorage => new Result.Base(ModuleFs, 6319); /// Error code: 2002-6320; Inner value: 0x316002 public static Result.Base UnsupportedOperateRangeForIntegrityVerificationStorage => new Result.Base(ModuleFs, 6320); /// Error code: 2002-6321; Inner value: 0x316202 public static Result.Base UnsupportedSetSizeForBlockCacheBufferedStorage => new Result.Base(ModuleFs, 6321); /// Error code: 2002-6322; Inner value: 0x316402 - public static Result.Base UnsupportedOperateRangeForNonSaveDataBlockCacheBufferedStorage => new Result.Base(ModuleFs, 6322); + public static Result.Base UnsupportedOperateRangeForWritableBlockCacheBufferedStorage => new Result.Base(ModuleFs, 6322); /// Error code: 2002-6323; Inner value: 0x316602 public static Result.Base UnsupportedOperateRangeForBlockCacheBufferedStorage => new Result.Base(ModuleFs, 6323); /// Error code: 2002-6324; Inner value: 0x316802 diff --git a/src/LibHac/FsSrv/Delegates.cs b/src/LibHac/FsSrv/Delegates.cs index 4ee0163c..725f9ec0 100644 --- a/src/LibHac/FsSrv/Delegates.cs +++ b/src/LibHac/FsSrv/Delegates.cs @@ -2,12 +2,10 @@ namespace LibHac.FsSrv; -public delegate Result RandomDataGenerator(Span buffer); - public delegate Result SaveTransferAesKeyGenerator(Span key, SaveDataTransferCryptoConfiguration.KeyIndex index, ReadOnlySpan keySource, int keyGeneration); public delegate Result SaveTransferCmacGenerator(Span mac, ReadOnlySpan data, SaveDataTransferCryptoConfiguration.KeyIndex index, int keyGeneration); -public delegate Result PatrolAllocateCountGetter(out long successCount, out long failureCount); +public delegate Result PatrolAllocateCountGetter(out long successCount, out long failureCount); \ No newline at end of file diff --git a/src/LibHac/FsSrv/SaveDataTransferCryptoConfiguration.cs b/src/LibHac/FsSrv/SaveDataTransferCryptoConfiguration.cs index d6b6b15d..21fdc3ef 100644 --- a/src/LibHac/FsSrv/SaveDataTransferCryptoConfiguration.cs +++ b/src/LibHac/FsSrv/SaveDataTransferCryptoConfiguration.cs @@ -1,5 +1,6 @@ using System; using LibHac.Common.FixedArrays; +using LibHac.FsSystem; namespace LibHac.FsSrv; diff --git a/src/LibHac/FsSystem/Delegates.cs b/src/LibHac/FsSystem/Delegates.cs new file mode 100644 index 00000000..89a10d26 --- /dev/null +++ b/src/LibHac/FsSystem/Delegates.cs @@ -0,0 +1,5 @@ +using System; + +namespace LibHac.FsSystem; + +public delegate Result RandomDataGenerator(Span buffer); \ No newline at end of file diff --git a/src/LibHac/FsSystem/DirectorySaveDataFileSystem.cs b/src/LibHac/FsSystem/DirectorySaveDataFileSystem.cs index ac1aca8d..7e610c01 100644 --- a/src/LibHac/FsSystem/DirectorySaveDataFileSystem.cs +++ b/src/LibHac/FsSystem/DirectorySaveDataFileSystem.cs @@ -4,7 +4,6 @@ using LibHac.Diag; using LibHac.Fs; using LibHac.Fs.Fsa; -using LibHac.FsSrv; using LibHac.Os; namespace LibHac.FsSystem; diff --git a/src/LibHac/FsSystem/IntegrityVerificationStorage.cs b/src/LibHac/FsSystem/IntegrityVerificationStorage.cs new file mode 100644 index 00000000..6f69e41d --- /dev/null +++ b/src/LibHac/FsSystem/IntegrityVerificationStorage.cs @@ -0,0 +1,623 @@ +using System; +using System.Runtime.CompilerServices; +using LibHac.Common; +using LibHac.Common.FixedArrays; +using LibHac.Crypto; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.Util; + +namespace LibHac.FsSystem; + +/// +/// An that can verify the integrity of its data by using a second +/// that contains hash digests of the main data. +/// +/// An consists of a data +/// and a hash . The main storage is split into blocks of a provided size and the hashes +/// of all these blocks are stored sequentially in the hash storage. Each time a data block is read its hash is +/// also read and used to verify the integrity of the main data. +/// An may be writable, updating the hash storage as required +/// when written to. Writable storages have some additional features compared to read-only storages:
+/// If the hash for a data block is all zeros then that block is treated as if the actual data is all zeros.
+/// To avoid collisions in the case where a block's actual hash is all zeros, a certain bit in all writable storage +/// hashes is always set to 1.
+/// An optional may be provided. This salt will be added to the beginning of each block of +/// data before it is hashed. +/// Based on FS 14.1.0 (nnSdk 14.3.0) +public class IntegrityVerificationStorage : IStorage +{ + public struct BlockHash + { + public Array32 Hash; + } + + public const int HashSize = Sha256.DigestSize; + + private ValueSubStorage _hashStorage; + private ValueSubStorage _dataStorage; + private int _verificationBlockSize; + private int _verificationBlockOrder; + private int _upperLayerVerificationBlockSize; + private int _upperLayerVerificationBlockOrder; + private IBufferManager _bufferManager; + private Optional _hashSalt; + private bool _isRealData; + private IHash256GeneratorFactory _hashGeneratorFactory; + private bool _isWritable; + private bool _allowClearedBlocks; + + public IntegrityVerificationStorage() + { + _hashStorage = new ValueSubStorage(); + _dataStorage = new ValueSubStorage(); + } + + public override void Dispose() + { + FinalizeObject(); + _dataStorage.Dispose(); + _hashStorage.Dispose(); + + base.Dispose(); + } + + /// + /// Sets the validation bit on the provided . The hashes of all writable blocks + /// have a certain bit set so we can tell the difference between a cleared block and a hash of all zeros. + /// + /// The to have its validation bit set. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SetValidationBit(ref BlockHash hash) + { + hash.Hash.Items[HashSize - 1] |= 0x80; + } + + /// + /// Checks if the provided has its validation bit set. The hashes of all writable blocks + /// have a certain bit set so we can tell the difference between a cleared block and a hash of all zeros. + /// + /// The to check. + /// if the validation bit is set; otherwise . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsValidationBit(in BlockHash hash) + { + return (hash.Hash.ItemsRo[HashSize - 1] & 0x80) != 0; + } + + public int GetBlockSize() + { + return _verificationBlockSize; + } + + public void Initialize(in ValueSubStorage hashStorage, in ValueSubStorage dataStorage, int sizeVerificationBlock, + int sizeUpperLayerVerificationBlock, IBufferManager bufferManager, + IHash256GeneratorFactory hashGeneratorFactory, in Optional hashSalt, bool isRealData, bool isWritable, + bool allowClearedBlocks) + { + Assert.SdkRequiresGreaterEqual(sizeVerificationBlock, HashSize); + Assert.SdkRequiresNotNull(bufferManager); + Assert.SdkRequiresNotNull(hashGeneratorFactory); + + _hashStorage.Set(in hashStorage); + _dataStorage.Set(in dataStorage); + + _hashGeneratorFactory = hashGeneratorFactory; + + _verificationBlockSize = sizeVerificationBlock; + _verificationBlockOrder = BitmapUtils.ILog2((uint)sizeVerificationBlock); + Assert.SdkRequiresEqual(1 << _verificationBlockOrder, _verificationBlockSize); + + _bufferManager = bufferManager; + + sizeUpperLayerVerificationBlock = Math.Max(sizeUpperLayerVerificationBlock, HashSize); + _upperLayerVerificationBlockSize = sizeUpperLayerVerificationBlock; + _upperLayerVerificationBlockOrder = BitmapUtils.ILog2((uint)sizeUpperLayerVerificationBlock); + Assert.SdkRequiresEqual(1 << _upperLayerVerificationBlockOrder, _upperLayerVerificationBlockSize); + + Assert.SdkAssert(_dataStorage.GetSize(out long dataSize).IsSuccess()); + Assert.SdkAssert(_hashStorage.GetSize(out long hashSize).IsSuccess()); + Assert.SdkAssert(hashSize / HashSize * _verificationBlockSize >= dataSize); + + _hashSalt = hashSalt; + + _isRealData = isRealData; + _isWritable = isWritable; + _allowClearedBlocks = allowClearedBlocks; + } + + public void FinalizeObject() + { + if (_bufferManager is not null) + { + using (var emptySubStorage = new ValueSubStorage()) + { + _hashStorage.Set(in emptySubStorage); + } + + using (var emptySubStorage = new ValueSubStorage()) + { + _dataStorage.Set(in emptySubStorage); + } + + _bufferManager = null; + } + } + + public override Result GetSize(out long size) + { + return _dataStorage.GetSize(out size); + } + + public override Result SetSize(long size) + { + return ResultFs.UnsupportedSetSizeForIntegrityVerificationStorage.Log(); + } + + public override Result Read(long offset, Span destination) + { + Assert.SdkRequiresNotEqual(0, destination.Length); + + Assert.SdkRequiresAligned(offset, _verificationBlockSize); + Assert.SdkRequiresAligned(destination.Length, _verificationBlockSize); + + if (destination.Length == 0) + return Result.Success; + + Result rc = _dataStorage.GetSize(out long dataSize); + if (rc.IsFailure()) return rc.Miss(); + + if (dataSize < offset) + return ResultFs.InvalidOffset.Log(); + + long alignedDataSize = Alignment.AlignUpPow2(dataSize, (uint)_verificationBlockSize); + rc = CheckAccessRange(offset, destination.Length, alignedDataSize); + if (rc.IsFailure()) return rc.Miss(); + + int readSize = destination.Length; + if (offset + readSize > dataSize) + { + // All reads to this storage must be aligned to the block size, but if the last data block is a partial block + // it will not be written to the base data storage. If that's the case, fill the unused portion of the block + // with zeros. The hash for the partial block is calculated using the padded, complete block. + int paddingOffset = (int)(dataSize - offset); + int paddingSize = _verificationBlockSize - (paddingOffset & (_verificationBlockSize - 1)); + Assert.SdkLess(paddingSize, _verificationBlockSize); + + // Clear the padding. + destination.Slice(paddingOffset, paddingSize).Clear(); + + // Set the new in-bounds size. + readSize = (int)(dataSize - offset); + } + + // Read all of the data to be validated. + rc = _dataStorage.Read(offset, destination.Slice(0, readSize)); + if (rc.IsFailure()) + { + destination.Clear(); + return rc.Log(); + } + + // Validate the hashes of the read data blocks. + Result verifyHashResult = Result.Success; + + using var hashGenerator = new UniqueRef(); + rc = _hashGeneratorFactory.Create(ref hashGenerator.Ref()); + if (rc.IsFailure()) return rc.Miss(); + + int signatureCount = destination.Length >> _verificationBlockOrder; + using var signatureBuffer = + new PooledBuffer(signatureCount * Unsafe.SizeOf(), Unsafe.SizeOf()); + int bufferCount = (int)Math.Min(signatureCount, signatureBuffer.GetSize() / (uint)Unsafe.SizeOf()); + + // Loop over each block while validating their signatures + int verifiedCount = 0; + while (verifiedCount < signatureCount) + { + int currentCount = Math.Min(bufferCount, signatureCount - verifiedCount); + + Result currentResult = ReadBlockSignature(signatureBuffer.GetBuffer(), + offset + (verifiedCount << _verificationBlockOrder), currentCount << _verificationBlockOrder); + + using var changePriority = new ScopedThreadPriorityChanger(1, ScopedThreadPriorityChanger.Mode.Relative); + + for (int i = 1; i < currentCount && currentResult.IsSuccess(); i++) + { + int verifiedSize = (verifiedCount + i) << _verificationBlockOrder; + ref BlockHash blockHash = ref signatureBuffer.GetBuffer()[i]; + currentResult = VerifyHash(destination.Slice(verifiedCount), ref blockHash, in hashGenerator); + + if (ResultFs.IntegrityVerificationStorageCorrupted.Includes(currentResult)) + { + // Don't output the corrupted block to the destination buffer + destination.Slice(verifiedSize, _verificationBlockSize).Clear(); + + if (!ResultFs.ClearedRealDataVerificationFailed.Includes(currentResult) && !_allowClearedBlocks) + { + verifyHashResult = currentResult; + } + + currentResult = Result.Success; + } + } + + if (currentResult.IsFailure()) + { + destination.Clear(); + return currentResult; + } + + verifiedCount += currentCount; + } + + return verifyHashResult; + } + + public override Result Write(long offset, ReadOnlySpan source) + { + if (source.Length == 0) + return Result.Success; + + Result rc = CheckOffsetAndSize(offset, source.Length); + if (rc.IsFailure()) return rc.Miss(); + + rc = _dataStorage.GetSize(out long dataSize); + if (rc.IsFailure()) return rc.Miss(); + + if (offset >= dataSize) + return ResultFs.InvalidOffset.Log(); + + rc = CheckAccessRange(offset, source.Length, Alignment.AlignUpPow2(dataSize, (uint)_verificationBlockSize)); + if (rc.IsFailure()) return rc.Miss(); + + Assert.SdkRequiresAligned(offset, _verificationBlockSize); + Assert.SdkRequiresAligned(source.Length, _verificationBlockSize); + Assert.SdkLessEqual(offset, dataSize); + Assert.SdkLess(offset + source.Length, dataSize + _verificationBlockSize); + + // When writing to a partial final block, the data past the end of the partial block should be all zeros. + if (offset + source.Length > dataSize) + { + Assert.SdkAssert(source.Slice((int)(dataSize - offset)).IsZeros()); + } + + // Determine the size of the unpadded data we're writing to the base data storage + int writeSize = source.Length; + if (offset + writeSize > dataSize) + { + writeSize = (int)(dataSize - offset); + + if (writeSize == 0) + return Result.Success; + } + + int alignedWriteSize = Alignment.AlignUpPow2(writeSize, (uint)_verificationBlockSize); + + Result updateResult = Result.Success; + int updatedSignatureCount = 0; + { + int signatureCount = alignedWriteSize >> _verificationBlockOrder; + + using var signatureBuffer = + new PooledBuffer(signatureCount * Unsafe.SizeOf(), Unsafe.SizeOf()); + int bufferCount = Math.Min(signatureCount, signatureBuffer.GetSize() / Unsafe.SizeOf()); + + using var hashGenerator = new UniqueRef(); + rc = _hashGeneratorFactory.Create(ref hashGenerator.Ref()); + if (rc.IsFailure()) return rc.Miss(); + + while (updatedSignatureCount < signatureCount) + { + int remainingCount = signatureCount - updatedSignatureCount; + int currentCount = Math.Min(bufferCount, remainingCount); + + using (new ScopedThreadPriorityChanger(1, ScopedThreadPriorityChanger.Mode.Relative)) + { + // Calculate the new hashes for the current set of blocks + for (int i = 0; i < currentCount; i++) + { + int updatedSize = (updatedSignatureCount + i) << _verificationBlockOrder; + CalcBlockHash(out signatureBuffer.GetBuffer()[i], source.Slice(updatedSize), + in hashGenerator); + } + } + + // Write the new block signatures. + updateResult = WriteBlockSignature(signatureBuffer.GetBuffer(), + offset: offset + (updatedSignatureCount << _verificationBlockOrder), + size: currentCount << _verificationBlockOrder); + + if (updateResult.IsFailure()) + break; + + updatedSignatureCount += currentCount; + } + } + + // The updated hash values have all been written. Now write the actual data. + // If there was an error writing the updated hashes, only the data for the blocks that were + // successfully updated will be written. + int dataWriteSize = Math.Min(writeSize, updatedSignatureCount << _verificationBlockOrder); + rc = _dataStorage.Write(offset, source.Slice(0, dataWriteSize)); + if (rc.IsFailure()) return rc.Miss(); + + return updateResult; + } + + public override Result Flush() + { + Result rc = _hashStorage.Flush(); + if (rc.IsFailure()) return rc.Miss(); + + rc = _dataStorage.Flush(); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + if (operationId != OperationId.InvalidateCache) + { + Assert.SdkRequiresAligned(offset, _verificationBlockSize); + Assert.SdkRequiresAligned(size, _verificationBlockSize); + } + + switch (operationId) + { + case OperationId.FillZero: + { + Assert.SdkRequires(_isWritable); + + Result rc = _dataStorage.GetSize(out long dataStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (offset < 0 || dataStorageSize < offset) + return ResultFs.InvalidOffset.Log(); + + // Get the range of the signatures for the blocks that will be cleared + long signOffset = (offset >> _verificationBlockOrder) * Unsafe.SizeOf(); + long signSize = Math.Min(size, dataStorageSize - offset) * Unsafe.SizeOf(); + + // Allocate a work buffer up to 4 times the size of the hash storage's verification block size. + int bufferSize = (int)Math.Min(signSize, 1 << (_upperLayerVerificationBlockOrder + 2)); + using var workBuffer = new RentedArray(bufferSize); + if (workBuffer.Array is null) + return ResultFs.AllocationMemoryFailedInIntegrityVerificationStorageA.Log(); + + workBuffer.Span.Clear(); + + long remainingSize = signSize; + + // Clear the hash storage in chunks. + while (remainingSize > 0) + { + int currentSize = (int)Math.Min(remainingSize, bufferSize); + + rc = _hashStorage.Write(signOffset + signSize - remainingSize, + workBuffer.Span.Slice(0, currentSize)); + if (rc.IsFailure()) return rc.Miss(); + + remainingSize -= currentSize; + } + + return Result.Success; + } + case OperationId.DestroySignature: + { + Assert.SdkRequires(_isWritable); + + Result rc = _dataStorage.GetSize(out long dataStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (offset < 0 || dataStorageSize < offset) + return ResultFs.InvalidOffset.Log(); + + // Get the range of the signatures for the blocks that will be cleared + long signOffset = (offset >> _verificationBlockOrder) * Unsafe.SizeOf(); + long signSize = Math.Min(size, dataStorageSize - offset) * Unsafe.SizeOf(); + + using var workBuffer = new RentedArray((int)signSize); + if (workBuffer.Array is null) + return ResultFs.AllocationMemoryFailedInIntegrityVerificationStorageB.Log(); + + // Read the existing signature. + rc = _hashStorage.Read(signOffset, workBuffer.Span); + if (rc.IsFailure()) return rc.Miss(); + + // Clear the signature. + // This flips all bits, leaving the verification bit cleared. + for (int i = 0; i < workBuffer.Span.Length; i++) + { + workBuffer.Span[i] ^= (byte)((i + 1) % (uint)HashSize == 0 ? 0x7F : 0xFF); + } + + // Write the cleared signature. + return _hashStorage.Write(signOffset, workBuffer.Span); + } + case OperationId.InvalidateCache: + { + // Only allow cache invalidation for read-only storages. + if (_isWritable) + return ResultFs.UnsupportedOperateRangeForWritableIntegrityVerificationStorage.Log(); + + Result rc = _hashStorage.OperateRange(operationId, 0, long.MaxValue); + if (rc.IsFailure()) return rc.Miss(); + + rc = _dataStorage.OperateRange(operationId, offset, size); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + case OperationId.QueryRange: + { + Result rc = _dataStorage.GetSize(out long dataStorageSize); + if (rc.IsFailure()) return rc.Miss(); + + if (offset < 0 || dataStorageSize < offset) + return ResultFs.InvalidOffset.Log(); + + long actualSize = Math.Min(size, dataStorageSize - offset); + rc = _dataStorage.OperateRange(outBuffer, operationId, offset, actualSize, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + default: + return ResultFs.UnsupportedOperateRangeForIntegrityVerificationStorage.Log(); + } + } + + private Result ReadBlockSignature(Span destination, long offset, int size) + { + Assert.SdkRequiresAligned(offset, _verificationBlockSize); + Assert.SdkRequiresAligned(size, _verificationBlockSize); + + // Calculate the range that contains the signatures. + long offsetSignData = (offset >> _verificationBlockOrder) * HashSize; + long sizeSignData = (size >> _verificationBlockOrder) * HashSize; + Assert.SdkGreaterEqual(destination.Length, sizeSignData); + + // Validate the hash storage contains the calculated range. + Result rc = _hashStorage.GetSize(out long sizeHash); + if (rc.IsFailure()) return rc.Miss(); + + Assert.SdkLessEqual(offsetSignData + sizeSignData, sizeHash); + + if (offsetSignData + sizeSignData > sizeHash) + return ResultFs.OutOfRange.Log(); + + // Read the signature. + rc = _hashStorage.Read(offsetSignData, destination.Slice(0, (int)sizeSignData)); + if (rc.IsFailure()) + { + // Clear any read signature data if something goes wrong. + destination.Slice(0, (int)sizeSignData); + return rc.Miss(); + } + + return Result.Success; + } + + private Result WriteBlockSignature(ReadOnlySpan source, long offset, int size) + { + Assert.SdkRequiresAligned(offset, _verificationBlockSize); + + long offsetSignData = (offset >> _verificationBlockOrder) * HashSize; + long sizeSignData = (size >> _verificationBlockOrder) * HashSize; + Assert.SdkGreaterEqual(source.Length, sizeSignData); + + Result rc = _hashStorage.Write(offsetSignData, source.Slice(0, (int)sizeSignData)); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public Result CalcBlockHash(out BlockHash outHash, ReadOnlySpan buffer, int verificationBlockSize) + { + UnsafeHelpers.SkipParamInit(out outHash); + + using var hashGenerator = new UniqueRef(); + Result rc = _hashGeneratorFactory.Create(ref hashGenerator.Ref()); + if (rc.IsFailure()) return rc.Miss(); + + CalcBlockHash(out outHash, buffer, verificationBlockSize, in hashGenerator); + return Result.Success; + } + + private void CalcBlockHash(out BlockHash outHash, ReadOnlySpan buffer, + in UniqueRef hashGenerator) + { + CalcBlockHash(out outHash, buffer, _verificationBlockSize, in hashGenerator); + } + + private void CalcBlockHash(out BlockHash outHash, ReadOnlySpan buffer, int verificationBlockSize, + in UniqueRef hashGenerator) + { + UnsafeHelpers.SkipParamInit(out outHash); + + if (_isWritable) + { + if (_hashSalt.HasValue) + { + // Calculate the hash using the salt if enabled. + hashGenerator.Get.Initialize(); + hashGenerator.Get.Update(_hashSalt.ValueRo.HashRo); + + hashGenerator.Get.Update(buffer.Slice(0, verificationBlockSize)); + hashGenerator.Get.GetHash(SpanHelpers.AsByteSpan(ref outHash)); + } + else + { + // Otherwise calculate the hash of just the data. + _hashGeneratorFactory.GenerateHash(SpanHelpers.AsByteSpan(ref outHash), + buffer.Slice(0, verificationBlockSize)); + } + + // The hashes of all writable blocks have the validation bit set. + SetValidationBit(ref outHash); + } + else + { + // Nothing special needed for read-only blocks. Just calculate the hash. + _hashGeneratorFactory.GenerateHash(SpanHelpers.AsByteSpan(ref outHash), + buffer.Slice(0, verificationBlockSize)); + } + } + + private Result VerifyHash(ReadOnlySpan buffer, ref BlockHash hash, + in UniqueRef hashGenerator) + { + Assert.SdkRequiresGreaterEqual(buffer.Length, HashSize); + + // Writable storages allow using an all-zeros hash to indicate an empty block. + if (_isWritable) + { + Result rc = IsCleared(out bool isCleared, in hash); + if (rc.IsFailure()) return rc.Miss(); + + if (isCleared) + return ResultFs.ClearedRealDataVerificationFailed.Log(); + } + + CalcBlockHash(out BlockHash actualHash, buffer, hashGenerator); + + if (!CryptoUtil.IsSameBytes(SpanHelpers.AsReadOnlyByteSpan(in hash), + SpanHelpers.AsReadOnlyByteSpan(in actualHash), Unsafe.SizeOf())) + { + hash = default; + + if (_isRealData) + { + return ResultFs.UnclearedRealDataVerificationFailed.Log(); + } + else + { + return ResultFs.NonRealDataVerificationFailed.Log(); + } + } + + return Result.Success; + } + + private Result IsCleared(out bool isCleared, in BlockHash hash) + { + Assert.SdkRequires(_isWritable); + + isCleared = false; + + if (IsValidationBit(in hash)) + return Result.Success; + + for (int i = 0; i < hash.Hash.ItemsRo.Length; i++) + { + if (hash.Hash.ItemsRo[i] != 0) + return ResultFs.InvalidZeroHash.Log(); + } + + isCleared = true; + return Result.Success; + } +} \ No newline at end of file diff --git a/src/LibHac/FsSystem/PooledBuffer.cs b/src/LibHac/FsSystem/PooledBuffer.cs index 154db9c9..7f6733ed 100644 --- a/src/LibHac/FsSystem/PooledBuffer.cs +++ b/src/LibHac/FsSystem/PooledBuffer.cs @@ -1,5 +1,6 @@ using System; using System.Buffers; +using System.Runtime.InteropServices; using LibHac.Diag; using LibHac.FsSrv; using LibHac.Os; @@ -116,6 +117,12 @@ public Span GetBuffer() return _array.AsSpan(0, _length); } + public Span GetBuffer() where T : unmanaged + { + Assert.SdkRequiresNotNull(_array); + return MemoryMarshal.Cast(_array.AsSpan(0, _length)); + } + public int GetSize() { Assert.SdkRequiresNotNull(_array); diff --git a/tests/LibHac.Tests/FsSystem/DirectorySaveDataFileSystemTests.cs b/tests/LibHac.Tests/FsSystem/DirectorySaveDataFileSystemTests.cs index 9f7f41fa..8707dfdb 100644 --- a/tests/LibHac.Tests/FsSystem/DirectorySaveDataFileSystemTests.cs +++ b/tests/LibHac.Tests/FsSystem/DirectorySaveDataFileSystemTests.cs @@ -4,7 +4,6 @@ using LibHac.Common; using LibHac.Fs; using LibHac.Fs.Fsa; -using LibHac.FsSrv; using LibHac.FsSystem; using LibHac.Tests.Fs; using LibHac.Tests.Fs.IFileSystemTestBase; From 6c9e6e5203455490ff198fef7a87c891233c0406 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 18 Apr 2022 18:01:38 -0700 Subject: [PATCH 20/24] Add HierarchicalIntegrityVerificationStorage --- src/LibHac/Crypto/HmacSha256.cs | 17 + src/LibHac/FsSrv/FileSystemServer.cs | 2 + src/LibHac/FsSystem/BitmapUtils.cs | 6 +- .../FsSystem/BlockCacheBufferedStorage.cs | 6 +- ...ierarchicalIntegrityVerificationStorage.cs | 817 ++++++++++++++++++ src/LibHac/Util/Optional.cs | 5 + .../LibHac.Tests/FsSystem/TypeLayoutTests.cs | 75 ++ 7 files changed, 922 insertions(+), 6 deletions(-) create mode 100644 src/LibHac/Crypto/HmacSha256.cs create mode 100644 src/LibHac/FsSystem/HierarchicalIntegrityVerificationStorage.cs diff --git a/src/LibHac/Crypto/HmacSha256.cs b/src/LibHac/Crypto/HmacSha256.cs new file mode 100644 index 00000000..8fe9dd2a --- /dev/null +++ b/src/LibHac/Crypto/HmacSha256.cs @@ -0,0 +1,17 @@ +using System; +using System.Security.Cryptography; +using LibHac.Diag; + +namespace LibHac.Crypto; + +public static class HmacSha256 +{ + public const int HashSize = Sha256.DigestSize; + + public static void GenerateHmacSha256(Span outMac, ReadOnlySpan data, ReadOnlySpan key) + { + bool success = HMACSHA256.TryHashData(key, data, outMac, out int bytesWritten); + + Abort.DoAbortUnless(success && bytesWritten == HashSize); + } +} \ No newline at end of file diff --git a/src/LibHac/FsSrv/FileSystemServer.cs b/src/LibHac/FsSrv/FileSystemServer.cs index cf0de519..9667c075 100644 --- a/src/LibHac/FsSrv/FileSystemServer.cs +++ b/src/LibHac/FsSrv/FileSystemServer.cs @@ -44,6 +44,7 @@ internal struct FileSystemServerGlobals : IDisposable public LocationResolverSetGlobals LocationResolverSet; public PooledBufferGlobals PooledBuffer; public GameCardServiceGlobals GameCardService; + public HierarchicalIntegrityVerificationStorageGlobals HierarchicalIntegrityVerificationStorage; public void Initialize(HorizonClient horizonClient, FileSystemServer fsServer) { @@ -55,6 +56,7 @@ public void Initialize(HorizonClient horizonClient, FileSystemServer fsServer) LocationResolverSet.Initialize(); PooledBuffer.Initialize(); GameCardService.Initialize(); + HierarchicalIntegrityVerificationStorage.Initialize(fsServer); } public void Dispose() diff --git a/src/LibHac/FsSystem/BitmapUtils.cs b/src/LibHac/FsSystem/BitmapUtils.cs index 0cdb4bfc..95ce63af 100644 --- a/src/LibHac/FsSystem/BitmapUtils.cs +++ b/src/LibHac/FsSystem/BitmapUtils.cs @@ -5,11 +5,11 @@ namespace LibHac.FsSystem; public static class BitmapUtils { // ReSharper disable once InconsistentNaming - public static uint ILog2(uint value) + public static int ILog2(uint value) { Assert.SdkRequiresGreater(value, 0u); - const uint intBitCount = 32; - return intBitCount - 1 - (uint)Util.BitUtil.CountLeadingZeros(value); + const int intBitCount = 32; + return intBitCount - 1 - Util.BitUtil.CountLeadingZeros(value); } } \ No newline at end of file diff --git a/src/LibHac/FsSystem/BlockCacheBufferedStorage.cs b/src/LibHac/FsSystem/BlockCacheBufferedStorage.cs index 1c339870..a572367e 100644 --- a/src/LibHac/FsSystem/BlockCacheBufferedStorage.cs +++ b/src/LibHac/FsSystem/BlockCacheBufferedStorage.cs @@ -59,8 +59,8 @@ public readonly long IsIncluded(long offset) private int _shiftBytesVerificationBlock; private int _flags; private int _bufferLevel; - private StorageType _storageType; private BlockCacheManager _cacheManager; + private bool _isWritable; public BlockCacheBufferedStorage() { @@ -93,7 +93,7 @@ public void SetRealDataCache(bool isEnabled) public Result Initialize(IBufferManager bufferManager, SdkRecursiveMutex mutex, IStorage data, long dataSize, int sizeBytesVerificationBlock, int maxCacheEntries, bool useRealDataCache, sbyte bufferLevel, - bool useKeepBurstMode, StorageType storageType) + bool useKeepBurstMode, bool isWritable) { Assert.SdkNotNull(data); Assert.SdkNotNull(mutex); @@ -111,7 +111,7 @@ public Result Initialize(IBufferManager bufferManager, SdkRecursiveMutex mutex, _lastResult = Result.Success; _flags = 0; _bufferLevel = bufferLevel; - _storageType = storageType; + _isWritable = isWritable; _shiftBytesVerificationBlock = (int)BitmapUtils.ILog2((uint)sizeBytesVerificationBlock); Assert.SdkEqual(1 << _shiftBytesVerificationBlock, _sizeBytesVerificationBlock); diff --git a/src/LibHac/FsSystem/HierarchicalIntegrityVerificationStorage.cs b/src/LibHac/FsSystem/HierarchicalIntegrityVerificationStorage.cs new file mode 100644 index 00000000..67270e15 --- /dev/null +++ b/src/LibHac/FsSystem/HierarchicalIntegrityVerificationStorage.cs @@ -0,0 +1,817 @@ +using System; +using System.Runtime.CompilerServices; +using LibHac.Common; +using LibHac.Common.FixedArrays; +using LibHac.Crypto; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.FsSrv; +using LibHac.Os; +using LibHac.Util; +using static LibHac.FsSystem.Constants; +using static LibHac.FsSystem.HierarchicalIntegrityVerificationStorage; +using static LibHac.Util.BitUtil; + +namespace LibHac.FsSystem; + +internal static class Constants +{ + public const int IntegrityMinLayerCount = 2; + public const int IntegrityMaxLayerCount = 7; +} + +internal struct HierarchicalIntegrityVerificationStorageGlobals +{ + public RandomDataGenerator GenerateRandom; + public Semaphore GlobalWriteSemaphore; + public Semaphore GlobalReadSemaphore; + + public void Initialize(FileSystemServer fsServer) + { + GlobalWriteSemaphore = new Semaphore(fsServer.Hos.Os, AccessCountMax, AccessCountMax); + GlobalReadSemaphore = new Semaphore(fsServer.Hos.Os, AccessCountMax, AccessCountMax); + } + + public void Dispose() + { + GlobalReadSemaphore.Dispose(); + } +} + +public class FileSystemBufferManagerSet +{ + public IBufferManager[] Buffers; + + public FileSystemBufferManagerSet() + { + Buffers = new IBufferManager[IntegrityMaxLayerCount]; + } +} + +public struct HierarchicalIntegrityVerificationLevelInformation +{ + public Fs.Int64 Offset; + public Fs.Int64 Size; + public int BlockOrder; + public uint Reserved; +} + +public struct HierarchicalIntegrityVerificationInformation +{ + public uint MaxLayers; + public Array6 Layers; + public HashSalt HashSalt; +} + +public struct HierarchicalIntegrityVerificationMetaInformation +{ + public uint Magic; + public uint Version; + public uint MasterHashSize; + public HierarchicalIntegrityVerificationInformation LevelHashInfo; + + public void Format() + { + var hashSalt = new Optional(); + Format(in hashSalt); + } + + public void Format(in Optional hashSalt) + { + Magic = IntegrityVerificationStorageMagic; + Version = IntegrityVerificationStorageVersion; + MasterHashSize = 0; + LevelHashInfo = default; + + if (hashSalt.HasValue) + { + LevelHashInfo.HashSalt = hashSalt.ValueRo; + } + } +} + +public struct HierarchicalIntegrityVerificationSizeSet +{ + public long ControlSize; + public long MasterHashSize; + public Array5 LayeredHashSizes; +} + +public class HierarchicalIntegrityVerificationStorageControlArea : IDisposable +{ + public struct InputParam + { + public Array6 LevelBlockSizes; + } + + public const int HashSize = Sha256.DigestSize; + + private ValueSubStorage _storage; + private HierarchicalIntegrityVerificationMetaInformation _meta; + + public HierarchicalIntegrityVerificationStorageControlArea() + { + _storage = new ValueSubStorage(); + } + + public void Dispose() + { + _storage.Dispose(); + } + + public static Result QuerySize(out HierarchicalIntegrityVerificationSizeSet outSizeSet, in InputParam inputParam, + int layerCount, long dataSize) + { + UnsafeHelpers.SkipParamInit(out outSizeSet); + + Assert.SdkRequires(layerCount >= IntegrityMinLayerCount && layerCount <= IntegrityMaxLayerCount); + + for (int level = 0; level < layerCount - 1; level++) + { + Assert.SdkRequires(inputParam.LevelBlockSizes[level] > 0 && IsPowerOfTwo(inputParam.LevelBlockSizes[level])); + } + + { + outSizeSet.ControlSize = Unsafe.SizeOf(); + + Span levelSize = stackalloc long[IntegrityMaxLayerCount]; + int level = layerCount - 1; + + levelSize[level] = Alignment.AlignUpPow2(dataSize, (uint)inputParam.LevelBlockSizes[level - 1]); + level--; + + for (; level > 0; level--) + { + // Calculate how much space is needed to store the hashes of the above level, rounding up to the next block size. + levelSize[level] = + Alignment.AlignUpPow2(levelSize[level + 1] / inputParam.LevelBlockSizes[level] * HashSize, + (uint)inputParam.LevelBlockSizes[level - 1]); + } + + // The size of the master hash does not get rounded up to the next block size. + levelSize[0] = levelSize[1] / inputParam.LevelBlockSizes[0] * HashSize; + outSizeSet.MasterHashSize = levelSize[0]; + + // Write the sizes of each level to the output struct. + for (level = 1; level < layerCount - 1; level++) + { + outSizeSet.LayeredHashSizes[level - 1] = levelSize[level]; + } + + return Result.Success; + } + } + + public static Result Format(in ValueSubStorage metaStorage, + in HierarchicalIntegrityVerificationMetaInformation metaInfo) + { + // Ensure the storage is large enough to hold the meta info. + Result rc = metaStorage.GetSize(out long metaSize); + if (rc.IsFailure()) return rc.Miss(); + + if (metaSize < Unsafe.SizeOf()) + return ResultFs.InvalidSize.Log(); + + // Validate the meta magic and version. + if (metaInfo.Magic != IntegrityVerificationStorageMagic) + return ResultFs.IncorrectIntegrityVerificationMagicCode.Log(); + + if ((metaInfo.Version & IntegrityVerificationStorageVersionMask) != IntegrityVerificationStorageVersion) + return ResultFs.UnsupportedVersion.Log(); + + // Write the meta info to the storage. + rc = metaStorage.Write(0, SpanHelpers.AsReadOnlyByteSpan(in metaInfo)); + if (rc.IsFailure()) return rc.Miss(); + + rc = metaStorage.Flush(); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public static Result Expand(in ValueSubStorage metaStorage, + in HierarchicalIntegrityVerificationMetaInformation newMeta) + { + // Ensure the storage is large enough to hold the meta info. + Result rc = metaStorage.GetSize(out long metaSize); + if (rc.IsFailure()) return rc.Miss(); + + if (metaSize < Unsafe.SizeOf()) + return ResultFs.InvalidSize.Log(); + + // Validate both the previous and new metas. + HierarchicalIntegrityVerificationMetaInformation previousMeta = default; + rc = metaStorage.Read(0, SpanHelpers.AsByteSpan(ref previousMeta)); + if (rc.IsFailure()) return rc.Miss(); + + if (newMeta.Magic != IntegrityVerificationStorageMagic || newMeta.Magic != previousMeta.Magic) + return ResultFs.IncorrectIntegrityVerificationMagicCode.Log(); + + if (newMeta.Version != IntegrityVerificationStorageVersion || newMeta.Version != previousMeta.Version) + return ResultFs.UnsupportedVersion.Log(); + + // Write the new meta. + rc = metaStorage.Write(0, SpanHelpers.AsReadOnlyByteSpan(in newMeta)); + if (rc.IsFailure()) return rc.Miss(); + + rc = metaStorage.Flush(); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public uint GetMasterHashSize() + { + return _meta.MasterHashSize; + } + + public void GetLevelHashInfo(out HierarchicalIntegrityVerificationInformation outInfo) + { + outInfo = _meta.LevelHashInfo; + } + + public Result Initialize(in ValueSubStorage metaStorage) + { + // Ensure the storage is large enough to hold the meta info. + Result rc = metaStorage.GetSize(out long metaSize); + if (rc.IsFailure()) return rc.Miss(); + + if (metaSize < Unsafe.SizeOf()) + return ResultFs.InvalidSize.Log(); + + // Set the storage and read the meta. + _storage.Set(in metaStorage); + rc = _storage.Read(0, SpanHelpers.AsByteSpan(ref _meta)); + if (rc.IsFailure()) return rc.Miss(); + + // Validate the meta magic and version. + if (_meta.Magic != IntegrityVerificationStorageMagic) + return ResultFs.IncorrectIntegrityVerificationMagicCode.Log(); + + if ((_meta.Version & IntegrityVerificationStorageVersionMask) != IntegrityVerificationStorageVersion) + return ResultFs.UnsupportedVersion.Log(); + + return Result.Success; + } + + public void FinalizeObject() + { + using var emptySubStorage = new ValueSubStorage(); + _storage.Set(in emptySubStorage); + } +} + +public class HierarchicalIntegrityVerificationStorage : IStorage +{ + [NonCopyableDisposable] + public struct HierarchicalStorageInformation : IDisposable + { + public enum Storage + { + MasterStorage = 0, + Layer1Storage = 1, + Layer2Storage = 2, + Layer3Storage = 3, + Layer4Storage = 4, + Layer5Storage = 5, + DataStorage = 6 + } + + private ValueSubStorage[] _storages; + + public HierarchicalStorageInformation(in HierarchicalStorageInformation other) + { + _storages = new ValueSubStorage[(int)Storage.DataStorage + 1]; + + for (int i = 0; i < _storages.Length; i++) + { + _storages[i] = new ValueSubStorage(in other._storages[i]); + } + } + + public HierarchicalStorageInformation() + { + _storages = new ValueSubStorage[(int)Storage.DataStorage + 1]; + } + + public void Dispose() + { + for (int i = 0; i < _storages.Length; i++) + { + _storages[i].Dispose(); + } + } + + public ref ValueSubStorage this[int index] + { + get + { + Assert.SdkRequiresInRange(index, (int)Storage.MasterStorage, (int)Storage.DataStorage + 1); + return ref _storages[index]; + } + } + + public void SetMasterHashStorage(in ValueSubStorage storage) => _storages[(int)Storage.MasterStorage].Set(in storage); + public void SetLayer1HashStorage(in ValueSubStorage storage) => _storages[(int)Storage.Layer1Storage].Set(in storage); + public void SetLayer2HashStorage(in ValueSubStorage storage) => _storages[(int)Storage.Layer2Storage].Set(in storage); + public void SetLayer3HashStorage(in ValueSubStorage storage) => _storages[(int)Storage.Layer3Storage].Set(in storage); + public void SetLayer4HashStorage(in ValueSubStorage storage) => _storages[(int)Storage.Layer4Storage].Set(in storage); + public void SetLayer5HashStorage(in ValueSubStorage storage) => _storages[(int)Storage.Layer5Storage].Set(in storage); + public void SetDataStorage(in ValueSubStorage storage) => _storages[(int)Storage.DataStorage].Set(in storage); + } + + internal const uint IntegrityVerificationStorageMagic = 0x43465649; // IVFC + internal const uint IntegrityVerificationStorageVersion = 0x00020000; + internal const uint IntegrityVerificationStorageVersionMask = 0xFFFF0000; + + internal const int HashSize = Sha256.DigestSize; + + internal const int AccessCountMax = 5; + internal TimeSpan AccessTimeout => TimeSpan.FromMilliSeconds(10); + + private const sbyte BaseBufferLevel = 0x10; + + private FileSystemBufferManagerSet _bufferManagers; + private SdkRecursiveMutex _mutex; + private IntegrityVerificationStorage[] _integrityStorages; + private BlockCacheBufferedStorage[] _bufferedStorages; + private Semaphore _readSemaphore; + private Semaphore _writeSemaphore; + private long _dataSize; + private int _layerCount; + + // LibHac addition + private FileSystemServer _fsServer; + + private static readonly byte[][] KeyArray = + { MasterKey.ToArray(), L1Key.ToArray(), L2Key.ToArray(), L3Key.ToArray(), L4Key.ToArray(), L5Key.ToArray() }; + + public HierarchicalIntegrityVerificationStorage(FileSystemServer fsServer) + { + _integrityStorages = new IntegrityVerificationStorage[IntegrityMaxLayerCount - 1]; + _bufferedStorages = new BlockCacheBufferedStorage[IntegrityMaxLayerCount - 1]; + + _dataSize = -1; + _fsServer = fsServer; + } + + public override void Dispose() + { + FinalizeObject(); + + if (_integrityStorages is not null) + { + foreach (IntegrityVerificationStorage storage in _integrityStorages) + { + storage.Dispose(); + } + } + + if (_bufferedStorages is not null) + { + foreach (BlockCacheBufferedStorage storage in _bufferedStorages) + { + storage.Dispose(); + } + } + + base.Dispose(); + } + + public FileSystemBufferManagerSet GetBuffers() + { + return _bufferManagers; + } + + public void GetParameters(out HierarchicalIntegrityVerificationStorageControlArea.InputParam outParam) + { + outParam = default; + + for (int i = 0; i < _layerCount - 2; i++) + { + outParam.LevelBlockSizes[i] = _integrityStorages[i].GetBlockSize(); + } + } + + public bool IsInitialized() + { + return _dataSize >= 0; + } + + public ValueSubStorage GetL1HashStorage() + { + return new ValueSubStorage(_bufferedStorages[_layerCount - 3], 0, + DivideUp(_dataSize, GetL1HashVerificationBlockSize())); + } + + public long GetL1HashVerificationBlockSize() + { + return _integrityStorages[_layerCount - 2].GetBlockSize(); + } + + public Result Initialize(in HierarchicalIntegrityVerificationInformation info, + ref HierarchicalStorageInformation storageInfo, FileSystemBufferManagerSet buffers, + IHash256GeneratorFactory hashGeneratorFactory, bool isHashSaltEnabled, SdkRecursiveMutex mutex, + int maxDataCacheEntries, int maxHashCacheEntries, sbyte bufferLevel, bool isWritable, bool allowClearedBlocks) + { + return Initialize(in info, ref storageInfo, buffers, hashGeneratorFactory, isHashSaltEnabled, mutex, null, null, + maxDataCacheEntries, maxHashCacheEntries, bufferLevel, isWritable, allowClearedBlocks); + } + + public Result Initialize(in HierarchicalIntegrityVerificationInformation info, + ref HierarchicalStorageInformation storageInfo, FileSystemBufferManagerSet buffers, + IHash256GeneratorFactory hashGeneratorFactory, bool isHashSaltEnabled, SdkRecursiveMutex mutex, + Semaphore readSemaphore, Semaphore writeSemaphore, int maxDataCacheEntries, int maxHashCacheEntries, + sbyte bufferLevel, bool isWritable, bool allowClearedBlocks) + { + // Validate preconditions. + Assert.SdkNotNull(buffers); + Assert.SdkAssert(info.MaxLayers >= IntegrityMinLayerCount && info.MaxLayers <= IntegrityMaxLayerCount); + + // Set member variables. + _layerCount = (int)info.MaxLayers; + _bufferManagers = buffers; + _mutex = mutex; + _readSemaphore = readSemaphore; + _writeSemaphore = writeSemaphore; + + { + // If hash salt is enabled, generate it. + var mac = new Optional(); + if (isHashSaltEnabled) + { + mac.Set(); + HmacSha256.GenerateHmacSha256(mac.Value.Hash, info.HashSalt.HashRo, KeyArray[0]); + } + + // Initialize the top level verification storage. + _integrityStorages[0] = new IntegrityVerificationStorage(); + _integrityStorages[0].Initialize(in storageInfo[(int)HierarchicalStorageInformation.Storage.MasterStorage], + in storageInfo[(int)HierarchicalStorageInformation.Storage.Layer1Storage], + 1 << info.Layers[0].BlockOrder, HashSize, _bufferManagers.Buffers[_layerCount - 2], + hashGeneratorFactory, in mac, false, isWritable, allowClearedBlocks); + } + + // Initialize the top level buffer storage. + _bufferedStorages[0] = new BlockCacheBufferedStorage(); + Result rc = _bufferedStorages[0].Initialize(_bufferManagers.Buffers[0], _mutex, _integrityStorages[0], + info.Layers[0].Size, 1 << info.Layers[0].BlockOrder, maxHashCacheEntries, false, BaseBufferLevel, false, + isWritable); + + if (!rc.IsFailure()) + { + int level; + + // Initialize the level storages. + for (level = 0; level < _layerCount - 3; level++) + { + // If hash salt is enabled, generate it. + var mac = new Optional(); + if (isHashSaltEnabled) + { + mac.Set(); + HmacSha256.GenerateHmacSha256(mac.Value.Hash, info.HashSalt.HashRo, KeyArray[level + 1]); + } + + // Initialize the verification storage. + using (var hashStorage = new ValueSubStorage(_bufferedStorages[level], 0, info.Layers[level].Size)) + { + _integrityStorages[level + 1] = new IntegrityVerificationStorage(); + _integrityStorages[level + 1].Initialize(in hashStorage, in storageInfo[level + 2], + 1 << info.Layers[level + 1].BlockOrder, 1 << info.Layers[level].BlockOrder, + _bufferManagers.Buffers[_layerCount - 2], hashGeneratorFactory, in mac, false, isWritable, + allowClearedBlocks); + } + + // Initialize the buffer storage. + _bufferedStorages[level + 1] = new BlockCacheBufferedStorage(); + rc = _bufferedStorages[level + 1].Initialize(_bufferManagers.Buffers[level + 1], _mutex, + _integrityStorages[level + 1], info.Layers[level + 1].Size, 1 << info.Layers[level + 1].BlockOrder, + maxHashCacheEntries, false, (sbyte)(BaseBufferLevel + (level + 1)), false, isWritable); + + if (rc.IsFailure()) + { + // Cleanup initialized storages if we failed. + _integrityStorages[level + 1].FinalizeObject(); + + for (; level > 0; level--) + { + _bufferedStorages[level].FinalizeObject(); + _integrityStorages[level].FinalizeObject(); + } + + break; + } + } + + if (!rc.IsFailure()) + { + // Initialize the final level storage. + // If hash salt is enabled, generate it. + var mac = new Optional(); + if (isHashSaltEnabled) + { + mac.Set(); + HmacSha256.GenerateHmacSha256(mac.Value.Hash, info.HashSalt.HashRo, KeyArray[level + 1]); + } + + // Initialize the verification storage. + using (var hashStorage = new ValueSubStorage(_bufferedStorages[level], 0, info.Layers[level].Size)) + { + _integrityStorages[level + 1] = new IntegrityVerificationStorage(); + _integrityStorages[level + 1].Initialize(in hashStorage, + in storageInfo[(int)HierarchicalStorageInformation.Storage.DataStorage], + 1 << info.Layers[level + 1].BlockOrder, 1 << info.Layers[level].BlockOrder, + _bufferManagers.Buffers[_layerCount - 2], hashGeneratorFactory, in mac, true, isWritable, + allowClearedBlocks); + } + + // Initialize the buffer storage. + _bufferedStorages[level + 1] = new BlockCacheBufferedStorage(); + rc = _bufferedStorages[level + 1].Initialize(_bufferManagers.Buffers[level + 1], _mutex, + _integrityStorages[level + 1], info.Layers[level + 1].Size, 1 << info.Layers[level + 1].BlockOrder, + maxDataCacheEntries, true, bufferLevel, true, isWritable); + + if (!rc.IsFailure()) + { + _dataSize = info.Layers[level + 1].Size; + return Result.Success; + } + + // Cleanup initialized storages if we failed. + _integrityStorages[level + 1].FinalizeObject(); + + for (; level > 0; level--) + { + _bufferedStorages[level].FinalizeObject(); + _integrityStorages[level].FinalizeObject(); + } + } + + _bufferedStorages[0].FinalizeObject(); + } + + _integrityStorages[0].FinalizeObject(); + + // Ensure we're uninitialized if we failed. + _dataSize = -1; + _bufferManagers = null; + _mutex = null; + + return rc; + } + + public void FinalizeObject() + { + if (_dataSize >= 0) + { + _dataSize = 0; + _bufferManagers = null; + _mutex = null; + + for (int level = _layerCount - 2; level >= 0; level--) + { + _bufferedStorages[level].FinalizeObject(); + _integrityStorages[level].FinalizeObject(); + } + + _dataSize = -1; + } + } + + public override Result GetSize(out long size) + { + Assert.SdkRequires(_dataSize >= 0); + + size = _dataSize; + return Result.Success; + } + + public override Result SetSize(long size) + { + return ResultFs.UnsupportedSetSizeForHierarchicalIntegrityVerificationStorage.Log(); + } + + public override Result Read(long offset, Span destination) + { + Assert.SdkRequires(_dataSize >= 0); + + if (destination.Length == 0) + return Result.Success; + + ref HierarchicalIntegrityVerificationStorageGlobals g = + ref _fsServer.Globals.HierarchicalIntegrityVerificationStorage; + + _readSemaphore?.Acquire(); + + try + { + if (!g.GlobalReadSemaphore.TimedAcquire(AccessTimeout)) + { + for (int level = _layerCount - 2; level >= 0; level--) + { + Result rc = _bufferedStorages[level].Flush(); + if (rc.IsFailure()) return rc.Miss(); + } + + g.GlobalReadSemaphore.Acquire(); + } + + try + { + Result rc = _bufferedStorages[_layerCount - 2].Read(offset, destination); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + finally + { + g.GlobalReadSemaphore.Release(); + } + } + finally + { + _readSemaphore?.Release(); + } + } + + public override Result Write(long offset, ReadOnlySpan source) + { + Assert.SdkRequires(_dataSize >= 0); + + if (source.Length == 0) + return Result.Success; + + ref HierarchicalIntegrityVerificationStorageGlobals g = + ref _fsServer.Globals.HierarchicalIntegrityVerificationStorage; + + _writeSemaphore?.Acquire(); + + try + { + if (!g.GlobalWriteSemaphore.TimedAcquire(AccessTimeout)) + { + for (int level = _layerCount - 2; level >= 0; level--) + { + Result rc = _bufferedStorages[level].Flush(); + if (rc.IsFailure()) return rc.Miss(); + } + + g.GlobalWriteSemaphore.Acquire(); + } + + try + { + Result rc = _bufferedStorages[_layerCount - 2].Write(offset, source); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + finally + { + g.GlobalWriteSemaphore.Release(); + } + } + finally + { + _writeSemaphore?.Release(); + } + } + + public override Result Flush() + { + return Result.Success; + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + switch (operationId) + { + case OperationId.FillZero: + case OperationId.DestroySignature: + { + Result rc = _bufferedStorages[_layerCount - 2] + .OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + case OperationId.InvalidateCache: + case OperationId.QueryRange: + { + Result rc = _bufferedStorages[_layerCount - 2] + .OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + default: + return ResultFs.UnsupportedOperateRangeForHierarchicalIntegrityVerificationStorage.Log(); + } + } + + public Result Commit() + { + for (int level = _layerCount - 2; level >= 0; level--) + { + Result rc = _bufferedStorages[level].Commit(); + if (rc.IsFailure()) return rc.Miss(); + } + + return Result.Success; + } + + public Result OnRollback() + { + for (int level = _layerCount - 2; level >= 0; level--) + { + Result rc = _bufferedStorages[level].OnRollback(); + if (rc.IsFailure()) return rc.Miss(); + } + + return Result.Success; + } + + public static void SetGenerateRandomFunction(FileSystemServer fsServer, RandomDataGenerator function) + { + fsServer.Globals.HierarchicalIntegrityVerificationStorage.GenerateRandom = function; + } + + public static sbyte GetDefaultDataCacheBufferLevel(int maxLayers) + { + return (sbyte)(BaseBufferLevel + maxLayers - 2); + } + + /// "HierarchicalIntegrityVerificationStorage::Master" + public static ReadOnlySpan MasterKey => // "HierarchicalIntegrityVerificationStorage::Master" + new[] + { + (byte)'H', (byte)'i', (byte)'e', (byte)'r', (byte)'a', (byte)'r', (byte)'c', (byte)'h', + (byte)'i', (byte)'c', (byte)'a', (byte)'l', (byte)'I', (byte)'n', (byte)'t', (byte)'e', + (byte)'g', (byte)'r', (byte)'i', (byte)'t', (byte)'y', (byte)'V', (byte)'e', (byte)'r', + (byte)'i', (byte)'f', (byte)'i', (byte)'c', (byte)'a', (byte)'t', (byte)'i', (byte)'o', + (byte)'n', (byte)'S', (byte)'t', (byte)'o', (byte)'r', (byte)'a', (byte)'g', (byte)'e', + (byte)':', (byte)':', (byte)'M', (byte)'a', (byte)'s', (byte)'t', (byte)'e', (byte)'r' + }; + + /// "HierarchicalIntegrityVerificationStorage::L1" + public static ReadOnlySpan L1Key => // "HierarchicalIntegrityVerificationStorage::L1" + new[] + { + (byte)'H', (byte)'i', (byte)'e', (byte)'r', (byte)'a', (byte)'r', (byte)'c', (byte)'h', + (byte)'i', (byte)'c', (byte)'a', (byte)'l', (byte)'I', (byte)'n', (byte)'t', (byte)'e', + (byte)'g', (byte)'r', (byte)'i', (byte)'t', (byte)'y', (byte)'V', (byte)'e', (byte)'r', + (byte)'i', (byte)'f', (byte)'i', (byte)'c', (byte)'a', (byte)'t', (byte)'i', (byte)'o', + (byte)'n', (byte)'S', (byte)'t', (byte)'o', (byte)'r', (byte)'a', (byte)'g', (byte)'e', + (byte)':', (byte)':', (byte)'L', (byte)'1' + }; + + /// "HierarchicalIntegrityVerificationStorage::L2" + public static ReadOnlySpan L2Key => // "HierarchicalIntegrityVerificationStorage::L2" + new[] + { + (byte)'H', (byte)'i', (byte)'e', (byte)'r', (byte)'a', (byte)'r', (byte)'c', (byte)'h', + (byte)'i', (byte)'c', (byte)'a', (byte)'l', (byte)'I', (byte)'n', (byte)'t', (byte)'e', + (byte)'g', (byte)'r', (byte)'i', (byte)'t', (byte)'y', (byte)'V', (byte)'e', (byte)'r', + (byte)'i', (byte)'f', (byte)'i', (byte)'c', (byte)'a', (byte)'t', (byte)'i', (byte)'o', + (byte)'n', (byte)'S', (byte)'t', (byte)'o', (byte)'r', (byte)'a', (byte)'g', (byte)'e', + (byte)':', (byte)':', (byte)'L', (byte)'2' + }; + + /// "HierarchicalIntegrityVerificationStorage::L3" + public static ReadOnlySpan L3Key => // "HierarchicalIntegrityVerificationStorage::L3" + new[] + { + (byte)'H', (byte)'i', (byte)'e', (byte)'r', (byte)'a', (byte)'r', (byte)'c', (byte)'h', + (byte)'i', (byte)'c', (byte)'a', (byte)'l', (byte)'I', (byte)'n', (byte)'t', (byte)'e', + (byte)'g', (byte)'r', (byte)'i', (byte)'t', (byte)'y', (byte)'V', (byte)'e', (byte)'r', + (byte)'i', (byte)'f', (byte)'i', (byte)'c', (byte)'a', (byte)'t', (byte)'i', (byte)'o', + (byte)'n', (byte)'S', (byte)'t', (byte)'o', (byte)'r', (byte)'a', (byte)'g', (byte)'e', + (byte)':', (byte)':', (byte)'L', (byte)'3' + }; + + /// "HierarchicalIntegrityVerificationStorage::L4" + public static ReadOnlySpan L4Key => // "HierarchicalIntegrityVerificationStorage::L4" + new[] + { + (byte)'H', (byte)'i', (byte)'e', (byte)'r', (byte)'a', (byte)'r', (byte)'c', (byte)'h', + (byte)'i', (byte)'c', (byte)'a', (byte)'l', (byte)'I', (byte)'n', (byte)'t', (byte)'e', + (byte)'g', (byte)'r', (byte)'i', (byte)'t', (byte)'y', (byte)'V', (byte)'e', (byte)'r', + (byte)'i', (byte)'f', (byte)'i', (byte)'c', (byte)'a', (byte)'t', (byte)'i', (byte)'o', + (byte)'n', (byte)'S', (byte)'t', (byte)'o', (byte)'r', (byte)'a', (byte)'g', (byte)'e', + (byte)':', (byte)':', (byte)'L', (byte)'4' + }; + + /// "HierarchicalIntegrityVerificationStorage::L5" + public static ReadOnlySpan L5Key => // "HierarchicalIntegrityVerificationStorage::L5" + new[] + { + (byte)'H', (byte)'i', (byte)'e', (byte)'r', (byte)'a', (byte)'r', (byte)'c', (byte)'h', + (byte)'i', (byte)'c', (byte)'a', (byte)'l', (byte)'I', (byte)'n', (byte)'t', (byte)'e', + (byte)'g', (byte)'r', (byte)'i', (byte)'t', (byte)'y', (byte)'V', (byte)'e', (byte)'r', + (byte)'i', (byte)'f', (byte)'i', (byte)'c', (byte)'a', (byte)'t', (byte)'i', (byte)'o', + (byte)'n', (byte)'S', (byte)'t', (byte)'o', (byte)'r', (byte)'a', (byte)'g', (byte)'e', + (byte)':', (byte)':', (byte)'L', (byte)'5' + }; +} \ No newline at end of file diff --git a/src/LibHac/Util/Optional.cs b/src/LibHac/Util/Optional.cs index e1549330..3dc6c74b 100644 --- a/src/LibHac/Util/Optional.cs +++ b/src/LibHac/Util/Optional.cs @@ -43,6 +43,11 @@ public Optional(T value) public static implicit operator Optional(in T value) => new Optional(in value); + public void Set() + { + _hasValue = true; + } + public void Set(T value) { _value = value; diff --git a/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs b/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs index 3e518524..c344430e 100644 --- a/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs +++ b/tests/LibHac.Tests/FsSystem/TypeLayoutTests.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using LibHac.FsSystem; using Xunit; using static LibHac.Tests.Common.Layout; @@ -279,4 +280,78 @@ public static void AesCtrCounterExtendedStorage_Entry_Layout() Assert.Equal(0x9, GetOffset(in s, in s.Reserved)); Assert.Equal(0xC, GetOffset(in s, in s.Generation)); } + + [Fact] + public static void HierarchicalIntegrityVerificationLevelInformation_Layout() + { + HierarchicalIntegrityVerificationLevelInformation s = default; + + Assert.Equal(0x18, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.Offset)); + Assert.Equal(0x08, GetOffset(in s, in s.Size)); + Assert.Equal(0x10, GetOffset(in s, in s.BlockOrder)); + Assert.Equal(0x14, GetOffset(in s, in s.Reserved)); + } + + [StructLayout(LayoutKind.Sequential)] + private struct HierarchicalIntegrityVerificationLevelInformationAlignmentTest + { + public byte A; + public HierarchicalIntegrityVerificationLevelInformation B; + } + + [Fact] + public static void HierarchicalIntegrityVerificationLevelInformation_Alignment() + { + var s = new HierarchicalIntegrityVerificationLevelInformationAlignmentTest(); + + Assert.Equal(0x1C, Unsafe.SizeOf()); + + Assert.Equal(0, GetOffset(in s, in s.A)); + Assert.Equal(4, GetOffset(in s, in s.B)); + } + + [Fact] + public static void HierarchicalIntegrityVerificationInformation_Layout() + { + HierarchicalIntegrityVerificationInformation s = default; + + Assert.Equal(0xB4, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.MaxLayers)); + Assert.Equal(0x04, GetOffset(in s, in s.Layers)); + Assert.Equal(0x94, GetOffset(in s, in s.HashSalt)); + + Assert.Equal(Constants.IntegrityMaxLayerCount - 1, s.Layers.ItemsRo.Length); + } + + [Fact] + public static void HierarchicalIntegrityVerificationMetaInformation_Layout() + { + HierarchicalIntegrityVerificationMetaInformation s = default; + + Assert.Equal(0xC0, Unsafe.SizeOf()); + + Assert.Equal(0x00, GetOffset(in s, in s.Magic)); + Assert.Equal(0x04, GetOffset(in s, in s.Version)); + Assert.Equal(0x08, GetOffset(in s, in s.MasterHashSize)); + Assert.Equal(0x0C, GetOffset(in s, in s.LevelHashInfo)); + } + + [Fact] + public static void HierarchicalIntegrityVerificationSizeSet_Layout() + { + HierarchicalIntegrityVerificationSizeSet s = default; + + Assert.Equal(Constants.IntegrityMaxLayerCount - 2, s.LayeredHashSizes.ItemsRo.Length); + } + + [Fact] + public static void HierarchicalIntegrityVerificationStorageControlArea_InputParam_Layout() + { + HierarchicalIntegrityVerificationStorageControlArea.InputParam s = default; + + Assert.Equal(Constants.IntegrityMaxLayerCount - 1, s.LevelBlockSizes.ItemsRo.Length); + } } \ No newline at end of file From 28a4deb93c1be04812737347558ced80e8d67168 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 18 Apr 2022 19:34:51 -0700 Subject: [PATCH 21/24] Add FatFileSystemCreator --- src/LibHac/Common/FixedArrays/Array68.cs | 31 ++++++++ src/LibHac/Fat/FatReport.cs | 15 ++++ src/LibHac/Fs/ErrorInfo.cs | 5 +- .../FsSrv/FsCreator/FatFileSystemCreator.cs | 73 +++++++++++++++++++ src/LibHac/FsSrv/StatusReportService.cs | 14 +++- tests/LibHac.Tests/Fs/TypeLayoutTests.cs | 5 +- 6 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 src/LibHac/Common/FixedArrays/Array68.cs create mode 100644 src/LibHac/Fat/FatReport.cs create mode 100644 src/LibHac/FsSrv/FsCreator/FatFileSystemCreator.cs diff --git a/src/LibHac/Common/FixedArrays/Array68.cs b/src/LibHac/Common/FixedArrays/Array68.cs new file mode 100644 index 00000000..2b4b9f43 --- /dev/null +++ b/src/LibHac/Common/FixedArrays/Array68.cs @@ -0,0 +1,31 @@ +#pragma warning disable CS0169, CS0649, IDE0051 // Field is never used, Field is never assigned to, Remove unused private members +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace LibHac.Common.FixedArrays; + +public struct Array68 +{ + public const int Length = 68; + + private Array64 _0; + private Array4 _64; + + public ref T this[int i] => ref Items[i]; + + public Span Items + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => SpanHelpers.CreateSpan(ref MemoryMarshal.GetReference(_0.Items), Length); + } + + public readonly ReadOnlySpan ItemsRo + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => SpanHelpers.CreateSpan(ref MemoryMarshal.GetReference(_0.ItemsRo), Length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static implicit operator ReadOnlySpan(in Array68 value) => value.ItemsRo; +} \ No newline at end of file diff --git a/src/LibHac/Fat/FatReport.cs b/src/LibHac/Fat/FatReport.cs new file mode 100644 index 00000000..216a056b --- /dev/null +++ b/src/LibHac/Fat/FatReport.cs @@ -0,0 +1,15 @@ +namespace LibHac.Fat; + +public struct FatReport +{ + public ushort FileCurrentOpenCount; + public ushort FilePeakOpenCount; + public ushort DirectoryCurrentOpenCount; + public ushort DirectoryPeakOpenCount; +} + +public struct FatReportInfo +{ + public ushort FilePeakOpenCount; + public ushort DirectoryPeakOpenCount; +} \ No newline at end of file diff --git a/src/LibHac/Fs/ErrorInfo.cs b/src/LibHac/Fs/ErrorInfo.cs index b3ef2385..3ca56178 100644 --- a/src/LibHac/Fs/ErrorInfo.cs +++ b/src/LibHac/Fs/ErrorInfo.cs @@ -10,7 +10,10 @@ public struct FileSystemProxyErrorInfo public FatError FatFsError; public int RecoveredByInvalidateCacheCount; public int SaveDataIndexCount; - public Array80 Reserved; + public FatReportInfo BisSystemFatReportInfo; + public FatReportInfo BisUserFatReport; + public FatReportInfo SdCardFatReport; + public Array68 Reserved; } public struct StorageErrorInfo diff --git a/src/LibHac/FsSrv/FsCreator/FatFileSystemCreator.cs b/src/LibHac/FsSrv/FsCreator/FatFileSystemCreator.cs new file mode 100644 index 00000000..c717f3bd --- /dev/null +++ b/src/LibHac/FsSrv/FsCreator/FatFileSystemCreator.cs @@ -0,0 +1,73 @@ +using System; +using LibHac.Common; +using LibHac.Fat; +using LibHac.Fs; +using LibHac.Fs.Fsa; +using LibHac.Os; + +namespace LibHac.FsSrv.FsCreator; + +/// +/// Handles creating FAT file systems. +/// +/// Based on FS 14.1.0 (nnSdk 14.3.0) +public class FatFileSystemCreator : IFatFileSystemCreator +{ + // ReSharper disable once NotAccessedField.Local + private MemoryResource _allocator; + private FatError _fatFsError; + private SdkMutexType _fatErrorMutex; + private FatReport _bisSystemReport; + private FatReport _bisUserReport; + private FatReport _sdCardReport; + private SdkMutexType _fatReportMutex; + + public FatFileSystemCreator(MemoryResource allocator) + { + _fatErrorMutex = new SdkMutexType(); + _fatReportMutex = new SdkMutexType(); + _allocator = allocator; + + // Missing: Call nn::fat::SetMemoryResource + } + + public Result Create(ref SharedRef outFileSystem, ref SharedRef baseStorage, + FatAttribute attribute, int driveId, Result invalidFatFormatResult, Result usableSpaceNotEnoughResult) + { + throw new NotImplementedException(); + } + + public Result Format(ref SharedRef partitionStorage, FatAttribute attribute, FatFormatParam formatParam, + int driveId, Result invalidFatFormatResult, Result usableSpaceNotEnoughResult) + { + throw new NotImplementedException(); + } + + public void GetAndClearFatFsError(out FatError outFatError) + { + using var scopedLock = new ScopedLock(ref _fatErrorMutex); + + outFatError = _fatFsError; + _fatFsError = default; + } + + public void GetAndClearFatReportInfo(out FatReportInfo outBisSystemFatReportInfo, + out FatReportInfo outBisUserFatReportInfo, out FatReportInfo outSdCardFatReportInfo) + { + using var scopedLock = new ScopedLock(ref _fatReportMutex); + + outBisSystemFatReportInfo.FilePeakOpenCount = _bisSystemReport.FilePeakOpenCount; + outBisSystemFatReportInfo.DirectoryPeakOpenCount = _bisSystemReport.DirectoryPeakOpenCount; + outBisUserFatReportInfo.FilePeakOpenCount = _bisUserReport.FilePeakOpenCount; + outBisUserFatReportInfo.DirectoryPeakOpenCount = _bisUserReport.DirectoryPeakOpenCount; + outSdCardFatReportInfo.FilePeakOpenCount = _sdCardReport.FilePeakOpenCount; + outSdCardFatReportInfo.DirectoryPeakOpenCount = _sdCardReport.DirectoryPeakOpenCount; + + _bisSystemReport.FilePeakOpenCount = _bisSystemReport.FileCurrentOpenCount; + _bisSystemReport.DirectoryPeakOpenCount = _bisSystemReport.DirectoryCurrentOpenCount; + _bisUserReport.FilePeakOpenCount = _bisUserReport.FileCurrentOpenCount; + _bisUserReport.DirectoryPeakOpenCount = _bisUserReport.DirectoryCurrentOpenCount; + _sdCardReport.FilePeakOpenCount = _sdCardReport.FileCurrentOpenCount; + _sdCardReport.DirectoryPeakOpenCount = _sdCardReport.DirectoryCurrentOpenCount; + } +} \ No newline at end of file diff --git a/src/LibHac/FsSrv/StatusReportService.cs b/src/LibHac/FsSrv/StatusReportService.cs index 4e6ccb0e..623ebc39 100644 --- a/src/LibHac/FsSrv/StatusReportService.cs +++ b/src/LibHac/FsSrv/StatusReportService.cs @@ -1,5 +1,6 @@ using LibHac.Diag; using LibHac.Fs; +using LibHac.FsSrv.FsCreator; using LibHac.FsSystem; using LibHac.Os; @@ -10,7 +11,7 @@ namespace LibHac.FsSrv; /// /// This struct handles forwarding calls to the object. /// No permissions are needed to call any of this struct's functions. -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public readonly struct StatusReportService { private readonly StatusReportServiceImpl _serviceImpl; @@ -40,7 +41,7 @@ public Result GetFsStackUsage(out uint stackUsage, FsStackUsageThreadType thread /// /// Manages getting and resetting various status reports and statistics about parts of the FS service. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class StatusReportServiceImpl { private Configuration _config; @@ -56,7 +57,7 @@ public struct Configuration { public NcaFileSystemServiceImpl NcaFileSystemServiceImpl; public SaveDataFileSystemServiceImpl SaveDataFileSystemServiceImpl; - // Missing: FatFileSystemCreator (Not an IFatFileSystemCreator) + public FatFileSystemCreator FatFileSystemCreator; public MemoryReport BufferManagerMemoryReport; public MemoryReport ExpHeapMemoryReport; public MemoryReport BufferPoolMemoryReport; @@ -79,7 +80,12 @@ public Result GetAndClearFileSystemProxyErrorInfo(out FileSystemProxyErrorInfo e out errorInfo.UnrecoverableDataCorruptionByRemountCount, out errorInfo.RecoveredByInvalidateCacheCount); - // Missing: GetFatInfo + if (_config.FatFileSystemCreator is not null) + { + _config.FatFileSystemCreator.GetAndClearFatFsError(out errorInfo.FatFsError); + _config.FatFileSystemCreator.GetAndClearFatReportInfo(out errorInfo.BisSystemFatReportInfo, + out errorInfo.BisUserFatReport, out errorInfo.SdCardFatReport); + } Assert.SdkRequiresNotNull(_config.SaveDataFileSystemServiceImpl); diff --git a/tests/LibHac.Tests/Fs/TypeLayoutTests.cs b/tests/LibHac.Tests/Fs/TypeLayoutTests.cs index 8035c0c6..4a6ef886 100644 --- a/tests/LibHac.Tests/Fs/TypeLayoutTests.cs +++ b/tests/LibHac.Tests/Fs/TypeLayoutTests.cs @@ -213,7 +213,10 @@ public static void FileSystemProxyErrorInfo_Layout() Assert.Equal(0x08, GetOffset(in s, in s.FatFsError)); Assert.Equal(0x28, GetOffset(in s, in s.RecoveredByInvalidateCacheCount)); Assert.Equal(0x2C, GetOffset(in s, in s.SaveDataIndexCount)); - Assert.Equal(0x30, GetOffset(in s, in s.Reserved)); + Assert.Equal(0x30, GetOffset(in s, in s.BisSystemFatReportInfo)); + Assert.Equal(0x34, GetOffset(in s, in s.BisUserFatReport)); + Assert.Equal(0x38, GetOffset(in s, in s.SdCardFatReport)); + Assert.Equal(0x3C, GetOffset(in s, in s.Reserved)); } [Fact] From b5ae21207b8bd58754e26b6e5c63de00c436f624 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 18 Apr 2022 19:41:10 -0700 Subject: [PATCH 22/24] Mark some classes as being updated for 14.1.0 --- src/LibHac/FsSrv/DebugConfigurationService.cs | 4 ++-- src/LibHac/FsSrv/Storage/GameCardService.cs | 2 +- src/LibHac/FsSrv/Storage/MmcService.cs | 4 ++-- src/LibHac/FsSrv/Storage/SdCardService.cs | 2 +- src/LibHac/FsSystem/BucketTree.cs | 2 +- src/LibHac/FsSystem/IndirectStorage.cs | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/LibHac/FsSrv/DebugConfigurationService.cs b/src/LibHac/FsSrv/DebugConfigurationService.cs index cd3593aa..dee37b2c 100644 --- a/src/LibHac/FsSrv/DebugConfigurationService.cs +++ b/src/LibHac/FsSrv/DebugConfigurationService.cs @@ -11,7 +11,7 @@ namespace LibHac.FsSrv; /// /// Handles debug configuration calls for . /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public struct DebugConfigurationService { private DebugConfigurationServiceImpl _serviceImpl; @@ -63,7 +63,7 @@ public Result Unregister(uint key) /// /// Manages a key-value list of debug settings. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class DebugConfigurationServiceImpl : IDisposable { private Configuration _config; diff --git a/src/LibHac/FsSrv/Storage/GameCardService.cs b/src/LibHac/FsSrv/Storage/GameCardService.cs index 6a7009ad..f2503958 100644 --- a/src/LibHac/FsSrv/Storage/GameCardService.cs +++ b/src/LibHac/FsSrv/Storage/GameCardService.cs @@ -35,7 +35,7 @@ public void Dispose() /// /// Contains functions for interacting with the game card storage device. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) internal static class GameCardService { private static ulong MakeAttributeId(OpenGameCardAttribute attribute) => (ulong)attribute; diff --git a/src/LibHac/FsSrv/Storage/MmcService.cs b/src/LibHac/FsSrv/Storage/MmcService.cs index dcb6b291..e205b974 100644 --- a/src/LibHac/FsSrv/Storage/MmcService.cs +++ b/src/LibHac/FsSrv/Storage/MmcService.cs @@ -14,7 +14,7 @@ namespace LibHac.FsSrv.Storage; /// /// Contains global MMC-storage-related functions. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public static class MmcServiceGlobalMethods { public static Result GetAndClearPatrolReadAllocateBufferCount(this FileSystemServer fsSrv, out long successCount, @@ -27,7 +27,7 @@ public static Result GetAndClearPatrolReadAllocateBufferCount(this FileSystemSer /// /// Contains functions for interacting with the MMC storage device. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) internal static class MmcService { private static int MakeOperationId(MmcManagerOperationIdValue operation) => (int)operation; diff --git a/src/LibHac/FsSrv/Storage/SdCardService.cs b/src/LibHac/FsSrv/Storage/SdCardService.cs index b0a63431..f18cda6f 100644 --- a/src/LibHac/FsSrv/Storage/SdCardService.cs +++ b/src/LibHac/FsSrv/Storage/SdCardService.cs @@ -17,7 +17,7 @@ namespace LibHac.FsSrv.Storage; /// /// Contains functions for interacting with the SD card storage device. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) internal static class SdCardService { private static int MakeOperationId(SdCardManagerOperationIdValue operation) => (int)operation; diff --git a/src/LibHac/FsSystem/BucketTree.cs b/src/LibHac/FsSystem/BucketTree.cs index 9ecd8b65..21b614d5 100644 --- a/src/LibHac/FsSystem/BucketTree.cs +++ b/src/LibHac/FsSystem/BucketTree.cs @@ -14,7 +14,7 @@ namespace LibHac.FsSystem; /// /// Allows searching and iterating the entries in a bucket tree data structure. /// -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public partial class BucketTree : IDisposable { private const uint ExpectedMagic = 0x52544B42; // BKTR diff --git a/src/LibHac/FsSystem/IndirectStorage.cs b/src/LibHac/FsSystem/IndirectStorage.cs index d5dcc966..d2f09c57 100644 --- a/src/LibHac/FsSystem/IndirectStorage.cs +++ b/src/LibHac/FsSystem/IndirectStorage.cs @@ -13,7 +13,7 @@ namespace LibHac.FsSystem; /// /// The 's contains /// values that describe how the created storage is to be built from the base storages. -/// Based on FS 13.1.0 (nnSdk 13.4.0) +/// Based on FS 14.1.0 (nnSdk 14.3.0) public class IndirectStorage : IStorage { public static readonly int StorageCount = 2; From f8b9c3557e510251a16f7c3d8f2fed95b80fb49c Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Tue, 19 Apr 2022 12:42:51 -0700 Subject: [PATCH 23/24] Add IntegrityRomFsStorage --- src/LibHac/FsSystem/IntegrityRomFsStorage.cs | 109 +++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 src/LibHac/FsSystem/IntegrityRomFsStorage.cs diff --git a/src/LibHac/FsSystem/IntegrityRomFsStorage.cs b/src/LibHac/FsSystem/IntegrityRomFsStorage.cs new file mode 100644 index 00000000..dc6dd6f8 --- /dev/null +++ b/src/LibHac/FsSystem/IntegrityRomFsStorage.cs @@ -0,0 +1,109 @@ +using System; +using System.Runtime.CompilerServices; +using LibHac.Common; +using LibHac.Diag; +using LibHac.Fs; +using LibHac.FsSrv; +using LibHac.Os; +using static LibHac.FsSystem.HierarchicalIntegrityVerificationStorage; + +namespace LibHac.FsSystem; + +/// +/// An that handles initializing a +/// from a RomFs. +/// +/// Based on FS 14.1.0 (nnSdk 14.3.0) +public class IntegrityRomFsStorage : IStorage +{ + private HierarchicalIntegrityVerificationStorage _integrityStorage; + private FileSystemBufferManagerSet _bufferManagerSet; + private SdkRecursiveMutex _mutex; + private byte[] _masterHash; + private UniqueRef _masterHashStorage; + + public IntegrityRomFsStorage(FileSystemServer fsServer) + { + _integrityStorage = new HierarchicalIntegrityVerificationStorage(fsServer); + _mutex = new SdkRecursiveMutex(); + + _bufferManagerSet = new FileSystemBufferManagerSet(); + } + + public override void Dispose() + { + FinalizeObject(); + + _masterHashStorage.Destroy(); + _integrityStorage.Dispose(); + + base.Dispose(); + } + + public Result Initialize(ref HierarchicalIntegrityVerificationInformation info, Hash masterHash, + ref HierarchicalStorageInformation storageInfo, IBufferManager bufferManager, int maxDataCacheEntries, + int maxHashCacheEntries, sbyte bufferLevel, IHash256GeneratorFactory hashGeneratorFactory) + { + // Validate preconditions. + Assert.SdkRequiresNotNull(bufferManager); + + // Set master hash. + _masterHash = SpanHelpers.AsReadOnlyByteSpan(in masterHash).ToArray(); + _masterHashStorage.Reset(new MemoryStorage(_masterHash)); + if (!_masterHashStorage.HasValue) + return ResultFs.AllocationMemoryFailedInIntegrityRomFsStorageA.Log(); + + // Set the master hash storage. + using var masterHashStorage = new ValueSubStorage(_masterHashStorage.Get, 0, Unsafe.SizeOf()); + storageInfo[(int)HierarchicalStorageInformation.Storage.MasterStorage].Set(in masterHashStorage); + + // Set buffers. + for (int i = 0; i < _bufferManagerSet.Buffers.Length; i++) + { + _bufferManagerSet.Buffers[i] = bufferManager; + } + + // Initialize our integrity storage. + Result rc = _integrityStorage.Initialize(in info, ref storageInfo, _bufferManagerSet, hashGeneratorFactory, + false, _mutex, maxDataCacheEntries, maxHashCacheEntries, bufferLevel, false, false); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public void FinalizeObject() + { + _integrityStorage.FinalizeObject(); + } + + public override Result Read(long offset, Span destination) + { + return _integrityStorage.Read(offset, destination); + } + + public override Result Write(long offset, ReadOnlySpan source) + { + return _integrityStorage.Write(offset, source); + } + + public override Result Flush() + { + return _integrityStorage.Flush(); + } + + public override Result SetSize(long size) + { + return ResultFs.UnsupportedSetSizeForIntegrityRomFsStorage.Log(); + } + + public override Result GetSize(out long size) + { + return _integrityStorage.GetSize(out size); + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + return _integrityStorage.OperateRange(outBuffer, operationId, offset, size, inBuffer); + } +} \ No newline at end of file From 99ad308b84a4511d87062adf3a126d92f0abdfc5 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Tue, 19 Apr 2022 22:31:34 -0700 Subject: [PATCH 24/24] Add SwitchStorage and RegionSwitchStorage --- build/CodeGen/results.csv | 3 + src/LibHac/Fs/ResultFs.cs | 6 + src/LibHac/FsSystem/SwitchStorage.cs | 321 +++++++++++++++++++++++++++ 3 files changed, 330 insertions(+) create mode 100644 src/LibHac/FsSystem/SwitchStorage.cs diff --git a/build/CodeGen/results.csv b/build/CodeGen/results.csv index e390c181..bba44f13 100644 --- a/build/CodeGen/results.csv +++ b/build/CodeGen/results.csv @@ -1064,6 +1064,9 @@ Module,DescriptionStart,DescriptionEnd,Flags,Namespace,Name,Summary 2,6386,,,,UnsupportedSetSizeForZeroBitmapHashStorageFile, 2,6387,,,,UnsupportedWriteForCompressedStorage, 2,6388,,,,UnsupportedOperateRangeForCompressedStorage, +2,6395,,,,UnsupportedRollbackOnlyModifiedForApplicationTemporaryFileSystem, +2,6396,,,,UnsupportedRollbackOnlyModifiedForDirectorySaveDataFileSystem, +2,6397,,,,UnsupportedOperateRangeForRegionSwitchStorage, 2,6400,6449,,,PermissionDenied, 2,6403,,,,PermissionDeniedForCreateHostFileSystem,Returned when opening a host FS on a retail device. diff --git a/src/LibHac/Fs/ResultFs.cs b/src/LibHac/Fs/ResultFs.cs index 50943f44..047d884b 100644 --- a/src/LibHac/Fs/ResultFs.cs +++ b/src/LibHac/Fs/ResultFs.cs @@ -1952,6 +1952,12 @@ public static class ResultFs public static Result.Base UnsupportedWriteForCompressedStorage => new Result.Base(ModuleFs, 6387); /// Error code: 2002-6388; Inner value: 0x31e802 public static Result.Base UnsupportedOperateRangeForCompressedStorage => new Result.Base(ModuleFs, 6388); + /// Error code: 2002-6395; Inner value: 0x31f602 + public static Result.Base UnsupportedRollbackOnlyModifiedForApplicationTemporaryFileSystem => new Result.Base(ModuleFs, 6395); + /// Error code: 2002-6396; Inner value: 0x31f802 + public static Result.Base UnsupportedRollbackOnlyModifiedForDirectorySaveDataFileSystem => new Result.Base(ModuleFs, 6396); + /// Error code: 2002-6397; Inner value: 0x31fa02 + public static Result.Base UnsupportedOperateRangeForRegionSwitchStorage => new Result.Base(ModuleFs, 6397); /// Error code: 2002-6400; Range: 6400-6449; Inner value: 0x320002 public static Result.Base PermissionDenied { [MethodImpl(MethodImplOptions.AggressiveInlining)] get => new Result.Base(ModuleFs, 6400, 6449); } diff --git a/src/LibHac/FsSystem/SwitchStorage.cs b/src/LibHac/FsSystem/SwitchStorage.cs new file mode 100644 index 00000000..342ee012 --- /dev/null +++ b/src/LibHac/FsSystem/SwitchStorage.cs @@ -0,0 +1,321 @@ +using System; +using System.Runtime.CompilerServices; +using LibHac.Common; +using LibHac.Fs; + +namespace LibHac.FsSystem; + +/// +/// An that will switch between forwarding requests to one of two different base +/// s. On each request the provided storage selection function will be called and the request +/// will be forwarded to the appropriate based on the return value. +/// +/// Based on FS 14.1.0 (nnSdk 14.3.0) +public class SwitchStorage : IStorage +{ + private SharedRef _trueStorage; + private SharedRef _falseStorage; + private Func _storageSelectionFunction; + + public SwitchStorage(in SharedRef trueStorage, in SharedRef falseStorage, + Func storageSelectionFunction) + { + _trueStorage = SharedRef.CreateCopy(in trueStorage); + _falseStorage = SharedRef.CreateCopy(in falseStorage); + _storageSelectionFunction = storageSelectionFunction; + } + + public override void Dispose() + { + _trueStorage.Destroy(); + _falseStorage.Destroy(); + + base.Dispose(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private IStorage SelectStorage() + { + return (_storageSelectionFunction() ? _trueStorage : _falseStorage).Get; + } + + public override Result Read(long offset, Span destination) + { + Result rc = SelectStorage().Read(offset, destination); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public override Result Write(long offset, ReadOnlySpan source) + { + Result rc = SelectStorage().Write(offset, source); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public override Result Flush() + { + Result rc = SelectStorage().Flush(); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public override Result GetSize(out long size) + { + Result rc = SelectStorage().GetSize(out size); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public override Result SetSize(long size) + { + Result rc = SelectStorage().SetSize(size); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, + ReadOnlySpan inBuffer) + { + switch (operationId) + { + case OperationId.InvalidateCache: + { + Result rc = _trueStorage.Get.OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + rc = _falseStorage.Get.OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + case OperationId.QueryRange: + { + Result rc = SelectStorage().OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + default: + return ResultFs.UnsupportedOperateRangeForSwitchStorage.Log(); + } + } +} + +/// +/// Takes a and two base s upon construction. Requests inside +/// the provided will be forwarded to one , and requests outside +/// will be forwarded to the other. +/// +/// Based on FS 14.1.0 (nnSdk 14.3.0) +public class RegionSwitchStorage : IStorage +{ + public struct Region + { + public long Offset; + public long Size; + } + + private SharedRef _insideRegionStorage; + private SharedRef _outsideRegionStorage; + private Region _region; + + public RegionSwitchStorage(in SharedRef insideRegionStorage, + in SharedRef outsideRegionStorage, Region region) + { + _insideRegionStorage = SharedRef.CreateCopy(in insideRegionStorage); + _outsideRegionStorage = SharedRef.CreateCopy(in outsideRegionStorage); + _region = region; + } + + public override void Dispose() + { + _insideRegionStorage.Destroy(); + _outsideRegionStorage.Destroy(); + + base.Dispose(); + } + + /// + /// Checks if the requested range is inside or outside the . + /// + /// The size past the start of the range until entering or exiting the . + /// The offset of the range to check. + /// The size of the range to check. + /// the start of the range is inside the ; + /// otherwise . + private bool CheckRegions(out long currentSize, long offset, long size) + { + if (_region.Offset > offset) + { + // The requested start offset is before the region's start offset. + // Check if the requested end offset is inside the region. + if (offset + size < _region.Offset) + { + // The request is completely outside the region. + currentSize = size; + } + else + { + // The request ends inside the region. Calculate the length of the request outside the region. + currentSize = _region.Offset - offset; + } + + return false; + } + + if (_region.Offset + _region.Size > offset) + { + // The requested start offset is inside the region. + // Check if the requested end offset is also inside the region. + if (offset + size < _region.Offset + _region.Size) + { + // The request is completely within the region. + currentSize = size; + } + else + { + // The request ends outside the region. Calculate the length of the request inside the region. + currentSize = _region.Offset + _region.Size - offset; + } + + return true; + } + + // The request starts after the end of the region. + currentSize = size; + return false; + } + + public override Result Read(long offset, Span destination) + { + int bytesRead = 0; + while (bytesRead < destination.Length) + { + if (CheckRegions(out long currentSize, offset + bytesRead, destination.Length - bytesRead)) + { + Result rc = _insideRegionStorage.Get.Read(offset + bytesRead, + destination.Slice(bytesRead, (int)currentSize)); + if (rc.IsFailure()) return rc.Miss(); + } + else + { + Result rc = _outsideRegionStorage.Get.Read(offset + bytesRead, + destination.Slice(bytesRead, (int)currentSize)); + if (rc.IsFailure()) return rc.Miss(); + } + + bytesRead += (int)currentSize; + } + + return Result.Success; + } + + public override Result Write(long offset, ReadOnlySpan source) + { + int bytesWritten = 0; + while (bytesWritten < source.Length) + { + if (CheckRegions(out long currentSize, offset + bytesWritten, source.Length - bytesWritten)) + { + Result rc = _insideRegionStorage.Get.Write(offset + bytesWritten, + source.Slice(bytesWritten, (int)currentSize)); + if (rc.IsFailure()) return rc.Miss(); + } + else + { + Result rc = _outsideRegionStorage.Get.Write(offset + bytesWritten, + source.Slice(bytesWritten, (int)currentSize)); + if (rc.IsFailure()) return rc.Miss(); + } + + bytesWritten += (int)currentSize; + } + + return Result.Success; + } + + public override Result Flush() + { + Result rc = _insideRegionStorage.Get.Flush(); + if (rc.IsFailure()) return rc.Miss(); + + rc = _outsideRegionStorage.Get.Flush(); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public override Result GetSize(out long size) + { + Result rc = _insideRegionStorage.Get.GetSize(out size); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public override Result SetSize(long size) + { + Result rc = _insideRegionStorage.Get.SetSize(size); + if (rc.IsFailure()) return rc.Miss(); + + rc = _outsideRegionStorage.Get.SetSize(size); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + + public override Result OperateRange(Span outBuffer, OperationId operationId, long offset, long size, ReadOnlySpan inBuffer) + { + switch (operationId) + { + case OperationId.InvalidateCache: + { + Result rc = _insideRegionStorage.Get.OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + rc = _outsideRegionStorage.Get.OperateRange(outBuffer, operationId, offset, size, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + + return Result.Success; + } + case OperationId.QueryRange: + { + Unsafe.SkipInit(out QueryRangeInfo mergedInfo); + mergedInfo.Clear(); + + long bytesProcessed = 0; + while (bytesProcessed < size) + { + Unsafe.SkipInit(out QueryRangeInfo currentInfo); + + if (CheckRegions(out long currentSize, offset + bytesProcessed, size - bytesProcessed)) + { + Result rc = _insideRegionStorage.Get.OperateRange(SpanHelpers.AsByteSpan(ref currentInfo), + operationId, offset + bytesProcessed, currentSize, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + } + else + { + Result rc = _outsideRegionStorage.Get.OperateRange(SpanHelpers.AsByteSpan(ref currentInfo), + operationId, offset + bytesProcessed, currentSize, inBuffer); + if (rc.IsFailure()) return rc.Miss(); + } + + mergedInfo.Merge(in currentInfo); + bytesProcessed += currentSize; + } + + SpanHelpers.AsByteSpan(ref mergedInfo).CopyTo(outBuffer); + return Result.Success; + } + default: + return ResultFs.UnsupportedOperateRangeForRegionSwitchStorage.Log(); + } + } +} \ No newline at end of file