From 785421deb815b5a2b0216ad0b5326ec19657d089 Mon Sep 17 00:00:00 2001 From: Kam Figy Date: Sun, 29 Jan 2017 22:39:26 -0800 Subject: [PATCH] Get rid of FastDirectoryEnumerator, as it was causing file locking errors on reserialization. Must be some sort of unmanaged resource disposal - it has a dispose on it, but it must be incomplete. Either way perf does not seem majorly impacted especially with DiSerialized. --- src/Rainbow/Rainbow.csproj | 1 - .../Storage/FastDirectoryEnumerator.cs | 517 ------------------ .../Storage/SerializationFileSystemTree.cs | 10 +- 3 files changed, 5 insertions(+), 523 deletions(-) delete mode 100644 src/Rainbow/Storage/FastDirectoryEnumerator.cs diff --git a/src/Rainbow/Rainbow.csproj b/src/Rainbow/Rainbow.csproj index f9b5e1b..d969705 100644 --- a/src/Rainbow/Rainbow.csproj +++ b/src/Rainbow/Rainbow.csproj @@ -81,7 +81,6 @@ - diff --git a/src/Rainbow/Storage/FastDirectoryEnumerator.cs b/src/Rainbow/Storage/FastDirectoryEnumerator.cs deleted file mode 100644 index 8524e29..0000000 --- a/src/Rainbow/Storage/FastDirectoryEnumerator.cs +++ /dev/null @@ -1,517 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Runtime.ConstrainedExecution; -using System.Runtime.InteropServices; -using System.Security; -using System.Security.Permissions; -using Microsoft.Win32.SafeHandles; - -// http://www.codeproject.com/Articles/38959/A-Faster-Directory-Enumerator -// Reduced directory iteration time by nearly 3x in Rainbow testing over Directory.GetFiles() -namespace Rainbow.Storage -{ - /// - /// Contains information about a file returned by the - /// class. - /// - [Serializable] - [DebuggerDisplay("{Name}")] - public class FileData - { - /// - /// Attributes of the file. - /// - public readonly FileAttributes Attributes; - - /// - /// File creation time in UTC - /// - public readonly DateTime CreationTimeUtc; - - /// - /// File last access time in UTC - /// - public readonly DateTime LastAccessTimeUtc; - - /// - /// File last write time in UTC - /// - public readonly DateTime LastWriteTimeUtc; - - /// - /// Name of the file - /// - public readonly string Name; - - /// - /// Full path to the file. - /// - public readonly string Path; - - /// - /// Size of the file in bytes - /// - public readonly long Size; - - /// - /// Initializes a new instance of the class. - /// - /// The directory that the file is stored at - /// - /// WIN32_FIND_DATA structure that this - /// object wraps. - /// - internal FileData(string dir, WIN32_FIND_DATA findData) - { - Attributes = findData.dwFileAttributes; - - - CreationTimeUtc = ConvertDateTime(findData.ftCreationTime_dwHighDateTime, - findData.ftCreationTime_dwLowDateTime); - - LastAccessTimeUtc = ConvertDateTime(findData.ftLastAccessTime_dwHighDateTime, - findData.ftLastAccessTime_dwLowDateTime); - - LastWriteTimeUtc = ConvertDateTime(findData.ftLastWriteTime_dwHighDateTime, - findData.ftLastWriteTime_dwLowDateTime); - - Size = CombineHighLowInts(findData.nFileSizeHigh, findData.nFileSizeLow); - - Name = findData.cFileName; - Path = System.IO.Path.Combine(dir, findData.cFileName); - } - - //public DateTime CreationTime - //{ - // get { return CreationTimeUtc.ToLocalTime(); } - //} - - ///// - ///// Gets the last access time in local time. - ///// - //public DateTime LastAccesTime - //{ - // get { return LastAccessTimeUtc.ToLocalTime(); } - //} - - ///// - ///// Gets the last access time in local time. - ///// - //public DateTime LastWriteTime - //{ - // get { return LastWriteTimeUtc.ToLocalTime(); } - //} - - private static long CombineHighLowInts(uint high, uint low) - { - return (((long) high) << 0x20) | low; - } - - private static DateTime ConvertDateTime(uint high, uint low) - { - var fileTime = CombineHighLowInts(high, low); - return DateTime.FromFileTimeUtc(fileTime); - } - } - - /// - /// Contains information about the file that is found - /// by the FindFirstFile or FindNextFile functions. - /// - [Serializable, StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto), BestFitMapping(false), ExcludeFromCodeCoverage] - internal class WIN32_FIND_DATA - { - public FileAttributes dwFileAttributes; - public uint ftCreationTime_dwLowDateTime; - public uint ftCreationTime_dwHighDateTime; - public uint ftLastAccessTime_dwLowDateTime; - public uint ftLastAccessTime_dwHighDateTime; - public uint ftLastWriteTime_dwLowDateTime; - public uint ftLastWriteTime_dwHighDateTime; - public uint nFileSizeHigh; - public uint nFileSizeLow; - public int dwReserved0; - public int dwReserved1; - [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 260)] public string cFileName; - [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 14)] public string cAlternateFileName; - - /// - /// Returns a that represents the current . - /// - /// - /// A that represents the current . - /// - public override string ToString() - { - return "File name=" + cFileName; - } - } - - /// - /// A fast enumerator of files in a directory. Use this if you need to get attributes for - /// all files in a directory. - /// - /// - /// This enumerator is substantially faster than using - /// and then creating a new FileInfo object for each path. Use this version when you - /// will need to look at the attibutes of each file returned (for example, you need - /// to check each file in a directory to see if it was modified after a specific date). - /// - [ExcludeFromCodeCoverage] - public static class FastDirectoryEnumerator - { - /// - /// Gets for all the files in a directory that - /// match a specific filter, optionally including all sub directories. - /// - /// The path to search. - /// The search string to match against files in the path. - /// - /// One of the SearchOption values that specifies whether the search - /// operation should include all subdirectories or only the current directory. - /// - /// - /// An object that implements and - /// allows you to enumerate the files in the given directory. - /// - /// - /// is a null reference (Nothing in VB) - /// - /// - /// is not one of the valid values of the - /// enumeration. - /// - public static IEnumerable EnumerateFiles(string path, string searchPattern, SearchOption searchOption) - { - if (path == null) - { - throw new ArgumentNullException(nameof(path)); - } - if (searchPattern == null) - { - throw new ArgumentNullException(nameof(searchPattern)); - } - if ((searchOption != SearchOption.TopDirectoryOnly) && (searchOption != SearchOption.AllDirectories)) - { - throw new ArgumentOutOfRangeException(nameof(searchOption)); - } - - var fullPath = Path.GetFullPath(path); - - return new FileEnumerable(fullPath, searchPattern, searchOption); - } - - /// - /// Gets for all the files in a directory that match a - /// specific filter. - /// - /// The path to search. - /// The search string to match against files in the path. - /// - /// - /// An object that implements and - /// allows you to enumerate the files in the given directory. - /// - /// - /// is a null reference (Nothing in VB) - /// - public static FileData[] GetFiles(string path, string searchPattern, SearchOption searchOption) - { - var e = EnumerateFiles(path, searchPattern, searchOption); - var list = new List(e); - - var retval = new FileData[list.Count]; - list.CopyTo(retval); - - return retval; - } - - /// - /// Provides the implementation of the - /// interface - /// - private class FileEnumerable : IEnumerable - { - private readonly string _mFilter; - private readonly string _mPath; - private readonly SearchOption _mSearchOption; - - /// - /// Initializes a new instance of the class. - /// - /// The path to search. - /// The search string to match against files in the path. - /// - /// One of the SearchOption values that specifies whether the search - /// operation should include all subdirectories or only the current directory. - /// - public FileEnumerable(string path, string filter, SearchOption searchOption) - { - _mPath = path; - _mFilter = filter; - _mSearchOption = searchOption; - } - - #region IEnumerable Members - - /// - /// Returns an enumerator that iterates through the collection. - /// - /// - /// A that can - /// be used to iterate through the collection. - /// - public IEnumerator GetEnumerator() - { - return new FileEnumerator(_mPath, _mFilter, _mSearchOption); - } - - #endregion - - #region IEnumerable Members - - /// - /// Returns an enumerator that iterates through a collection. - /// - /// - /// An object that can be - /// used to iterate through the collection. - /// - IEnumerator IEnumerable.GetEnumerator() - { - return new FileEnumerator(_mPath, _mFilter, _mSearchOption); - } - - #endregion - } - - /// - /// Wraps a FindFirstFile handle. - /// - // ReSharper disable once ClassNeverInstantiated.Local - private sealed class SafeFindHandle : SafeHandleZeroOrMinusOneIsInvalid - { - /// - /// Initializes a new instance of the class. - /// - [SecurityPermission(SecurityAction.LinkDemand, UnmanagedCode = true)] - internal SafeFindHandle() - : base(true) - { - } - - [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)] - [DllImport("kernel32.dll")] - private static extern bool FindClose(IntPtr handle); - - /// - /// When overridden in a derived class, executes the code required to free the handle. - /// - /// - /// true if the handle is released successfully; otherwise, in the - /// event of a catastrophic failure, false. In this case, it - /// generates a releaseHandleFailed MDA Managed Debugging Assistant. - /// - protected override bool ReleaseHandle() - { - return FindClose(handle); - } - } - - /// - /// Provides the implementation of the - /// interface - /// - [SuppressUnmanagedCodeSecurity, ExcludeFromCodeCoverage] - private class FileEnumerator : IEnumerator - { - private readonly Stack _mContextStack; - private readonly string _mFilter; - private readonly SearchOption _mSearchOption; - private readonly WIN32_FIND_DATA _mWinFindData = new WIN32_FIND_DATA(); - private SearchContext _mCurrentContext; - private SafeFindHandle _mHndFindFile; - private string _mPath; - - /// - /// Initializes a new instance of the class. - /// - /// The path to search. - /// The search string to match against files in the path. - /// - /// One of the SearchOption values that specifies whether the search - /// operation should include all subdirectories or only the current directory. - /// - public FileEnumerator(string path, string filter, SearchOption searchOption) - { - _mPath = path; - _mFilter = filter; - _mSearchOption = searchOption; - _mCurrentContext = new SearchContext(path); - - if (_mSearchOption == SearchOption.AllDirectories) - { - _mContextStack = new Stack(); - } - } - - #region IEnumerator Members - - /// - /// Gets the element in the collection at the current position of the enumerator. - /// - /// - /// - /// The element in the collection at the current position of the enumerator. - /// - public FileData Current => new FileData(_mPath, _mWinFindData); - - #endregion - - #region IDisposable Members - - /// - /// Performs application-defined tasks associated with freeing, releasing, - /// or resetting unmanaged resources. - /// - public void Dispose() - { - _mHndFindFile?.Dispose(); - } - - #endregion - - [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] - private static extern SafeFindHandle FindFirstFile(string fileName, - [In, Out] WIN32_FIND_DATA data); - - [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] - private static extern bool FindNextFile(SafeFindHandle hndFindFile, - [In, Out, MarshalAs(UnmanagedType.LPStruct)] WIN32_FIND_DATA lpFindFileData); - - /// - /// Hold context information about where we current are in the directory search. - /// - private class SearchContext - { - public readonly string Path; - public Stack SubdirectoriesToProcess; - - public SearchContext(string path) - { - Path = path; - } - } - - #region IEnumerator Members - - /// - /// Gets the element in the collection at the current position of the enumerator. - /// - /// - /// - /// The element in the collection at the current position of the enumerator. - /// - object IEnumerator.Current => new FileData(_mPath, _mWinFindData); - - /// - /// Advances the enumerator to the next element of the collection. - /// - /// - /// true if the enumerator was successfully advanced to the next element; - /// false if the enumerator has passed the end of the collection. - /// - /// - /// The collection was modified after the enumerator was created. - /// - public bool MoveNext() - { - var retval = false; - - //If the handle is null, this is first call to MoveNext in the current - // directory. In that case, start a new search. - if (_mCurrentContext.SubdirectoriesToProcess == null) - { - if (_mHndFindFile == null) - { - new FileIOPermission(FileIOPermissionAccess.PathDiscovery, _mPath).Demand(); - - var searchPath = Path.Combine(_mPath, _mFilter); - _mHndFindFile = FindFirstFile(searchPath, _mWinFindData); - retval = !_mHndFindFile.IsInvalid; - } - else - { - //Otherwise, find the next item. - retval = FindNextFile(_mHndFindFile, _mWinFindData); - } - } - - //If the call to FindNextFile or FindFirstFile succeeded... - if (retval) - { - if ((_mWinFindData.dwFileAttributes & FileAttributes.Directory) == FileAttributes.Directory) - { - //Ignore folders for now. We call MoveNext recursively here to - // move to the next item that FindNextFile will return. - return MoveNext(); - } - } - else if (_mSearchOption == SearchOption.AllDirectories) - { - if (_mCurrentContext.SubdirectoriesToProcess == null) - { - var subDirectories = Directory.GetDirectories(_mPath); - _mCurrentContext.SubdirectoriesToProcess = new Stack(subDirectories); - } - - if (_mCurrentContext.SubdirectoriesToProcess.Count > 0) - { - var subDir = _mCurrentContext.SubdirectoriesToProcess.Pop(); - - _mContextStack.Push(_mCurrentContext); - _mPath = subDir; - _mHndFindFile = null; - _mCurrentContext = new SearchContext(_mPath); - return MoveNext(); - } - - //If there are no more files in this directory and we are - // in a sub directory, pop back up to the parent directory and - // continue the search from there. - if (_mContextStack.Count > 0) - { - _mCurrentContext = _mContextStack.Pop(); - _mPath = _mCurrentContext.Path; - if (_mHndFindFile != null) - { - _mHndFindFile.Close(); - _mHndFindFile = null; - } - - return MoveNext(); - } - } - - return retval; - } - - /// - /// Sets the enumerator to its initial position, which is before the first element in the collection. - /// - /// - /// The collection was modified after the enumerator was created. - /// - public void Reset() - { - _mHndFindFile = null; - } - - #endregion - } - } -} \ No newline at end of file diff --git a/src/Rainbow/Storage/SerializationFileSystemTree.cs b/src/Rainbow/Storage/SerializationFileSystemTree.cs index 75d264c..52987cb 100644 --- a/src/Rainbow/Storage/SerializationFileSystemTree.cs +++ b/src/Rainbow/Storage/SerializationFileSystemTree.cs @@ -550,21 +550,21 @@ protected virtual string[] GetChildPaths(IItemMetadata item) if (serializedItem == null) throw new InvalidOperationException("Item {0} does not exist on disk.".FormatWith(item.Path)); - IEnumerable children = Enumerable.Empty(); + IEnumerable children = Enumerable.Empty(); var childrenPath = Path.ChangeExtension(serializedItem.SerializedItemId, null); if (Directory.Exists(childrenPath)) { - children = FastDirectoryEnumerator.GetFiles(childrenPath, "*" + _formatter.FileExtension, SearchOption.TopDirectoryOnly); + children = Directory.EnumerateFiles(childrenPath, "*" + _formatter.FileExtension, SearchOption.TopDirectoryOnly); } var shortPath = Path.Combine(_physicalRootPath, item.Id.ToString()); if (Directory.Exists(shortPath)) - children = children.Concat(FastDirectoryEnumerator.GetFiles(shortPath, "*" + _formatter.FileExtension, SearchOption.TopDirectoryOnly)); + children = children.Concat(Directory.EnumerateFiles(shortPath, "*" + _formatter.FileExtension, SearchOption.TopDirectoryOnly)); - return children.Select(result => result.Path).ToArray(); + return children.ToArray(); } protected virtual string PrepareItemNameForFileSystem(string name) @@ -616,7 +616,7 @@ protected virtual IItemMetadata GetItemForGlobalPath(string globalPath, Guid exp var result = GetPhysicalFilePathsForVirtualPath(localPath) .Select(ReadItemMetadata) - .FirstOrDefault(candidateItem => candidateItem.Id == expectedItemId); + .FirstOrDefault(candidateItem => candidateItem != null && candidateItem.Id == expectedItemId); if (result == null) return null;