Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Add image cache metadata #50

Merged
merged 8 commits into from
Jan 7, 2019
Merged
5 changes: 3 additions & 2 deletions src/ImageSharp.Web/Caching/IImageCache.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public interface IImageCache
/// </summary>
/// <param name="key">The cache key.</param>
/// <returns>The <see cref="IImageResolver"/>.</returns>
IImageResolver Get(string key);
Task<IImageResolver> GetAsync(string key);

/// <summary>
/// Returns a value indicating whether the current cached item is expired.
Expand All @@ -47,7 +47,8 @@ public interface IImageCache
/// </summary>
/// <param name="key">The cache key.</param>
/// <param name="stream">The stream containing the image to store.</param>
/// <param name="contentType">The content type of the image to store.</param>
/// <returns>The <see cref="Task{DateTimeOffset}"/>.</returns>
Task<DateTimeOffset> SetAsync(string key, Stream stream);
Task<DateTimeOffset> SetAsync(string key, Stream stream, string contentType);
}
}
93 changes: 89 additions & 4 deletions src/ImageSharp.Web/Caching/PhysicalFileSystemCache.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.FileProviders;
using Microsoft.Extensions.Options;
using SixLabors.ImageSharp.Web.Helpers;
using SixLabors.ImageSharp.Web.Middleware;
using SixLabors.ImageSharp.Web.Resolvers;
using SixLabors.Memory;
Expand All @@ -35,6 +36,16 @@ public class PhysicalFileSystemCache : IImageCache
/// </summary>
public const string DefaultCheckSourceChanged = "false";

/// <summary>
/// Filename extension for the metadata files.
/// </summary>
private const string MetaFileExtension = ".meta";

/// <summary>
/// Key for the Content-Type value in the metadata files.
/// </summary>
private const string ContentTypeKey = "Content-Type";

/// <summary>
/// The hosting environment the application is running in.
/// </summary>
Expand All @@ -55,6 +66,11 @@ public class PhysicalFileSystemCache : IImageCache
/// </summary>
private readonly ImageSharpMiddlewareOptions options;

/// <summary>
/// Contains various helper methods based on the current configuration.
/// </summary>
private readonly FormatHelper formatHelper;

/// <summary>
/// Initializes a new instance of the <see cref="PhysicalFileSystemCache"/> class.
/// </summary>
Expand All @@ -76,6 +92,7 @@ public PhysicalFileSystemCache(IHostingEnvironment environment, MemoryAllocator
this.fileProvider = this.environment.WebRootFileProvider;
this.memoryAllocator = memoryAllocator;
this.options = options.Value;
this.formatHelper = new FormatHelper(this.options.Configuration);
}

/// <inheritdoc/>
Expand All @@ -86,17 +103,40 @@ public PhysicalFileSystemCache(IHostingEnvironment environment, MemoryAllocator
};

/// <inheritdoc/>
public IImageResolver Get(string key)
public async Task<IImageResolver> GetAsync(string key)
{
IFileInfo fileInfo = this.fileProvider.GetFileInfo(this.ToFilePath(key));
string path = this.ToFilePath(key);
IFileInfo fileInfo = this.fileProvider.GetFileInfo(path);

// Check to see if the file exists.
if (!fileInfo.Exists)
{
return null;
}

return new PhysicalFileSystemResolver(fileInfo);
string contentType = null;

// If a metadata file exists, then try to load it and obtain the content type from the saved metadata
IFileInfo metaFileInfo = this.fileProvider.GetFileInfo($"{path}{MetaFileExtension}");
if (metaFileInfo.Exists)
{
Dictionary<string, string> metadata;
using (Stream metaStream = metaFileInfo.CreateReadStream())
{
metadata = await this.ReadMetadataFile(metaStream);
}

metadata.TryGetValue(ContentTypeKey, out contentType);
}

// If content type metadata was not available, then fallback on guessing the content type
// based on the filename.
if (contentType == null)
{
contentType = this.formatHelper.GetContentType(key);
}

return new PhysicalFileSystemResolver(fileInfo, contentType);
}

/// <inheritdoc/>
Expand All @@ -122,9 +162,10 @@ public Task<CachedInfo> IsExpiredAsync(HttpContext context, string key, DateTime
}

/// <inheritdoc/>
public async Task<DateTimeOffset> SetAsync(string key, Stream stream)
public async Task<DateTimeOffset> SetAsync(string key, Stream stream, string contentType)
{
string path = Path.Combine(this.environment.WebRootPath, this.ToFilePath(key));
string metaPath = $"{path}{MetaFileExtension}";
string directory = Path.GetDirectoryName(path);

if (!Directory.Exists(directory))
Expand All @@ -137,6 +178,15 @@ public async Task<DateTimeOffset> SetAsync(string key, Stream stream)
await stream.CopyToAsync(fileStream).ConfigureAwait(false);
}

using (FileStream fileStream = File.Create(metaPath))
{
var metadata = new Dictionary<string, string>
{
{ ContentTypeKey, contentType }
};
await this.WriteMetadataFile(metadata, fileStream);
}

return File.GetLastWriteTimeUtc(path);
}

Expand All @@ -146,5 +196,40 @@ public async Task<DateTimeOffset> SetAsync(string key, Stream stream)
/// <param name="key">The cache key.</param>
/// <returns>The <see cref="string"/>.</returns>
private string ToFilePath(string key) => $"{this.Settings[Folder]}/{string.Join("/", key.Substring(0, (int)this.options.CachedNameLength).ToCharArray())}/{key}";

private async Task<Dictionary<string, string>> ReadMetadataFile(Stream stream)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would have to be implementation agnostic.

I'm thinking we create a custom struct that can be easily serialized deserialized into raw bytes that can be used for storing both the content type and last modified date.

CacheMeta
8 bytes | ModifiedDate
2 bytes | Content Type string length
n bytes | Content Type

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My thinking with this implementation was to use a file format that provides easy forward/backward compatibility so that additional metadata fields could be easily added in the future (I already have one such use that I would like to discuss if you end up deciding to proceed with this metadata concept). I would have used JSON, but didn't want to take a dependency on a JSON library, so I went with a really simple text-based key/value pair file format. I wasn't too worried about wasting a few extra bytes by having the data stored as text as the size should be trivial relative to the actual image data that it is annotating.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But thinking about it now, I do like the idea of having the metadata contained in a struct, as that would make it possible to add new metadata fields without needing to add any additional methods or change function signatures. Something like:

public struct ImageMetadata
{
    public DateTime LastModifiedUtc;
    public string ContentType;
}

public interface IImageResolver
{
    Task<Stream> OpenReadAsync();
    Task<ImageMetadata> GetImageMetadata();
}

public interface IImageCache
{
    Task<IImageResolver> GetAsync(string key);
    Task<DateTimeOffset> SetAsync(string key, Stream stream, ImageMetadata metadata);
}

{
Dictionary<string, string> metadata = new Dictionary<string, string>();

using (var reader = new StreamReader(stream, System.Text.Encoding.UTF8))
{
string line;
while ((line = await reader.ReadLineAsync()) != null)
{
int idx = line.IndexOf(':');
if (idx > 0)
{
string key = line.Substring(0, idx).Trim();
string value = line.Substring(idx + 1).Trim();
metadata[key] = value;
}
}
}

return metadata;
}

private async Task WriteMetadataFile(Dictionary<string, string> metadata, Stream stream)
{
using (var writer = new StreamWriter(stream, System.Text.Encoding.UTF8))
{
foreach (KeyValuePair<string, string> keyValuePair in metadata)
{
await writer.WriteLineAsync($"{keyValuePair.Key}: {keyValuePair.Value}");
}

await writer.FlushAsync();
}
}
}
}
16 changes: 9 additions & 7 deletions src/ImageSharp.Web/Middleware/ImageSharpMiddleware.cs
Original file line number Diff line number Diff line change
Expand Up @@ -190,11 +190,12 @@ public async Task Invoke(HttpContext context)
if (!info.Expired)
{
// We're pulling the image from the cache.
IImageResolver cachedImage = this.cache.Get(key);
IImageResolver cachedImage = await this.cache.GetAsync(key).ConfigureAwait(false);
using (Stream cachedBuffer = await cachedImage.OpenReadAsync().ConfigureAwait(false))
{
// Image is a cached image. Return the correct response now.
await this.SendResponse(imageContext, key, info.LastModifiedUtc, cachedBuffer).ConfigureAwait(false);
string contentType = await cachedImage.GetContentTypeAsync();
await this.SendResponse(imageContext, key, info.LastModifiedUtc, cachedBuffer, contentType).ConfigureAwait(false);
}

return;
Expand All @@ -211,6 +212,8 @@ public async Task Invoke(HttpContext context)
// This reduces the overheads of unnecessary processing plus avoids file locks.
using (await AsyncLock.WriterLockAsync(key).ConfigureAwait(false))
{
string contentType;

// No allocations here for inStream since we are passing the raw input stream.
// outStream allocation depends on the memory allocator used.
outStream = new ChunkedMemoryStream(this.memoryAllocator);
Expand All @@ -219,6 +222,7 @@ public async Task Invoke(HttpContext context)
{
image.Process(this.logger, this.processors, commands);
this.options.OnBeforeSave?.Invoke(image);
contentType = image.Format.DefaultMimeType;
image.Save(outStream);
}

Expand All @@ -227,8 +231,8 @@ public async Task Invoke(HttpContext context)
this.options.OnProcessed?.Invoke(new ImageProcessingContext(context, outStream, commands, Path.GetExtension(key)));
outStream.Position = 0;

DateTimeOffset cachedDate = await this.cache.SetAsync(key, outStream).ConfigureAwait(false);
await this.SendResponse(imageContext, key, cachedDate, outStream).ConfigureAwait(false);
DateTimeOffset cachedDate = await this.cache.SetAsync(key, outStream, contentType).ConfigureAwait(false);
await this.SendResponse(imageContext, key, cachedDate, outStream, contentType).ConfigureAwait(false);
}
}
}
Expand All @@ -252,12 +256,10 @@ public async Task Invoke(HttpContext context)
}
}

private async Task SendResponse(ImageContext imageContext, string key, DateTimeOffset lastModified, Stream stream)
private async Task SendResponse(ImageContext imageContext, string key, DateTimeOffset lastModified, Stream stream, string contentType)
{
imageContext.ComprehendRequestHeaders(lastModified, stream.Length);

string contentType = this.formatHelper.GetContentType(key);

switch (imageContext.GetPreconditionState())
{
case ImageContext.PreconditionState.Unspecified:
Expand Down
5 changes: 4 additions & 1 deletion src/ImageSharp.Web/Providers/PhysicalFileSystemProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,10 @@ public Task<IImageResolver> GetAsync(HttpContext context)
return Task.FromResult<IImageResolver>(null);
}

return Task.FromResult<IImageResolver>(new PhysicalFileSystemResolver(fileInfo));
// Make an educated guess of the contentType based on the filename.
string contentType = this.formatHelper.GetContentType(fileInfo.Name);

return Task.FromResult<IImageResolver>(new PhysicalFileSystemResolver(fileInfo, contentType));
}
}
}
6 changes: 6 additions & 0 deletions src/ImageSharp.Web/Resolvers/IImageResolver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@ public interface IImageResolver
/// <returns>The <see cref="DateTime"/>.</returns>
Task<DateTime> GetLastWriteTimeUtcAsync();

/// <summary>
/// Gets the content type of the image data.
/// </summary>
/// <returns>The content type.</returns>
Task<string> GetContentTypeAsync();

/// <summary>
/// Gets the input image stream in an asynchronous manner.
/// </summary>
Expand Down
11 changes: 10 additions & 1 deletion src/ImageSharp.Web/Resolvers/PhysicalFileSystemResolver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,25 @@ namespace SixLabors.ImageSharp.Web.Resolvers
public class PhysicalFileSystemResolver : IImageResolver
{
private readonly IFileInfo fileInfo;
private readonly string contentType;

/// <summary>
/// Initializes a new instance of the <see cref="PhysicalFileSystemResolver"/> class.
/// </summary>
/// <param name="fileInfo">The input file info.</param>
public PhysicalFileSystemResolver(IFileInfo fileInfo) => this.fileInfo = fileInfo;
/// <param name="contentType">The content type of this file.</param>
public PhysicalFileSystemResolver(IFileInfo fileInfo, string contentType)
{
this.fileInfo = fileInfo;
this.contentType = contentType;
}

/// <inheritdoc/>
public Task<DateTime> GetLastWriteTimeUtcAsync() => Task.FromResult(this.fileInfo.LastModified.UtcDateTime);

/// <inheritdoc/>
public Task<string> GetContentTypeAsync() => Task.FromResult(this.contentType);

/// <inheritdoc/>
public Task<Stream> OpenReadAsync() => Task.FromResult(this.fileInfo.CreateReadStream());
}
Expand Down