Skip to content

Commit

Permalink
removes stat api calls and collects memtmadata with list api
Browse files Browse the repository at this point in the history
  • Loading branch information
Ersan Bozduman committed May 22, 2024
1 parent 9329a37 commit 827f38f
Show file tree
Hide file tree
Showing 9 changed files with 65 additions and 94 deletions.
8 changes: 0 additions & 8 deletions Minio.Examples/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,14 +111,6 @@ public static async Task Main()
var destBucketName = GetRandomName();
var destObjectName = GetRandomName();
var lockBucketName = GetRandomName();
var progress = new SyncProgress<ProgressReport>(progressReport =>
{
// Console.WriteLine(
// $"Percentage: {progressReport.Percentage}% TotalBytesTransferred: {progressReport.TotalBytesTransferred} bytes");
// if (progressReport.Percentage != 100)
// Console.SetCursorPosition(0, Console.CursorTop - 1);
// else Console.WriteLine();
});
var objectsList = new List<string>();
for (var i = 0; i < 10; i++) objectsList.Add(objectName + i);

Expand Down
19 changes: 17 additions & 2 deletions Minio.Functional.Tests/FunctionalTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3492,10 +3492,16 @@ internal static async Task PutObject_Test9(IMinioClient minio)
const string contentType = "application/octet-stream";
var percentage = 0;
var totalBytesTransferred = 0L;
// Display progress as a percentage
var progress = new SyncProgress<ProgressReport>(progressReport =>
{
percentage = progressReport.Percentage;
totalBytesTransferred = progressReport.TotalBytesTransferred;
// Console.WriteLine(
// $"Percentage: {progressReport.Percentage}% TotalBytesTransferred: {progressReport.TotalBytesTransferred} bytes");
// if (progressReport.Percentage != 100)
// Console.SetCursorPosition(0, Console.CursorTop - 1);
// else Console.WriteLine();
});
var args = new Dictionary<string, string>
(StringComparer.Ordinal)
Expand Down Expand Up @@ -3541,10 +3547,19 @@ internal static async Task PutObject_Test10(IMinioClient minio)
var contentType = "binary/octet-stream";
var percentage = 0;
var totalBytesTransferred = 0L;
// Display progress as a percentage
var progress = new SyncProgress<ProgressReport>(progressReport =>
{
percentage = progressReport.Percentage;
totalBytesTransferred = progressReport.TotalBytesTransferred;
// Console.WriteLine(
// $"PutObject_Test10 - Percentage: {progressReport.Percentage}% TotalBytesTransferred: {progressReport.TotalBytesTransferred} bytes");
// if (progressReport.Percentage != 100)
// {
// var topPosition = Console.CursorTop > 0 ? Console.CursorTop - 1 : Console.CursorTop;
// Console.SetCursorPosition(0, topPosition);
// }
// else Console.WriteLine();
});
var args = new Dictionary<string, string>
(StringComparer.Ordinal)
Expand Down Expand Up @@ -5347,7 +5362,6 @@ internal static async Task ListObjectVersions_Test1(IMinioClient minio)
{
items.Add(item);
Assert.IsTrue(item.Key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase));
var s = string.Join("; ", item.UserMetadata.Select(x => x.Key + "=" + x.Value));
count++;
}

Expand All @@ -5357,7 +5371,8 @@ internal static async Task ListObjectVersions_Test1(IMinioClient minio)
var indxdSfx = i.ToString(CultureInfo.InvariantCulture) + suffix;
var userMDataCount = 0;
foreach (var mtDt in items[i].UserMetadata)
if (mtDt.Key.EndsWith(indxdSfx, StringComparison.Ordinal) &&
if (mtDt.Key.StartsWith("X-Amz-Meta-", StringComparison.Ordinal) &&
mtDt.Key.EndsWith(indxdSfx, StringComparison.Ordinal) &&
mtDt.Value.EndsWith(indxdSfx, StringComparison.Ordinal))
userMDataCount++;
Assert.AreEqual(customMetadataSizeList[i], userMDataCount);
Expand Down
6 changes: 0 additions & 6 deletions Minio.Functional.Tests/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,6 @@ public static async Task Main(string[] args)
.WithSSL(isSecure)
.Build();

// // Assign parameters before starting the test
// var bucketName = FunctionalTest.GetRandomName();
// var objectName = FunctionalTest.GetRandomName();
// var destBucketName = FunctionalTest.GetRandomName();
// var destObjectName = FunctionalTest.GetRandomName();

// Set app Info
minioClient.SetAppInfo("app-name", "app-version");
// Set HTTP Tracing On
Expand Down
39 changes: 0 additions & 39 deletions Minio/ApiEndpoints/BucketOperations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,28 +231,6 @@ public IObservable<Item> ListObjectsAsync(ListObjectsArgs args, CancellationToke
{
var objectList = await GetObjectVersionsListAsync(goArgs, cts.Token).ConfigureAwait(false);
if (objectList is null) return;
// Add user metadata information
if (goArgs.IncludeUserMetadata)
{
var indx = 0;
foreach (var itm in objectList.Item2)
{
var statObjectArgs = new StatObjectArgs()
.WithBucket(args.BucketName)
.WithObject(itm.Key);
var objStat = await StatObjectAsync(statObjectArgs, cancellationToken)
.ConfigureAwait(false);
objectList.Item1.UserMetadata = [];
itm.UserMetadata = [];
foreach (var pair in objStat.MetaData)
{
objectList.Item1.UserMetadata.Add(new MetadataItem(pair.Key, pair.Value));
itm.UserMetadata.Add(new MetadataItem(pair.Key, pair.Value));
}
indx++;
}
}
var listObjectsItemResponse = new ListObjectVersionResponse(objectList, obs);
if (objectList.Item2.Count == 0 && count == 0) return;
Expand All @@ -271,23 +249,6 @@ public IObservable<Item> ListObjectsAsync(ListObjectsArgs args, CancellationToke
count == 0))
return;
// Add user metadata information
if (goArgs.IncludeUserMetadata)
{
var indx = 0;
foreach (var itm in objectList.Item2)
{
var statObjectArgs = new StatObjectArgs()
.WithBucket(args.BucketName)
.WithObject(itm.Key);
var objStat = await StatObjectAsync(statObjectArgs, cancellationToken)
.ConfigureAwait(false);
foreach (var pair in objStat.MetaData)
objectList.Item1.UserMetadata.Add(new MetadataItem(pair.Key, pair.Value));
indx++;
}
}
var listObjectsItemResponse = new ListObjectsItemResponse(args, objectList, obs);
marker = listObjectsItemResponse.NextMarker;
isRunning = objectList.Item1.IsTruncated;
Expand Down
15 changes: 5 additions & 10 deletions Minio/ApiEndpoints/ObjectOperations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -593,7 +593,7 @@ public async Task<PutObjectResponse> PutObjectAsync(PutObjectArgs args,
args = args.WithRequestBody(bytes)
.WithStreamData(null)
.WithObjectSize(bytesRead);
return await PutObjectSinglePartAsync(args, cancellationToken, true).ConfigureAwait(false);
return await PutObjectSinglePartAsync(args, cancellationToken).ConfigureAwait(false);
}

// For all sizes greater than 5MiB do multipart.
Expand Down Expand Up @@ -830,11 +830,7 @@ await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder,
/// Headers, SSE Headers
/// </param>
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>
/// <param name="singleFile">
/// This boolean parameter differentiates single part file upload and
/// multi part file upload as this function is shared by both.
/// </param>
/// <returns></returns>
/// <returns>"PutObjectResponse"</returns>
/// <exception cref="AuthorizationException">When access or secret key is invalid</exception>
/// <exception cref="InvalidBucketNameException">When bucket name is invalid</exception>
/// <exception cref="InvalidObjectNameException">When object name is invalid</exception>
Expand All @@ -844,19 +840,18 @@ await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder,
/// <exception cref="InvalidOperationException">The file stream is currently in a read operation</exception>
/// <exception cref="AccessDeniedException">For encrypted PUT operation, Access is denied if the key is wrong</exception>
private async Task<PutObjectResponse> PutObjectSinglePartAsync(PutObjectArgs args,
CancellationToken cancellationToken = default,
bool singleFile = false)
CancellationToken cancellationToken = default)
{
//Skipping validate as we need the case where stream sends 0 bytes
var progressReport = new ProgressReport();
if (singleFile) args.Progress?.Report(progressReport);
args.Progress?.Report(progressReport);
var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false);
using var response =
await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder,
cancellationToken: cancellationToken)
.ConfigureAwait(false);

if (singleFile && args.Progress is not null)
if (args.Progress is not null)
{
var statArgs = new StatObjectArgs()
.WithBucket(args.BucketName)
Expand Down
2 changes: 2 additions & 0 deletions Minio/DataModel/Args/GetObjectListArgs.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ internal override HttpRequestMessageBuilder BuildRequest(HttpRequestMessageBuild
requestMessageBuilder.AddQueryParameter("max-keys", "1000");
requestMessageBuilder.AddQueryParameter("encoding-type", "url");
requestMessageBuilder.AddQueryParameter("prefix", Prefix);
if (IncludeUserMetadata) requestMessageBuilder.AddQueryParameter("metadata", "true");

if (Versions)
{
requestMessageBuilder.AddQueryParameter("versions", "");
Expand Down
49 changes: 23 additions & 26 deletions Minio/DataModel/Response/GetObjectsVersionsListResponse.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,45 +34,42 @@ internal GetObjectsVersionsListResponse(HttpStatusCode statusCode, string respon
BucketResult = Utils.DeserializeXml<ListVersionsResult>(responseContent);

List<Item> items = [];
List<MetadataItem> userMetadata = [];
var root = XDocument.Parse(responseContent);
XNamespace ns = Utils.DetermineNamespace(root);

var versNodes = root.Root.Descendants(ns + "Version");
var userMtdt = versNodes.Descendants(ns + "UserMetadata");
if (userMtdt.Any())

for (var indx = 0; versNodes.Skip(indx).Any(); indx++)
{
var i = 0;
foreach (var mtData in userMtdt)
var item = new Item
{
userMetadata[i].Key = mtData.Element(ns + "MetadataItem").Element(ns + "Key").Value;
userMetadata[i].Value = mtData.Element(ns + "MetadataItem").Element(ns + "Value").Value;
i++;
}
}
Key = versNodes.ToList()[indx].Element(ns + "Key").Value,
LastModified = versNodes.ToList()[indx].Element(ns + "LastModified").Value,
ETag = versNodes.ToList()[indx].Element(ns + "ETag").Value,
VersionId = versNodes.ToList()[indx].Element(ns + "VersionId").Value,
StorageClass = versNodes.ToList()[indx].Element(ns + "StorageClass").Value,
Size = ulong.Parse(versNodes.ToList()[indx].Element(ns + "Size").Value),
IsLatest = bool.Parse(versNodes.ToList()[indx].Element(ns + "IsLatest").Value),
IsDir = BucketResult.Prefix is not null
};

if (versNodes.Any())
for (var indx = 0; versNodes.Skip(indx).Any(); indx++)
if (userMtdt.Any())
{
var item = new Item
List<MetadataItem> userMetadata = [];
foreach (var el in userMtdt.ToList()[indx].Elements())
{
Key = versNodes.ToList()[indx].Element(ns + "Key").Value,
LastModified = versNodes.ToList()[indx].Element(ns + "LastModified").Value,
ETag = versNodes.ToList()[indx].Element(ns + "ETag").Value,
VersionId = versNodes.ToList()[indx].Element(ns + "VersionId").Value,
StorageClass = versNodes.ToList()[indx].Element(ns + "StorageClass").Value,
Size = ulong.Parse(versNodes.ToList()[indx].Element(ns + "Size").Value),
IsLatest = bool.Parse(versNodes.ToList()[indx].Element(ns + "IsLatest").Value),
UserMetadata = userMetadata,
IsDir = BucketResult.Prefix is not null
};
var strippedEl = Utils.RemoveAllNamespaces(el);
if (strippedEl.Name.ToString().StartsWith("X-Amz-Meta-", StringComparison.Ordinal))
userMetadata.Add(new MetadataItem(strippedEl.Name.ToString(), (string)strippedEl));
}

items.Add(item);
if (userMetadata.Count > 0)
item.UserMetadata = userMetadata;
}

// TO DO
// Is DeleteMarker = bool.Parse(c.Element(ns + "IsDeleteMarker").Value)
// Usertags = ...
items.Add(item);
}
ObjectsTuple = new Tuple<ListVersionsResult, List<Item>>(BucketResult, items);
}
}
5 changes: 2 additions & 3 deletions Minio/Helper/AmazonAwsS3XmlReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,9 @@ namespace Minio.Helper;

public class AmazonAwsS3XmlReader : XmlTextReader
{
public new string NamespaceURI = "http://s3.amazonaws.com/doc/2006-03-01/";

public AmazonAwsS3XmlReader(Stream stream) : base(stream)
{
NamespaceURI = "http://s3.amazonaws.com/doc/2006-03-01/";
}

public override string NamespaceURI => "http://s3.amazonaws.com/doc/2006-03-01/";
}
16 changes: 16 additions & 0 deletions Minio/Helper/Utils.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,22 @@ public static class Utils

private static readonly Lazy<IDictionary<string, string>> contentTypeMap = new(AddContentTypeMappings);

/// <summary>
/// Remove all xmlns/namespace values from an xml document
/// </summary>
/// <param name="e"> XElement</param>
public static XElement RemoveAllNamespaces(XElement e)
{
return new XElement(e.Name.LocalName, from n in e.Nodes()
select n is XElement ? RemoveAllNamespaces(n as XElement) : n,
e.HasAttributes
? from a in e.Attributes()
where !a.IsNamespaceDeclaration
select new XAttribute(a.Name.LocalName, a.Value)
: null);
}


/// <summary>
/// IsValidBucketName - verify bucket name in accordance with
/// http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
Expand Down

0 comments on commit 827f38f

Please sign in to comment.