From 827f38ff334e8431015cc6893dbf9292bcf275fe Mon Sep 17 00:00:00 2001 From: Ersan Bozduman Date: Tue, 21 May 2024 22:42:05 -0700 Subject: [PATCH] removes stat api calls and collects memtmadata with list api --- Minio.Examples/Program.cs | 8 --- Minio.Functional.Tests/FunctionalTest.cs | 19 ++++++- Minio.Functional.Tests/Program.cs | 6 --- Minio/ApiEndpoints/BucketOperations.cs | 39 --------------- Minio/ApiEndpoints/ObjectOperations.cs | 15 ++---- Minio/DataModel/Args/GetObjectListArgs.cs | 2 + .../GetObjectsVersionsListResponse.cs | 49 +++++++++---------- Minio/Helper/AmazonAwsS3XmlReader.cs | 5 +- Minio/Helper/Utils.cs | 16 ++++++ 9 files changed, 65 insertions(+), 94 deletions(-) diff --git a/Minio.Examples/Program.cs b/Minio.Examples/Program.cs index fbf81eb2b..76bd6e1b9 100644 --- a/Minio.Examples/Program.cs +++ b/Minio.Examples/Program.cs @@ -111,14 +111,6 @@ public static async Task Main() var destBucketName = GetRandomName(); var destObjectName = GetRandomName(); var lockBucketName = GetRandomName(); - var progress = new SyncProgress(progressReport => - { - // Console.WriteLine( - // $"Percentage: {progressReport.Percentage}% TotalBytesTransferred: {progressReport.TotalBytesTransferred} bytes"); - // if (progressReport.Percentage != 100) - // Console.SetCursorPosition(0, Console.CursorTop - 1); - // else Console.WriteLine(); - }); var objectsList = new List(); for (var i = 0; i < 10; i++) objectsList.Add(objectName + i); diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index 37958ed35..557c3571e 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -3492,10 +3492,16 @@ internal static async Task PutObject_Test9(IMinioClient minio) const string contentType = "application/octet-stream"; var percentage = 0; var totalBytesTransferred = 0L; + // Display progress as a percentage var progress = new SyncProgress(progressReport => { percentage = progressReport.Percentage; totalBytesTransferred = progressReport.TotalBytesTransferred; + // Console.WriteLine( + // $"Percentage: {progressReport.Percentage}% TotalBytesTransferred: {progressReport.TotalBytesTransferred} bytes"); + // if (progressReport.Percentage != 100) + // Console.SetCursorPosition(0, Console.CursorTop - 1); + // else Console.WriteLine(); }); var args = new Dictionary (StringComparer.Ordinal) @@ -3541,10 +3547,19 @@ internal static async Task PutObject_Test10(IMinioClient minio) var contentType = "binary/octet-stream"; var percentage = 0; var totalBytesTransferred = 0L; + // Display progress as a percentage var progress = new SyncProgress(progressReport => { percentage = progressReport.Percentage; totalBytesTransferred = progressReport.TotalBytesTransferred; + // Console.WriteLine( + // $"PutObject_Test10 - Percentage: {progressReport.Percentage}% TotalBytesTransferred: {progressReport.TotalBytesTransferred} bytes"); + // if (progressReport.Percentage != 100) + // { + // var topPosition = Console.CursorTop > 0 ? Console.CursorTop - 1 : Console.CursorTop; + // Console.SetCursorPosition(0, topPosition); + // } + // else Console.WriteLine(); }); var args = new Dictionary (StringComparer.Ordinal) @@ -5347,7 +5362,6 @@ internal static async Task ListObjectVersions_Test1(IMinioClient minio) { items.Add(item); Assert.IsTrue(item.Key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)); - var s = string.Join("; ", item.UserMetadata.Select(x => x.Key + "=" + x.Value)); count++; } @@ -5357,7 +5371,8 @@ internal static async Task ListObjectVersions_Test1(IMinioClient minio) var indxdSfx = i.ToString(CultureInfo.InvariantCulture) + suffix; var userMDataCount = 0; foreach (var mtDt in items[i].UserMetadata) - if (mtDt.Key.EndsWith(indxdSfx, StringComparison.Ordinal) && + if (mtDt.Key.StartsWith("X-Amz-Meta-", StringComparison.Ordinal) && + mtDt.Key.EndsWith(indxdSfx, StringComparison.Ordinal) && mtDt.Value.EndsWith(indxdSfx, StringComparison.Ordinal)) userMDataCount++; Assert.AreEqual(customMetadataSizeList[i], userMDataCount); diff --git a/Minio.Functional.Tests/Program.cs b/Minio.Functional.Tests/Program.cs index 027e31f3d..40439f90c 100644 --- a/Minio.Functional.Tests/Program.cs +++ b/Minio.Functional.Tests/Program.cs @@ -79,12 +79,6 @@ public static async Task Main(string[] args) .WithSSL(isSecure) .Build(); - // // Assign parameters before starting the test - // var bucketName = FunctionalTest.GetRandomName(); - // var objectName = FunctionalTest.GetRandomName(); - // var destBucketName = FunctionalTest.GetRandomName(); - // var destObjectName = FunctionalTest.GetRandomName(); - // Set app Info minioClient.SetAppInfo("app-name", "app-version"); // Set HTTP Tracing On diff --git a/Minio/ApiEndpoints/BucketOperations.cs b/Minio/ApiEndpoints/BucketOperations.cs index 4bfa4cfd2..ba803d6bd 100644 --- a/Minio/ApiEndpoints/BucketOperations.cs +++ b/Minio/ApiEndpoints/BucketOperations.cs @@ -231,28 +231,6 @@ public IObservable ListObjectsAsync(ListObjectsArgs args, CancellationToke { var objectList = await GetObjectVersionsListAsync(goArgs, cts.Token).ConfigureAwait(false); if (objectList is null) return; - // Add user metadata information - if (goArgs.IncludeUserMetadata) - { - var indx = 0; - foreach (var itm in objectList.Item2) - { - var statObjectArgs = new StatObjectArgs() - .WithBucket(args.BucketName) - .WithObject(itm.Key); - var objStat = await StatObjectAsync(statObjectArgs, cancellationToken) - .ConfigureAwait(false); - objectList.Item1.UserMetadata = []; - itm.UserMetadata = []; - foreach (var pair in objStat.MetaData) - { - objectList.Item1.UserMetadata.Add(new MetadataItem(pair.Key, pair.Value)); - itm.UserMetadata.Add(new MetadataItem(pair.Key, pair.Value)); - } - - indx++; - } - } var listObjectsItemResponse = new ListObjectVersionResponse(objectList, obs); if (objectList.Item2.Count == 0 && count == 0) return; @@ -271,23 +249,6 @@ public IObservable ListObjectsAsync(ListObjectsArgs args, CancellationToke count == 0)) return; - // Add user metadata information - if (goArgs.IncludeUserMetadata) - { - var indx = 0; - foreach (var itm in objectList.Item2) - { - var statObjectArgs = new StatObjectArgs() - .WithBucket(args.BucketName) - .WithObject(itm.Key); - var objStat = await StatObjectAsync(statObjectArgs, cancellationToken) - .ConfigureAwait(false); - foreach (var pair in objStat.MetaData) - objectList.Item1.UserMetadata.Add(new MetadataItem(pair.Key, pair.Value)); - indx++; - } - } - var listObjectsItemResponse = new ListObjectsItemResponse(args, objectList, obs); marker = listObjectsItemResponse.NextMarker; isRunning = objectList.Item1.IsTruncated; diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index b424bae0f..e45ac8f2e 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -593,7 +593,7 @@ public async Task PutObjectAsync(PutObjectArgs args, args = args.WithRequestBody(bytes) .WithStreamData(null) .WithObjectSize(bytesRead); - return await PutObjectSinglePartAsync(args, cancellationToken, true).ConfigureAwait(false); + return await PutObjectSinglePartAsync(args, cancellationToken).ConfigureAwait(false); } // For all sizes greater than 5MiB do multipart. @@ -830,11 +830,7 @@ await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, /// Headers, SSE Headers /// /// Optional cancellation token to cancel the operation - /// - /// This boolean parameter differentiates single part file upload and - /// multi part file upload as this function is shared by both. - /// - /// + /// "PutObjectResponse" /// When access or secret key is invalid /// When bucket name is invalid /// When object name is invalid @@ -844,19 +840,18 @@ await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, /// The file stream is currently in a read operation /// For encrypted PUT operation, Access is denied if the key is wrong private async Task PutObjectSinglePartAsync(PutObjectArgs args, - CancellationToken cancellationToken = default, - bool singleFile = false) + CancellationToken cancellationToken = default) { //Skipping validate as we need the case where stream sends 0 bytes var progressReport = new ProgressReport(); - if (singleFile) args.Progress?.Report(progressReport); + args.Progress?.Report(progressReport); var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false); using var response = await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, cancellationToken: cancellationToken) .ConfigureAwait(false); - if (singleFile && args.Progress is not null) + if (args.Progress is not null) { var statArgs = new StatObjectArgs() .WithBucket(args.BucketName) diff --git a/Minio/DataModel/Args/GetObjectListArgs.cs b/Minio/DataModel/Args/GetObjectListArgs.cs index f5a4d2041..0d17b81a8 100644 --- a/Minio/DataModel/Args/GetObjectListArgs.cs +++ b/Minio/DataModel/Args/GetObjectListArgs.cs @@ -96,6 +96,8 @@ internal override HttpRequestMessageBuilder BuildRequest(HttpRequestMessageBuild requestMessageBuilder.AddQueryParameter("max-keys", "1000"); requestMessageBuilder.AddQueryParameter("encoding-type", "url"); requestMessageBuilder.AddQueryParameter("prefix", Prefix); + if (IncludeUserMetadata) requestMessageBuilder.AddQueryParameter("metadata", "true"); + if (Versions) { requestMessageBuilder.AddQueryParameter("versions", ""); diff --git a/Minio/DataModel/Response/GetObjectsVersionsListResponse.cs b/Minio/DataModel/Response/GetObjectsVersionsListResponse.cs index e6fb0ad2a..0f0d50819 100644 --- a/Minio/DataModel/Response/GetObjectsVersionsListResponse.cs +++ b/Minio/DataModel/Response/GetObjectsVersionsListResponse.cs @@ -34,45 +34,42 @@ internal GetObjectsVersionsListResponse(HttpStatusCode statusCode, string respon BucketResult = Utils.DeserializeXml(responseContent); List items = []; - List userMetadata = []; var root = XDocument.Parse(responseContent); XNamespace ns = Utils.DetermineNamespace(root); var versNodes = root.Root.Descendants(ns + "Version"); var userMtdt = versNodes.Descendants(ns + "UserMetadata"); - if (userMtdt.Any()) + + for (var indx = 0; versNodes.Skip(indx).Any(); indx++) { - var i = 0; - foreach (var mtData in userMtdt) + var item = new Item { - userMetadata[i].Key = mtData.Element(ns + "MetadataItem").Element(ns + "Key").Value; - userMetadata[i].Value = mtData.Element(ns + "MetadataItem").Element(ns + "Value").Value; - i++; - } - } + Key = versNodes.ToList()[indx].Element(ns + "Key").Value, + LastModified = versNodes.ToList()[indx].Element(ns + "LastModified").Value, + ETag = versNodes.ToList()[indx].Element(ns + "ETag").Value, + VersionId = versNodes.ToList()[indx].Element(ns + "VersionId").Value, + StorageClass = versNodes.ToList()[indx].Element(ns + "StorageClass").Value, + Size = ulong.Parse(versNodes.ToList()[indx].Element(ns + "Size").Value), + IsLatest = bool.Parse(versNodes.ToList()[indx].Element(ns + "IsLatest").Value), + IsDir = BucketResult.Prefix is not null + }; - if (versNodes.Any()) - for (var indx = 0; versNodes.Skip(indx).Any(); indx++) + if (userMtdt.Any()) { - var item = new Item + List userMetadata = []; + foreach (var el in userMtdt.ToList()[indx].Elements()) { - Key = versNodes.ToList()[indx].Element(ns + "Key").Value, - LastModified = versNodes.ToList()[indx].Element(ns + "LastModified").Value, - ETag = versNodes.ToList()[indx].Element(ns + "ETag").Value, - VersionId = versNodes.ToList()[indx].Element(ns + "VersionId").Value, - StorageClass = versNodes.ToList()[indx].Element(ns + "StorageClass").Value, - Size = ulong.Parse(versNodes.ToList()[indx].Element(ns + "Size").Value), - IsLatest = bool.Parse(versNodes.ToList()[indx].Element(ns + "IsLatest").Value), - UserMetadata = userMetadata, - IsDir = BucketResult.Prefix is not null - }; + var strippedEl = Utils.RemoveAllNamespaces(el); + if (strippedEl.Name.ToString().StartsWith("X-Amz-Meta-", StringComparison.Ordinal)) + userMetadata.Add(new MetadataItem(strippedEl.Name.ToString(), (string)strippedEl)); + } - items.Add(item); + if (userMetadata.Count > 0) + item.UserMetadata = userMetadata; } - // TO DO - // Is DeleteMarker = bool.Parse(c.Element(ns + "IsDeleteMarker").Value) - // Usertags = ... + items.Add(item); + } ObjectsTuple = new Tuple>(BucketResult, items); } } diff --git a/Minio/Helper/AmazonAwsS3XmlReader.cs b/Minio/Helper/AmazonAwsS3XmlReader.cs index 42060783f..7ddafe913 100644 --- a/Minio/Helper/AmazonAwsS3XmlReader.cs +++ b/Minio/Helper/AmazonAwsS3XmlReader.cs @@ -20,10 +20,9 @@ namespace Minio.Helper; public class AmazonAwsS3XmlReader : XmlTextReader { - public new string NamespaceURI = "http://s3.amazonaws.com/doc/2006-03-01/"; - public AmazonAwsS3XmlReader(Stream stream) : base(stream) { - NamespaceURI = "http://s3.amazonaws.com/doc/2006-03-01/"; } + + public override string NamespaceURI => "http://s3.amazonaws.com/doc/2006-03-01/"; } diff --git a/Minio/Helper/Utils.cs b/Minio/Helper/Utils.cs index bd5d953b4..2685e8370 100644 --- a/Minio/Helper/Utils.cs +++ b/Minio/Helper/Utils.cs @@ -45,6 +45,22 @@ public static class Utils private static readonly Lazy> contentTypeMap = new(AddContentTypeMappings); + /// + /// Remove all xmlns/namespace values from an xml document + /// + /// XElement + public static XElement RemoveAllNamespaces(XElement e) + { + return new XElement(e.Name.LocalName, from n in e.Nodes() + select n is XElement ? RemoveAllNamespaces(n as XElement) : n, + e.HasAttributes + ? from a in e.Attributes() + where !a.IsNamespaceDeclaration + select new XAttribute(a.Name.LocalName, a.Value) + : null); + } + + /// /// IsValidBucketName - verify bucket name in accordance with /// http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html