diff --git a/.gitignore b/.gitignore index 88bda8272..0d595d7e6 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ bld/ [Bb]in/ [Oo]bj/ +.history # Visual Studio 2015 cache/options directory .vs/ diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index f7c971811..100c74d2f 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -338,7 +338,7 @@ internal static Task RunCoreTests(MinioClient minioClient) GetBucketPolicy_Test1(minioClient) }; - return Utils.RunInParallel(coreTestsTasks, async (task, _) => { await task.ConfigureAwait(false); }); + return Utils.RunInParallel(coreTestsTasks, async (task, _) => await task.ConfigureAwait(false)); } internal static async Task BucketExists_Test(MinioClient minio) @@ -587,7 +587,7 @@ internal static async Task TearDown(MinioClient minio, string bucketName) if (!bktExists) return; var taskList = new List(); - var getVersions = false; + var versioningOnBuckets = new Dictionary(); // Get Versioning/Retention Info. var lockConfigurationArgs = new GetObjectLockConfigurationArgs() @@ -600,8 +600,9 @@ internal static async Task TearDown(MinioClient minio, string bucketName) .WithBucket(bucketName)).ConfigureAwait(false); if (versioningConfig is not null && (versioningConfig.Status.Contains("Enabled") || versioningConfig.Status.Contains("Suspended"))) - getVersions = true; - + versioningOnBuckets.Add(bucketName, true); + else + versioningOnBuckets.Add(bucketName, false); lockConfig = await minio.GetObjectLockConfigurationAsync(lockConfigurationArgs).ConfigureAwait(false); } catch (MissingObjectLockConfigurationException) @@ -617,17 +618,17 @@ internal static async Task TearDown(MinioClient minio, string bucketName) var listObjectsArgs = new ListObjectsArgs() .WithBucket(bucketName) .WithRecursive(true) - .WithVersions(getVersions); + .WithVersions(versioningOnBuckets[bucketName]); var objectNamesVersions = new List>(); var objectNames = new List(); + var exceptionList = new List(); var observable = minio.ListObjectsAsync(listObjectsArgs); - var exceptionList = new List(); var subscription = observable.Subscribe( item => { - if (getVersions) + if (versioningOnBuckets[bucketName]) objectNamesVersions.Add(new Tuple(item.Key, item.VersionId)); else objectNames.Add(item.Key); @@ -659,6 +660,7 @@ internal static async Task TearDown(MinioClient minio, string bucketName) } else { + // No retention/lock settings here if (objectNamesVersions.Count > 0) { var removeObjectArgs = new RemoveObjectsArgs() @@ -667,18 +669,18 @@ internal static async Task TearDown(MinioClient minio, string bucketName) Task t = minio.RemoveObjectsAsync(removeObjectArgs); tasks.Add(t); } + } - if (objectNames.Count > 0) - { - var removeObjectArgs = new RemoveObjectsArgs() - .WithBucket(bucketName) - .WithObjects(objectNames); - - Task t = minio.RemoveObjectsAsync(removeObjectArgs); - tasks.Add(t); - } + if (objectNames.Count > 0) + { + var removeObjectArgs = new RemoveObjectsArgs() + .WithBucket(bucketName) + .WithObjects(objectNames); + Task t = minio.RemoveObjectsAsync(removeObjectArgs); + tasks.Add(t); } + // Remove all the objects await Task.WhenAll(tasks).ConfigureAwait(false); var rbArgs = new RemoveBucketArgs() .WithBucket(bucketName); @@ -785,6 +787,7 @@ internal static async Task PutGetStatEncryptedObject_Test1(MinioClient minio) } finally { + if (File.Exists(tempFileName)) File.Delete(tempFileName); await TearDown(minio, bucketName).ConfigureAwait(false); } } @@ -1329,6 +1332,10 @@ internal static async Task RemoveObjects_Test3(MinioClient minio) DateTime.Now - startTime, "", ex.Message, ex.ToString(), args).Log(); throw; } + finally + { + await TearDown(minio, bucketName).ConfigureAwait(false); + } } internal static async Task DownloadObjectAsync(MinioClient minio, string url, string filePath) @@ -2231,18 +2238,15 @@ internal static async Task ObjectLockConfigurationAsync_Test1(MinioClient minio) { { "bucketName", bucketName } }; - var setLockNotImplemented = false; - var getLockNotImplemented = false; try { await Setup_WithLock_Test(minio, bucketName).ConfigureAwait(false); - //TODO: Use it for testing and remove { var objectRetention = new ObjectRetentionConfiguration(DateTime.Today.AddDays(3)); using (var filestream = rsg.GenerateStreamFromSeed(1 * KB)) { - // Twice, for 2 versions. + // "PUT" the object twice, to have 2 versions. var putObjectArgs1 = new PutObjectArgs() .WithBucket(bucketName) .WithObject(objectName) @@ -2253,6 +2257,7 @@ internal static async Task ObjectLockConfigurationAsync_Test1(MinioClient minio) await minio.PutObjectAsync(putObjectArgs1).ConfigureAwait(false); } + // Generates the 2nd version using (var filestream = rsg.GenerateStreamFromSeed(1 * KB)) { var putObjectArgs2 = new PutObjectArgs() @@ -2265,26 +2270,6 @@ internal static async Task ObjectLockConfigurationAsync_Test1(MinioClient minio) await minio.PutObjectAsync(putObjectArgs2).ConfigureAwait(false); } } - } - catch (NotImplementedException ex) - { - new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), setObjectLockConfigurationSignature, - "Tests whether SetObjectLockConfigurationAsync passes", TestStatus.NA, DateTime.Now - startTime, - ex.Message, ex.ToString(), args: args).Log(); - await TearDown(minio, bucketName).ConfigureAwait(false); - return; - } - catch (Exception ex) - { - new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), setObjectLockConfigurationSignature, - "Tests whether SetObjectLockConfigurationAsync passes", TestStatus.FAIL, DateTime.Now - startTime, - ex.Message, ex.ToString(), args: args).Log(); - await TearDown(minio, bucketName).ConfigureAwait(false); - throw; - } - - try - { var objectLockArgs = new SetObjectLockConfigurationArgs() .WithBucket(bucketName) .WithLockConfiguration( @@ -2295,13 +2280,6 @@ internal static async Task ObjectLockConfigurationAsync_Test1(MinioClient minio) "Tests whether SetObjectLockConfigurationAsync passes", TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); } - catch (NotImplementedException ex) - { - setLockNotImplemented = true; - new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), setObjectLockConfigurationSignature, - "Tests whether SetObjectLockConfigurationAsync passes", TestStatus.NA, DateTime.Now - startTime, - ex.Message, ex.ToString(), args: args).Log(); - } catch (Exception ex) { new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), setObjectLockConfigurationSignature, @@ -2322,55 +2300,33 @@ internal static async Task ObjectLockConfigurationAsync_Test1(MinioClient minio) Assert.IsNotNull(config.Rule.DefaultRetention); Assert.AreEqual(config.Rule.DefaultRetention.Days, 33); new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), getObjectLockConfigurationSignature, - "Tests whether GetObjectLockConfigurationAsync passes", TestStatus.PASS, DateTime.Now - startTime, + "Tests whether Set and Get ...ObjectLockConfigurationAsync apis pass", TestStatus.PASS, + DateTime.Now - startTime, args: args).Log(); } - catch (NotImplementedException ex) - { - getLockNotImplemented = true; - new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), getObjectLockConfigurationSignature, - "Tests whether GetObjectLockConfigurationAsync passes", TestStatus.NA, DateTime.Now - startTime, - ex.Message, ex.ToString(), args: args).Log(); - } catch (Exception ex) { - await TearDown(minio, bucketName).ConfigureAwait(false); new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), getObjectLockConfigurationSignature, - "Tests whether GetObjectLockConfigurationAsync passes", TestStatus.FAIL, DateTime.Now - startTime, + "Tests whether Set and Get ...ObjectLockConfigurationAsync passes", TestStatus.FAIL, + DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); throw; } try { - if (setLockNotImplemented || getLockNotImplemented) - { - // Cannot test Remove Object Lock with Set & Get Object Lock implemented. - new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, - "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.NA, DateTime.Now - startTime, - "Functionality that is not implemented", "", args: args).Log(); - await TearDown(minio, bucketName).ConfigureAwait(false); - return; - } - var objectLockArgs = new RemoveObjectLockConfigurationArgs() .WithBucket(bucketName); await minio.RemoveObjectLockConfigurationAsync(objectLockArgs).ConfigureAwait(false); var getObjectLockArgs = new GetObjectLockConfigurationArgs() .WithBucket(bucketName); var config = await minio.GetObjectLockConfigurationAsync(getObjectLockArgs).ConfigureAwait(false); - Assert.IsNotNull(config); - Assert.IsNull(config.Rule); + Assert.AreEqual(ObjectLockConfiguration.LockEnabled, config.ObjectLockEnabled); + Assert.IsNull(config?.Rule); new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); } - catch (NotImplementedException ex) - { - new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, - "Tests whether RemoveObjectLockConfigurationAsync passes", TestStatus.NA, DateTime.Now - startTime, - ex.Message, ex.ToString(), args: args).Log(); - } catch (Exception ex) { new MintLogger(nameof(ObjectLockConfigurationAsync_Test1), deleteObjectLockConfigurationSignature, @@ -2380,7 +2336,6 @@ internal static async Task ObjectLockConfigurationAsync_Test1(MinioClient minio) } finally { - await Task.Delay(1500).ConfigureAwait(false); await TearDown(minio, bucketName).ConfigureAwait(false); } } @@ -2652,35 +2607,35 @@ internal static async Task GetObjectS3Zip_Test1(MinioClient minio) // ListObject api test with different prefix values // prefix value="", expected number of files listed=1 - var prefix = ""; - await ListObjects_Test(minio, bucketName, prefix, 1, headers: extractHeader).ConfigureAwait(false); + var prefix1 = ""; + await ListObjects_Test(minio, bucketName, prefix1, 1, headers: extractHeader).ConfigureAwait(false); // prefix value="/", expected number of files listed=nFiles - prefix = objectName + "/"; - await ListObjects_Test(minio, bucketName, prefix, nFiles, headers: extractHeader) + var prefix2 = objectName + "/"; + await ListObjects_Test(minio, bucketName, prefix2, nFiles, headers: extractHeader) .ConfigureAwait(false); // prefix value="/test", expected number of files listed=nFiles - prefix = objectName + "/test"; - await ListObjects_Test(minio, bucketName, prefix, nFiles, headers: extractHeader) + var prefix3 = objectName + "/test"; + await ListObjects_Test(minio, bucketName, prefix3, nFiles, headers: extractHeader) .ConfigureAwait(false); // prefix value="/test/", expected number of files listed=nFiles - prefix = objectName + "/test/"; - await ListObjects_Test(minio, bucketName, prefix, nFiles, headers: extractHeader) + var prefix4 = objectName + "/test/"; + await ListObjects_Test(minio, bucketName, prefix4, nFiles, headers: extractHeader) .ConfigureAwait(false); - // prefix value="/test", expected number of files listed=nFiles - prefix = objectName + "/test/small"; - await ListObjects_Test(minio, bucketName, prefix, nFiles, headers: extractHeader) + // prefix value="/test/small", expected number of files listed=nFiles + var prefix5 = objectName + "/test/small"; + await ListObjects_Test(minio, bucketName, prefix5, nFiles, headers: extractHeader) .ConfigureAwait(false); - // prefix value="/test", expected number of files listed=nFiles - prefix = objectName + "/test/small/"; - await ListObjects_Test(minio, bucketName, prefix, nFiles, headers: extractHeader) + // prefix value="/test/small/", expected number of files listed=nFiles + var prefix6 = objectName + "/test/small/"; + await ListObjects_Test(minio, bucketName, prefix6, nFiles, headers: extractHeader) .ConfigureAwait(false); - // prefix value="/test", expected number of files listed=1 + // singleObjectName, expected number of files listed=1 await ListObjects_Test(minio, bucketName, singleObjectName, 1, headers: extractHeader) .ConfigureAwait(false); @@ -2763,7 +2718,7 @@ await PutObject_Tester(minio, bucketName, objectName, null, contentType, // data. This is required to match and convert json data // "receivedJson" into class "ErrorResponse" var len = "{'Error':".Length; - var trimmedFront = receivedJson.Substring(len); + var trimmedFront = receivedJson[len..]; var trimmedFull = trimmedFront.Substring(0, trimmedFront.Length - 1); var err = JsonSerializer.Deserialize(trimmedFull); @@ -3649,11 +3604,11 @@ internal static async Task PutObject_Test10(MinioClient minio) { percentage = progressReport.Percentage; totalBytesTransferred = progressReport.TotalBytesTransferred; - //Console.WriteLine( + // Console.WriteLine( // $"Percentage: {progressReport.Percentage}% TotalBytesTransferred: {progressReport.TotalBytesTransferred} bytes"); - //if (progressReport.Percentage != 100) + // if (progressReport.Percentage != 100) // Console.SetCursorPosition(0, Console.CursorTop - 1); - //else Console.WriteLine(); + // else Console.WriteLine(); }); var args = new Dictionary (StringComparer.Ordinal) @@ -4838,7 +4793,6 @@ internal static async Task GetObject_Test2(MinioClient minio) { new MintLogger("GetObject_Test2", getObjectSignature, "Test setup for GetObject with a filename", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); - await TearDown(minio, bucketName).ConfigureAwait(false); throw; } @@ -5328,7 +5282,6 @@ internal static async Task ListObjects_Test5(MinioClient minio) await Task.WhenAll(tasks).ConfigureAwait(false); await ListObjects_Test(minio, bucketName, objectNamePrefix, numObjects, false).ConfigureAwait(false); - await Task.Delay(5000).ConfigureAwait(false); new MintLogger("ListObjects_Test5", listObjectsSignature, "Tests whether ListObjects lists all objects when number of objects == 100", TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); @@ -5520,7 +5473,7 @@ internal static async Task ListObjects_Test(MinioClient minio, string bucketName () => { }); } - await Task.Delay(5000).ConfigureAwait(false); + await Task.Delay(15000 + (numObjects * 110)).ConfigureAwait(false); Assert.AreEqual(numObjects, count); } @@ -5951,7 +5904,7 @@ internal static async Task ListIncompleteUpload_Test2(MinioClient minio) { await Setup_Test(minio, bucketName).ConfigureAwait(false); using var cts = new CancellationTokenSource(); - cts.CancelAfter(TimeSpan.FromMilliseconds(15)); + cts.CancelAfter(TimeSpan.FromMilliseconds(65)); try { using var filestream = rsg.GenerateStreamFromSeed(50 * MB); @@ -6362,7 +6315,6 @@ internal static async Task BucketLifecycleAsync_Test2(MinioClient minio) } catch (Exception ex) { - await TearDown(minio, bucketName).ConfigureAwait(false); new MintLogger(nameof(BucketLifecycleAsync_Test2) + ".1", setBucketLifecycleSignature, "Tests whether SetBucketLifecycleAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); diff --git a/Minio.Functional.Tests/Program.cs b/Minio.Functional.Tests/Program.cs index daf64e01a..170a319bb 100644 --- a/Minio.Functional.Tests/Program.cs +++ b/Minio.Functional.Tests/Program.cs @@ -33,6 +33,7 @@ public static async Task Main(string[] args) var kmsEnabled = "0"; var port = 80; + AppContext.SetSwitch("System.Net.Http.UseSocketsHttpHandler", false); var useAWS = Environment.GetEnvironmentVariable("AWS_ENDPOINT") is not null; if (Environment.GetEnvironmentVariable("SERVER_ENDPOINT") is not null) { @@ -40,9 +41,9 @@ public static async Task Main(string[] args) var posColon = endPoint.LastIndexOf(':'); if (posColon != -1) { - port = int.Parse(endPoint.Substring(posColon + 1, endPoint.Length - posColon - 1), NumberStyles.Integer, + port = int.Parse(endPoint.AsSpan(posColon + 1, endPoint.Length - posColon - 1), NumberStyles.Integer, CultureInfo.InvariantCulture); - endPoint = endPoint.Substring(0, posColon); + endPoint = endPoint[..posColon]; } accessKey = Environment.GetEnvironmentVariable("ACCESS_KEY"); @@ -205,10 +206,10 @@ public static async Task Main(string[] args) // FunctionalTest.PresignedPostPolicy_Test1(minioClient).Wait(); // Test incomplete uploads - functionalTestTasks.Add(FunctionalTest.ListIncompleteUpload_Test1(minioClient)); - functionalTestTasks.Add(FunctionalTest.ListIncompleteUpload_Test2(minioClient)); - functionalTestTasks.Add(FunctionalTest.ListIncompleteUpload_Test3(minioClient)); - functionalTestTasks.Add(FunctionalTest.RemoveIncompleteUpload_Test(minioClient)); + await FunctionalTest.ListIncompleteUpload_Test1(minioClient); + await FunctionalTest.ListIncompleteUpload_Test2(minioClient); + await FunctionalTest.ListIncompleteUpload_Test3(minioClient); + await FunctionalTest.RemoveIncompleteUpload_Test(minioClient); // Test GetBucket policy functionalTestTasks.Add(FunctionalTest.GetBucketPolicy_Test1(minioClient)); diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index 0673f83a7..e9d1dd448 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -869,6 +869,13 @@ private async Task> PutObjectPartAsync(PutObjectPartArg double partSize = multiPartInfo.partSize; double partCount = multiPartInfo.partCount; double lastPartSize = multiPartInfo.lastPartSize; + // for small size objects when object's size < part size + if (args.ObjectSize < partSize && args.ObjectSize != -1) + { + partSize = args.ObjectSize; + lastPartSize = 0; + } + var totalParts = new Part[(int)partCount]; var expectedReadSize = partSize; diff --git a/Minio/MinioClient.cs b/Minio/MinioClient.cs index e0ffe3635..8416430c4 100644 --- a/Minio/MinioClient.cs +++ b/Minio/MinioClient.cs @@ -206,7 +206,7 @@ private async Task GetRegion(string bucketName) /// Expected to be called from CreateRequest /// /// The child object of Args class - private void ArgsCheck(RequestArgs args) + private static void ArgsCheck(RequestArgs args) { if (args is null) throw new ArgumentNullException(nameof(args), @@ -403,12 +403,6 @@ private async Task ExecuteTaskCoreAsync( CancellationToken cancellationToken = default) { var startTime = DateTime.Now; - // Logs full url when HTTPtracing is enabled. - if (trace) - { - var fullUrl = requestMessageBuilder.RequestUri; - } - var v4Authenticator = new V4Authenticator(Secure, AccessKey, SecretKey, Region, SessionToken); @@ -611,8 +605,10 @@ private static void ParseErrorFromContent(ResponseResult response) if (response.StatusCode.Equals(HttpStatusCode.NotImplemented) && errResponse.Code.Equals("NotImplemented", StringComparison.OrdinalIgnoreCase)) + { #pragma warning disable MA0025 // Implement the functionality instead of throwing NotImplementedException throw new NotImplementedException(errResponse.Message); + } #pragma warning restore MA0025 // Implement the functionality instead of throwing NotImplementedException if (response.StatusCode.Equals(HttpStatusCode.BadRequest) @@ -723,6 +719,7 @@ protected virtual void Dispose(bool disposing) if (disposing) if (DisposeHttpClient) HttpClient?.Dispose(); + disposedValue = true; } }