From 739a7f8b24cc3f458d88ff759d8b03c2b8d3e5a6 Mon Sep 17 00:00:00 2001 From: Ken Faulkner Date: Wed, 28 Feb 2018 08:03:36 +1100 Subject: [PATCH] updated AWS SDK --- azurecopy/Handlers/AzureBlobCopyHandler.cs | 1 + azurecopy/Handlers/AzureHandler.cs | 82 +++++++++++---------- azurecopy/Helpers/S3Helper.cs | 1 + azurecopy/Properties/AssemblyInfo.cs | 4 +- azurecopy/azurecopy.csproj | 9 ++- azurecopy/packages.config | 4 +- azurecopycommand/Properties/AssemblyInfo.cs | 4 +- 7 files changed, 57 insertions(+), 48 deletions(-) diff --git a/azurecopy/Handlers/AzureBlobCopyHandler.cs b/azurecopy/Handlers/AzureBlobCopyHandler.cs index 51f32e8..40eef51 100644 --- a/azurecopy/Handlers/AzureBlobCopyHandler.cs +++ b/azurecopy/Handlers/AzureBlobCopyHandler.cs @@ -93,6 +93,7 @@ private static string GeneratedAccessibleUrl( BasicBlobContainer origBlob) throw new NotImplementedException("Blobcopy against onedrive is not implemented yet"); } + Console.WriteLine("shared url is {0}", url); return url; } diff --git a/azurecopy/Handlers/AzureHandler.cs b/azurecopy/Handlers/AzureHandler.cs index 4b724df..d3704de 100644 --- a/azurecopy/Handlers/AzureHandler.cs +++ b/azurecopy/Handlers/AzureHandler.cs @@ -501,57 +501,63 @@ private void ParallelWriteBlockBlob(Stream stream, CloudBlockBlob blob, int para chunkSize = length / parallelFactor; } - var numberOfBlocks = (length / chunkSize) + 1; - var blockIdList = new string[numberOfBlocks]; - var chunkSizeList = new int[numberOfBlocks]; var taskList = new List(); + var blockIdList = new string[0]; - var count = numberOfBlocks - 1; - - // read the data... spawn a task to launch... then wait for all. - while (count >= 0) + // chunksize is 0 if the blob is 0 in size. It really can happen (blob placeholder) + if (chunkSize > 0) { - while (count >= 0 && taskList.Count < parallelFactor) - { - var index = (numberOfBlocks - count - 1); - var chunkSizeToUpload = (int)Math.Min(chunkSize, length - (index * chunkSize)); + var numberOfBlocks = (length / chunkSize) + 1; + blockIdList = new string[numberOfBlocks]; + var chunkSizeList = new int[numberOfBlocks]; + var count = numberOfBlocks - 1; - // only upload if we have data to give. - // edge case where we already have uploaded all the data. - if (chunkSizeToUpload > 0) + // read the data... spawn a task to launch... then wait for all. + while (count >= 0) + { + while (count >= 0 && taskList.Count < parallelFactor) { - chunkSizeList[index] = chunkSizeToUpload; - var dataBuffer = new byte[chunkSizeToUpload]; - stream.Seek(index * chunkSize, SeekOrigin.Begin); - stream.Read(dataBuffer, 0, chunkSizeToUpload); + var index = (numberOfBlocks - count - 1); + var chunkSizeToUpload = (int)Math.Min(chunkSize, length - (index * chunkSize)); - var t = Task.Factory.StartNew(() => + // only upload if we have data to give. + // edge case where we already have uploaded all the data. + if (chunkSizeToUpload > 0) { - var tempCount = index; - var uploadSize = chunkSizeList[tempCount]; - - var newBuffer = new byte[uploadSize]; - Array.Copy(dataBuffer, newBuffer, dataBuffer.Length); - - var blockId = Convert.ToBase64String(Guid.NewGuid().ToByteArray()); + chunkSizeList[index] = chunkSizeToUpload; + var dataBuffer = new byte[chunkSizeToUpload]; + stream.Seek(index * chunkSize, SeekOrigin.Begin); + stream.Read(dataBuffer, 0, chunkSizeToUpload); - using (var memStream = new MemoryStream(newBuffer, 0, uploadSize)) + var t = Task.Factory.StartNew(() => { - blob.PutBlock(blockId, memStream, null); - } - blockIdList[tempCount] = blockId; - }); - taskList.Add(t); + var tempCount = index; + var uploadSize = chunkSizeList[tempCount]; + + var newBuffer = new byte[uploadSize]; + Array.Copy(dataBuffer, newBuffer, dataBuffer.Length); + + var blockId = Convert.ToBase64String(Guid.NewGuid().ToByteArray()); + + using (var memStream = new MemoryStream(newBuffer, 0, uploadSize)) + { + blob.PutBlock(blockId, memStream, null); + } + blockIdList[tempCount] = blockId; + }); + taskList.Add(t); + } + count--; } - count--; - } - var waitedIndex = Task.WaitAny(taskList.ToArray()); - if (waitedIndex >= 0) - { - taskList.RemoveAt(waitedIndex); + var waitedIndex = Task.WaitAny(taskList.ToArray()); + if (waitedIndex >= 0) + { + taskList.RemoveAt(waitedIndex); + } } } + Task.WaitAll(taskList.ToArray()); blob.PutBlockList(blockIdList.Where(t => t != null)); } diff --git a/azurecopy/Helpers/S3Helper.cs b/azurecopy/Helpers/S3Helper.cs index 4eaaf63..2afd30a 100644 --- a/azurecopy/Helpers/S3Helper.cs +++ b/azurecopy/Helpers/S3Helper.cs @@ -62,6 +62,7 @@ static S3Helper() rd["ap-southeast-2"] = Amazon.RegionEndpoint.APSoutheast2; rd["ap-northeast-1"] = Amazon.RegionEndpoint.APNortheast1; rd["eu-central-1"] = Amazon.RegionEndpoint.EUCentral1; + rd["ca-central-1"] = Amazon.RegionEndpoint.CACentral1; rd["eu-west-1"] = Amazon.RegionEndpoint.EUWest1; rd["sa-east-1"] = Amazon.RegionEndpoint.SAEast1; rd["eu-west-1"] = Amazon.RegionEndpoint.EUWest1; diff --git a/azurecopy/Properties/AssemblyInfo.cs b/azurecopy/Properties/AssemblyInfo.cs index b511f78..b5eb275 100644 --- a/azurecopy/Properties/AssemblyInfo.cs +++ b/azurecopy/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.5.0")] -[assembly: AssemblyFileVersion("1.5.0")] +[assembly: AssemblyVersion("1.5.2")] +[assembly: AssemblyFileVersion("1.5.2")] diff --git a/azurecopy/azurecopy.csproj b/azurecopy/azurecopy.csproj index 3e0a398..174ef7e 100644 --- a/azurecopy/azurecopy.csproj +++ b/azurecopy/azurecopy.csproj @@ -83,12 +83,10 @@ - ..\packages\AWSSDK.Core.3.3.5\lib\net45\AWSSDK.Core.dll - True + ..\packages\AWSSDK.Core.3.3.21.6\lib\net45\AWSSDK.Core.dll - ..\packages\AWSSDK.S3.3.3.4\lib\net45\AWSSDK.S3.dll - True + ..\packages\AWSSDK.S3.3.3.17\lib\net45\AWSSDK.S3.dll ..\packages\DropNet.1.10.23.0\lib\net40\DropNet.dll @@ -220,6 +218,9 @@ true + + +