Skip to content

Commit

Permalink
updated AWS SDK
Browse files Browse the repository at this point in the history
  • Loading branch information
kpfaulkner committed Feb 27, 2018
1 parent a27d8c1 commit 739a7f8
Show file tree
Hide file tree
Showing 7 changed files with 57 additions and 48 deletions.
1 change: 1 addition & 0 deletions azurecopy/Handlers/AzureBlobCopyHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ private static string GeneratedAccessibleUrl( BasicBlobContainer origBlob)
throw new NotImplementedException("Blobcopy against onedrive is not implemented yet");
}

Console.WriteLine("shared url is {0}", url);
return url;

}
Expand Down
82 changes: 44 additions & 38 deletions azurecopy/Handlers/AzureHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -501,57 +501,63 @@ private void ParallelWriteBlockBlob(Stream stream, CloudBlockBlob blob, int para
chunkSize = length / parallelFactor;
}

var numberOfBlocks = (length / chunkSize) + 1;
var blockIdList = new string[numberOfBlocks];
var chunkSizeList = new int[numberOfBlocks];
var taskList = new List<Task>();
var blockIdList = new string[0];

var count = numberOfBlocks - 1;

// read the data... spawn a task to launch... then wait for all.
while (count >= 0)
// chunksize is 0 if the blob is 0 in size. It really can happen (blob placeholder)
if (chunkSize > 0)
{
while (count >= 0 && taskList.Count < parallelFactor)
{
var index = (numberOfBlocks - count - 1);
var chunkSizeToUpload = (int)Math.Min(chunkSize, length - (index * chunkSize));
var numberOfBlocks = (length / chunkSize) + 1;
blockIdList = new string[numberOfBlocks];
var chunkSizeList = new int[numberOfBlocks];
var count = numberOfBlocks - 1;

// only upload if we have data to give.
// edge case where we already have uploaded all the data.
if (chunkSizeToUpload > 0)
// read the data... spawn a task to launch... then wait for all.
while (count >= 0)
{
while (count >= 0 && taskList.Count < parallelFactor)
{
chunkSizeList[index] = chunkSizeToUpload;
var dataBuffer = new byte[chunkSizeToUpload];
stream.Seek(index * chunkSize, SeekOrigin.Begin);
stream.Read(dataBuffer, 0, chunkSizeToUpload);
var index = (numberOfBlocks - count - 1);
var chunkSizeToUpload = (int)Math.Min(chunkSize, length - (index * chunkSize));

var t = Task.Factory.StartNew(() =>
// only upload if we have data to give.
// edge case where we already have uploaded all the data.
if (chunkSizeToUpload > 0)
{
var tempCount = index;
var uploadSize = chunkSizeList[tempCount];
var newBuffer = new byte[uploadSize];
Array.Copy(dataBuffer, newBuffer, dataBuffer.Length);
var blockId = Convert.ToBase64String(Guid.NewGuid().ToByteArray());
chunkSizeList[index] = chunkSizeToUpload;
var dataBuffer = new byte[chunkSizeToUpload];
stream.Seek(index * chunkSize, SeekOrigin.Begin);
stream.Read(dataBuffer, 0, chunkSizeToUpload);

using (var memStream = new MemoryStream(newBuffer, 0, uploadSize))
var t = Task.Factory.StartNew(() =>
{
blob.PutBlock(blockId, memStream, null);
}
blockIdList[tempCount] = blockId;
});
taskList.Add(t);
var tempCount = index;
var uploadSize = chunkSizeList[tempCount];
var newBuffer = new byte[uploadSize];
Array.Copy(dataBuffer, newBuffer, dataBuffer.Length);
var blockId = Convert.ToBase64String(Guid.NewGuid().ToByteArray());
using (var memStream = new MemoryStream(newBuffer, 0, uploadSize))
{
blob.PutBlock(blockId, memStream, null);
}
blockIdList[tempCount] = blockId;
});
taskList.Add(t);
}
count--;
}
count--;
}

var waitedIndex = Task.WaitAny(taskList.ToArray());
if (waitedIndex >= 0)
{
taskList.RemoveAt(waitedIndex);
var waitedIndex = Task.WaitAny(taskList.ToArray());
if (waitedIndex >= 0)
{
taskList.RemoveAt(waitedIndex);
}
}
}

Task.WaitAll(taskList.ToArray());
blob.PutBlockList(blockIdList.Where(t => t != null));
}
Expand Down
1 change: 1 addition & 0 deletions azurecopy/Helpers/S3Helper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ static S3Helper()
rd["ap-southeast-2"] = Amazon.RegionEndpoint.APSoutheast2;
rd["ap-northeast-1"] = Amazon.RegionEndpoint.APNortheast1;
rd["eu-central-1"] = Amazon.RegionEndpoint.EUCentral1;
rd["ca-central-1"] = Amazon.RegionEndpoint.CACentral1;
rd["eu-west-1"] = Amazon.RegionEndpoint.EUWest1;
rd["sa-east-1"] = Amazon.RegionEndpoint.SAEast1;
rd["eu-west-1"] = Amazon.RegionEndpoint.EUWest1;
Expand Down
4 changes: 2 additions & 2 deletions azurecopy/Properties/AssemblyInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,5 @@
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.5.0")]
[assembly: AssemblyFileVersion("1.5.0")]
[assembly: AssemblyVersion("1.5.2")]
[assembly: AssemblyFileVersion("1.5.2")]
9 changes: 5 additions & 4 deletions azurecopy/azurecopy.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,10 @@
</PropertyGroup>
<ItemGroup>
<Reference Include="AWSSDK.Core, Version=3.3.0.0, Culture=neutral, PublicKeyToken=885c28607f98e604, processorArchitecture=MSIL">
<HintPath>..\packages\AWSSDK.Core.3.3.5\lib\net45\AWSSDK.Core.dll</HintPath>
<Private>True</Private>
<HintPath>..\packages\AWSSDK.Core.3.3.21.6\lib\net45\AWSSDK.Core.dll</HintPath>
</Reference>
<Reference Include="AWSSDK.S3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=885c28607f98e604, processorArchitecture=MSIL">
<HintPath>..\packages\AWSSDK.S3.3.3.4\lib\net45\AWSSDK.S3.dll</HintPath>
<Private>True</Private>
<HintPath>..\packages\AWSSDK.S3.3.3.17\lib\net45\AWSSDK.S3.dll</HintPath>
</Reference>
<Reference Include="DropNet, Version=1.10.23.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\DropNet.1.10.23.0\lib\net40\DropNet.dll</HintPath>
Expand Down Expand Up @@ -220,6 +218,9 @@
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup>
<ItemGroup>
<Analyzer Include="..\packages\AWSSDK.S3.3.3.17\analyzers\dotnet\cs\AWSSDK.S3.CodeAnalysis.dll" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Expand Down
4 changes: 2 additions & 2 deletions azurecopy/packages.config
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="AWSSDK.Core" version="3.3.5" targetFramework="net451" />
<package id="AWSSDK.S3" version="3.3.4" targetFramework="net451" />
<package id="AWSSDK.Core" version="3.3.21.6" targetFramework="net451" />
<package id="AWSSDK.S3" version="3.3.17" targetFramework="net451" />
<package id="DropNet" version="1.10.23.0" targetFramework="net451" />
<package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net451" />
<package id="Microsoft.Data.Edm" version="5.6.4" targetFramework="net451" />
Expand Down
4 changes: 2 additions & 2 deletions azurecopycommand/Properties/AssemblyInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,5 @@
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.5.0")]
[assembly: AssemblyFileVersion("1.5.0")]
[assembly: AssemblyVersion("1.5.2")]
[assembly: AssemblyFileVersion("1.5.2")]

0 comments on commit 739a7f8

Please sign in to comment.