Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable arcade support #113

Merged
merged 13 commits into from
May 24, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ BenchmarkDotNet.Artifacts/
project.lock.json
project.fragment.lock.json
artifacts/
.dotnet/

# StyleCop
StyleCopReport.xml
Expand Down
7 changes: 7 additions & 0 deletions NuGet.config
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<clear />
<add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
</packageSources>
</configuration>
128 changes: 29 additions & 99 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ trigger:
- master

variables:
solution: '**/*.sln'
buildConfiguration: 'Release'
dotnetCoreVersion: '2.2.105'
_SignType: real
_TeamName: DotNetSpark

Expand All @@ -23,31 +21,11 @@ jobs:
displayName: Build and Test Sources
pool: Hosted VS2017

steps:
- task: DotNetCoreInstaller@0
inputs:
version: '$(dotnetCoreVersion)'

- task: DotNetCoreCLI@2
displayName: '.NET build'
inputs:
command: build
projects: '$(solution)'
arguments: '--configuration $(buildConfiguration)'

- task: BatchScript@1
displayName: Publish Microsoft.Spark.Worker
inputs:
filename: script\publish-workers.cmd
arguments: $(Build.SourcesDirectory) $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker $(buildConfiguration)

- task: DotNetCoreCLI@2
displayName: '.NET unit tests'
inputs:
command: test
projects: '**/*UnitTest/*.csproj'
arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
variables:
${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
_OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)

steps:
- task: Maven@3
displayName: 'Maven build src'
inputs:
Expand All @@ -64,6 +42,21 @@ jobs:
filename: script\download-spark-distros.cmd
arguments: $(Build.BinariesDirectory)

- script: build.cmd -pack
eerhardt marked this conversation as resolved.
Show resolved Hide resolved
-c $(buildConfiguration)
-ci
$(_OfficialBuildIdArgs)
/p:PublishSparkWorker=true
/p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker
displayName: '.NET build'

- task: DotNetCoreCLI@2
displayName: '.NET unit tests'
inputs:
command: test
safern marked this conversation as resolved.
Show resolved Hide resolved
projects: '**/*UnitTest/*.csproj'
arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'

- task: DotNetCoreCLI@2
displayName: 'E2E tests for Spark 2.3.0'
inputs:
Expand Down Expand Up @@ -134,19 +127,12 @@ jobs:
- task: CopyFiles@2
displayName: Stage .NET artifacts
inputs:
sourceFolder: $(Build.SourcesDirectory)/src/csharp/Microsoft.Spark/bin/$(buildConfiguration)
sourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(buildConfiguration)/Shipping
contents: |
**/*.dll
**/*.pdb
targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/src/csharp/Microsoft.Spark/bin/$(buildConfiguration)
**/*.nupkg
**/*.snupkg
targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/packages/$(buildConfiguration)/Shipping
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the target folder is $(Build.ArtifactStagingDirectory)/BuildArtifacts and we make the contents be: **/Shipping/**/*.nupkg and **/Shipping/**/*.snupkg and then the sourceFolder being (Build.SourcesDirectory) it should preserve the folder structure, right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes I think so. I can try changing it that way tomorrow morning.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not required to merge but it’s a nice cleanup.


- task: CopyFiles@2
displayName: Stage scala artifacts
inputs:
sourceFolder: $(Build.SourcesDirectory)/src/scala
contents: '**/*.jar'
targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/src/scala

- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Expand All @@ -162,32 +148,12 @@ jobs:
queue: buildpool.windows.10.amd64.vs2017

steps:
- task: DotNetCoreInstaller@0
inputs:
version: '$(dotnetCoreVersion)'

- task: DownloadBuildArtifacts@0
displayName: Download Build Artifacts
inputs:
artifactName: Microsoft.Spark.Binaries
downloadPath: $(Build.ArtifactStagingDirectory)

- task: CopyFiles@2
displayName: Copy .NET artifacts
inputs:
sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/BuildArtifacts/src/csharp/Microsoft.Spark/bin/$(buildConfiguration)
contents: |
**/*.dll
**/*.pdb
targetFolder: $(Build.SourcesDirectory)/src/csharp/Microsoft.Spark/bin/$(buildConfiguration)

- task: CopyFiles@2
displayName: Copy scala artifacts
inputs:
sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/BuildArtifacts/src/scala
contents: '**/*.jar'
targetFolder: $(Build.SourcesDirectory)/src/scala

- task: MicroBuildSigningPlugin@2
displayName: Install MicroBuild plugin
inputs:
Expand All @@ -205,26 +171,6 @@ jobs:
msbuildArguments: /t:Restore
msbuildVersion: 15.0

- task: MSBuild@1
displayName: 'Sign .NET binaries'
inputs:
solution: eng/Sign.proj
msbuildArguments: /t:SignBinaries
/p:SignSparkBinaries=true
/p:SignAssetsDir=$(Build.SourcesDirectory)\src\csharp\Microsoft.Spark\bin\$(buildConfiguration)\
/p:SignType=$(_SignType)
msbuildVersion: 15.0

- task: MSBuild@1
displayName: 'Sign .jar binaries'
inputs:
solution: eng/Sign.proj
msbuildArguments: /t:SignBinaries
/p:SignJarBinaries=true
/p:SignAssetsDir=$(Build.SourcesDirectory)\src\scala\
/p:SignType=$(_SignType)
msbuildVersion: 15.0

- task: MSBuild@1
displayName: 'Sign worker binaries'
inputs:
Expand All @@ -235,36 +181,25 @@ jobs:
/p:SignType=$(_SignType)
msbuildVersion: 15.0

- task: DotNetCoreCLI@2
displayName: 'Create NuGet packages'
inputs:
command: pack
projects: '$(solution)'
arguments: '--no-build --configuration $(buildConfiguration) --output $(Build.ArtifactStagingDirectory)'

- task: MSBuild@1
displayName: 'Sign nuget/snupkg packages'
inputs:
solution: eng/Sign.proj
msbuildArguments: /t:SignBinaries
/p:SignAssetsDir=$(Build.ArtifactStagingDirectory)\
/p:SignAssetsDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\
/p:SignNugetPackages=true
/p:SignType=$(_SignType)
msbuildVersion: 15.0

- task: CopyFiles@2
displayName: Copy nupkg to publish
inputs:
sourceFolder: $(Build.ArtifactStagingDirectory)
contents: 'Microsoft.Spark*.nupkg'
sourceFolder: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries
contents: |
**/*.nupkg
**/*.snupkg
targetFolder: $(Build.ArtifactStagingDirectory)/Packages

- task: CopyFiles@2
displayName: Copy snupkg to publish
inputs:
sourceFolder: $(Build.ArtifactStagingDirectory)
contents: 'Microsoft.Spark*.snupkg'
targetFolder: $(Build.ArtifactStagingDirectory)/SymbolPackages
flattenFolders: true
safern marked this conversation as resolved.
Show resolved Hide resolved

- task: PowerShell@2
displayName: Package Microsoft.Spark.Worker
Expand All @@ -283,8 +218,3 @@ jobs:
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/Packages'
artifactName: Microsoft.Spark.Binaries

- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/SymbolPackages'
artifactName: Microsoft.Spark.Binaries
4 changes: 4 additions & 0 deletions benchmark/csharp/Directory.Build.props
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" TreatAsLocalProperty="ExcludeRestorePackageImports">
<Import Project="..\..\src\csharp\Directory.Build.props" />

<PropertyGroup>
<IsShipping>false</IsShipping>
</PropertyGroup>
</Project>
4 changes: 4 additions & 0 deletions benchmark/csharp/Directory.Build.targets
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<Project>
<Import Project="..\..\src\csharp\Directory.Build.targets" />
</Project>
3 changes: 3 additions & 0 deletions build.cmd
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
@echo off
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build %*"
exit /b %ErrorLevel%
16 changes: 16 additions & 0 deletions build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/usr/bin/env bash

source="${BASH_SOURCE[0]}"

# resolve $SOURCE until the file is no longer a symlink
while [[ -h $source ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"

# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done

scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
"$scriptroot/eng/common/build.sh" --build --restore $@
4 changes: 2 additions & 2 deletions docs/building/ubuntu-instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,14 +190,14 @@ You should see JARs created for the supported Spark versions:
</details>
3. Manually copy Worker binaries into the Samples output location.
```
cp ~/dotnet.spark/src/csharp/Microsoft.Spark.Worker/bin/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish/* ~/dotnet.spark/examples/Microsoft.Spark.CSharp.Examples/bin/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish/
cp ~/dotnet.spark/artifacts/bin/Microsoft.Spark.Worker/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish/* ~/dotnet.spark/artifacts/bin/Microsoft.Spark.CSharp.Examples/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish/
```

# Run Samples

Once you build the samples, you can use `spark-submit` to submit your .NET Core apps. Make sure you have followed the [pre-requisites](#pre-requisites) section and installed Apache Spark.

1. Open a terminal and go to the directory where your app binary has been generated (e.g., `~/dotnet.spark/examples/Microsoft.Spark.CSharp.Examples/bin/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish`)
1. Open a terminal and go to the directory where your app binary has been generated (e.g., `~/dotnet.spark/artifacts/bin/Microsoft.Spark.CSharp.Examples/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish`)
2. Running your app follows the basic structure:
```bash
spark-submit \
Expand Down
4 changes: 2 additions & 2 deletions docs/building/windows-instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -210,14 +210,14 @@ You should see JARs created for the supported Spark versions:
</details>
3. Manually copy Worker binaries into the Samples output location.
```
cp c:\github\dotnet-spark\src\csharp\Microsoft.Spark.Worker\bin\Debug\netcoreapp2.1\win10-x64\publish\* C:\github\dotnet-spark\examples\Microsoft.Spark.CSharp.Examples\bin\Debug\netcoreapp2.1\win10-x64\publish\
cp c:\github\dotnet-spark\artifacts\bin\Microsoft.Spark.Worker\Debug\netcoreapp2.1\win10-x64\publish\* C:\github\dotnet-spark\artifacts\bin\Microsoft.Spark.CSharp.Examples\Debug\netcoreapp2.1\win10-x64\publish\
```

# Run Samples

Once you build the samples, running them will be through `spark-submit` regardless of whether you are targeting .NET Framework or .NET Core apps. Make sure you have followed the [pre-requisites](#pre-requisites) section and installed Apache Spark.

1. Open Powershell and go to the directory where your app binary has been generated (e.g., `c:\github\dotnet\spark\examples\Microsoft.Spark.CSharp.Examples\bin\Debug\net461` for .NET Framework, `c:\github\dotnet-spark\examples\Microsoft.Spark.CSharp.Examples\bin\Debug\netcoreapp2.1\win10-x64\publish` for .NET Core)
1. Open Powershell and go to the directory where your app binary has been generated (e.g., `c:\github\dotnet\spark\artifacts\bin\Microsoft.Spark.CSharp.Examples\Debug\net461` for .NET Framework, `c:\github\dotnet-spark\artifacts\bin\Microsoft.Spark.CSharp.Examples\Debug\netcoreapp2.1\win10-x64\publish` for .NET Core)
2. Running your app follows the basic structure:
```powershell
spark-submit.cmd `
Expand Down
28 changes: 28 additions & 0 deletions eng/AfterSolutionBuild.targets
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
<Project>

<ItemGroup>
<_PublishProject Include="$(RepoRoot)src\csharp\Microsoft.Spark.Worker\Microsoft.Spark.Worker.csproj" />
</ItemGroup>

<ItemGroup>
<_PublishTarget Include="FullFramework" Framework="net461" RuntimeId="win-x64" />
<_PublishTarget Include="WindowsCore" Framework="netcoreapp2.1" RuntimeId="win-x64" />
<_PublishTarget Include="LinuxCore" Framework="netcoreapp2.1" RuntimeId="linux-x64" />
</ItemGroup>

<Target Name="PublishSparkWorker"
AfterTargets="Build"
Condition="'$(PublishSparkWorker)' == 'true'">

<Error Condition="'$(SparkWorkerPublishDir)' == ''"
Text="SparkWorkerPublishDir variable is not set." />

<Exec Command="dotnet publish &quot;@(_PublishProject)&quot; ^
-c $(Configuration) ^
-f %(_PublishTarget.Framework) ^
-r %(_PublishTarget.RuntimeId) ^
-o &quot;$(SparkWorkerPublishDir)\%(_PublishTarget.Framework)\%(_PublishTarget.RuntimeId)&quot; ^
/p:OfficialBuildId=$(OfficialBuildId)" />

</Target>
</Project>
7 changes: 7 additions & 0 deletions eng/Build.props
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<Project>
<ItemGroup>
<ProjectToBuild Include="$(RepoRoot)src\csharp\**\*.csproj"/>
<ProjectToBuild Include="$(RepoRoot)examples\**\*.csproj;$(RepoRoot)examples\**\*.fsproj" />
<ProjectToBuild Include="$(RepoRoot)benchmark\csharp\**\*.csproj" />
</ItemGroup>
</Project>
Binary file removed eng/Open.snk
Binary file not shown.
16 changes: 2 additions & 14 deletions eng/Sign.proj
Original file line number Diff line number Diff line change
Expand Up @@ -36,23 +36,11 @@
</ItemGroup>

<ItemGroup Condition="'$(SignNugetPackages)' == 'true'">
<FilesToSign Include="$(OutDir)*.nupkg;$(OutDir)*.snupkg">
<FilesToSign Include="$(OutDir)**/*.nupkg;$(OutDir)**/*.snupkg">
<Authenticode>NuGet</Authenticode>
</FilesToSign>
</ItemGroup>

<ItemGroup Condition="'$(SignSparkBinaries)' == 'true'">
<FilesToSign Include="$(OutDir)**/Microsoft.Spark.dll">
<Authenticode>Microsoft</Authenticode>
</FilesToSign>
</ItemGroup>

<ItemGroup Condition="'$(SignJarBinaries)' == 'true'">
<FilesToSign Include="$(OutDir)**/microsoft-spark-*.jar">
<Authenticode>MicrosoftJARSHA2</Authenticode>
</FilesToSign>
</ItemGroup>

<Error Condition="'@(FilesToSign)' == ''" Text="There are no files to sign. FilesToSign group is empty."/>
</Target>
</Project>
</Project>
Binary file removed eng/Test.snk
Binary file not shown.
11 changes: 11 additions & 0 deletions eng/Version.Details.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<Dependencies>
<ProductDependencies>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19257.7">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>bda52d7619f9420de46f2c39ffc972864bbcab63</Sha>
</Dependency>
</ToolsetDependencies>
</Dependencies>
11 changes: 11 additions & 0 deletions eng/Versions.props
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<VersionPrefix>0.2.0</VersionPrefix>
<PreReleaseVersionLabel>prerelease</PreReleaseVersionLabel>
<RestoreSources>
$(RestoreSources);
https://api.nuget.org/v3/index.json;
</RestoreSources>
</PropertyGroup>
</Project>
2 changes: 2 additions & 0 deletions eng/common/CIBuild.cmd
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
@echo off
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
Loading