Skip to content

Commit

Permalink
Refactor and Rename Extractor (#104)
Browse files Browse the repository at this point in the history
* Refactor and Rename MultiExtractor to RecursiveExtractor
* Fix Broken Test Data
  • Loading branch information
daalcant authored Jun 8, 2020
1 parent 41be2e1 commit 2430c8f
Show file tree
Hide file tree
Showing 45 changed files with 276 additions and 43 deletions.
7 changes: 6 additions & 1 deletion .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,14 @@ jobs:
with:
languages: csharp

- name: Setup NuGet.exe for use with actions
uses: NuGet/setup-nuget@v1.0.2
with:
nuget-version: latest

- name: Build OSS Gadget
run: |
dotnet restore
nuget restore OSSGadget.sln -ConfigFile NuGet.Config
dotnet build -c Release
working-directory: ./src

Expand Down
30 changes: 15 additions & 15 deletions Pipelines/multiextractor-pipeline.yml
Original file line number Diff line number Diff line change
@@ -1,29 +1,29 @@
# Azure Pipelines
# https://aka.ms/yaml

name: MultiExtractor_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)
name: RecursiveExtractor_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)
trigger:
batch: true
branches:
include:
- master
paths:
include:
- src/MultiExtractor
- src/RecursiveExtractor
pr:
branches:
include:
- master
paths:
include:
- Pipelines
- src/MultiExtractor
- src/RecursiveExtractor

stages:
- stage: Test
jobs:
- job: build_test
displayName: Run MultiExtractor Tests
displayName: Run RecursiveExtractor Tests
pool:
vmImage: 'windows-latest'
steps:
Expand All @@ -39,15 +39,15 @@ stages:
displayName: NuGet Restore
inputs:
command: 'restore'
restoreSolution: 'src/MultiExtractor.Tests/MultiExtractor.Tests.csproj'
restoreSolution: 'src/RecursiveExtractor.Tests/RecursiveExtractor.Tests.csproj'
feedsToUse: 'config'
nugetConfigPath: 'src/MultiExtractor/NuGet.Config'
nugetConfigPath: 'src/NuGet.Config'
verbosityRestore: 'Normal'
- task: DotNetCoreCLI@2
displayName: Dotnet Test
inputs:
command: 'test'
projects: 'src/MultiExtractor.Tests/MultiExtractor.Tests.csproj'
projects: 'src/RecursiveExtractor.Tests/RecursiveExtractor.Tests.csproj'

- stage: Build
jobs:
Expand All @@ -70,12 +70,12 @@ stages:
$version = (nbgv get-version -v AssemblyInformationalVersion).split('+')[0]
Write-Host "##vso[task.setvariable variable=ReleaseVersion;]$version"
- task: DotNetCoreCLI@2
displayName: Pack MultiExtractor Nupkg (Lib)
displayName: Pack RecursiveExtractor Nupkg (Lib)
inputs:
command: 'custom'
custom: 'pack'
arguments: '-c Release -o $(Build.BinariesDirectory)\nuget\MultiExtractor_$(ReleaseVersion)'
workingDirectory: '$(SolutionDirectory)\MultiExtractor'
arguments: '-c Release -o $(Build.BinariesDirectory)\nuget\RecursiveExtractor_$(ReleaseVersion)'
workingDirectory: '$(SolutionDirectory)\RecursiveExtractor'
- task: AntiMalware@3
displayName: Anti-Malware Scan
inputs:
Expand All @@ -93,7 +93,7 @@ stages:
rootFolderOrFile: '$(Build.BinariesDirectory)\nuget'
includeRootFolder: true
archiveType: 'zip'
archiveFile: '$(Build.StagingDirectory)\MultiExtractor_nuget_$(ReleaseVersion).zip'
archiveFile: '$(Build.StagingDirectory)\RecursiveExtractor_nuget_$(ReleaseVersion).zip'
replaceExistingArchive: true
- task: PublishBuildArtifacts@1
displayName: Store Windows, NetCoreApp, Nupkg Archives
Expand Down Expand Up @@ -155,7 +155,7 @@ stages:
displayName: Code Sign Nuget Packages
inputs:
ConnectedServiceName: 'OSSGadget_CodeSign'
FolderPath: '$(Build.BinariesDirectory)/nuget/MultiExtractor_$(ReleaseVersion)'
FolderPath: '$(Build.BinariesDirectory)/nuget/RecursiveExtractor_$(ReleaseVersion)'
Pattern: '*.nupkg, *.snupkg'
signConfigType: 'inlineSignParams'
inlineOperation: |
Expand Down Expand Up @@ -194,12 +194,12 @@ stages:
displayName: Move Nuget Packages
inputs:
targetType: 'inline'
script: 'mv $env:BUILD_BINARIESDIRECTORY/nuget/MultiExtractor_$(ReleaseVersion)/*.nupkg $env:BUILD_STAGINGDIRECTORY/'
script: 'mv $env:BUILD_BINARIESDIRECTORY/nuget/RecursiveExtractor_$(ReleaseVersion)/*.nupkg $env:BUILD_STAGINGDIRECTORY/'
- task: PowerShell@2
displayName: Move Nuget Symbols
inputs:
targetType: 'inline'
script: 'mv $env:BUILD_BINARIESDIRECTORY/nuget/MultiExtractor_$(ReleaseVersion)/*.snupkg $env:BUILD_STAGINGDIRECTORY/'
script: 'mv $env:BUILD_BINARIESDIRECTORY/nuget/RecursiveExtractor_$(ReleaseVersion)/*.snupkg $env:BUILD_STAGINGDIRECTORY/'
- task: PublishPipelineArtifact@1
displayName: Publish Signed Artifacts to Pipeline
inputs:
Expand All @@ -214,7 +214,7 @@ stages:
target: '$(Build.SourceVersion)'
tagSource: 'userSpecifiedTag'
tag: 'v$(ReleaseVersion)'
title: 'Release v$(ReleaseVersion) - MultiExtractor'
title: 'Release v$(ReleaseVersion) - RecursiveExtractor'
assets: |
$(Build.StagingDirectory)/*.zip
$(Build.StagingDirectory)/HASHES.txt
Expand Down
230 changes: 230 additions & 0 deletions Pipelines/recursiveextractor-pipeline.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
# Azure Pipelines
# https://aka.ms/yaml

name: RecursiveExtractor_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)
trigger:
batch: true
branches:
include:
- master
paths:
include:
- src/RecursiveExtractor
pr:
branches:
include:
- master
paths:
include:
- Pipelines
- src/RecursiveExtractor

stages:
- stage: Test
jobs:
- job: build_test
displayName: Run RecursiveExtractor Tests
pool:
vmImage: 'windows-latest'
steps:
- task: UseDotNet@2
inputs:
packageType: 'sdk'
version: '3.1.x'
- task: NuGetToolInstaller@1
displayName: Install Nuget Tool
inputs:
versionSpec: '5.4.x'
- task: NuGetCommand@2
displayName: NuGet Restore
inputs:
command: 'restore'
restoreSolution: 'src/RecursiveExtractor.Tests/RecursiveExtractor.Tests.csproj'
feedsToUse: 'config'
nugetConfigPath: 'src/NuGet.Config'
verbosityRestore: 'Normal'
- task: DotNetCoreCLI@2
displayName: Dotnet Test
inputs:
command: 'test'
projects: 'src/RecursiveExtractor.Tests/RecursiveExtractor.Tests.csproj'

- stage: Build
jobs:
- job: build_win
displayName: Build Nupkg
pool:
vmImage: 'windows-latest'
steps:
- task: UseDotNet@2
inputs:
packageType: 'sdk'
version: '3.1.x'
- script: 'dotnet tool install -g nbgv'
displayName: 'Install GitVersioning'
- task: PowerShell@2
displayName: Set Release Version
inputs:
targetType: 'inline'
script: |
$version = (nbgv get-version -v AssemblyInformationalVersion).split('+')[0]
Write-Host "##vso[task.setvariable variable=ReleaseVersion;]$version"
- task: DotNetCoreCLI@2
displayName: Pack RecursiveExtractor Nupkg (Lib)
inputs:
command: 'custom'
custom: 'pack'
arguments: '-c Release -o $(Build.BinariesDirectory)\nuget\RecursiveExtractor_$(ReleaseVersion)'
workingDirectory: '$(SolutionDirectory)\RecursiveExtractor'
- task: AntiMalware@3
displayName: Anti-Malware Scan
inputs:
InputType: 'Basic'
ScanType: 'CustomScan'
FileDirPath: '$(Build.BinariesDirectory)'
EnableServices: true
SupportLogOnError: false
TreatSignatureUpdateFailureAs: 'Warning'
SignatureFreshness: 'UpToDate'
TreatStaleSignatureAs: 'Warning'
- task: ArchiveFiles@2
displayName: Archive Artifact - Nupkg
inputs:
rootFolderOrFile: '$(Build.BinariesDirectory)\nuget'
includeRootFolder: true
archiveType: 'zip'
archiveFile: '$(Build.StagingDirectory)\RecursiveExtractor_nuget_$(ReleaseVersion).zip'
replaceExistingArchive: true
- task: PublishBuildArtifacts@1
displayName: Store Windows, NetCoreApp, Nupkg Archives
inputs:
PathtoPublish: '$(Build.StagingDirectory)'
ArtifactName: 'Unsigned_WinNuget'
publishLocation: 'Container'

- job: sign_hash_release
displayName: Code Sign, Generate Hashes, Publish Public Releases
dependsOn:
- build_win
condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'))
pool:
vmImage: 'windows-latest'
steps:
- task: UseDotNet@2
inputs:
packageType: 'sdk'
version: '3.1.x'
- script: 'dotnet tool install -g nbgv'
displayName: 'Install GitVersioning'
- task: PowerShell@2
displayName: Set Release Version
inputs:
targetType: 'inline'
script: |
$version = (nbgv get-version -v AssemblyInformationalVersion).split('+')[0]
Write-Host "##vso[task.setvariable variable=ReleaseVersion;]$version"
- task: DownloadBuildArtifacts@0
displayName: Retrieve Windows, NetCoreApp, Nupkg Archives
inputs:
buildType: 'current'
downloadType: 'specific'
itemPattern: 'Unsigned_WinNuget/*.zip'
downloadPath: '$(Build.BinariesDirectory)'
- task: ExtractFiles@1
displayName: Extract Artifacts for Signing
inputs:
archiveFilePatterns: '$(Build.BinariesDirectory)\*.zip'
destinationFolder: '$(Build.BinariesDirectory)'
cleanDestinationFolder: false
- task: AntiMalware@3
displayName: Anti-Malware Scan
inputs:
InputType: 'Basic'
ScanType: 'CustomScan'
FileDirPath: '$(Build.BinariesDirectory)'
EnableServices: true
SupportLogOnError: true
TreatSignatureUpdateFailureAs: 'Warning'
SignatureFreshness: 'UpToDate'
TreatStaleSignatureAs: 'Warning'
- task: UseDotNet@2
inputs:
packageType: 'sdk'
version: '2.1.804'
- task: EsrpCodeSigning@1
displayName: Code Sign Nuget Packages
inputs:
ConnectedServiceName: 'OSSGadget_CodeSign'
FolderPath: '$(Build.BinariesDirectory)/nuget/RecursiveExtractor_$(ReleaseVersion)'
Pattern: '*.nupkg, *.snupkg'
signConfigType: 'inlineSignParams'
inlineOperation: |
[
{
"KeyCode" : "CP-401405",
"OperationCode" : "NuGetSign",
"Parameters" : {},
"ToolName" : "sign",
"ToolVersion" : "1.0"
},
{
"KeyCode" : "CP-401405",
"OperationCode" : "NuGetVerify",
"Parameters" : {},
"ToolName" : "sign",
"ToolVersion" : "1.0"
}
]
SessionTimeout: '60'
MaxConcurrency: '50'
MaxRetryAttempts: '5'
- powershell: 'Get-ChildItem -Path ''$(Build.BinariesDirectory)'' -Recurse CodeSign* | foreach { Remove-Item -Path $_.FullName }'
displayName: 'Delete Code Sign Summaries'
- task: PowerShell@2
displayName: Generate Hashes
inputs:
targetType: 'inline'
script: |
Get-ChildItem $(Build.StagingDirectory) | Foreach-Object {
$name = $_.Name
$tmp = (Get-FileHash "$(Build.StagingDirectory)\$name").Hash
Add-Content $(Build.StagingDirectory)\HASHES.txt "$tmp`t$name"
}
- task: PowerShell@2
displayName: Move Nuget Packages
inputs:
targetType: 'inline'
script: 'mv $env:BUILD_BINARIESDIRECTORY/nuget/RecursiveExtractor_$(ReleaseVersion)/*.nupkg $env:BUILD_STAGINGDIRECTORY/'
- task: PowerShell@2
displayName: Move Nuget Symbols
inputs:
targetType: 'inline'
script: 'mv $env:BUILD_BINARIESDIRECTORY/nuget/RecursiveExtractor_$(ReleaseVersion)/*.snupkg $env:BUILD_STAGINGDIRECTORY/'
- task: PublishPipelineArtifact@1
displayName: Publish Signed Artifacts to Pipeline
inputs:
targetPath: '$(Build.StagingDirectory)'
artifact: 'Signed_Binaries'
- task: GitHubRelease@1
displayName: Release to GitHub
inputs:
gitHubConnection: 'OSSGadget'
repositoryName: 'microsoft/OSSGadget'
action: 'create'
target: '$(Build.SourceVersion)'
tagSource: 'userSpecifiedTag'
tag: 'v$(ReleaseVersion)'
title: 'Release v$(ReleaseVersion) - RecursiveExtractor'
assets: |
$(Build.StagingDirectory)/*.zip
$(Build.StagingDirectory)/HASHES.txt
changeLogCompareToRelease: 'lastNonDraftRelease'
changeLogType: 'commitBased'
- task: NuGetCommand@2
displayName: Publish NuGet Packages
inputs:
command: 'push'
packagesToPush: '$(Build.StagingDirectory)/*.nupkg'
nuGetFeedType: 'external'
publishFeedCredentials: 'CST-E Nuget CI'
verbosityPush: 'Normal'
File renamed without changes.
6 changes: 3 additions & 3 deletions src/OSSGadget.sln
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "oss-detect-cryptography", "
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "oss-risk-calculator", "oss-risk-calculator\oss-risk-calculator.csproj", "{84189BD7-D6F4-4E84-83A8-4FA1B811CDDD}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MultiExtractor", "MultiExtractor\MultiExtractor.csproj", "{AB7A75CB-6FD8-4854-A6EE-0470867CACBB}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RecursiveExtractor", "RecursiveExtractor\RecursiveExtractor.csproj", "{AB7A75CB-6FD8-4854-A6EE-0470867CACBB}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MultiExtractor.Tests", "MultiExtractor.Tests\MultiExtractor.Tests.csproj", "{4811485D-2338-4DE5-8A02-428662CE14D4}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RecursiveExtractor.Tests", "RecursiveExtractor.Tests\RecursiveExtractor.Tests.csproj", "{4811485D-2338-4DE5-8A02-428662CE14D4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "oss-tests", "oss-tests\oss-tests.csproj", "{812DF4A5-057B-4227-A3E8-4DFF67DBE07A}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "oss-tests", "oss-tests\oss-tests.csproj", "{812DF4A5-057B-4227-A3E8-4DFF67DBE07A}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// Licensed under the MIT License.

using Microsoft.VisualStudio.TestTools.UnitTesting;
using Microsoft.CST.OpenSource.MultiExtractor;
using Microsoft.CST.OpenSource.RecursiveExtractor;
using System.IO;
using System.Linq;
using System;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
Expand All @@ -15,7 +15,7 @@
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\MultiExtractor\MultiExtractor.csproj" />
<ProjectReference Include="..\RecursiveExtractor\RecursiveExtractor.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="TestData\10GB.7z.bz2">
Expand Down
File renamed without changes.
File renamed without changes.
Loading

0 comments on commit 2430c8f

Please sign in to comment.