From 6dfa7ac2112dc57b7f33120e143d43fb1dbc4399 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 5 Jan 2022 10:54:58 -0800 Subject: [PATCH] Remove graph file generation functionality Removes the functionality from the templates that generates graph files. These aren't really useful anymore, as BARViz subsumed this functionality a long while back. We also need to move to the 1ES pools, so this removes another place where they aren't in use. Repos that happen to still specify should not see a break, and the additional parameters (graphFileGeneration) should just get ignored. --- .../AzureDevOps/AzureDevOpsOnboarding.md | 5 -- .../AzureDevOps/GeneratingGraphFiles.md | 60 ------------- Documentation/AzureDevOps/TemplateSchema.md | 3 - eng/common/generate-graph-files.ps1 | 86 ------------------- .../templates/job/generate-graph-files.yml | 48 ----------- eng/common/templates/jobs/jobs.yml | 10 --- 6 files changed, 212 deletions(-) delete mode 100644 Documentation/AzureDevOps/GeneratingGraphFiles.md delete mode 100644 eng/common/generate-graph-files.ps1 delete mode 100644 eng/common/templates/job/generate-graph-files.yml diff --git a/Documentation/AzureDevOps/AzureDevOpsOnboarding.md b/Documentation/AzureDevOps/AzureDevOpsOnboarding.md index a7f4acc9be8..9462bf086e1 100644 --- a/Documentation/AzureDevOps/AzureDevOpsOnboarding.md +++ b/Documentation/AzureDevOps/AzureDevOpsOnboarding.md @@ -13,7 +13,6 @@ - [Internal : (Pool Provider: NetCoreInternal-Pool)](#internal--pool-provider-netcoreinternal-pool) - [CI badge link](#ci-badge-link) - [Signed Builds](#signed-builds) - - [Generate Graph Files](#generate-graph-files) - [Security](#security) - [Notes about Yaml](#notes-about-yaml) - [Troubleshooting](#troubleshooting) @@ -155,10 +154,6 @@ https://dev.azure.com/dnceng/public/_build?definitionId=208&branchName=master dev.azure.com/dnceng now has support for signed builds. Code should be mirrored to dev.azure.com/dnceng/internal as outlined in the [Azure DevOps Guidance](./AzureDevOpsGuidance.md#projects). See [MovingFromDevDivToDncEng.md](./MovingFromDevDivToDncEng.md) for information about moving signed builds from DevDiv to DncEng. -## Generate Graph Files - -Generation of graph files as part of official builds is now supported. See [GeneratingGraphFiles.md](GeneratingGraphFiles.md) for information on how to opt-in to this feature. - ## Security [Security documentation](https://docs.microsoft.com/en-us/azure/devops/build-release/actions/ci-build-github?view=vsts#security-considerations) diff --git a/Documentation/AzureDevOps/GeneratingGraphFiles.md b/Documentation/AzureDevOps/GeneratingGraphFiles.md deleted file mode 100644 index 8cac3bdf603..00000000000 --- a/Documentation/AzureDevOps/GeneratingGraphFiles.md +++ /dev/null @@ -1,60 +0,0 @@ -# Generating Graph Files - -What are the dependencies in this build? What commits are included and what how far are they from -those in HEAD? These were some of the common questions asked during meetings. In order to answer them -we had to go check different files and child repos and make sure we had the right idea. - -Today, with the help of `darc` and the data stored in the Build Asset Registry we can generate different -kind of graph files which contain the data needed to take the right decisions and answer the questions -mentioned above. - -## The command - -Under the scenes the thing generating the graph files is `darc` by running the `get-dependency-graph` command. - -## Shiproom data - -### Enabling graph generation - -Graph generation is off by default so you'd need to enable it in your official builds. As well, you'd need to define -whether you want toolset dependencies included in your graph. This is also off by default. - -If your build definition relies on `/eng/common/templates/jobs/jobs.yml` to execute the jobs you just need to -define the following under template parameters: - -```yaml -- template: /eng/common/templates/jobs/jobs.yml - parameters: - graphFileGeneration: - enabled: true - includeToolset: true -``` - -The above enables graph generation where the graphs will include toolset dependencies. If toolset dependencies are not needed -just remove `includeToolset: true` - -### Generated Files - -* graph-flat.txt: includes the contents of a dependency graph with unique sets of repository+branch -* graph-full.txt: includes the contents of a complete dependency graph -* graphviz.txt: includes the contents of a graph that can be then pasted in http://www.webgraphviz.com -* graph.png: the same image that would be generated by graphviz.txt in http://www.webgraphviz.com - -## Generating the graph locally - -As mentioned above, we use darc to generate the different graphs. The base command is the same and we just need to pass a different -set of options. - -* To generate a full graph: `darc get-dependency-graph --output-file ` -* To generate a flat graph: `darc get-dependency-graph --flat --output-file ` -* To generate a graphviz graph: `darc get-dependency-graph --graphviz ` - -There are more options for this command for you to include toolset dependencies, coherency information, etc. For a list of all the -options execute `darc get-dependency-graph -h`. - -## Samples - -* [Arcade](https://github.com/dotnet/arcade/blob/master/azure-pipelines.yml#L39) - Graph generation is deferred to -`/eng/common/templates/jobs/jobs.yml` -* [Core-SDK](https://github.com/dotnet/core-sdk/blob/master/.vsts-ci.yml#L280) - Graph generation is defined in the main -build definition \ No newline at end of file diff --git a/Documentation/AzureDevOps/TemplateSchema.md b/Documentation/AzureDevOps/TemplateSchema.md index c304c87dd86..cc1c511781e 100644 --- a/Documentation/AzureDevOps/TemplateSchema.md +++ b/Documentation/AzureDevOps/TemplateSchema.md @@ -16,9 +16,6 @@ parameters: continueOnError: boolean # 'true' if future jobs should run even if this job fails; defaults to 'false' enablePublishBuildArtifacts: boolean # Enables publishing build logs as an Azure DevOps artifact. enablePublishUsingPipelines: boolean # Enable publishing using release pipelines - graphFileGeneration: - enabled: boolean # Enable generating the graph files at the end of the build - includeToolset: boolean # Include toolset dependencies in the generated graph files jobs: [ jobSchema ] # see "Job schema" below publishBuildAssetsDependsOn: [ string ] # Override automatically derived dependsOn value for "publish build assets" job runAsPublic: boolean # Specify if job should run as a public build even in the internal project diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 deleted file mode 100644 index 0728b1a8b57..00000000000 --- a/eng/common/generate-graph-files.ps1 +++ /dev/null @@ -1,86 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens - [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) - [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) - [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created - [string] $darcVersion, # darc's version - [string] $graphvizVersion = '2.38', # GraphViz version - [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about - # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies -) - -function CheckExitCode ([string]$stage) -{ - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode - } -} - -try { - $ErrorActionPreference = 'Stop' - . $PSScriptRoot\tools.ps1 - - Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') - - Push-Location $PSScriptRoot - - Write-Host 'Installing darc...' - . .\darc-init.ps1 -darcVersion $darcVersion - CheckExitCode 'Running darc-init' - - $engCommonBaseDir = Join-Path $PSScriptRoot 'native\' - $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory - $nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external' - $installBin = Join-Path $graphvizInstallDir 'bin' - - Write-Host 'Installing dot...' - .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose - - $darcExe = "$env:USERPROFILE\.dotnet\tools" - $darcExe = Resolve-Path "$darcExe\darc.exe" - - Create-Directory $outputFolder - - # Generate 3 graph descriptions: - # 1. Flat with coherency information - # 2. Graphviz (dot) file - # 3. Standard dependency graph - $graphVizFilePath = "$outputFolder\graphviz.txt" - $graphVizImageFilePath = "$outputFolder\graph.png" - $normalGraphFilePath = "$outputFolder\graph-full.txt" - $flatGraphFilePath = "$outputFolder\graph-flat.txt" - $baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" ) - - if ($includeToolset) { - Write-Host 'Toolsets will be included in the graph...' - $baseOptions += @( '--include-toolset' ) - } - - Write-Host 'Generating standard dependency graph...' - & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath - CheckExitCode 'Generating normal dependency graph' - - Write-Host 'Generating flat dependency graph and graphviz file...' - & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath - CheckExitCode 'Generating flat and graphviz dependency graph' - - Write-Host "Generating graph image $graphVizFilePath" - $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" - & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" - CheckExitCode 'Generating graphviz image' - - Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" -} -catch { - if (!$includeToolset) { - Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow - Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow - } - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Arcade' -Message $_ - ExitWithExitCode 1 -} finally { - Pop-Location -} \ No newline at end of file diff --git a/eng/common/templates/job/generate-graph-files.yml b/eng/common/templates/job/generate-graph-files.yml deleted file mode 100644 index e54ce956f90..00000000000 --- a/eng/common/templates/job/generate-graph-files.yml +++ /dev/null @@ -1,48 +0,0 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - -jobs: -- job: Generate_Graph_Files - - dependsOn: ${{ parameters.dependsOn }} - - displayName: Generate Graph Files - - pool: ${{ parameters.pool }} - - variables: - # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT - # DotNet-AllOrgs-Darc-Pats provides: dn-bot-devdiv-dnceng-rw-code-pat - - group: Publish-Build-Assets - - group: DotNet-AllOrgs-Darc-Pats - - name: _GraphArguments - value: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT) - -azdoPat $(dn-bot-devdiv-dnceng-rw-code-pat) - -barToken $(MaestroAccessToken) - -outputFolder '$(Build.StagingDirectory)/GraphFiles/' - - ${{ if ne(parameters.includeToolset, 'false') }}: - - name: _GraphArguments - value: ${{ variables._GraphArguments }} -includeToolset - - steps: - - task: PowerShell@2 - displayName: Generate Graph Files - inputs: - filePath: eng\common\generate-graph-files.ps1 - arguments: $(_GraphArguments) - continueOnError: true - - task: PublishBuildArtifacts@1 - displayName: Publish Graph to Artifacts - inputs: - PathtoPublish: '$(Build.StagingDirectory)/GraphFiles' - PublishLocation: Container - ArtifactName: GraphFiles - continueOnError: true - condition: always() diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 8dd1fdbd144..ff4ab75c886 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -87,13 +87,3 @@ jobs: runAsPublic: ${{ parameters.runAsPublic }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - - - ${{ if eq(parameters.graphFileGeneration.enabled, true) }}: - - template: ../job/generate-graph-files.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - includeToolset: ${{ parameters.graphFileGeneration.includeToolset }} - dependsOn: - - Asset_Registry_Publish - pool: - vmImage: 'windows-2019'