diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index b2ca199d455e..8f2117ecc74a 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -1,5 +1,4 @@ -Import-Module "${PSScriptRoot}/modules/ChangeLog-Operations.psm1" -. (Join-Path $PSScriptRoot SemVer.ps1) +. (Join-Path $PSScriptRoot common.ps1) $SDIST_PACKAGE_REGEX = "^(?.*)\-(?$([AzureEngSemanticVersion]::SEMVER_REGEX))" @@ -40,336 +39,6 @@ function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) { } } -# Parse out package publishing information given a maven POM file -function ParseMavenPackage($pkg, $workingDirectory) { - [xml]$contentXML = Get-Content $pkg - - $pkgId = $contentXML.project.artifactId - $pkgVersion = $contentXML.project.version - $groupId = if ($contentXML.project.groupId -eq $null) { $contentXML.project.parent.groupId } else { $contentXML.project.groupId } - $releaseNotes = "" - $readmeContent = "" - - # if it's a snapshot. return $null (as we don't want to create tags for this, but we also don't want to fail) - if ($pkgVersion.Contains("SNAPSHOT")) { - return $null - } - - $changeLogLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-changelog.md")[0] - if ($changeLogLoc) { - $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion - } - - $readmeContentLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-readme.md")[0] - if ($readmeContentLoc) { - $readmeContent = Get-Content -Raw $readmeContentLoc - } - - return New-Object PSObject -Property @{ - PackageId = $pkgId - GroupId = $groupId - PackageVersion = $pkgVersion - ReleaseTag = "$($pkgId)_$($pkgVersion)" - Deployable = $forceCreate -or !(IsMavenPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion -groupId $groupId.Replace(".", "/")) - ReleaseNotes = $releaseNotes - ReadmeContent = $readmeContent - } -} - -# Returns the maven (really sonatype) publish status of a package id and version. -function IsMavenPackageVersionPublished($pkgId, $pkgVersion, $groupId) { - try { - - $uri = "https://oss.sonatype.org/content/repositories/releases/$groupId/$pkgId/$pkgVersion/$pkgId-$pkgVersion.pom" - $pomContent = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -Method "GET" -uri $uri - - if ($pomContent -ne $null -or $pomContent.Length -eq 0) { - return $true - } - else { - return $false - } - } - catch { - $statusCode = $_.Exception.Response.StatusCode.value__ - $statusDescription = $_.Exception.Response.StatusDescription - - # if this is 404ing, then this pkg has never been published before - if ($statusCode -eq 404) { - return $false - } - - Write-Host "VersionCheck to maven for packageId $pkgId failed with statuscode $statusCode" - Write-Host $statusDescription - exit(1) - } -} - -# make certain to always take the package json closest to the top -function ResolvePkgJson($workFolder) { - $pathsWithComplexity = @() - foreach ($file in (Get-ChildItem -Path $workFolder -Recurse -Include "package.json")) { - $complexity = ($file.FullName -Split { $_ -eq "/" -or $_ -eq "\" }).Length - $pathsWithComplexity += New-Object PSObject -Property @{ - Path = $file - Complexity = $complexity - } - } - - return ($pathsWithComplexity | Sort-Object -Property Complexity)[0].Path -} - -# Parse out package publishing information given a .tgz npm artifact -function ParseNPMPackage($pkg, $workingDirectory) { - $workFolder = "$workingDirectory$($pkg.Basename)" - $origFolder = Get-Location - $releaseNotes = "" - $readmeContent = "" - - New-Item -ItemType Directory -Force -Path $workFolder - cd $workFolder - - tar -xzf $pkg - - $packageJSON = ResolvePkgJson -workFolder $workFolder | Get-Content | ConvertFrom-Json - $pkgId = $packageJSON.name - $pkgVersion = $packageJSON.version - - $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] - if ($changeLogLoc) { - $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion - } - - $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1 - if ($readmeContentLoc) { - $readmeContent = Get-Content -Raw $readmeContentLoc - } - - cd $origFolder - Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue - - $resultObj = New-Object PSObject -Property @{ - PackageId = $pkgId - PackageVersion = $pkgVersion - ReleaseTag = "$($pkgId)_$($pkgVersion)" - Deployable = $forceCreate -or !(IsNPMPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion) - ReleaseNotes = $releaseNotes - ReadmeContent = $readmeContent - } - - return $resultObj -} - -# Returns the npm publish status of a package id and version. -function IsNPMPackageVersionPublished($pkgId, $pkgVersion) { - $npmVersions = (npm show $pkgId versions) - - if ($LastExitCode -ne 0) { - npm ping - - if ($LastExitCode -eq 0) { - return $False - } - - Write-Host "Could not find a deployed version of $pkgId, and NPM connectivity check failed." - exit(1) - } - - $npmVersionList = $npmVersions.split(",") | % { return $_.replace("[", "").replace("]", "").Trim() } - return $npmVersionList.Contains($pkgVersion) -} - -# Parse out package publishing information given a nupkg ZIP format. -function ParseNugetPackage($pkg, $workingDirectory) { - $workFolder = "$workingDirectory$($pkg.Basename)" - $origFolder = Get-Location - $zipFileLocation = "$workFolder/$($pkg.Basename).zip" - $releaseNotes = "" - $readmeContent = "" - - New-Item -ItemType Directory -Force -Path $workFolder - - Copy-Item -Path $pkg -Destination $zipFileLocation - Expand-Archive -Path $zipFileLocation -DestinationPath $workFolder - [xml] $packageXML = Get-ChildItem -Path "$workFolder/*.nuspec" | Get-Content - $pkgId = $packageXML.package.metadata.id - $pkgVersion = $packageXML.package.metadata.version - - $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] - if ($changeLogLoc) { - $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion - } - - $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0] - if ($readmeContentLoc) { - $readmeContent = Get-Content -Raw $readmeContentLoc - } - - Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue - - return New-Object PSObject -Property @{ - PackageId = $pkgId - PackageVersion = $pkgVersion - ReleaseTag = "$($pkgId)_$($pkgVersion)" - Deployable = $forceCreate -or !(IsNugetPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion) - ReleaseNotes = $releaseNotes - ReadmeContent = $readmeContent - } -} - -# Returns the nuget publish status of a package id and version. -function IsNugetPackageVersionPublished($pkgId, $pkgVersion) { - - $nugetUri = "https://api.nuget.org/v3-flatcontainer/$($pkgId.ToLowerInvariant())/index.json" - - try { - $nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -uri $nugetUri -Method "GET" - - return $nugetVersions.versions.Contains($pkgVersion) - } - catch { - $statusCode = $_.Exception.Response.StatusCode.value__ - $statusDescription = $_.Exception.Response.StatusDescription - - # if this is 404ing, then this pkg has never been published before - if ($statusCode -eq 404) { - return $False - } - - Write-Host "Nuget Invocation failed:" - Write-Host "StatusCode:" $statusCode - Write-Host "StatusDescription:" $statusDescription - exit(1) - } - -} - -# Parse out package publishing information given a python sdist of ZIP format. -function ParsePyPIPackage($pkg, $workingDirectory) { - $pkg.Basename -match $SDIST_PACKAGE_REGEX | Out-Null - - $pkgId = $matches["package"] - $pkgVersion = $matches["versionstring"] - - $workFolder = "$workingDirectory$($pkg.Basename)" - $origFolder = Get-Location - $releaseNotes = "" - $readmeContent = "" - - New-Item -ItemType Directory -Force -Path $workFolder - Expand-Archive -Path $pkg -DestinationPath $workFolder - - $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] - if ($changeLogLoc) { - $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion - } - - $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1 - - if ($readmeContentLoc) { - $readmeContent = Get-Content -Raw $readmeContentLoc - } - - Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue - - return New-Object PSObject -Property @{ - PackageId = $pkgId - PackageVersion = $pkgVersion - ReleaseTag = "$($pkgId)_$($pkgVersion)" - Deployable = $forceCreate -or !(IsPythonPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion) - ReleaseNotes = $releaseNotes - ReadmeContent = $readmeContent - } -} - -function ParseCArtifact($pkg, $workingDirectory) { - $packageInfo = Get-Content -Raw -Path $pkg | ConvertFrom-JSON - $packageArtifactLocation = (Get-ItemProperty $pkg).Directory.FullName - $releaseNotes = "" - $readmeContent = "" - - $pkgVersion = $packageInfo.version - - $changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0] - if ($changeLogLoc) - { - $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion - } - - $readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0] - if ($readmeContentLoc) { - $readmeContent = Get-Content -Raw $readmeContentLoc - } - - return New-Object PSObject -Property @{ - PackageId = '' - PackageVersion = $pkgVersion - ReleaseTag = $pkgVersion - # Artifact info is always considered deployable for C becasue it is not - # deployed anywhere. Dealing with duplicate tags happens downstream in - # CheckArtifactShaAgainstTagsList - Deployable = $true - ReleaseNotes = $releaseNotes - } -} - -function ParseCppArtifact($pkg, $workingDirectory) { - $packageInfo = Get-Content -Raw -Path $pkg | ConvertFrom-JSON - $packageArtifactLocation = (Get-ItemProperty $pkg).Directory.FullName - $releaseNotes = "" - $readmeContent = "" - - $pkgVersion = $packageInfo.version - $pkgName = $packageInfo.name - - $changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0] - if ($changeLogLoc) - { - $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion - } - - $readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0] - if ($readmeContentLoc) { - $readmeContent = Get-Content -Raw $readmeContentLoc - } - - return New-Object PSObject -Property @{ - PackageId = $pkgName - PackageVersion = $pkgVersion - ReleaseTag = "${pkgName}_${pkgVersion}" - # Artifact info is always considered deployable for now becasue it is not - # deployed anywhere. Dealing with duplicate tags happens downstream in - # CheckArtifactShaAgainstTagsList - Deployable = $true - ReleaseNotes = $releaseNotes - } -} - - -# Returns the pypi publish status of a package id and version. -function IsPythonPackageVersionPublished($pkgId, $pkgVersion) { - try { - $existingVersion = (Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -Method "Get" -uri "https://pypi.org/pypi/$pkgId/$pkgVersion/json").info.version - - # if existingVersion exists, then it's already been published - return $True - } - catch { - $statusCode = $_.Exception.Response.StatusCode.value__ - $statusDescription = $_.Exception.Response.StatusDescription - - # if this is 404ing, then this pkg has never been published before - if ($statusCode -eq 404) { - return $False - } - - Write-Host "PyPI Invocation failed:" - Write-Host "StatusCode:" $statusCode - Write-Host "StatusDescription:" $statusDescription - exit(1) - } -} - # Retrieves the list of all tags that exist on the target repository function GetExistingTags($apiUrl) { try { @@ -394,12 +63,12 @@ function GetExistingTags($apiUrl) { } # Retrieve release tag for artiface package. If multiple packages, then output the first one. -function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError = $true) { +function RetrieveReleaseTag($artifactLocation, $continueOnError = $true) { if (!$artifactLocation) { return "" } try { - $pkgs, $parsePkgInfoFn = RetrievePackages -pkgRepository $pkgRepository -artifactLocation $artifactLocation + $pkgs, $parsePkgInfoFn = RetrievePackages -artifactLocation $artifactLocation if (!$pkgs -or !$pkgs[0]) { Write-Host "No packages retrieved from artifact location." return "" @@ -421,52 +90,23 @@ function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError Write-Error "No release tag retrieved from $artifactLocation" } } -function RetrievePackages($pkgRepository, $artifactLocation) { - $parsePkgInfoFn = "" - $packagePattern = "" - $pkgRepository = $pkgRepository.Trim() - switch ($pkgRepository) { - "Maven" { - $parsePkgInfoFn = "ParseMavenPackage" - $packagePattern = "*.pom" - break - } - "Nuget" { - $parsePkgInfoFn = "ParseNugetPackage" - $packagePattern = "*.nupkg" - break - } - "NPM" { - $parsePkgInfoFn = "ParseNPMPackage" - $packagePattern = "*.tgz" - break - } - "PyPI" { - $parsePkgInfoFn = "ParsePyPIPackage" - $packagePattern = "*.zip" - break - } - "C" { - $parsePkgInfoFn = "ParseCArtifact" - $packagePattern = "*.json" - } - "CPP" { - $parsePkgInfoFn = "ParseCppArtifact" - $packagePattern = "*.json" - } - default { - Write-Host "Unrecognized Language: $language" - exit(1) - } - } + +function RetrievePackages($artifactLocation) { $pkgs = Get-ChildItem -Path $artifactLocation -Include $packagePattern -Recurse -File - return $pkgs, $parsePkgInfoFn + if ((Get-ChildItem -Path Function: | ? { $_.Name -eq $GetPackageInfoFromPackageFileFn }).Count -gt 0) + { + return $pkgs, $GetPackageInfoFromPackageFileFn + } + else + { + LogError "The function '$GetPackageInfoFromPackageFileFn' was not found." + } } # Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases -function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $continueOnError = $false) { +function VerifyPackages($artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $continueOnError = $false) { $pkgList = [array]@() - $pkgs, $parsePkgInfoFn = RetrievePackages -pkgRepository $pkgRepository -artifactLocation $artifactLocation + $pkgs, $parsePkgInfoFn = RetrievePackages -artifactLocation $artifactLocation foreach ($pkg in $pkgs) { try { diff --git a/eng/common/scripts/create-tags-and-git-release.ps1 b/eng/common/scripts/create-tags-and-git-release.ps1 index 83f2caa5cf41..07d8cd0ed5bc 100644 --- a/eng/common/scripts/create-tags-and-git-release.ps1 +++ b/eng/common/scripts/create-tags-and-git-release.ps1 @@ -18,13 +18,13 @@ param ( [switch]$continueOnError = $false ) -. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) +. (Join-Path $PSScriptRoot common.ps1) $apiUrl = "https://api.github.com/repos/$repoId" Write-Host "Using API URL $apiUrl" # VERIFY PACKAGES -$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha -continueOnError $continueOnError +$pkgList = VerifyPackages -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha -continueOnError $continueOnError if ($pkgList) { Write-Host "Given the visible artifacts, github releases will be created for the following:" diff --git a/eng/common/scripts/update-docs-metadata.ps1 b/eng/common/scripts/update-docs-metadata.ps1 index 2f7f3343de43..662544854dae 100644 --- a/eng/common/scripts/update-docs-metadata.ps1 +++ b/eng/common/scripts/update-docs-metadata.ps1 @@ -14,41 +14,22 @@ param ( $DocRepoContentLocation = "docs-ref-services/" # within the doc repo, where does our readme go? ) -. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) -. (Join-Path $PSScriptRoot SemVer.ps1) +. (Join-Path $PSScriptRoot common.ps1) $releaseReplaceRegex = "(https://github.com/$RepoId/(?:blob|tree)/)master" -function GetMetaData($lang){ - switch ($lang) { - "java" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv" - break - } - ".net" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/dotnet-packages.csv" - break - } - "python" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/python-packages.csv" - break - } - "javascript" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv" - break - } - default { - Write-Host "Unrecognized Language: $language" - exit(1) - } +function GetMetaData { + if (Test-Path Variable:MetadataUri) { + $metadataResponse = Invoke-RestMethod -Uri $MetadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv + } + else { + LogError "The variable '$MetadataUri' was not found." } - - $metadataResponse = Invoke-RestMethod -Uri $metadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv return $metadataResponse } -function GetAdjustedReadmeContent($pkgInfo, $lang){ +function GetAdjustedReadmeContent($pkgInfo){ $date = Get-Date -Format "MM/dd/yyyy" $service = "" @@ -56,7 +37,7 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){ $pkgId = $pkgInfo.PackageId.Replace("@azure/", "") try { - $metadata = GetMetaData -lang $lang + $metadata = GetMetaData $service = $metadata | ? { $_.Package -eq $pkgId } @@ -93,8 +74,7 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){ } $apiUrl = "https://api.github.com/repos/$repoId" -$pkgs = VerifyPackages -pkgRepository $Repository ` - -artifactLocation $ArtifactLocation ` +$pkgs = VerifyPackages -artifactLocation $ArtifactLocation ` -workingDirectory $WorkDirectory ` -apiUrl $apiUrl ` -releaseSha $ReleaseSHA ` @@ -116,7 +96,7 @@ if ($pkgs) { $readmeLocation = Join-Path $DocRepoLocation $DocRepoContentLocation $readmeName if ($packageInfo.ReadmeContent) { - $adjustedContent = GetAdjustedReadmeContent -pkgInfo $packageInfo -lang $Language + $adjustedContent = GetAdjustedReadmeContent -pkgInfo $packageInfo } if ($adjustedContent) {