From c3c9405f119043fefa099d04c433c2f31af3a86c Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 3 Apr 2019 16:43:27 +0200 Subject: [PATCH] azdevops: #58 --- mk/xamarin.mk | 4 +- tools/devops/build-samples.csx | 75 ++++++++-------------------------- tools/devops/build-samples.yml | 56 +++++++++++++++++-------- tools/devops/utils.csx | 73 +++++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 77 deletions(-) create mode 100644 tools/devops/utils.csx diff --git a/mk/xamarin.mk b/mk/xamarin.mk index bcdc3ed4e0d9..fd92ade0648a 100644 --- a/mk/xamarin.mk +++ b/mk/xamarin.mk @@ -1,6 +1,6 @@ ifdef ENABLE_XAMARIN -NEEDED_MACCORE_VERSION := 1695dfb1dabe8fd27716978edc5aa929a8dbfbff -NEEDED_MACCORE_BRANCH := distribution-certificates +NEEDED_MACCORE_VERSION := bf02d63234c682d428c599e891c3f5475a80369e +NEEDED_MACCORE_BRANCH := master MACCORE_DIRECTORY := maccore MACCORE_MODULE := git@github.com:xamarin/maccore.git diff --git a/tools/devops/build-samples.csx b/tools/devops/build-samples.csx index 58a165c1f3f4..87439999a803 100644 --- a/tools/devops/build-samples.csx +++ b/tools/devops/build-samples.csx @@ -10,7 +10,8 @@ using Newtonsoft.Json.Linq; using Xamarin.Provisioning; using Xamarin.Provisioning.Model; -// Provision Mono, XI and XM. +// Provision Mono, XI, XM, Mono, Objective-Sharpie, Xcode, provisioning profiles. +// // We get Mono from the current commit's MIN_MONO_URL value in Make.config // We get XI and XM from the current commit's manifest from GitHub's statuses // @@ -20,68 +21,13 @@ using Xamarin.Provisioning.Model; var commit = Environment.GetEnvironmentVariable ("BUILD_SOURCEVERSION"); var provision_from_commit = Environment.GetEnvironmentVariable ("PROVISION_FROM_COMMIT") ?? commit; -var statuses = string.Empty; -string manifest_url = null; -string[] manifest = null; -IEnumerable make_config = null; - -public static string DownloadWithGithubAuth (string uri) -{ - var downloader = new Downloader (); - var path = Path.GetTempFileName (); - var headers = new List<(string, string)> (); - var authToken = AuthToken ("github.com"); - if (!string.IsNullOrEmpty (authToken)) - headers.Add (("Authorization", $"token {authToken}")); - path = downloader - .DownloadItemAsync ( - uri, - headers.ToArray (), - Path.GetDirectoryName (path), - Path.GetFileName (path), - options: Downloader.Options.Default.WithUseCache (false)) - .GetAwaiter () - .GetResult (); - try { - return File.ReadAllText (path); - } finally { - File.Delete (path); - } -} - -string GetManifestUrl () -{ - if (manifest_url == null) { - var url = $"https://api.github.com/repos/xamarin/xamarin-macios/statuses/{provision_from_commit}"; - var json = JToken.Parse (DownloadWithGithubAuth (url)); - var value = (JValue) ((JArray) json).Where ((v) => v ["context"].ToString () == "manifest").Select ((v) => v ["target_url"]).FirstOrDefault (); - manifest_url = (string) value?.Value; - if (manifest_url == null) - throw new Exception ($"Could not find the manifest for {provision_from_commit}. Is the commit already built by CI?"); - } - return manifest_url; -} - -string[] GetManifest () -{ - if (manifest == null) - manifest = ReadAllText (GetManifestUrl ()).Split ('\n'); - return manifest; -} string FindVariable (string variable) { - var value = Environment.GetEnvironmentVariable (variable); + var value = FindConfigurationVariable (variable, provision_from_commit); if (!string.IsNullOrEmpty (value)) return value; - if (make_config == null) - make_config = Exec ("git", "show", $"{provision_from_commit}:Make.config"); - foreach (var line in make_config) { - if (line.StartsWith (variable + "=", StringComparison.Ordinal)) - return line.Substring (variable.Length + 1); - } - switch (variable) { case "XI_PACKAGE": value = GetManifest ().Where ((v) => v.Contains ("xamarin.ios-") && v.EndsWith (".pkg", StringComparison.Ordinal)).FirstOrDefault (); @@ -99,7 +45,7 @@ string FindVariable (string variable) string GetVersion (string url) { - return Regex.Match (mono_package, "[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*").Value; + return Regex.Match (mono_package, "[0-9]+[.][0-9]+[.][0-9]+([.][0-9]+)?").Value; } if (string.IsNullOrEmpty (provision_from_commit)) { @@ -112,10 +58,12 @@ Console.WriteLine ($"Provisioning from {provision_from_commit}..."); var mono_package = FindVariable ("MIN_MONO_URL"); var xi_package = FindVariable ("XI_PACKAGE"); var xm_package = FindVariable ("XM_PACKAGE"); +var sharpie_package = FindVariable ("MIN_SHARPIE_URL"); Console.WriteLine ($"Mono: {mono_package}"); Console.WriteLine ($"Xamarin.iOS: {xi_package}"); Console.WriteLine ($"Xamarin.Mac: {xm_package}"); +Console.WriteLine ($"Objective-Sharpie: {sharpie_package}"); Item ("Mono", GetVersion (mono_package)) .Source (mono => mono_package); @@ -125,3 +73,14 @@ Item ("Xamarin.iOS", GetVersion (xi_package)) Item ("Xamarin.Mac", GetVersion (xm_package)) .Source (xm => xm_package); + +Item ("Objective-Sharpie", GetVersion (sharpie_package)) + .Source (sharpie => sharpie_package); + +// Xcode +#load "../../../maccore/tools/devops/external-deps.csx" +var xcode_path = Path.GetDirectoryName (Path.GetDirectoryName (FindVariable ("XCODE_DEVELOPER_ROOT"))); +Exec ($"ln -Fhs {xcode_path} /Applications/Xcode.app"); + +// Provisioning profiles +Exec ($"../../../maccore/tools/install-qa-provisioning-profiles.sh"); diff --git a/tools/devops/build-samples.yml b/tools/devops/build-samples.yml index 9da72eb04106..c4dfca5916f8 100644 --- a/tools/devops/build-samples.yml +++ b/tools/devops/build-samples.yml @@ -83,30 +83,26 @@ jobs: git config -l | sort displayName: System Info - # FIXME: this should figure out the exact Xcode version from PROVISION_FROM_COMMIT. - bash: | set -x set -e - rm -f /Applications/Xcode.app - ln -Fhs /Applications/Xcode_10.1.app /Applications/Xcode101.app - ln -Fhs /Applications/Xcode_10.1.app /Applications/Xcode.app - sudo xcode-select -s /Applications/Xcode.app - ./system-dependencies.sh --ignore-all --provision-sharpie - displayName: Provision some dependencies + ./configure --enable-xamarin + # the github auth we use only works with https, so change maccore's url to be https:// instead of git@ + make reset-maccore MACCORE_MODULE=$(grep ^MACCORE_MODULE mk/xamarin.mk | sed -e 's/.*:= //' -e 's_git@github.com:_https://github.com/_' -e 's/[.]git//') V=1 + displayName: Fetch maccore - task: xamops.azdevex.provisionator-task.provisionator@1 - displayName: Provision more dependencies + displayName: Provision XI/XM/Mono/Xcode/Objective-Sharpie inputs: provisioning_script: $(System.DefaultWorkingDirectory)/tools/devops/build-samples.csx - bash: | set -x set -e - ./configure --enable-xamarin - # the github auth we use only works with https, so change maccore's url to be https:// instead of git@ - make reset-maccore MACCORE_MODULE=$(grep ^MACCORE_MODULE mk/xamarin.mk | sed -e 's/.*:= //' -e 's_git@github.com:_https://github.com/_' -e 's/[.]git//') V=1 - ../maccore/tools/install-qa-provisioning-profiles.sh - displayName: Provision provisioning profiles + env | sort + ls -lad /Applications/Xcode* + xcode-select -p + displayName: System Info post provisioning - bash: | set -x @@ -126,21 +122,47 @@ jobs: publishRunAttachments: true mergeTestResults: true + - bash: "echo ##vso[task.setvariable variable=JobStatus;isOutput=true]$(Agent.JobStatus)" + name: ExportedVariables + displayName: Export status + condition: always() + + - job: AddGitHubComment dependsOn: macOS condition: always() pool: vmImage: 'macOS-10.13' + variables: + jobResultDebugiPhoneAF: $[dependencies.macOS.outputs['Debug|iPhone|A-F.ExportedVariables.JobStatus']] + jobResultDebugiPhoneGR: $[dependencies.macOS.outputs['Debug|iPhone|G-R.ExportedVariables.JobStatus']] + jobResultDebugiPhoneSZ: $[dependencies.macOS.outputs['Debug|iPhone|S-Z.ExportedVariables.JobStatus']] + jobResultDebugiPhoneSimulator: $[dependencies.macOS.outputs['Debug|iPhoneSimulator.ExportedVariables.JobStatus']] + jobResultReleaseiPhoneAF: $[dependencies.macOS.outputs['Release|iPhone|A-F.ExportedVariables.JobStatus']] + jobResultReleaseiPhoneGR: $[dependencies.macOS.outputs['Release|iPhone|G-R.ExportedVariables.JobStatus']] + jobResultReleaseiPhoneSZ: $[dependencies.macOS.outputs['Release|iPhone|S-Z.ExportedVariables.JobStatus']] + jobResultReleaseiPhoneSimulator: $[dependencies.macOS.outputs['Release|iPhoneSimulator.ExportedVariables.JobStatus']] + jobResultDebugMac: $[dependencies.macOS.outputs['Debug|Mac.ExportedVariables.JobStatus']] + jobResultReleaseMac: $[dependencies.macOS.outputs['Release|Mac.ExportedVariables.JobStatus']] steps: - bash: | set -x set -e - EMOJI="❌" - if [[ "$AGENT_JOBSTATUS" == "Succeeded" ]]; then + env | sort + statuses="JOBRESULTDEBUGIPHONEAF JOBRESULTDEBUGIPHONEGR JOBRESULTDEBUGIPHONESZ JOBRESULTDEBUGIPHONESIMULATOR JOBRESULTRELEASEIPHONEAF JOBRESULTRELEASEIPHONEGR JOBRESULTRELEASEIPHONESZ JOBRESULTRELEASEIPHONESIMULATOR JOBRESULTDEBUGMAC JOBRESULTRELEASEMAC" + EMOJI= + for name in $statuses; do + status=${!name} + if ! [[ "$status" ~= Succeeded* + ]]; then + EMOJII="❌" + elif [[ "$status" == "SucceededWithIssues" && "$EMOJII" == "" ]]; then + EMOJII="⚠️" + fi + done + if [[ "$EMOJII" == "" ]]; then EMOJII="✅" - elif [[ "$AGENT_JOBSTATUS" == "SucceededWithIssues" ]]; then - EMOJII="⚠️" fi echo "$EMOJII Status for '$BUILD_DEFINITIONNAME': $AGENT_JOBSTATUS. [View results](${SYSTEM_COLLECTIONURI}/${SYSTEM_TEAMPROJECT}/_build/results?buildId=${BUILD_BUILDID})" > commit-comment.md ./jenkins/add-commit-comment.sh "--token=$(github-pat)" "--hash=$(Build.SourceVersion)" "--file=commit-comment.md" diff --git a/tools/devops/utils.csx b/tools/devops/utils.csx new file mode 100644 index 000000000000..86cde31337b4 --- /dev/null +++ b/tools/devops/utils.csx @@ -0,0 +1,73 @@ +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +using Newtonsoft.Json.Linq; + +string DownloadWithGithubAuth (string uri) +{ + var downloader = new Downloader (); + var path = Path.GetTempFileName (); + var headers = new List<(string, string)> (); + var authToken = AuthToken ("github.com"); + if (!string.IsNullOrEmpty (authToken)) + headers.Add (("Authorization", $"token {authToken}")); + path = downloader + .DownloadItemAsync ( + uri, + headers.ToArray (), + Path.GetDirectoryName (path), + Path.GetFileName (path), + options: Downloader.Options.Default.WithUseCache (false)) + .GetAwaiter () + .GetResult (); + try { + return File.ReadAllText (path); + } finally { + File.Delete (path); + } +} + +string manifest_url = null; +string GetManifestUrl (string org, string repo, string hash) +{ + if (manifest_url == null) { + var url = $"https://api.github.com/repos/{org}/{repo}/statuses/{hash}"; + var json = JToken.Parse (DownloadWithGithubAuth (url)); + var value = (JValue) ((JArray) json).Where ((v) => v ["context"].ToString () == "manifest").Select ((v) => v ["target_url"]).FirstOrDefault (); + manifest_url = (string) value?.Value; + if (manifest_url == null) + throw new Exception ($"Could not find the manifest for {hash}. Is the commit already built by CI?"); + } + return manifest_url; +} + +string[] manifest = null; +string[] GetManifest (string org, string repo, string hash) +{ + if (manifest == null) + manifest = ReadAllText (GetManifestUrl (org, repo, hash)).Split ('\n'); + return manifest; +} + +// Looks for a variable either in the environment, or in current repo's Make.config. +// Returns null if the variable couldn't be found. +IEnumerable make_config = null; +string FindConfigurationVariable (string variable, string hash = "HEAD") +{ + var value = Environment.GetEnvironmentVariable (variable); + if (!string.IsNullOrEmpty (value)) + return value; + + if (make_config == null) + make_config = Exec ("git", "show", $"{hash}:Make.config"); + foreach (var line in make_config) { + if (line.StartsWith (variable + "=", StringComparison.Ordinal)) + return line.Substring (variable.Length + 1); + } + + return null; +}