Williamhe/1853186 update automation to use staging account (#2072)

* Revert "Add new dotnet SDK versions" This reverts commit a45589e. * Update automation to use the staging storage account * Remove log * Add sas token env var * Remove displaying environment variables
microsoft · Jul 20, 2023 · 8190134 · 8190134
1 parent 02b4249
commit 8190134
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 8 deletions.
diff --git a/.github/workflows/automationTemplate.yaml b/.github/workflows/automationTemplate.yaml
@@ -25,6 +25,7 @@ env:
   PYTHON_MIN_RELEASE_VERSION: ${{ vars.PYTHON_MIN_RELEASE_VERSION }}
   PYTHON_MAX_RELEASE_VERSION: ${{ vars.PYTHON_MAX_RELEASE_VERSION }}
   PYTHON_BLOCKED_VERSIONS_ARRAY: ${{ vars.PYTHON_BLOCKED_VERSIONS_ARRAY }}
+  ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: ${{ secrets.ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN }}
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -41,7 +42,6 @@ jobs:
           declare -A platformNamesAsInBasePipeline
           platformNamesAsInBasePipeline=( ["dotnet"]="DotNetCore" ["python"]="Python" )
           echo "PLATFORM_NAME_CAMEL_CASE=${platformNamesAsInBasePipeline[${{ inputs.platformName }}]}" >> $GITHUB_ENV
-          printenv
 
       - name: Run SDK Automation
         run: |

diff --git a/build/tools/Automation/Constants.cs b/build/tools/Automation/Constants.cs
@@ -9,7 +9,9 @@ namespace Microsoft.Oryx.Automation
     public class Constants
     {
         public const string OryxSdkStorageBaseUrl = "https://oryx-cdn.microsoft.io";
+        public const string OryxSdkStagingStorageBaseUrl = "https://oryxsdksstaging.blob.core.windows.net";
         public const string OryxSdkStorageBaseUrlEnvVar = "ORYX_SDK_STORAGE_BASE_URL";
+        public const string OryxSdkStagingPrivateSasTokenEnvVar = "ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN";
         public const string VersionsToBuildTxtFileName = "versionsToBuild.txt";
         public const string ConstantsYaml = "constants.yaml";
         public static readonly HashSet<string> DebianFlavors = new HashSet<string>()

diff --git a/build/tools/Automation/DotNet/DotNetAutomator.cs b/build/tools/Automation/DotNet/DotNetAutomator.cs
@@ -58,8 +58,22 @@ public async Task RunAsync()
                 this.oryxSdkStorageBaseUrl = Constants.OryxSdkStorageBaseUrl;
             }
 
-            this.oryxDotNetSdkVersions = await this.httpService.GetOryxSdkVersionsAsync(
-                this.oryxSdkStorageBaseUrl + DotNetConstants.DotNetSuffixUrl);
+            string sdkVersionsUrl = this.oryxSdkStorageBaseUrl + DotNetConstants.DotNetSuffixUrl;
+
+            // A SAS token is required for the staging account.
+            if (this.oryxSdkStorageBaseUrl == Constants.OryxSdkStagingStorageBaseUrl)
+            {
+                string sasToken = Environment.GetEnvironmentVariable(Constants.OryxSdkStagingPrivateSasTokenEnvVar);
+                if (string.IsNullOrEmpty(sasToken))
+                {
+                    throw new ArgumentException($"The environment variable {Constants.OryxSdkStagingPrivateSasTokenEnvVar} " +
+                        $"must be provided in order to access {Constants.OryxSdkStagingStorageBaseUrl}");
+                }
+
+                sdkVersionsUrl += "&" + sasToken;
+            }
+
+            this.oryxDotNetSdkVersions = await this.httpService.GetOryxSdkVersionsAsync(sdkVersionsUrl);
             this.dotNetMinReleaseVersion = Environment.GetEnvironmentVariable(DotNetConstants.DotNetMinReleaseVersionEnvVar);
             this.dotNetMaxReleaseVersion = Environment.GetEnvironmentVariable(DotNetConstants.DotNetMaxReleaseVersionEnvVar);
             var blockedVersions = Environment.GetEnvironmentVariable(

diff --git a/build/tools/Automation/Python/PythonAutomator.cs b/build/tools/Automation/Python/PythonAutomator.cs
@@ -26,6 +26,7 @@ public class PythonAutomator
         private string pythonMinReleaseVersion;
         private string pythonMaxReleaseVersion;
         private List<string> pythonBlockedVersions = new List<string>();
+        private HashSet<string> oryxPythonSdkVersions;
 
         public PythonAutomator(
             IHttpService httpService,
@@ -47,6 +48,22 @@ public async Task RunAsync()
                 this.oryxSdkStorageBaseUrl = Constants.OryxSdkStorageBaseUrl;
             }
 
+            string sdkVersionsUrl = this.oryxSdkStorageBaseUrl + PythonConstants.PythonSuffixUrl;
+
+            // A SAS token is required for the staging account.
+            if (this.oryxSdkStorageBaseUrl == Constants.OryxSdkStagingStorageBaseUrl)
+            {
+                string sasToken = Environment.GetEnvironmentVariable(Constants.OryxSdkStagingPrivateSasTokenEnvVar);
+                if (string.IsNullOrEmpty(sasToken))
+                {
+                    throw new ArgumentException($"The environment variable {Constants.OryxSdkStagingPrivateSasTokenEnvVar} " +
+                        $"must be provided in order to access {Constants.OryxSdkStagingStorageBaseUrl}");
+                }
+
+                sdkVersionsUrl += "&" + sasToken;
+            }
+
+            this.oryxPythonSdkVersions = await this.httpService.GetOryxSdkVersionsAsync(sdkVersionsUrl);
             this.pythonMinReleaseVersion = Environment.GetEnvironmentVariable(PythonConstants.PythonMinReleaseVersionEnvVar);
             this.pythonMaxReleaseVersion = Environment.GetEnvironmentVariable(PythonConstants.PythonMaxReleaseVersionEnvVar);
             var blockedVersions = Environment.GetEnvironmentVariable(PythonConstants.PythonBlockedVersionsEnvVar);
@@ -84,9 +101,6 @@ public async Task<List<PythonVersion>> GetNewPythonVersionsAsync()
             var response = await this.httpService.GetDataAsync(PythonConstants.PythonReleaseUrl);
             var releases = JsonConvert.DeserializeObject<List<Release>>(response);
 
-            HashSet<string> oryxSdkVersions = await this.httpService.GetOryxSdkVersionsAsync(
-                this.oryxSdkStorageBaseUrl + PythonConstants.PythonSuffixUrl);
-
             var pythonVersions = new List<PythonVersion>();
             foreach (var release in releases)
             {
@@ -98,7 +112,7 @@ public async Task<List<PythonVersion>> GetNewPythonVersionsAsync()
                         minVersion: this.pythonMinReleaseVersion,
                         maxVersion: this.pythonMaxReleaseVersion,
                         this.pythonBlockedVersions) &&
-                    !oryxSdkVersions.Contains(newVersion))
+                    !this.oryxPythonSdkVersions.Contains(newVersion))
                 {
                     pythonVersions.Add(new PythonVersion
                     {

diff --git a/build/tools/Automation/Services/HttpService.cs b/build/tools/Automation/Services/HttpService.cs
@@ -33,7 +33,6 @@ public async Task<string> GetDataAsync(string url)
         {
             try
             {
-                Console.WriteLine("Making GET request to: " + url);
                 HttpResponseMessage response = await this.httpClient.GetAsync(url);
                 Console.WriteLine($"Response received: {response.StatusCode}");
                 if (!response.IsSuccessStatusCode)