From 4a46ece2fc0e92496f7f6d2505efc1ee214bdafd Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Wed, 10 Jul 2024 12:21:54 +1000 Subject: [PATCH] Support configuring script pod resource requirements (#977) * Support configuring script pod resource requirements * Tweak error message Co-authored-by: Kevin Tchang <151479559+kevjt@users.noreply.github.com> * Log deserilization error to script log --------- Co-authored-by: Kevin Tchang <151479559+kevjt@users.noreply.github.com> --- .../Kubernetes/KubernetesConfig.cs | 3 + .../KubernetesRawScriptPodCreator.cs | 12 ++-- .../Kubernetes/KubernetesScriptPodCreator.cs | 58 ++++++++++++++----- 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs index f25045238..66d49bd7c 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs @@ -43,6 +43,9 @@ public static class KubernetesConfig .Select(str => str.Trim()) .WhereNotNullOrWhiteSpace() .ToArray() ?? Array.Empty(); + + public static readonly string PodResourceJsonVariableName = $"{EnvVarPrefix}__PODRESOURCEJSON"; + public static string? PodResourceJson => Environment.GetEnvironmentVariable(PodResourceJsonVariableName); public static string MetricsEnableVariableName => $"{EnvVarPrefix}__ENABLEMETRICSCAPTURE"; public static bool MetricsIsEnabled diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs index 8694b157e..8d8ad3d42 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs @@ -40,7 +40,7 @@ protected override async Task> CreateInitContainers(StartKube Name = $"{podName}-init", Image = command.PodImageConfiguration?.Image ?? await containerResolver.GetContainerImageForCluster(), Command = new List { "sh", "-c", GetInitExecutionScript("/nfs-mount", homeDir, workspacePath) }, - VolumeMounts = new List{new("/nfs-mount", "init-nfs-volume"), new(homeDir, "tentacle-home")}, + VolumeMounts = new List { new("/nfs-mount", "init-nfs-volume"), new(homeDir, "tentacle-home") }, Resources = new V1ResourceRequirements { Requests = new Dictionary @@ -54,11 +54,11 @@ protected override async Task> CreateInitContainers(StartKube return new List { container }; } - protected override async Task> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments) + protected override async Task> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments, InMemoryTentacleScriptLog tentacleScriptLog) { return new List { - await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments) + await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments, tentacleScriptLog) }; } @@ -66,17 +66,17 @@ protected override IList CreateVolumes(StartKubernetesScriptCommandV1 { return new List { - new () + new() { Name = "tentacle-home", EmptyDir = new V1EmptyDirVolumeSource() }, - new () + new() { Name = "init-nfs-volume", PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource { - ClaimName = KubernetesConfig.PodVolumeClaimName + ClaimName = KubernetesConfig.PodVolumeClaimName } } }; diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs index 29e0045f8..57cef50f9 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using k8s; using k8s.Models; using Newtonsoft.Json; using Octopus.Diagnostics; @@ -43,7 +44,7 @@ public KubernetesScriptPodCreator( IKubernetesPodContainerResolver containerResolver, IApplicationInstanceSelector appInstanceSelector, ISystemLog log, - ITentacleScriptLogProvider scriptLogProvider, + ITentacleScriptLogProvider scriptLogProvider, IHomeConfiguration homeConfiguration, KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem) { @@ -159,7 +160,7 @@ static string CreateImagePullSecretName(string feedUrl, string? username) async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace workspace, string? imagePullSecretName, InMemoryTentacleScriptLog tentacleScriptLog, CancellationToken cancellationToken) { var homeDir = homeConfiguration.HomeDirectory ?? throw new InvalidOperationException("Home directory is not set."); - + var podName = command.ScriptTicket.ToKubernetesScriptPodName(); LogVerboseToBothLogs($"Creating Kubernetes Pod '{podName}'.", tentacleScriptLog); @@ -196,7 +197,7 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo Spec = new V1PodSpec { InitContainers = await CreateInitContainers(command, podName, homeDir, workspacePath), - Containers = await CreateScriptContainers(command, podName, scriptName, homeDir, workspacePath, workspace.ScriptArguments), + Containers = await CreateScriptContainers(command, podName, scriptName, homeDir, workspacePath, workspace.ScriptArguments, tentacleScriptLog), ImagePullSecrets = imagePullSecretNames, ServiceAccountName = serviceAccountName, RestartPolicy = "Never", @@ -206,8 +207,8 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo { new(matchExpressions: new List { - new("kubernetes.io/os", "In", new List{"linux"}), - new("kubernetes.io/arch", "In", new List{"arm64","amd64"}) + new("kubernetes.io/os", "In", new List { "linux" }), + new("kubernetes.io/arch", "In", new List { "arm64", "amd64" }) }) }))) } @@ -218,11 +219,11 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo LogVerboseToBothLogs($"Executing script in Kubernetes Pod '{podName}'.", tentacleScriptLog); } - protected virtual async Task> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments) + protected virtual async Task> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments, InMemoryTentacleScriptLog tentacleScriptLog) { return new List { - await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments) + await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments, tentacleScriptLog) }.AddIfNotNull(CreateWatchdogContainer(homeDir)); } @@ -253,9 +254,12 @@ void LogVerboseToBothLogs(string message, InMemoryTentacleScriptLog tentacleScri tentacleScriptLog.Verbose(message); } - protected async Task CreateScriptContainer(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments) + protected async Task CreateScriptContainer(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments, InMemoryTentacleScriptLog tentacleScriptLog) { var spaceInformation = kubernetesPhysicalFileSystem.GetStorageInformation(); + + var resourceRequirements = GetScriptPodResourceRequirements(tentacleScriptLog); + return new V1Container { Name = podName, @@ -267,7 +271,7 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom Path.Combine(homeDir, workspacePath, scriptName) }.Concat(scriptArguments ?? Array.Empty()) .ToList(), - VolumeMounts = new List{new(homeDir, "tentacle-home")}, + VolumeMounts = new List { new(homeDir, "tentacle-home") }, Env = new List { new(KubernetesConfig.NamespaceVariableName, KubernetesConfig.Namespace), @@ -284,14 +288,36 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom //We intentionally exclude setting "TentacleJournal" since it doesn't make sense to keep a Deployment Journal for Kubernetes deployments }, - Resources = new V1ResourceRequirements + Resources = resourceRequirements + }; + } + + V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLog tentacleScriptLog) + { + var json = KubernetesConfig.PodResourceJson; + if (!string.IsNullOrWhiteSpace(json)) + { + try { - //set resource requests to be quite low for now as the scripts tend to run fairly quickly - Requests = new Dictionary - { - ["cpu"] = new("25m"), - ["memory"] = new("100Mi") - } + return KubernetesJson.Deserialize(json); + } + catch (Exception e) + { + var message = $"Failed to deserialize env.{KubernetesConfig.PodResourceJsonVariableName} into valid pod resource requirements.{Environment.NewLine}JSON value: {json}{Environment.NewLine}Using default resource requests for script pod."; + //if we can't parse the JSON, fall back to the defaults below and warn the user + log.WarnFormat(e, message); + //write a verbose message to the script log. + tentacleScriptLog.Verbose(message); + } + } + + return new V1ResourceRequirements + { + //set resource requests to be quite low for now as the scripts tend to run fairly quickly + Requests = new Dictionary + { + ["cpu"] = new("25m"), + ["memory"] = new("100Mi") } }; }