Skip to content

Commit

Permalink
Support configuring script pod resource requirements (#977)
Browse files Browse the repository at this point in the history 
* Support configuring script pod resource requirements

* Tweak error message

Co-authored-by: Kevin Tchang <151479559+kevjt@users.noreply.github.com>

* Log deserilization error to script log

---------

Co-authored-by: Kevin Tchang <151479559+kevjt@users.noreply.github.com>
  • Loading branch information
@APErebus @kevjt
APErebus and kevjt authored Jul 10, 2024
1 parent a3b04b3 commit 4a46ece
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 22 deletions.
3 changes: 3 additions & 0 deletions source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ public static class KubernetesConfig
.Select(str => str.Trim())
.WhereNotNullOrWhiteSpace()
.ToArray() ?? Array.Empty<string>();

public static readonly string PodResourceJsonVariableName = $"{EnvVarPrefix}__PODRESOURCEJSON";
public static string? PodResourceJson => Environment.GetEnvironmentVariable(PodResourceJsonVariableName);

public static string MetricsEnableVariableName => $"{EnvVarPrefix}__ENABLEMETRICSCAPTURE";
public static bool MetricsIsEnabled
Expand Down
12 changes: 6 additions & 6 deletions source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ protected override async Task<IList<V1Container>> CreateInitContainers(StartKube
Name = $"{podName}-init",
Image = command.PodImageConfiguration?.Image ?? await containerResolver.GetContainerImageForCluster(),
Command = new List<string> { "sh", "-c", GetInitExecutionScript("/nfs-mount", homeDir, workspacePath) },
VolumeMounts = new List<V1VolumeMount>{new("/nfs-mount", "init-nfs-volume"), new(homeDir, "tentacle-home")},
VolumeMounts = new List<V1VolumeMount> { new("/nfs-mount", "init-nfs-volume"), new(homeDir, "tentacle-home") },
Resources = new V1ResourceRequirements
{
Requests = new Dictionary<string, ResourceQuantity>
Expand All @@ -54,29 +54,29 @@ protected override async Task<IList<V1Container>> CreateInitContainers(StartKube
return new List<V1Container> { container };
}

protected override async Task<IList<V1Container>> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments)
protected override async Task<IList<V1Container>> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments, InMemoryTentacleScriptLog tentacleScriptLog)
{
return new List<V1Container>
{
await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments)
await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments, tentacleScriptLog)
};
}

protected override IList<V1Volume> CreateVolumes(StartKubernetesScriptCommandV1 command)
{
return new List<V1Volume>
{
new ()
new()
{
Name = "tentacle-home",
EmptyDir = new V1EmptyDirVolumeSource()
},
new ()
new()
{
Name = "init-nfs-volume",
PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource
{
ClaimName = KubernetesConfig.PodVolumeClaimName
ClaimName = KubernetesConfig.PodVolumeClaimName
}
}
};
Expand Down
58 changes: 42 additions & 16 deletions source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using k8s;
using k8s.Models;
using Newtonsoft.Json;
using Octopus.Diagnostics;
Expand Down Expand Up @@ -43,7 +44,7 @@ public KubernetesScriptPodCreator(
IKubernetesPodContainerResolver containerResolver,
IApplicationInstanceSelector appInstanceSelector,
ISystemLog log,
ITentacleScriptLogProvider scriptLogProvider,
ITentacleScriptLogProvider scriptLogProvider,
IHomeConfiguration homeConfiguration,
KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem)
{
Expand Down Expand Up @@ -159,7 +160,7 @@ static string CreateImagePullSecretName(string feedUrl, string? username)
async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace workspace, string? imagePullSecretName, InMemoryTentacleScriptLog tentacleScriptLog, CancellationToken cancellationToken)
{
var homeDir = homeConfiguration.HomeDirectory ?? throw new InvalidOperationException("Home directory is not set.");

var podName = command.ScriptTicket.ToKubernetesScriptPodName();

LogVerboseToBothLogs($"Creating Kubernetes Pod '{podName}'.", tentacleScriptLog);
Expand Down Expand Up @@ -196,7 +197,7 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo
Spec = new V1PodSpec
{
InitContainers = await CreateInitContainers(command, podName, homeDir, workspacePath),
Containers = await CreateScriptContainers(command, podName, scriptName, homeDir, workspacePath, workspace.ScriptArguments),
Containers = await CreateScriptContainers(command, podName, scriptName, homeDir, workspacePath, workspace.ScriptArguments, tentacleScriptLog),
ImagePullSecrets = imagePullSecretNames,
ServiceAccountName = serviceAccountName,
RestartPolicy = "Never",
Expand All @@ -206,8 +207,8 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo
{
new(matchExpressions: new List<V1NodeSelectorRequirement>
{
new("kubernetes.io/os", "In", new List<string>{"linux"}),
new("kubernetes.io/arch", "In", new List<string>{"arm64","amd64"})
new("kubernetes.io/os", "In", new List<string> { "linux" }),
new("kubernetes.io/arch", "In", new List<string> { "arm64", "amd64" })
})
})))
}
Expand All @@ -218,11 +219,11 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo
LogVerboseToBothLogs($"Executing script in Kubernetes Pod '{podName}'.", tentacleScriptLog);
}

protected virtual async Task<IList<V1Container>> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments)
protected virtual async Task<IList<V1Container>> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments, InMemoryTentacleScriptLog tentacleScriptLog)
{
return new List<V1Container>
{
await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments)
await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments, tentacleScriptLog)
}.AddIfNotNull(CreateWatchdogContainer(homeDir));
}

Expand Down Expand Up @@ -253,9 +254,12 @@ void LogVerboseToBothLogs(string message, InMemoryTentacleScriptLog tentacleScri
tentacleScriptLog.Verbose(message);
}

protected async Task<V1Container> CreateScriptContainer(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments)
protected async Task<V1Container> CreateScriptContainer(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments, InMemoryTentacleScriptLog tentacleScriptLog)
{
var spaceInformation = kubernetesPhysicalFileSystem.GetStorageInformation();

var resourceRequirements = GetScriptPodResourceRequirements(tentacleScriptLog);

return new V1Container
{
Name = podName,
Expand All @@ -267,7 +271,7 @@ protected async Task<V1Container> CreateScriptContainer(StartKubernetesScriptCom
Path.Combine(homeDir, workspacePath, scriptName)
}.Concat(scriptArguments ?? Array.Empty<string>())
.ToList(),
VolumeMounts = new List<V1VolumeMount>{new(homeDir, "tentacle-home")},
VolumeMounts = new List<V1VolumeMount> { new(homeDir, "tentacle-home") },
Env = new List<V1EnvVar>
{
new(KubernetesConfig.NamespaceVariableName, KubernetesConfig.Namespace),
Expand All @@ -284,14 +288,36 @@ protected async Task<V1Container> CreateScriptContainer(StartKubernetesScriptCom

//We intentionally exclude setting "TentacleJournal" since it doesn't make sense to keep a Deployment Journal for Kubernetes deployments
},
Resources = new V1ResourceRequirements
Resources = resourceRequirements
};
}

V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLog tentacleScriptLog)
{
var json = KubernetesConfig.PodResourceJson;
if (!string.IsNullOrWhiteSpace(json))
{
try
{
//set resource requests to be quite low for now as the scripts tend to run fairly quickly
Requests = new Dictionary<string, ResourceQuantity>
{
["cpu"] = new("25m"),
["memory"] = new("100Mi")
}
return KubernetesJson.Deserialize<V1ResourceRequirements>(json);
}
catch (Exception e)
{
var message = $"Failed to deserialize env.{KubernetesConfig.PodResourceJsonVariableName} into valid pod resource requirements.{Environment.NewLine}JSON value: {json}{Environment.NewLine}Using default resource requests for script pod.";
//if we can't parse the JSON, fall back to the defaults below and warn the user
log.WarnFormat(e, message);
//write a verbose message to the script log.
tentacleScriptLog.Verbose(message);
}
}

return new V1ResourceRequirements
{
//set resource requests to be quite low for now as the scripts tend to run fairly quickly
Requests = new Dictionary<string, ResourceQuantity>
{
["cpu"] = new("25m"),
["memory"] = new("100Mi")
}
};
}
Expand Down

0 comments on commit 4a46ece

Please sign in to comment.