Skip to content

Commit

Permalink
Refactor pipeline generation triggers (#1708)
Browse files Browse the repository at this point in the history 
- Use common logic for updating triggers
- Override all triggers except the PR triggers for ci which come from yaml
- Allow the PR triggers for that we override to be triggered via comment on any branch
- Renable adding CI triggers for PR and Unified pipelines
- Add some caching of the pipeline references to help cut down on some network calls
  • Loading branch information
@weshaggard
weshaggard authored Jun 21, 2021
1 parent 1c5c932 commit 08ad132
Show file tree
Hide file tree
Showing 4 changed files with 139 additions and 204 deletions.
65 changes: 2 additions & 63 deletions ...tor/Azure.Sdk.Tools.PipelineGenerator/Conventions/IntegrationTestingPipelineConvention.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ namespace PipelineGenerator.Conventions
public class IntegrationTestingPipelineConvention : PipelineConvention
{
public override string SearchPattern => "tests.yml";
public override bool IsScheduled => !Context.NoSchedule;
public override bool RemoveCITriggers => true;

public IntegrationTestingPipelineConvention(ILogger logger, PipelineGenerationContext context) : base(logger, context)
{
Expand All @@ -27,72 +25,13 @@ protected override async Task<bool> ApplyConventionAsync(BuildDefinition definit
{
var hasChanges = await base.ApplyConventionAsync(definition, component);

// Ensure PR trigger
var prTriggers = definition.Triggers.OfType<PullRequestTrigger>();
if (prTriggers == default || !prTriggers.Any())
if (EnsureDefautPullRequestTrigger(definition, overrideYaml: true))
{
var newTrigger = GetDefaultPrTrigger();
definition.Triggers.Add(newTrigger);
hasChanges = true;
}
else
{
foreach (var trigger in prTriggers)
{
if (EnsurePrTriggerDefaults(trigger))
{
hasChanges = true;
}
}
}

return hasChanges;
}

private PullRequestTrigger GetDefaultPrTrigger()
{
var newTrigger = new PullRequestTrigger
{
Forks = new Forks { AllowSecrets = true, Enabled = true },
RequireCommentsForNonTeamMembersOnly = false,
IsCommentRequiredForPullRequest = true,
};
newTrigger.BranchFilters.Add($"+{Context.Branch}");

return newTrigger;
}

private bool EnsurePrTriggerDefaults(PullRequestTrigger target)
{
var hasChanges = false;

if (!target.Forks.AllowSecrets)
{
target.Forks.AllowSecrets = true;
hasChanges = true;
}

if (!target.Forks.Enabled)
{
target.Forks.Enabled = true;
hasChanges = true;
}

if (target.RequireCommentsForNonTeamMembersOnly)
{
target.RequireCommentsForNonTeamMembersOnly = false;
hasChanges = true;
}

if (!target.IsCommentRequiredForPullRequest)
{
target.IsCommentRequiredForPullRequest = true;
hasChanges = true;
}

if (!target.BranchFilters.Contains($"+{Context.Branch}"))
if (!Context.NoSchedule && EnsureDefaultScheduledTrigger(definition))
{
target.BranchFilters.Add($"+{Context.Branch}");
hasChanges = true;
}

Expand Down
173 changes: 130 additions & 43 deletions tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/PipelineConvention.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,12 @@ public PipelineConvention(ILogger logger, PipelineGenerationContext context)
}

private const string ReportBuildStatusKey = "reportBuildStatus";
private Dictionary<string, BuildDefinitionReference> pipelineReferences;

protected ILogger Logger { get; }
protected PipelineGenerationContext Context { get; }

public abstract string SearchPattern { get; }
public abstract bool IsScheduled { get; }

public abstract bool RemoveCITriggers { get; }

protected abstract string GetDefinitionName(SdkComponent component);

Expand Down Expand Up @@ -113,25 +111,31 @@ private async Task<BuildDefinition> GetExistingDefinitionAsync(string definition
var projectReference = await Context.GetProjectReferenceAsync(cancellationToken);
var sourceRepository = await Context.GetSourceRepositoryAsync(cancellationToken);
var buildClient = await Context.GetBuildHttpClientAsync(cancellationToken);
var definitionReferences = await buildClient.GetDefinitionsAsync(
project: projectReference.Id,
name: definitionName,
path: Context.DevOpsPath,
repositoryId: sourceRepository.Id,
repositoryType: "github"
);

if (definitionReferences.Count() > 1)

if (pipelineReferences == default)
{
Logger.LogError("More than one definition with name '{0}' found - this is an error!", definitionName);
var definitionReferences = await buildClient.GetDefinitionsAsync(
project: projectReference.Id,
path: Context.DevOpsPath
);

foreach (var duplicationDefinitionReference in definitionReferences)
pipelineReferences = new Dictionary<string, BuildDefinitionReference>();
foreach (var definition in definitionReferences)
{
Logger.LogDebug("Definition '{0}' at: {1}", definitionName, duplicationDefinitionReference.GetWebUrl());
if (pipelineReferences.ContainsKey(definition.Name))
{
Logger.LogDebug($"Found more then one definition with name {definition.Name}, picking the first one {pipelineReferences[definition.Name].Id} and not {definition.Id}");
}
else
{
pipelineReferences.Add(definition.Name, definition);
}
}
Logger.LogDebug($"Cached {definitionReferences.Count} pipelines.");
}

var definitionReference = definitionReferences.SingleOrDefault();
BuildDefinitionReference definitionReference = null;
pipelineReferences.TryGetValue(definitionName, out definitionReference);

if (definitionReference != null)
{
Expand Down Expand Up @@ -286,58 +290,141 @@ protected virtual Task<bool> ApplyConventionAsync(BuildDefinition definition, Sd
hasChanges = true;
}


if (IsScheduled)
if (definition.Path != this.Context.DevOpsPath)
{
var scheduleTriggers = definition.Triggers.OfType<ScheduleTrigger>();
definition.Path = this.Context.DevOpsPath;
hasChanges = true;
}

// Only add the schedule trigger if one doesn't exist.
if (scheduleTriggers == default || !scheduleTriggers.Any())
if (definition.Repository.Properties.TryGetValue(ReportBuildStatusKey, out var reportBuildStatusString))
{
if (!bool.TryParse(reportBuildStatusString, out var reportBuildStatusValue) || !reportBuildStatusValue)
{
var computedSchedule = CreateScheduleFromDefinition(definition);

definition.Triggers.Add(new ScheduleTrigger
{
Schedules = new List<Schedule> { computedSchedule }
});

definition.Repository.Properties[ReportBuildStatusKey] = "true";
hasChanges = true;
}
}
else
{
definition.Repository.Properties.Add(ReportBuildStatusKey, "true");
hasChanges = true;
}

if (RemoveCITriggers)
return Task.FromResult(hasChanges);
}

protected bool EnsureDefautPullRequestTrigger(BuildDefinition definition, bool overrideYaml = true)
{
bool hasChanges = false;
var prTriggers = definition.Triggers.OfType<PullRequestTrigger>();
if (prTriggers == default || !prTriggers.Any())
{
for (int i = definition.Triggers.Count - 1; i >= 0; i--)
var newTrigger = new PullRequestTrigger();

if (overrideYaml)
{
if (definition.Triggers[i] is ContinuousIntegrationTrigger)
newTrigger.SettingsSourceType = 1; // Override what is in the yaml file and use what is in the pipeline definition
newTrigger.BranchFilters.Add("+*");
}
else
{
newTrigger.SettingsSourceType = 2; // Pull settings from yaml
}

newTrigger.Forks = new Forks
{
AllowSecrets = true,
Enabled = true
};
newTrigger.RequireCommentsForNonTeamMembersOnly = false;
newTrigger.IsCommentRequiredForPullRequest = true;

definition.Triggers.Add(newTrigger);
hasChanges = true;
}
else
{
foreach (var trigger in prTriggers)
{
if (overrideYaml)
{
if (trigger.SettingsSourceType != 1 ||
trigger.BranchFilters.Contains("+*"))
{
// Override what is in the yaml file and use what is in the pipeline definition
trigger.SettingsSourceType = 1;
trigger.BranchFilters.Add("+*");
}
}
else
{
definition.Triggers.RemoveAt(i);
if (trigger.SettingsSourceType != 2)
{
// Pull settings from yaml
trigger.SettingsSourceType = 2;
hasChanges = true;
}

}
if (trigger.RequireCommentsForNonTeamMembersOnly ||
!trigger.Forks.AllowSecrets ||
!trigger.Forks.Enabled ||
!trigger.IsCommentRequiredForPullRequest
)
{
trigger.Forks.AllowSecrets = true;
trigger.Forks.Enabled = true;
trigger.RequireCommentsForNonTeamMembersOnly = false;
trigger.IsCommentRequiredForPullRequest = true;

hasChanges = true;
}
}
}
return hasChanges;
}

if (definition.Path != this.Context.DevOpsPath)
protected bool EnsureDefaultScheduledTrigger(BuildDefinition definition)
{
bool hasChanges = false;
var scheduleTriggers = definition.Triggers.OfType<ScheduleTrigger>();

// Only add the schedule trigger if one doesn't exist.
if (scheduleTriggers == default || !scheduleTriggers.Any())
{
definition.Path = this.Context.DevOpsPath;
var computedSchedule = CreateScheduleFromDefinition(definition);

definition.Triggers.Add(new ScheduleTrigger
{
Schedules = new List<Schedule> { computedSchedule }
});

hasChanges = true;
}
return hasChanges;
}

if (definition.Repository.Properties.TryGetValue(ReportBuildStatusKey, out var reportBuildStatusString))
protected bool EnsureDefaultCITrigger(BuildDefinition definition)
{
bool hasChanges = false;
var ciTrigger = definition.Triggers.OfType<ContinuousIntegrationTrigger>().SingleOrDefault();
if (ciTrigger == null)
{
if (!bool.TryParse(reportBuildStatusString, out var reportBuildStatusValue) || !reportBuildStatusValue)
definition.Triggers.Add(new ContinuousIntegrationTrigger()
{
definition.Repository.Properties[ReportBuildStatusKey] = "true";
hasChanges = true;
}
SettingsSourceType = 2 // Get CI trigger data from yaml file
});
hasChanges = true;
}
else
{
definition.Repository.Properties.Add(ReportBuildStatusKey, "true");
hasChanges = true;
if (ciTrigger.SettingsSourceType != 2)
{
ciTrigger.SettingsSourceType = 2;
hasChanges = true;
}
}

return Task.FromResult(hasChanges);
return hasChanges;
}
}
}
Expand Down
46 changes: 2 additions & 44 deletions .../Azure.Sdk.Tools.PipelineGenerator/Conventions/PullRequestValidationPipelineConvention.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,62 +20,20 @@ protected override string GetDefinitionName(SdkComponent component)
}

public override string SearchPattern => "ci.yml";
public override bool IsScheduled => false;
public override bool RemoveCITriggers => true;

protected override async Task<bool> ApplyConventionAsync(BuildDefinition definition, SdkComponent component)
{
// NOTE: Not happy with this code at all, I'm going to look for a reasonable
// API that can do equality comparisons (without having to do all the checks myself).

var hasChanges = await base.ApplyConventionAsync(definition, component);

var prTrigger = definition.Triggers.OfType<PullRequestTrigger>().SingleOrDefault();

if (prTrigger == null)
if (EnsureDefautPullRequestTrigger(definition, overrideYaml: false))
{
// TODO: We should probably be more complete here.
definition.Triggers.Add(new PullRequestTrigger()
{
SettingsSourceType = 2, // HACK: See above.
Forks = new Forks()
{
AllowSecrets = false,
Enabled = true
}
});
hasChanges = true;
}
else
{
// TODO: We should probably be more complete here.
if (prTrigger.SettingsSourceType != 2 || prTrigger.Forks.AllowSecrets != false || prTrigger.Forks.Enabled != true)
{
prTrigger.SettingsSourceType = 2;
prTrigger.Forks.AllowSecrets = false;
prTrigger.Forks.Enabled = true;
hasChanges = true;
}
}

var ciTrigger = definition.Triggers.OfType<ContinuousIntegrationTrigger>().SingleOrDefault();

if (ciTrigger == null)
if (EnsureDefaultCITrigger(definition))
{
definition.Triggers.Add(new ContinuousIntegrationTrigger()
{
SettingsSourceType = 2
});
hasChanges = true;
}
else
{
if (ciTrigger.SettingsSourceType != 2)
{
ciTrigger.SettingsSourceType = 2;
hasChanges = true;
}
}

return hasChanges;
}
Expand Down
Loading

0 comments on commit 08ad132

Please sign in to comment.