Skip to content

Commit

Permalink
Merge pull request #1396 from BlythMeister/ParallelOrder
Browse files Browse the repository at this point in the history 
Partial fix for #1395
  • Loading branch information
@forki
forki authored Mar 27, 2017
2 parents 0cca21d + 87bdde1 commit 2fb55a7
Show file tree
Hide file tree
Showing 2 changed files with 251 additions and 61 deletions.
107 changes: 78 additions & 29 deletions src/app/FakeLib/TargetHelper.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ type private DependencyType =
| Hard = 1
| Soft = 2

type private DependencyLevel =
{
level:int;
dependants: string list;
}

/// [omit]
let mutable PrintStackTraceOnError = false

Expand Down Expand Up @@ -63,6 +69,8 @@ let reset() =
ExecutedTargetTimes.Clear()
FinalTargets.Clear()

let mutable CurrentTargetOrder = []

/// Returns a list with all target names.
let getAllTargetsNames() = TargetDict |> Seq.map (fun t -> t.Key) |> Seq.toList

Expand Down Expand Up @@ -349,14 +357,16 @@ let private visitDependencies fVisit targetName =
let visit fGetDependencies fVisit targetName =
let visited = new HashSet<_>()
let ordered = new List<_>()
let rec visitDependenciesAux level (depType,targetName) =
let rec visitDependenciesAux level (dependentTarget:option<TargetTemplate<unit>>) (depType,targetName) =
let target = getTarget targetName
let isVisited = visited.Contains targetName
visited.Add targetName |> ignore
fVisit (target, depType, level, isVisited)
(fGetDependencies target) |> Seq.iter (visitDependenciesAux (level + 1))
fVisit (dependentTarget, target, depType, level, isVisited)

(fGetDependencies target) |> Seq.iter (visitDependenciesAux (level + 1) (Some target))

if not isVisited then ordered.Add targetName
visitDependenciesAux 0 (DependencyType.Hard, targetName)
visitDependenciesAux 0 None (DependencyType.Hard, targetName)
visited, ordered

// First pass is to accumulate targets in (hard) dependency graph
Expand All @@ -382,7 +392,7 @@ let PrintDependencyGraph verbose target =
| true,target ->
logfn "%sDependencyGraph for Target %s:" (if verbose then String.Empty else "Shortened ") target.Name

let logDependency ((t: TargetTemplate<unit>), depType, level, isVisited) =
let logDependency (_, (t: TargetTemplate<unit>), depType, level, isVisited) =
if verbose || not isVisited then
let indent = (String(' ', level * 3))
if depType = DependencyType.Soft then
Expand All @@ -394,7 +404,11 @@ let PrintDependencyGraph verbose target =

log ""
log "The resulting target order is:"
Seq.iter (logfn " - %s") ordered
CurrentTargetOrder
|> List.iteri (fun index x ->
if (environVarOrDefault "parallel-jobs" "1" |> int > 1) then
logfn "Group - %d" (index + 1)
Seq.iter (logfn " - %s") x)

/// <summary>Writes a dependency graph of all targets in the DOT format.</summary>
let PrintDotDependencyGraph () =
Expand Down Expand Up @@ -477,29 +491,63 @@ let determineBuildOrder (target : string) =

let t = getTarget target

let targetLevels = new Dictionary<_,_>()
let addTargetLevel ((target: TargetTemplate<unit>), _, level, _ ) =
match targetLevels.TryGetValue target.Name with
| true, mapLevel when mapLevel >= level -> ()
| _ -> targetLevels.[target.Name] <- level
let targetLevels = new Dictionary<string,DependencyLevel>()

let appendDepentantOption (currentList:string list) (dependantTarget:option<TargetTemplate<unit>>) =
match dependantTarget with
| None -> currentList
| Some x -> List.append currentList [x.Name] |> List.distinct

let rec SetTargetLevel newLevel target =
match targetLevels.TryGetValue target with
| true, exDependencyLevel ->
if exDependencyLevel.level < newLevel then
exDependencyLevel.dependants |> List.iter (fun x -> SetTargetLevel (newLevel - 1) x)
if exDependencyLevel.dependants.Length > 0 then
targetLevels.[target] <- {level = newLevel; dependants = exDependencyLevel.dependants}
| _ -> ()

let addTargetLevel ((dependantTarget:option<TargetTemplate<unit>>), (target: TargetTemplate<unit>), _, level, _ ) =
let (|LevelIncreaseWithDependantTarget|_|) = function
| (true, exDependencyLevel), Some dt when exDependencyLevel.level > level -> Some (exDependencyLevel, dt)
| _ -> None

let (|LevelIncreaseWithNoDependantTarget|_|) = function
| (true, exDependencyLevel), None when exDependencyLevel.level > level -> Some (exDependencyLevel)
| _ -> None

let (|LevelDecrease|_|) = function
| (true, exDependencyLevel), _ when exDependencyLevel.level < level -> Some (exDependencyLevel)
| _ -> None

let (|NewTarget|_|) = function
| (false, _), _ -> Some ()
| _ -> None

match targetLevels.TryGetValue target.Name, dependantTarget with
| LevelIncreaseWithDependantTarget (exDependencyLevel, dt) ->
SetTargetLevel (exDependencyLevel.level - 1) dt.Name
targetLevels.[target.Name] <- {level = exDependencyLevel.level; dependants = (appendDepentantOption exDependencyLevel.dependants dependantTarget)}
| LevelIncreaseWithNoDependantTarget (exDependencyLevel) ->
targetLevels.[target.Name] <- {level = exDependencyLevel.level; dependants = (appendDepentantOption exDependencyLevel.dependants dependantTarget)}
| LevelDecrease (exDependencyLevel) ->
exDependencyLevel.dependants |> List.iter (fun x -> SetTargetLevel (level - 1) x)
targetLevels.[target.Name] <- {level = level; dependants = (appendDepentantOption exDependencyLevel.dependants dependantTarget)}
| NewTarget ->
targetLevels.[target.Name] <- {level = level; dependants=(appendDepentantOption [] dependantTarget)}
| _ -> ()

let visited, ordered = visitDependencies addTargetLevel target

// the results are grouped by their level, sorted descending (by level) and
// finally grouped together in a list<TargetTemplate<unit>[]>
let result =
targetLevels
|> Seq.map (fun pair -> pair.Key, pair.Value)
|> Seq.groupBy snd
|> Seq.sortBy (fun (l,_) -> -l)
|> Seq.map snd
|> Seq.map (fun v -> v |> Seq.map fst |> Seq.distinct |> Seq.map getTarget |> Seq.toArray)
|> Seq.toList

// Note that this build order cannot be considered "optimal"
// since it may introduce order where actually no dependencies
// exist. However it yields a "good" execution order in practice.
result
// finally grouped together in a list<TargetTemplate<unit>[]
targetLevels
|> Seq.map (fun pair -> pair.Key, pair.Value.level)
|> Seq.groupBy snd
|> Seq.sortBy (fun (l,_) -> -l)
|> Seq.map snd
|> Seq.map (fun v -> v |> Seq.map fst |> Seq.distinct |> Seq.map getTarget |> Seq.toArray)
|> Seq.toList

/// Runs a single target without its dependencies
let runSingleTarget (target : TargetTemplate<unit>) =
Expand All @@ -522,8 +570,6 @@ let runTargetsParallel (count : int) (targets : Target[]) =
.ToArray()
|> ignore

let mutable CurrentTargetOrder = []

/// Runs a target and its dependencies.
let run targetName =
if doesTargetMeanPrintDotGraph targetName then PrintDotDependencyGraph() else
Expand Down Expand Up @@ -557,20 +603,23 @@ let run targetName =
order
|> List.map (fun targets -> targets |> Array.map (fun t -> t.Name) |> Array.toList)

PrintDependencyGraph false targetName

// run every level in parallel
for par in order do
runTargetsParallel parallelJobs par

else
// single threaded build.
PrintDependencyGraph false targetName


// Note: we could use the ordering resulting from flattening the result of determineBuildOrder
// for a single threaded build (thereby centralizing the algorithm for build order), but that
// ordering is inconsistent with earlier versions of FAKE (and PrintDependencyGraph).
let _, ordered = visitDependencies ignore targetName
CurrentTargetOrder <- ordered |> Seq.map (fun t -> [t]) |> Seq.toList

PrintDependencyGraph false targetName

runTargets (ordered |> Seq.map getTarget |> Seq.toArray)

finally
Expand Down
Loading

0 comments on commit 2fb55a7

Please sign in to comment.