Merge pull request #588 from AlgebraicJulia/struct-acsets-benchmarks

Benchmarks for paper

benchmark/Graphs.jl

@@ -11,6 +11,33 @@ using Catlab.Graphs.BasicGraphs: TheoryGraph
 using Catlab.WiringDiagrams: query
 using Catlab.Programs: @relation
 
+testdatadir = joinpath(dirname(@__FILE__), "..", "test", "testdata")
+
+# Example Graphs
+#
+################
+
+# Stolen from the Lightgraphs benchmark suite
+
+dg1fn = joinpath(testdatadir, "graph-50-500.jgz")
+
+LG_GRAPHS = Dict{String,LG.DiGraph}(
+    "complete100"   => LG.complete_digraph(100),
+    # "5000-50000"    => LG.loadgraph(dg1fn)["graph-5000-50000"],
+    "path500"       => LG.path_digraph(500)
+)
+
+GRAPHS = Dict(k => from_lightgraph(g) for (k,g) in LG_GRAPHS)
+
+LG_SYMGRAPHS = Dict{String,LG.Graph}(
+    "complete100"   => LG.complete_graph(100),
+    "tutte"         => LG.smallgraph(:tutte),
+    "path500"       => LG.path_graph(500),
+    # "5000-49947"    => LG.SimpleGraph(DIGRAPHS["5000-50000"])
+)
+
+SYMGRAPHS = Dict(k => from_lightgraph(g) for (k,g) in LG_SYMGRAPHS)
+
 # Helpers
 #########
 
@@ -49,7 +76,7 @@ end
 function bench_iter_neighbors(g)
   count = 0
   for v in vertices(g)
-    count += length(neighbors(g, v))
+    count += length(neighbors(g,v))
   end
   count
 end
@@ -65,6 +92,8 @@ end
 function Graphs.connected_component_projection(g::LG.AbstractGraph)
   label = Vector{Int}(undef, LG.nv(g))
   LG.connected_components!(label, g)
+  normalized = searchsortedfirst.(Ref(unique!(sort(label))), label)
+  FinFunction(normalized)
 end
 
 abstract type FindTrianglesAlgorithm end
@@ -116,29 +145,43 @@ lgbench["has-edge"] = @benchmarkable bench_has_edge($lg)
 clbench["iter-neighbors"] = @benchmarkable bench_iter_neighbors($g)
 lgbench["iter-neighbors"] = @benchmarkable bench_iter_neighbors($lg)
 
+
+bench = SUITE["GraphConnComponents"] = BenchmarkGroup()
+clbench = bench["Catlab"] = BenchmarkGroup()
+lgbench = bench["LightGraphs"] = BenchmarkGroup()
+
 n₀ = 2000
 g₀ = path_graph(Graph, n₀)
 g = ob(coproduct(fill(g₀, 5)))
 lg = LG.DiGraph(g)
-clbench["path-graph-components"] =
-  @benchmarkable connected_component_projection($g)
-lgbench["path-graph-components"] =
+clbench["path-graph"] =
+  @benchmarkable connected_component_projection_bfs($g)
+lgbench["path-graph"] =
   @benchmarkable connected_component_projection($lg)
 
 g₀ = star_graph(Graph, n₀)
 g = ob(coproduct(fill(g₀, 5)))
 lg = LG.DiGraph(g)
-clbench["star-graph-components"] =
-  @benchmarkable connected_component_projection($g)
-lgbench["star-graph-components"] =
+clbench["star-graph"] =
+  @benchmarkable connected_component_projection_bfs($g)
+lgbench["star-graph"] =
   @benchmarkable connected_component_projection($lg)
 
+for gn in keys(GRAPHS)
+  clbench[gn] = @benchmarkable connected_component_projection_bfs($(GRAPHS[gn]))
+  lgbench[gn] = @benchmarkable connected_component_projection($(LG_GRAPHS[gn]))
+end
+
+bench = SUITE["GraphTriangles"] = BenchmarkGroup()
+clbench = bench["Catlab"] = BenchmarkGroup()
+lgbench = bench["LightGraphs"] = BenchmarkGroup()
+
 n = 100
 g = wheel_graph(Graph, n)
 lg = LG.DiGraph(g)
 clbench["wheel-graph-triangles-hom"] =
   @benchmarkable ntriangles($g, TriangleBacktrackingSearch())
-clbench["wheel-graph-triangles-query"] =
+clbench["wheel-graph-triangles"] =
   @benchmarkable ntriangles($g, TriangleQuery())
 
 # Symmetric graphs
@@ -170,6 +213,10 @@ lgbench["has-edge"] = @benchmarkable bench_has_edge($lg)
 clbench["iter-neighbors"] = @benchmarkable bench_iter_neighbors($g)
 lgbench["iter-neighbors"] = @benchmarkable bench_iter_neighbors($lg)
 
+bench = SUITE["SymmetricGraphConnComponent"] = BenchmarkGroup()
+clbench = bench["Catlab"] = BenchmarkGroup()
+lgbench = bench["LightGraphs"] = BenchmarkGroup()
+
 n₀ = 2000
 g₀ = path_graph(SymmetricGraph, n₀)
 g = ob(coproduct(fill(g₀, 5)))
@@ -187,12 +234,22 @@ clbench["star-graph-components"] =
 lgbench["star-graph-components"] =
   @benchmarkable connected_component_projection($lg)
 
+for gn in keys(SYMGRAPHS)
+  clbench[gn] = @benchmarkable connected_component_projection_bfs($(SYMGRAPHS[gn]))
+  lgbench[gn] = @benchmarkable connected_component_projection($(LG_SYMGRAPHS[gn]))
+end
+
+bench = SUITE["SymmetricGraphTriangles"] = BenchmarkGroup()
+clbench = bench["Catlab"] = BenchmarkGroup()
+lgbench = bench["LightGraphs"] = BenchmarkGroup()
+
 n = 100
 g = wheel_graph(SymmetricGraph, n)
 lg = LG.Graph(g)
 clbench["wheel-graph-triangles-hom"] =
   @benchmarkable ntriangles($g, TriangleBacktrackingSearch())
-clbench["wheel-graph-triangles-query"] =
+# clbench["wheel-graph-triangles-query"] =
+clbench["wheel-graph-triangles"] =
   @benchmarkable ntriangles($g, TriangleQuery())
 lgbench["wheel-graph-triangles"] = @benchmarkable sum(LG.triangles($lg))
 
@@ -202,7 +259,7 @@ lgbench["wheel-graph-triangles"] = @benchmarkable sum(LG.triangles($lg))
 bench = SUITE["WeightedGraph"] = BenchmarkGroup()
 clbench = bench["Catlab"] = BenchmarkGroup()
 clvecbench = bench["Catlab-vectorized"] = BenchmarkGroup()
-lgbench = bench["LightGraphs"] = BenchmarkGroup()
+lgbench = bench["MetaGraphs"] = BenchmarkGroup()
 
 n = 10000
 g = path_graph(WeightedGraph{Float64}, n; E=(weight=range(0,1,length=n-1),))
@@ -305,3 +362,51 @@ lgbench["indexed-lookup"] = @benchmarkable begin
     @assert $mg["v$i", :label] == i
   end
 end
+
+# Random Graphs
+###############
+
+bench = SUITE["RandomGraph"] = BenchmarkGroup()
+clbench = bench["Catlab"] = BenchmarkGroup()
+lgbench = bench["LightGraphs"] = BenchmarkGroup()
+
+sizes = [10000]
+ps = [0.001]
+for size in sizes, p in ps
+  clbench["erdos_renyi-$size-$p"] =
+    @benchmarkable erdos_renyi($Graph, $size, $(p/2))
+  lgbench["erdos_renyi-$size-$p"] =
+    @benchmarkable LightGraphs.erdos_renyi($size, $p)
+end
+
+ks = [10]
+
+for size in sizes, k in ks
+  clbench["expected_degree_graph-$size-$k"] =
+    @benchmarkable expected_degree_graph($Graph, $([min(k,size-1) for _ in 1:size]))
+  lgbench["expected_degree_graph-$size-$k"] =
+    @benchmarkable LightGraphs.expected_degree_graph($([min(k,size-1) for _ in 1:size]))
+end
+
+for size in sizes, k in ks
+  clbench["watts_strogatz-$size-$k"] =
+    @benchmarkable watts_strogatz($Graph, $size, $(min(k,size-1)), 0.5)
+  lgbench["watts_strogatz-$size-$k"] =
+    @benchmarkable LightGraphs.watts_strogatz($size, $(min(k,size-1)), 0.5)
+end
+
+# Searching
+###########
+
+bench = SUITE["Searching"] = BenchmarkGroup()
+clbench = bench["Catlab"] = BenchmarkGroup()
+lgbench = bench["LightGraphs"] = BenchmarkGroup()
+
+for size in sizes, p in ps
+  local g = erdos_renyi(Graph, size, p)
+  local lg = LightGraphs.SimpleDiGraph(g)
+  clbench["bfs_erdos_renyi-$size-$p"] = @benchmarkable bfs_parents($g,1)
+  lgbench["bfs_erdos_renyi-$size-$p"] = @benchmarkable LightGraphs.bfs_parents($lg,1)
+  clbench["dfs_erdos_renyi-$size-$p"] = @benchmarkable dfs_parents($g,1)
+  lgbench["dfs_erdos_renyi-$size-$p"] = @benchmarkable LightGraphs.dfs_parents($lg,1)
+end

benchmark/make_plots.jl

@@ -0,0 +1,7 @@
+include("benchmarks.jl")
+
+include("plots.jl")
+
+results = run(SUITE)
+data = graphbench_data(results)
+plot_all_subcats(data)

benchmark/plots.jl

@@ -0,0 +1,57 @@
+using Plots, StatsPlots
+using Plots.PlotMeasures
+using DataFrames, Query
+using BenchmarkTools
+
+function graphbench_data(suite)
+  data = DataFrame(subcat=String[],bench=String[],platform=String[],
+                   mt_normalized=Float64[],mediantime=Float64[])
+  graphbenches = suite["Graphs"]
+  noncatlab_times = Dict{Tuple{String,String},Float64}()
+  for (subcat,subsuite) in graphbenches
+    for (platform,results) in subsuite
+      for (bench,result) in results
+        if platform ∈ ["LightGraphs", "MetaGraphs"]
+            noncatlab_times[(subcat,bench)] = median(result).time
+        end
+        new_row = (subcat=subcat,
+                    bench=bench,
+                    platform=platform,
+                    mt_normalized=0.,
+                    mediantime=median(result).time)
+        push!(data, new_row)
+      end
+    end
+  end
+  for i in 1:length(data.subcat)
+    key = (data[i,:subcat], data[i,:bench])
+    if key ∈ keys(noncatlab_times)
+      data[i,:mt_normalized] = data[i,:mediantime] / noncatlab_times[key]
+    end
+  end
+  data
+end
+
+function subcat_data(dat,subcat)
+  dat |>
+    @filter(_.subcat==subcat) |>
+    @filter(_.platform == ["Catlab"]) |>
+    @orderby((_.bench,_.platform)) |>
+    @select(-:subcat) |>
+    DataFrame
+end
+
+function plot_subcat(dat,subcat,yscale=:linear)
+  subcat_data(dat,subcat) |>
+    @df groupedbar(:bench,:mt_normalized,group=:platform,
+                   xrotation=45,legend=:outerright,bar_width=0.5,
+                   yscale=yscale, yguide="Rel. time", bottom_margin=50px)
+end
+
+function plot_all_subcats(dat)
+  for subcat in unique(dat[!,:subcat])
+    yscale = subcat ∈ ["WeightedGraph", "LabeledGraph"] ? :log : :linear
+    fig = plot_subcat(dat,subcat,yscale)
+    savefig(fig, string("figures/",subcat,".pdf"))
+  end
+end

src/categorical_algebra/ACSetInterface.jl

@@ -62,12 +62,12 @@ of `[:src,:vattr]` as two independent columns does not even make sense, since
 they have different domains (belong to different tables).
 """
 function  subpart end
-@inline subpart(acs, part, name) = view_or_slice(subpart(acs, name), part)
+@inline Base.@propagate_inbounds subpart(acs, part, name) = view_or_slice(subpart(acs, name), part)
 
 function view_or_slice end
 @inline view_or_slice(x::AbstractVector, i::Union{Integer,StaticArray}) = x[i]
 @inline view_or_slice(x::AbstractVector, ::Colon) = x
-@inline view_or_slice(x::AbstractVector, i) = @view x[i]
+@inline Base.@propagate_inbounds view_or_slice(x::AbstractVector, i) = @view x[i]
 
 @inline subpart(acs, expr::GATExpr{:generator}) = subpart(acs, first(expr))
 @inline subpart(acs, expr::GATExpr{:id}) = parts(acs, first(dom(expr)))
@@ -120,16 +120,18 @@ end
 incident(acs, part, expr::GATExpr; kw...) =
   incident(acs, part, subpart_names(expr); kw...)
 
+@inline add_part!(acs, type; kw...) = add_part!(acs, type, (;kw...))
+
 """ Add part of given type to acset, optionally setting its subparts.
 
 Returns the ID of the added part.
 
 See also: [`add_parts!`](@ref).
 """
-@inline function add_part!(acs, type::Symbol, args...; kw...)
+@inline function add_part!(acs, type::Symbol, kw)
   part = only(add_parts!(acs,type,1))
   try
-    set_subparts!(acs, part, args...; kw...)
+    set_subparts!(acs, part, kw)
   catch e
     rem_part!(acs, type, part)
     rethrow(e)
@@ -145,10 +147,12 @@ See also: [`add_part!`](@ref).
 """
 function add_parts! end
 
-@inline function add_parts!(acs, type::Symbol, n::Int, args...; kw...)
+@inline add_parts!(acs, type::Symbol, n::Int; kw...) = add_parts!(acs, type, n, (;kw...))
+
+@inline function add_parts!(acs, type::Symbol, n::Int, kw)
   parts = add_parts!(acs, type, n)
   try
-    set_subparts!(acs, parts, args...; kw...)
+    set_subparts!(acs, parts, kw)
   catch e
     rem_parts!(acs, type, parts)
     rethrow(e)
@@ -183,10 +187,8 @@ Both single and vectorized assignment are supported.
 
 See also: [`set_subpart!`](@ref).
 """
-@inline function set_subparts!(acs, part, kw::NamedTuple)
-  for name in keys(kw)
-    set_subpart!(acs, part, name, kw[name])
-  end
+@inline @generated function set_subparts!(acs, part, kw::NamedTuple{keys}) where {keys}
+  Expr(:block,[:(set_subpart!(acs, part, $(Expr(:quote, name)), kw.$name)) for name in keys]...)
 end
 
 @inline set_subparts!(acs, part; kw...) = set_subparts!(acs, part, (;kw...))