Problem running preconditioned stokes in parrallel with periodic boundary conditions

[dhrichards]

If I introduce periodic bc's in the x-direction to test/Applications/Stokes.jl through model = CartesianDiscreteModel(parts,np,domain,nc,isperiodic = (true,false)) and run the code in mpi, it fails when the number of divisions in the x direction is greater than 1 (i.e. np = (1,2) runs but np = (2,1) fails). The error is:

┌ Error: 
│   exception =
│    BoundsError: attempt to access 209-element Vector{Int32} at index [0]
│    Stacktrace:
│      [1] getindex
│        @ ./essentials.jl:13 [inlined]
│      [2] getindex
│        @ ./abstractarray.jl:1297 [inlined]
│      [3] #132
│        @ ~/.julia/packages/GridapDistributed/z6aPu/src/Algebra.jl:972 [inlined]
│      [4] iterate
│        @ ./generator.jl:47 [inlined]
│      [5] collect_to!(dest::Vector{Int32}, itr::Base.Generator{Vector{Int64}, GridapDistributed.var"#132#134"{Vector{Int32}, PartitionedArrays.VectorFromDict{Int64, Int32}}}, offs::Int64, st::Int64)
│        @ Base ./array.jl:840
│      [6] collect_to_with_first!
│        @ ./array.jl:818 [inlined]
│      [7] _collect(c::Vector{Int64}, itr::Base.Generator{Vector{Int64}, GridapDistributed.var"#132#134"{Vector{Int32}, PartitionedArrays.VectorFromDict{Int64, Int32}}}, #unused#::Base.EltypeUnknown, isz::Base.HasShape{1})
│        @ Base ./array.jl:812
│      [8] collect_similar
│        @ ./array.jl:711 [inlined]
│      [9] map
│        @ ./abstractarray.jl:3261 [inlined]
│     [10] #131
│        @ ~/.julia/packages/GridapDistributed/z6aPu/src/Algebra.jl:972 [inlined]
│     [11] map(::GridapDistributed.var"#131#133", ::MPIArray{Vector{Int64}, 1}, ::MPIArray{LocalIndices, 1})
│        @ PartitionedArrays ~/.julia/packages/PartitionedArrays/py6uo/src/mpi_array.jl:229
│     [12] #get_gid_owners#130
│        @ ~/.julia/packages/GridapDistributed/z6aPu/src/Algebra.jl:969 [inlined]
│     [13] get_gid_owners(I::MPIArray{Vector{Int64}, 1}, ids::PRange{MPIArray{LocalIndices, 1}})
│        @ GridapDistributed ~/.julia/packages/GridapDistributed/z6aPu/src/Algebra.jl:968
│     [14] (::GridapDistributed.var"#163#166"{Symbol, Bool, Matrix{MPIArray{Vector{Int32}, 1}}, Matrix{MPIArray{Vector{Int64}, 1}}})(id::Int64, prange::PRange{MPIArray{LocalIndices, 1}})
│        @ GridapDistributed ~/.julia/packages/GridapDistributed/z6aPu/src/Algebra.jl:1144
│     [15] #4
│        @ ./generator.jl:36 [inlined]
│     [16] iterate
│        @ ./generator.jl:47 [inlined]
│     [17] collect_to!(dest::Vector{Tuple{MPIArray{Vector{Int64}, 1}, MPIArray{Vector{Int32}, 1}}}, itr::Base.Generator{Base.Iterators.Zip{Tuple{LinearIndices{1, Tuple{Base.OneTo{Int64}}}, Vector{PRange{MPIArray{LocalIndices, 1}}}}}, Base.var"#4#5"{GridapDistributed.var"#163#166"{Symbol, Bool, Matrix{MPIArray{Vector{Int32}, 1}}, Matrix{MPIArray{Vector{Int64}, 1}}}}}, offs::Int64, st::Tuple{Int64, Int64})
│        @ Base ./array.jl:840
│     [18] collect_to_with_first!(dest::Vector{Tuple{MPIArray{Vector{Int64}, 1}, MPIArray{Vector{Int32}, 1}}}, v1::Tuple{MPIArray{Vector{Int64}, 1}, MPIArray{Vector{Int32}, 1}}, itr::Base.Generator{Base.Iterators.Zip{Tuple{LinearIndices{1, Tuple{Base.OneTo{Int64}}}, Vector{PRange{MPIArray{LocalIndices, 1}}}}}, Base.var"#4#5"{GridapDistributed.var"#163#166"{Symbol, Bool, Matrix{MPIArray{Vector{Int32}, 1}}, Matrix{MPIArray{Vector{Int64}, 1}}}}}, st::Tuple{Int64, Int64})
│        @ Base ./array.jl:818
│     [19] collect(itr::Base.Generator{Base.Iterators.Zip{Tuple{LinearIndices{1, Tuple{Base.OneTo{Int64}}}, Vector{PRange{MPIArray{LocalIndices, 1}}}}}, Base.var"#4#5"{GridapDistributed.var"#163#166"{Symbol, Bool, Matrix{MPIArray{Vector{Int32}, 1}}, Matrix{MPIArray{Vector{Int64}, 1}}}}})
│        @ Base ./array.jl:792
│     [20] map
│        @ ./abstractarray.jl:3383 [inlined]
│     [21] _setup_prange(dofs_gids_prange::Vector{PRange{MPIArray{LocalIndices, 1}}}, gids::Matrix{MPIArray{Vector{Int64}, 1}}; ax::Symbol, ghost::Bool, owners::Matrix{MPIArray{Vector{Int32}, 1}})
│        @ GridapDistributed ~/.julia/packages/GridapDistributed/z6aPu/src/Algebra.jl:1138
│     [22] _sa_create_from_nz_with_callback(callback::GridapDistributed.var"#f#113", async_callback::GridapDistributed.var"#f#113", a::Gridap.Fields.MatrixBlock{GridapDistributed.DistributedAllocationCOO{SubAssembledRows, MPIArray{Gridap.Algebra.AllocationCOO{SparseArrays.SparseMatrixCSC{Float64, Int64}, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}}, Vector{Int64}, Vector{Float64}}, 1}, PRange{MPIArray{LocalIndices, 1}}, PRange{MPIArray{LocalIndices, 1}}}}, b::Nothing)
│        @ GridapDistributed ~/.julia/packages/GridapDistributed/z6aPu/src/Algebra.jl:609
│     [23] create_from_nz
│        @ ~/.julia/packages/GridapDistributed/z6aPu/src/Algebra.jl:575 [inlined]
│     [24] create_from_nz(a::Gridap.Fields.MatrixBlock{GridapDistributed.DistributedAllocationCOO{SubAssembledRows, MPIArray{Gridap.Algebra.AllocationCOO{SparseArrays.SparseMatrixCSC{Float64, Int64}, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}}, Vector{Int64}, Vector{Float64}}, 1}, PRange{MPIArray{LocalIndices, 1}}, PRange{MPIArray{LocalIndices, 1}}}}, b::Gridap.Fields.VectorBlock{GridapDistributed.PVectorAllocationTrackTouchedAndValues{MPIArray{GridapDistributed.ArrayAllocationTrackTouchedAndValues{Vector{Float64}}, 1}, MPIArray{Vector{Float64}, 1}, PRange{MPIArray{LocalIndices, 1}}}})
│        @ Gridap.Algebra ~/.julia/packages/Gridap/5nlYf/src/Algebra/AlgebraInterfaces.jl:367
│     [25] assemble_matrix_and_vector(a::Gridap.MultiField.BlockSparseMatrixAssembler{2, 2, (1, 1), (1, 2), GridapDistributed.DistributedSparseMatrixAssembler{SubAssembledRows, MPIArray{GenericSparseMatrixAssembler, 1}, GridapDistributed.PSparseMatrixBuilderCOO{SparseArrays.SparseMatrixCSC{Float64, Int64}, SubAssembledRows}, GridapDistributed.PVectorBuilder{BlockVector{Float64, Vector{Vector{Float64}}, Tuple{BlockedUnitRange{Vector{Int64}}}}, SubAssembledRows}, PRange{MPIArray{LocalIndices, 1}}, PRange{MPIArray{LocalIndices, 1}}}}, data::MPIArray{Tuple{Tuple{Vector{Any}, Vector{Any}, Vector{Any}}, Tuple{Vector{Any}, Vector{Any}, Vector{Any}}, Tuple{Vector{Any}, Vector{Any}}}, 1})
│        @ Gridap.FESpaces ~/.julia/packages/Gridap/5nlYf/src/FESpaces/SparseMatrixAssemblers.jl:102
│     [26] AffineFEOperator(weakform::Gridap.FESpaces.var"#60#61"{var"#a#9"{GridapDistributed.DistributedMeasure{MPIArray{Gridap.CellData.GenericMeasure, 1}, GridapDistributed.DistributedTriangulation{2, 2, MPIArray{BodyFittedTriangulation{2, 2, CartesianDiscreteModel{2, Float64, typeof(identity)}, Gridap.Geometry.GridView{2, 2, CartesianGrid{2, Float64, typeof(identity)}, Vector{Int64}}, Gridap.Arrays.LazyArray{Fill{Reindex{Gridap.Arrays.IdentityVector{Int64}}, 1, Tuple{Base.OneTo{Int64}}}, Int64, 1, Tuple{Vector{Int64}}}}, 1}, GridapDistributed.GenericDistributedDiscreteModel{2, 2, MPIArray{CartesianDiscreteModel{2, Float64, typeof(identity)}, 1}, Vector{PRange}}}}, var"#biform#7"{var"#biform_u#6"{var"#graddiv#5"{LocalProjectionMap{GenericLagrangianRefFE{L2Conformity, 2}, GenericQuadrature{2, Float64, Vector{VectorValue{2, Float64}}, Vector{Float64}}, Cholesky{Float64, Matrix{Float64}}}, Float64}}}}, var"#l#10"{GridapDistributed.DistributedMeasure{MPIArray{Gridap.CellData.GenericMeasure, 1}, GridapDistributed.DistributedTriangulation{2, 2, MPIArray{BodyFittedTriangulation{2, 2, CartesianDiscreteModel{2, Float64, typeof(identity)}, Gridap.Geometry.GridView{2, 2, CartesianGrid{2, Float64, typeof(identity)}, Vector{Int64}}, Gridap.Arrays.LazyArray{Fill{Reindex{Gridap.Arrays.IdentityVector{Int64}}, 1, Tuple{Base.OneTo{Int64}}}, Int64, 1, Tuple{Vector{Int64}}}}, 1}, GridapDistributed.GenericDistributedDiscreteModel{2, 2, MPIArray{CartesianDiscreteModel{2, Float64, typeof(identity)}, 1}, Vector{PRange}}}}, var"#liform#8"{VectorValue{2, Float64}}}}, trial::GridapDistributed.DistributedMultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, Vector{GridapDistributed.DistributedSingleFieldFESpace{A, PRange{MPIArray{LocalIndices, 1}}, GridapDistributed.DistributedTriangulation{2, 2, MPIArray{BodyFittedTriangulation{2, 2, CartesianDiscreteModel{2, Float64, typeof(identity)}, CartesianGrid{2, Float64, typeof(identity)}, Gridap.Arrays.IdentityVector{Int64}}, 1}, GridapDistributed.GenericDistributedDiscreteModel{2, 2, MPIArray{CartesianDiscreteModel{2, Float64, typeof(identity)}, 1}, Vector{PRange}}}, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}} where A}, MPIArray{MultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, UnConstrained, BlockVector{Float64, Vector{Vector{Float64}}, Tuple{BlockedUnitRange{Vector{Int64}}}}}, 1}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}, GridapDistributed.BlockPArray{Vector{Float64}, Float64, 1, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}}}, test::GridapDistributed.DistributedMultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, Vector{GridapDistributed.DistributedSingleFieldFESpace{A, PRange{MPIArray{LocalIndices, 1}}, GridapDistributed.DistributedTriangulation{2, 2, MPIArray{BodyFittedTriangulation{2, 2, CartesianDiscreteModel{2, Float64, typeof(identity)}, CartesianGrid{2, Float64, typeof(identity)}, Gridap.Arrays.IdentityVector{Int64}}, 1}, GridapDistributed.GenericDistributedDiscreteModel{2, 2, MPIArray{CartesianDiscreteModel{2, Float64, typeof(identity)}, 1}, Vector{PRange}}}, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}} where A}, MPIArray{MultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, UnConstrained, BlockVector{Float64, Vector{Vector{Float64}}, Tuple{BlockedUnitRange{Vector{Int64}}}}}, 1}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}, GridapDistributed.BlockPArray{Vector{Float64}, Float64, 1, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}}}, assem::Gridap.MultiField.BlockSparseMatrixAssembler{2, 2, (1, 1), (1, 2), GridapDistributed.DistributedSparseMatrixAssembler{SubAssembledRows, MPIArray{GenericSparseMatrixAssembler, 1}, GridapDistributed.PSparseMatrixBuilderCOO{SparseArrays.SparseMatrixCSC{Float64, Int64}, SubAssembledRows}, GridapDistributed.PVectorBuilder{BlockVector{Float64, Vector{Vector{Float64}}, Tuple{BlockedUnitRange{Vector{Int64}}}}, SubAssembledRows}, PRange{MPIArray{LocalIndices, 1}}, PRange{MPIArray{LocalIndices, 1}}}})
│        @ Gridap.FESpaces ~/.julia/packages/Gridap/5nlYf/src/FESpaces/AffineFEOperators.jl:38
│     [27] AffineFEOperator(::Function, ::GridapDistributed.DistributedMultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, Vector{GridapDistributed.DistributedSingleFieldFESpace{A, PRange{MPIArray{LocalIndices, 1}}, GridapDistributed.DistributedTriangulation{2, 2, MPIArray{BodyFittedTriangulation{2, 2, CartesianDiscreteModel{2, Float64, typeof(identity)}, CartesianGrid{2, Float64, typeof(identity)}, Gridap.Arrays.IdentityVector{Int64}}, 1}, GridapDistributed.GenericDistributedDiscreteModel{2, 2, MPIArray{CartesianDiscreteModel{2, Float64, typeof(identity)}, 1}, Vector{PRange}}}, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}} where A}, MPIArray{MultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, UnConstrained, BlockVector{Float64, Vector{Vector{Float64}}, Tuple{BlockedUnitRange{Vector{Int64}}}}}, 1}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}, GridapDistributed.BlockPArray{Vector{Float64}, Float64, 1, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}}}, ::Vararg{GridapDistributed.DistributedMultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, Vector{GridapDistributed.DistributedSingleFieldFESpace{A, PRange{MPIArray{LocalIndices, 1}}, GridapDistributed.DistributedTriangulation{2, 2, MPIArray{BodyFittedTriangulation{2, 2, CartesianDiscreteModel{2, Float64, typeof(identity)}, CartesianGrid{2, Float64, typeof(identity)}, Gridap.Arrays.IdentityVector{Int64}}, 1}, GridapDistributed.GenericDistributedDiscreteModel{2, 2, MPIArray{CartesianDiscreteModel{2, Float64, typeof(identity)}, 1}, Vector{PRange}}}, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}} where A}, MPIArray{MultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, UnConstrained, BlockVector{Float64, Vector{Vector{Float64}}, Tuple{BlockedUnitRange{Vector{Int64}}}}}, 1}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}, GridapDistributed.BlockPArray{Vector{Float64}, Float64, 1, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}}}})
│        @ Gridap.FESpaces ~/.julia/packages/Gridap/5nlYf/src/FESpaces/AffineFEOperators.jl:47
│     [28] AffineFEOperator(::var"#a#9"{GridapDistributed.DistributedMeasure{MPIArray{Gridap.CellData.GenericMeasure, 1}, GridapDistributed.DistributedTriangulation{2, 2, MPIArray{BodyFittedTriangulation{2, 2, CartesianDiscreteModel{2, Float64, typeof(identity)}, Gridap.Geometry.GridView{2, 2, CartesianGrid{2, Float64, typeof(identity)}, Vector{Int64}}, Gridap.Arrays.LazyArray{Fill{Reindex{Gridap.Arrays.IdentityVector{Int64}}, 1, Tuple{Base.OneTo{Int64}}}, Int64, 1, Tuple{Vector{Int64}}}}, 1}, GridapDistributed.GenericDistributedDiscreteModel{2, 2, MPIArray{CartesianDiscreteModel{2, Float64, typeof(identity)}, 1}, Vector{PRange}}}}, var"#biform#7"{var"#biform_u#6"{var"#graddiv#5"{LocalProjectionMap{GenericLagrangianRefFE{L2Conformity, 2}, GenericQuadrature{2, Float64, Vector{VectorValue{2, Float64}}, Vector{Float64}}, Cholesky{Float64, Matrix{Float64}}}, Float64}}}}, ::var"#l#10"{GridapDistributed.DistributedMeasure{MPIArray{Gridap.CellData.GenericMeasure, 1}, GridapDistributed.DistributedTriangulation{2, 2, MPIArray{BodyFittedTriangulation{2, 2, CartesianDiscreteModel{2, Float64, typeof(identity)}, Gridap.Geometry.GridView{2, 2, CartesianGrid{2, Float64, typeof(identity)}, Vector{Int64}}, Gridap.Arrays.LazyArray{Fill{Reindex{Gridap.Arrays.IdentityVector{Int64}}, 1, Tuple{Base.OneTo{Int64}}}, Int64, 1, Tuple{Vector{Int64}}}}, 1}, GridapDistributed.GenericDistributedDiscreteModel{2, 2, MPIArray{CartesianDiscreteModel{2, Float64, typeof(identity)}, 1}, Vector{PRange}}}}, var"#liform#8"{VectorValue{2, Float64}}}, ::GridapDistributed.DistributedMultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, Vector{GridapDistributed.DistributedSingleFieldFESpace{A, PRange{MPIArray{LocalIndices, 1}}, GridapDistributed.DistributedTriangulation{2, 2, MPIArray{BodyFittedTriangulation{2, 2, CartesianDiscreteModel{2, Float64, typeof(identity)}, CartesianGrid{2, Float64, typeof(identity)}, Gridap.Arrays.IdentityVector{Int64}}, 1}, GridapDistributed.GenericDistributedDiscreteModel{2, 2, MPIArray{CartesianDiscreteModel{2, Float64, typeof(identity)}, 1}, Vector{PRange}}}, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}} where A}, MPIArray{MultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, UnConstrained, BlockVector{Float64, Vector{Vector{Float64}}, Tuple{BlockedUnitRange{Vector{Int64}}}}}, 1}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}, GridapDistributed.BlockPArray{Vector{Float64}, Float64, 1, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}}}, ::Vararg{GridapDistributed.DistributedMultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, Vector{GridapDistributed.DistributedSingleFieldFESpace{A, PRange{MPIArray{LocalIndices, 1}}, GridapDistributed.DistributedTriangulation{2, 2, MPIArray{BodyFittedTriangulation{2, 2, CartesianDiscreteModel{2, Float64, typeof(identity)}, CartesianGrid{2, Float64, typeof(identity)}, Gridap.Arrays.IdentityVector{Int64}}, 1}, GridapDistributed.GenericDistributedDiscreteModel{2, 2, MPIArray{CartesianDiscreteModel{2, Float64, typeof(identity)}, 1}, Vector{PRange}}}, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}} where A}, MPIArray{MultiFieldFESpace{BlockMultiFieldStyle{2, (1, 1), (1, 2)}, UnConstrained, BlockVector{Float64, Vector{Vector{Float64}}, Tuple{BlockedUnitRange{Vector{Int64}}}}}, 1}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}, GridapDistributed.BlockPArray{Vector{Float64}, Float64, 1, PVector{Vector{Float64}, MPIArray{Vector{Float64}, 1}, MPIArray{LocalIndices, 1}, MPIArray{PartitionedArrays.VectorAssemblyCache{Float64}, 1}, Float64}, GridapDistributed.BlockPRange{MPIArray{LocalIndices, 1}}}}})
│        @ Gridap.FESpaces ~/.julia/packages/Gridap/5nlYf/src/FESpaces/AffineFEOperators.jl:51
│     [29] main(distribute::PartitionedArrays.var"#88#89"{MPI.Comm, Bool}, np::Tuple{Int64, Int64}, nc::Tuple{Int64, Int64})
│        @ Main ~/julia/GridapIce/src/Precondition/stokesapplication.jl:71
│     [30] (::var"#13#14")(distribute::Function)
│        @ Main ~/julia/GridapIce/src/Precondition/stokesapplication.jl:101
│     [31] with_mpi(f::var"#13#14"; comm::MPI.Comm, duplicate_comm::Bool)
│        @ PartitionedArrays ~/.julia/packages/PartitionedArrays/py6uo/src/mpi_array.jl:73
│     [32] with_mpi(f::Function)
│        @ PartitionedArrays ~/.julia/packages/PartitionedArrays/py6uo/src/mpi_array.jl:64
│     [33] top-level scope
│        @ ~/julia/GridapIce/src/Precondition/stokesapplication.jl:100
│     [34] include(mod::Module, _path::String)
│        @ Base ./Base.jl:457
│     [35] exec_options(opts::Base.JLOptions)
│        @ Base ./client.jl:307
│     [36] _start()
│        @ Base ./client.jl:522
└ @ PartitionedArrays ~/.julia/packages/PartitionedArrays/py6uo/src/mpi_array.jl:75
application called MPI_Abort(MPI_COMM_WORLD, 1) - process 1
ERROR: failed process: Process(`/home/daniel/.julia/artifacts/b7a943fb6a811908b073b8af69d955f16703ca2b/bin/mpiexec -n 2 julia ./src/Precondition/stokesapplication.jl`, ProcessExited(1)) [1]

Stacktrace:
  [1] pipeline_error
    @ ./process.jl:565 [inlined]
  [2] run(::Cmd; wait::Bool)
    @ Base ./process.jl:480
  [3] run(::Cmd)
    @ Base process.jl:477
  [4] (::var"#3#4")(exe::String)
    @ Main none:4
  [5] (::JLLWrappers.var"#2#3"{var"#3#4", String})()
    @ JLLWrappers ~/.julia/packages/JLLWrappers/pG9bm/src/runtime.jl:49
  [6] withenv(::JLLWrappers.var"#2#3"{var"#3#4", String}, ::Pair{String, String}, ::Vararg{Pair{String, String}})
    @ Base env.jl:197
  [7] withenv_executable_wrapper(f::Function, executable_path::String, PATH::String, LIBPATH::String, adjust_PATH::Bool, adjust_LIBPATH::Bool)
    @ JLLWrappers ~/.julia/packages/JLLWrappers/pG9bm/src/runtime.jl:48
  [8] invokelatest(::Any, ::Any, ::Vararg{Any}; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Base ./essentials.jl:816
  [9] invokelatest(::Any, ::Any, ::Vararg{Any})
    @ Base ./essentials.jl:813
 [10] mpiexec(f::Function; adjust_PATH::Bool, adjust_LIBPATH::Bool)
    @ MPICH_jll ~/.julia/packages/JLLWrappers/pG9bm/src/products/executable_generators.jl:22
 [11] mpiexec(f::Function)
    @ MPICH_jll ~/.julia/packages/JLLWrappers/pG9bm/src/products/executable_generators.jl:19
 [12] top-level scope
    @ none:4

[JordiManyer]

Hi @dhrichards , thank you for reporting this! I believe this is related to a similar issue I've been tracking. I've openend a PR in GridapDistributed that should fix it.
It will hopefully get merged and published in a couple days, but you can already give it a try by pointing to the corresponding branch, i.e

(GridapSolvers) pkg> add GridapDistributed#bugfix-block-assembly

If you do, let me know if this fixes the issue.

[dhrichards]

Hi @JordiManyer , yes that seems to fix the issue. Thanks!

[JordiManyer]

Ok great! I'll close this issue and let you know whenever we merge.